Subscribe to release notesStatus pageCommunity
Developer guides

Upload image predictions

How to upload predictions on image data in a model run and sample upload formats.

Overview

To upload predictions in Labelbox, you need to create a prediction payload. In this section, we provide this payload for every supported annotation type.

Predictions Payload Types

Labelbox supports two formats for the annotations payload:

  • Python annotation types (recommended)
    • Provides a seamless transition between third-party platforms, machine learning pipelines, and Labelbox.
    • Allows you to build annotations locally with local file paths, numpy arrays, or URLs
    • Easily convert Python Annotation Type format to NDJSON format to quickly import annotations to Labelbox
    • Supports one-level nested classification (radio, checklist, or free-form text) under a tool or classification annotation.
  • JSON
    • Skips formatting annotation payload in the Labelbox Python annotation type
    • Supports any levels of nested classification (radio, checklist, or free-form text) under a tool or classification annotation.

Confidence Score

You can include confidence scores and custom metrics when you upload your model predictions to a model run. However, given the predictions and annotations in a model run, Labelbox will automatically calculate some auto-generated metrics upon upload.

📘

Uploading confidence scores is optional

If you do not specify a confidence score, the prediction will be treated as if it had a confidence score of 1.

Supported Prediction

The following predictions are supported for an image data row:

  • Radio
  • Checklist
  • Free-form text
  • Bounding box
  • Point
  • Polyline
  • Polygon
  • Segmentation masks

Classification

Radio (single-choice)

radio_prediction = lb_types.ClassificationAnnotation(
    name="radio_question", 
    value=lb_types.Radio(answer = lb_types.ClassificationAnswer(name = "second_radio_answer", confidence=0.5))
)

radio_prediction_ndjson = {
  "name": "radio_question",
  "answer": {"name": "second_radio_answer", "confidence": 0.5}
}

Classification: Nested radio

nested_radio_prediction = lb_types.ClassificationAnnotation(
  name="nested_radio_question",
  value=lb_types.Radio(
    answer=lb_types.ClassificationAnswer(
      name="first_radio_answer",
      confidence=0.5,
      classifications=[
        lb_types.ClassificationAnnotation(
          name="sub_radio_question",
          value=lb_types.Radio(
            answer=lb_types.ClassificationAnswer(
              name="first_sub_radio_answer",
              confidence=0.5
            )
          )
        )
      ]
    )
  )
)


nested_radio_prediction_ndjson = {
    "name": "nested_radio_question",
    "answer": {
        "name": "first_radio_answer",
        "confidence": 0.5,
        "classifications": [{
            "name": "sub_radio_question",
            "answer": {
                "name": "first_sub_radio_answer",
                "confidence": 0.5
            }
        }]
    }
}

Classification: Nested checklist


nested_checklist_prediction = lb_types.ClassificationAnnotation(
  name="nested_checklist_question",
  value=lb_types.Checklist(
    answer=[lb_types.ClassificationAnswer(
      name="first_checklist_answer",
      confidence=0.5,
      classifications=[
        lb_types.ClassificationAnnotation(
          name="sub_checklist_question",
          value=lb_types.Checklist(
            answer=[lb_types.ClassificationAnswer(
            name="first_sub_checklist_answer",
            confidence=0.5
          )]
        ))
      ]
    )]
  )
)

nested_checklist_prediction_ndjson = {
    "name": "nested_checklist_question",
    "answer": [{
        "name": "first_checklist_answer",
        "confidence": 0.5,
        "classifications": [{
            "name": "sub_checklist_question",
            "answer": {
                "name": "first_sub_checklist_answer",
                "confidence": 0.5
            }
        }]
    }]
}

Checklist (multiple choice)

checklist_prediction = lb_types.ClassificationAnnotation(
  name="checklist_question", 
  value=lb_types.Checklist(
      answer = [
        lb_types.ClassificationAnswer(
            name = "first_checklist_answer", 
            confidence=0.5
        ), 
        lb_types.ClassificationAnswer(
            name = "second_checklist_answer", 
            confidence=0.5
        )
      ]
    )
 )

checklist_prediction_ndjson = {
  "name": "checklist_question",
  "answer": [
    {"name": "first_checklist_answer" , "confidence": 0.5},
    {"name": "second_checklist_answer", "confidence": 0.5}
  ]
}

Bounding box

bbox_prediction = lb_types.ObjectAnnotation(
  name = "bounding_box",  
  confidence=0.5, 
  value=lb_types.Rectangle(
    start=lb_types.Point(x=1690, y=977),  #  x = left, y = top 
    end=lb_types.Point(x=1915, y=1307),  # x= left + width , y = top + height
  )
)

bbox_prediction_ndjson = {
  "name": "bounding_box", 
  "confidence": 0.5,
  "bbox": {
          "top": 977,
          "left": 1690,
          "height": 330,
          "width": 225
      }
}

Bounding box with nested classification

bbox_with_radio_subclass_prediction = lb_types.ObjectAnnotation(
    name="bbox_with_radio_subclass",
    confidence=0.5, 
    value=lb_types.Rectangle(
        start=lb_types.Point(x=541, y=933),  #  x = left, y = top 
        end=lb_types.Point(x=871, y=1124),  # x= left + width , y = top + height
    ),
    classifications=[
    	lb_types.ClassificationAnnotation(
        	name="sub_radio_question",
      		value=lb_types.Radio(answer=lb_types.ClassificationAnswer(name="first_sub_radio_answer", confidence=0.5))
    )
  ]
)


bbox_with_radio_subclass_prediction_ndjson = {
    "name": "bbox_with_radio_subclass",
  	"confidence": 0.5
    "classifications": [{
        "name": "sub_radio_question",
        "answer": 
            { "name":"first_sub_radio_answer", "confidence": 0.5}
         
    }],
    "bbox": {
          "top": 933,
          "left": 541,
          "height": 191,
          "width": 330
        }
}

Polygon

polygon_prediction = lb_types.ObjectAnnotation(
  name = "polygon",  
  confidence = 0.5, 
  value=lb_types.Polygon( 
        points=[lb_types.Point(x=1489.581,y=183.934), lb_types.Point(x=2278.306,y=256.885), lb_types.Point(x=2428.197,y=200.437), lb_types.Point(x=2560.0,y=335.419),
                lb_types.Point(x=2557.386,y=503.165), lb_types.Point(x=2320.596,y=503.103), lb_types.Point(x=2156.083, y=628.943), lb_types.Point(x=2161.111,y=785.519),
                lb_types.Point(x=2002.115, y=894.647), lb_types.Point(x=1838.456,y=877.874), lb_types.Point(x=1436.53,y=874.636), lb_types.Point(x=1411.403,y=758.579),
                lb_types.Point(x=1353.853,y=751.74), lb_types.Point(x=1345.264, y=453.461), lb_types.Point(x=1426.011,y=421.129)]
    ),
)

polygon_prediction_ndjson = {
  "name": "polygon",
  "confidence": 0.5,
  "polygon": [
    {"x": 1489.581, "y": 183.934},
    {"x": 2278.306, "y": 256.885},
    {"x": 2428.197, "y": 200.437},
    {"x": 2560.0, "y": 335.419},
    {"x": 2557.386, "y": 503.165},
    {"x": 2320.596, "y": 503.103},
    {"x": 2156.083, "y": 628.943},
    {"x": 2161.111, "y": 785.519},
    {"x": 2002.115, "y": 894.647},
    {"x": 1838.456, "y": 877.874},
    {"x": 1436.53, "y": 874.636},
    {"x": 1411.403, "y": 758.579},
    {"x": 1353.853, "y": 751.74},
    {"x": 1345.264, "y": 453.461},
    {"x": 1426.011, "y": 421.129},
    {"x": 1489.581, "y": 183.934}
  ]
}

Classification: free-form text


text_annotation = lb_types.ClassificationAnnotation(
  name="free_text", 
  value=lb_types.Text(answer="sample text", confidence=0.5)
)

text_annotation_ndjson = {
  "name": "free_text",
  "confidence": 0.5,
  "answer": "sample text",
}

Segmentation mask

# We recommend to use byte array
url = "https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg.png"
response = requests.get(url)
mask_data = lb.types.MaskData(im_bytes=response.content) # You can also use "url" instead of img_bytes to pass the PNG mask url.

mask_prediction = lb_types.ObjectAnnotation(
    name="mask",
    confidence=0.5,
    value=lb_types.Mask(
        mask=mask_data,
        color=(255, 255, 255))
)



# using instanceURI, bytes array is not fully supported.
mask_prediction_ndjson = {
  "name": "mask",
  "confidence": 0.5,
  "classifications": [],
  "mask": {"instanceURI": url,
  "colorRGB": (255, 255, 255)}
}

Segmentation mask with nested classification

url_2 = "https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg_with_subclass.png"
response_2 = requests.get(url_2)
mask_data_2 = lb_types.MaskData(im_bytes=response_2.content)

# Python annotation
mask_with_text_subclass_prediction = lb_types.ObjectAnnotation(
  name = "mask_with_text_subclass", # must match your ontology feature"s name
  value=lb_types.Mask(
    mask=mask_data_2,
    confidence=0.5,
    color=(255, 255, 255)),
  classifications=[
    lb_types.ClassificationAnnotation(
      name="sub_free_text",
      value=lb_types.Text(answer="free text answer")
    )]
)

mask_with_text_subclass_prediction_ndjson = {
  "name": "mask_with_text_subclass",
  "mask": {"instanceURI": url_2,
  "colorRGB": (255, 255, 255)},
  "classifications":[{
    "name": "sub_free_text",
    "answer": "free text answer"
  }]
}

Point

point_prediction = lb_types.ObjectAnnotation(
  name = "point",  
  confidence=0.5,
  value = lb_types.Point(x=1166.606, y=1441.768),
)

point_prediction_ndjson = {
  "name": "point",
  "confidence": 0.5,
  "classifications": [],
  "point": {"x": 1166.606, "y": 1441.768}
}

Polyline

polyline_prediction = lb_types.ObjectAnnotation(
  name = "polyline", 
  confidence=0.5, 
  value=lb_types.Line( 
        points=[lb_types.Point(x=2534.353, y=249.471), lb_types.Point(x=2429.492, y=182.092), lb_types.Point(x=2294.322, y=221.962), lb_types.Point(x=2224.491, y=180.463), lb_types.Point(x=2136.123, y=204.716),
                lb_types.Point(x=1712.247, y=173.949), lb_types.Point(x=1703.838, y=84.438), lb_types.Point(x=1579.772, y=82.61), lb_types.Point(x=1583.442, y=167.552),
                lb_types.Point(x=1478.869, y=164.903), lb_types.Point(x=1418.941, y=318.149), lb_types.Point(x=1243.128, y=400.815), lb_types.Point(x=1022.067, y=319.007),
                lb_types.Point(x=892.367, y=379.216), lb_types.Point(x=670.273, y=364.408), lb_types.Point(x=613.114, y=288.16), lb_types.Point(x=377.559, y=238.251),
                lb_types.Point(x=368.087, y=185.064), lb_types.Point(x=246.557, y=167.286), lb_types.Point(x=236.648, y=285.61), lb_types.Point(x=90.929, y=326.412)]
    ),
)


polyline_prediction_ndjson = {
  "name": "polyline",
  "confidence":0.5,
  "classifications": [],
  "line": [
    {"x": 2534.353, "y": 249.471},
    {"x": 2429.492, "y": 182.092},
    {"x": 2294.322, "y": 221.962},
    {"x": 2224.491, "y": 180.463},
    {"x": 2136.123, "y": 204.716},
    {"x": 1712.247, "y": 173.949},
    {"x": 1703.838, "y": 84.438},
    {"x": 1579.772, "y": 82.61},
    {"x": 1583.442, "y": 167.552},
    {"x": 1478.869, "y": 164.903},
    {"x": 1418.941, "y": 318.149},
    {"x": 1243.128, "y": 400.815},
    {"x": 1022.067, "y": 319.007},
    {"x": 892.367, "y": 379.216},
    {"x": 670.273, "y": 364.408},
    {"x": 613.114, "y": 288.16},
    {"x": 377.559, "y": 238.251},
    {"x": 368.087, "y": 185.064},
    {"x": 246.557, "y": 167.286},
    {"x": 236.648, "y": 285.61},
    {"x": 90.929, "y": 326.412}
  ]
}

Example: Upload predictions to model run

To upload predictions to a model run:

Before you start

These examples require the following libraries:

import labelbox as lb
import labelbox.types as lb_types
import uuid
import requests

Replace the value of API_KEY with a valid API key to connect to the Labelbox client.

API_KEY = None
client = lb.Client(API_KEY)

Step 1: Import data rows into Catalog

# send a sample image as batch to the project
global_key = "2560px-Kitano_Street_Kobe01s5s4110.jpeg"
test_img_url = {
    "row_data": "https://storage.googleapis.com/labelbox-datasets/image_sample_data/2560px-Kitano_Street_Kobe01s5s4110.jpeg",
    "global_key": global_key
}
dataset = client.create_dataset(name="image_prediction_demo")
task = dataset.create_data_rows([test_img_url])
print("Errors:",task.errors)
print("Failed data rows:", task.failed_data_rows)

Step 2: Set up ontology for predictions

Your model run ontology should support all tools and classifications used in your predictions.

This example shows how to create an ontology with all supported prediction types .

ontology_builder = lb.OntologyBuilder(
  classifications=[ # List of Classification objects
    lb.Classification( 
      class_type=lb.Classification.Type.RADIO,
      name="radio_question", 
      options=[
        lb.Option(value="first_radio_answer"),
        lb.Option(value="second_radio_answer")
      ]
    ),
    lb.Classification( 
      class_type=lb.Classification.Type.CHECKLIST,
      name="checklist_question", 
      options=[
        lb.Option(value="first_checklist_answer"),
        lb.Option(value="second_checklist_answer")
      ]
    ), 
    lb.Classification( 
      class_type=lb.Classification.Type.TEXT,
      name="free_text"
    ),
    lb.Classification(
        class_type=lb.Classification.Type.RADIO,
        name="nested_radio_question",
        options=[
            lb.Option("first_radio_answer",
                options=[
                    lb.Classification(
                        class_type=lb.Classification.Type.RADIO,
                        name="sub_radio_question",
                        options=[lb.Option("first_sub_radio_answer")]
                    )
                ]
            )
          ] 
        ),
    lb.Classification(
      class_type=lb.Classification.Type.CHECKLIST,
      name="nested_checklist_question",
      options=[
          lb.Option("first_checklist_answer",
            options=[
              lb.Classification(
                  class_type=lb.Classification.Type.CHECKLIST,
                  name="sub_checklist_question", 
                  options=[lb.Option("first_sub_checklist_answer")]
              )
          ]
        )
      ]
    ),      
  ],
  tools=[ # List of tools
    lb.Tool(
      tool=lb.Tool.Type.BBOX,
      name="bounding_box"), 
    lb.Tool( 
      tool=lb.Tool.Type.BBOX,
      name="bbox_with_radio_subclass",
      classifications=[
            lb.Classification(
                class_type=lb.Classification.Type.RADIO,
                name="sub_radio_question",
                options=[
                  lb.Option(value="first_sub_radio_answer")
                ]
              ),
        ]
      ), 
    lb.Tool( 
      tool=lb.Tool.Type.POLYGON,
      name="polygon"),
    lb.Tool( 
      tool=lb.Tool.Type.RASTER_SEGMENTATION,
      name="mask"),
    lb.Tool(
      tool=lb.Tool.Type.RASTER_SEGMENTATION,
      name="mask_with_text_subclass",
      classifications=[
          lb.Classification(
              class_type=lb.Classification.Type.TEXT,
              name="sub_free_text")
          ]
      ),
 	  lb.Tool( 
      tool=lb.Tool.Type.POINT,
      name="point"), 
    lb.Tool( 
      tool=lb.Tool.Type.LINE,
      name="polyline")]
)

ontology = client.create_ontology("Image Prediction Import Demo", 
                                  ontology_builder.asdict(), 
                                  media_type=lb.MediaType.Image
                                  )

Step 3: Create model and model run

Create a Model using the ontology and a model run.

# create Model
model = client.create_model(name="image_model_run_" + str(uuid.uuid4()),
                            ontology_id=ontology.uid)
# create Model Run
model_run = model.create_model_run("iteration 1")

Step 4: Send data rows to model run

model_run.upsert_data_rows(global_keys=[global_key])

Step 5: Create prediction payloads

For help creating prediction payloads, see supported predictions. You can declare payloads as Python annotation types (preferred) or as NDJSON objects. This example demonstrates each format and shows how to compose annotations into labels attached to the data rows.

The resulting label_prediction_ndjson and label_prediction payloads should have exactly the same prediction content (except for the value of generated uuid string values).

label_prediction = []
label_prediction.append(lb_types.Label(
    data={"global_key" : global_key },
    annotations = [
      radio_prediction,
      nested_radio_prediction,
      checklist_prediction, 
      nested_checklist_prediction,
      bbox_prediction, 
      bbox_with_radio_subclass_prediction, 
      polyline_prediction,
      polygon_prediction, 
      mask_prediction, 
      mask_with_text_subclass_prediction,
      point_prediction,
      text_annotation
      ]
  )
)

label_prediction_ndjson = []

for annot in [
    radio_prediction_ndjson,
    checklist_prediction_ndjson, 
    bbox_prediction_ndjson, 
    bbox_with_radio_subclass_prediction_ndjson, 
    polygon_prediction_ndjson, 
    mask_prediction_ndjson, 
    mask_with_text_subclass_prediction_ndjson,
    point_prediction_ndjson,
    polyline_prediction_ndjson,
    text_annotation_ndjson, 
    nested_radio_prediction_ndjson,
    nested_checklist_prediction_ndjson
]:
  annot.update({
      "dataRow": {"globalKey": global_key}
  })
  label_prediction_ndjson.append(annot)

Step 6: Upload prediction payloads to model run

# Upload the prediction label to the Model Run
upload_job_prediction = model_run.add_predictions(
    name="prediction_upload_job"+str(uuid.uuid4()),
    predictions=label_prediction)

# Errors will appear for prediction uploads that failed.
print("Errors:",  upload_job_prediction.errors)
print("Status of uploads: ", upload_job_prediction.statuses)

Step 7: Send annotations to a model run

This step is optional.

This example creates a project with ground truth annotations to visualize both annotations and predictions in the model run.

To send annotations to a model run:

  1. Import them into a project
  2. Create a label payload
  3. Send them to the model run.
# 7.1 Create a labelbox project 
project = client.create_project(name="Image Prediction Demo",
                                    media_type=lb.MediaType.Image)
project.connect_ontology(ontology)


# 7.2 Create a batch 
project.create_batch(
  "batch_predictions_demo", # Each batch in a project must have a unique name
  global_keys=[global_key], # Paginated collection of data row objects, list of data row ids or global keys
  priority=5 # priority between 1(Highest) - 5(lowest)
)


# 7.3 Create your annotation payload as explained in:
# https://docs.labelbox.com/reference/import-image-annotations#supported-annotations
radio_annotation ...
nested_radio_annotation ... 
nested_checklist_annotation ...
checklist_annotation  ... 
bbox_annotation ...
bbox_with_radio_subclass_annotation ...
polygon_annotation ... 
text_annotation ...
mask_annotation  ...
mask_with_text_subclass_annotation ...
point_annotation ...
polyline_annotation ...


# 7.4  Create a Label object by identifying the applicable data row in Labelbox and providing a list of annotations
labels = []
annotations = [
    radio_annotation,
    nested_radio_annotation,
    checklist_annotation,
    nested_checklist_annotation,
    text_annotation,
    bbox_annotation,
    bbox_with_radio_subclass_annotation,
    polygon_annotation,
    mask_annotation,
    mask_with_text_subclass_annotation,
    point_annotation,
    polyline_annotation
]
labels.append(
    lb_types.Label(
        data={"global_key" : global_key },
        annotations=annotations))


# 7.5 Upload annotations to the project using Label import 
upload_job_annotation = lb.LabelImport.create_from_objects(
    client = client,
    project_id = project.uid,
    name="annotation_import_" + str(uuid.uuid4()),
    labels=labels)

upload_job_annotation.wait_until_done()
# Errors will appear for annotation uploads that failed.
print("Errors:", upload_job_annotation.errors)
print("Status of uploads: ", upload_job_annotation.statuses)


# 7.6 Send the annotations to the model run 

model_run.upsert_labels(project_id=project.uid)