Subscribe to release notesStatus pageCommunity

Upload image predictions

How to upload predictions on image data in a model run and sample upload formats.

Open this Colab for an interactive tutorial about uploading predictions and annotations on images in a model run.

Supported predictions

To upload predictions in Labelbox, you need to create a predictions payload. In this section, we provide this payload for every supported prediction type.

Labelbox supports two formats for the predictions payload:

  • Python Annotation Types (recommended)
  • NDJSON

Both are described below.

📘

Uploading confidence scores is optional

If you do not specify a confidence score, the prediction will be treated as if it had a confidence score of 1.

Classification: Radio (single-choice)

radio_prediction = lb_types.ClassificationAnnotation(
    name="radio_question", 
    value=lb_types.Radio(answer = lb_types.ClassificationAnswer(name = "second_radio_answer", confidence=0.5))
)

radio_prediction_ndjson = {
  "name": "radio_question",
  "answer": {"name": "second_radio_answer", "confidence": 0.5}
}

Classification: Nested radio

nested_radio_prediction = lb_types.ClassificationAnnotation(
  name="nested_radio_question",
  value=lb_types.Radio(
    answer=lb_types.ClassificationAnswer(
      name="first_radio_answer",
      confidence=0.5,
      classifications=[
        lb_types.ClassificationAnnotation(
          name="sub_radio_question",
          value=lb_types.Radio(
            answer=lb_types.ClassificationAnswer(
              name="first_sub_radio_answer",
              confidence=0.5
            )
          )
        )
      ]
    )
  )
)


nested_radio_prediction_ndjson = {
    "name": "nested_radio_question",
    "answer": {
        "name": "first_radio_answer",
        "confidence": 0.5,
        "classifications": [{
            "name": "sub_radio_question",
            "answer": {
                "name": "first_sub_radio_answer",
                "confidence": 0.5
            }
        }]
    }
}

Classification: Nested checklist


nested_checklist_prediction = lb_types.ClassificationAnnotation(
  name="nested_checklist_question",
  value=lb_types.Checklist(
    answer=[lb_types.ClassificationAnswer(
      name="first_checklist_answer",
      confidence=0.5,
      classifications=[
        lb_types.ClassificationAnnotation(
          name="sub_checklist_question",
          value=lb_types.Checklist(
            answer=[lb_types.ClassificationAnswer(
            name="first_sub_checklist_answer",
            confidence=0.5
          )]
        ))
      ]
    )]
  )
)

nested_checklist_prediction_ndjson = {
    "name": "nested_checklist_question",
    "answer": [{
        "name": "first_checklist_answer",
        "confidence": 0.5,
        "classifications": [{
            "name": "sub_checklist_question",
            "answer": {
                "name": "first_sub_checklist_answer",
                "confidence": 0.5
            }
        }]
    }]
}

Classification: Checklist (multi-choice)

checklist_prediction = lb_types.ClassificationAnnotation(
  name="checklist_question", 
  value=lb_types.Checklist(
      answer = [
        lb_types.ClassificationAnswer(
            name = "first_checklist_answer", 
            confidence=0.5
        ), 
        lb_types.ClassificationAnswer(
            name = "second_checklist_answer", 
            confidence=0.5
        )
      ]
    )
 )

checklist_prediction_ndjson = {
  "name": "checklist_question",
  "answer": [
    {"name": "first_checklist_answer" , "confidence": 0.5},
    {"name": "second_checklist_answer", "confidence": 0.5}
  ]
}

Bounding box

bbox_prediction = lb_types.ObjectAnnotation(
  name = "bounding_box",  
  confidence=0.5, 
  value=lb_types.Rectangle(
    start=lb_types.Point(x=1690, y=977),  #  x = left, y = top 
    end=lb_types.Point(x=1915, y=1307),  # x= left + width , y = top + height
  )
)

bbox_prediction_ndjson = {
  "name": "bounding_box", 
  "confidence": 0.5,
  "bbox": {
          "top": 977,
          "left": 1690,
          "height": 330,
          "width": 225
      }
}

Bounding box with nested classification

bbox_with_radio_subclass_prediction = lb_types.ObjectAnnotation(
    name="bbox_with_radio_subclass",
    confidence=0.5, 
    value=lb_types.Rectangle(
        start=lb_types.Point(x=541, y=933),  #  x = left, y = top 
        end=lb_types.Point(x=871, y=1124),  # x= left + width , y = top + height
    ),
    classifications=[
    	lb_types.ClassificationAnnotation(
        	name="sub_radio_question",
      		value=lb_types.Radio(answer=lb_types.ClassificationAnswer(name="first_sub_radio_answer", confidence=0.5))
    )
  ]
)


bbox_with_radio_subclass_prediction_ndjson = {
    "name": "bbox_with_radio_subclass",
  	"confidence": 0.5
    "classifications": [{
        "name": "sub_radio_question",
        "answer": 
            { "name":"first_sub_radio_answer", "confidence": 0.5}
         
    }],
    "bbox": {
          "top": 933,
          "left": 541,
          "height": 191,
          "width": 330
        }
}

Polygon

polygon_prediction = lb_types.ObjectAnnotation(
  name = "polygon",  
  confidence = 0.5, 
  value=lb_types.Polygon( 
        points=[lb_types.Point(x=1489.581,y=183.934), lb_types.Point(x=2278.306,y=256.885), lb_types.Point(x=2428.197,y=200.437), lb_types.Point(x=2560.0,y=335.419),
                lb_types.Point(x=2557.386,y=503.165), lb_types.Point(x=2320.596,y=503.103), lb_types.Point(x=2156.083, y=628.943), lb_types.Point(x=2161.111,y=785.519),
                lb_types.Point(x=2002.115, y=894.647), lb_types.Point(x=1838.456,y=877.874), lb_types.Point(x=1436.53,y=874.636), lb_types.Point(x=1411.403,y=758.579),
                lb_types.Point(x=1353.853,y=751.74), lb_types.Point(x=1345.264, y=453.461), lb_types.Point(x=1426.011,y=421.129)]
    ),
)

polygon_prediction_ndjson = {
  "name": "polygon",
  "confidence": 0.5,
  "polygon": [
    {"x": 1489.581, "y": 183.934},
    {"x": 2278.306, "y": 256.885},
    {"x": 2428.197, "y": 200.437},
    {"x": 2560.0, "y": 335.419},
    {"x": 2557.386, "y": 503.165},
    {"x": 2320.596, "y": 503.103},
    {"x": 2156.083, "y": 628.943},
    {"x": 2161.111, "y": 785.519},
    {"x": 2002.115, "y": 894.647},
    {"x": 1838.456, "y": 877.874},
    {"x": 1436.53, "y": 874.636},
    {"x": 1411.403, "y": 758.579},
    {"x": 1353.853, "y": 751.74},
    {"x": 1345.264, "y": 453.461},
    {"x": 1426.011, "y": 421.129},
    {"x": 1489.581, "y": 183.934}
  ]
}

Classification: Free-form text


text_annotation = lb_types.ClassificationAnnotation(
  name="free_text", 
  value=lb_types.Text(answer="sample text", confidence=0.5)
)

text_annotation_ndjson = {
  "name": "free_text",
  "confidence": 0.5,
  "answer": "sample text",
}

Segmentation mask

# We recommend to use byte array
url = "https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg.png"
response = requests.get(url)
mask_data = lb.types.MaskData(im_bytes=response.content) # You can also use "url" instead of img_bytes to pass the PNG mask url.

mask_prediction = lb_types.ObjectAnnotation(
    name="mask",
    confidence=0.5,
    value=lb_types.Mask(
        mask=mask_data,
        color=(255, 255, 255))
)



# using instanceURI, bytes array is not fully supported.
mask_prediction_ndjson = {
  "name": "mask",
  "confidence": 0.5,
  "classifications": [],
  "mask": {"instanceURI": url,
  "colorRGB": (255, 255, 255)}
}

Segmentation mask with nested classification

url_2 = "https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg_with_subclass.png"
response_2 = requests.get(url_2)
mask_data_2 = lb_types.MaskData(im_bytes=response_2.content)

# Python annotation
mask_with_text_subclass_prediction = lb_types.ObjectAnnotation(
  name = "mask_with_text_subclass", # must match your ontology feature"s name
  value=lb_types.Mask(
    mask=mask_data_2,
    confidence=0.5,
    color=(255, 255, 255)),
  classifications=[
    lb_types.ClassificationAnnotation(
      name="sub_free_text",
      value=lb_types.Text(answer="free text answer")
    )]
)

mask_with_text_subclass_prediction_ndjson = {
  "name": "mask_with_text_subclass",
  "mask": {"instanceURI": url_2,
  "colorRGB": (255, 255, 255)},
  "classifications":[{
    "name": "sub_free_text",
    "answer": "free text answer"
  }]
}

Point

point_prediction = lb_types.ObjectAnnotation(
  name = "point",  
  confidence=0.5,
  value = lb_types.Point(x=1166.606, y=1441.768),
)

point_prediction_ndjson = {
  "name": "point",
  "confidence": 0.5,
  "classifications": [],
  "point": {"x": 1166.606, "y": 1441.768}
}

Polyline

polyline_prediction = lb_types.ObjectAnnotation(
  name = "polyline", 
  confidence=0.5, 
  value=lb_types.Line( 
        points=[lb_types.Point(x=2534.353, y=249.471), lb_types.Point(x=2429.492, y=182.092), lb_types.Point(x=2294.322, y=221.962), lb_types.Point(x=2224.491, y=180.463), lb_types.Point(x=2136.123, y=204.716),
                lb_types.Point(x=1712.247, y=173.949), lb_types.Point(x=1703.838, y=84.438), lb_types.Point(x=1579.772, y=82.61), lb_types.Point(x=1583.442, y=167.552),
                lb_types.Point(x=1478.869, y=164.903), lb_types.Point(x=1418.941, y=318.149), lb_types.Point(x=1243.128, y=400.815), lb_types.Point(x=1022.067, y=319.007),
                lb_types.Point(x=892.367, y=379.216), lb_types.Point(x=670.273, y=364.408), lb_types.Point(x=613.114, y=288.16), lb_types.Point(x=377.559, y=238.251),
                lb_types.Point(x=368.087, y=185.064), lb_types.Point(x=246.557, y=167.286), lb_types.Point(x=236.648, y=285.61), lb_types.Point(x=90.929, y=326.412)]
    ),
)


polyline_prediction_ndjson = {
  "name": "polyline",
  "confidence":0.5,
  "classifications": [],
  "line": [
    {"x": 2534.353, "y": 249.471},
    {"x": 2429.492, "y": 182.092},
    {"x": 2294.322, "y": 221.962},
    {"x": 2224.491, "y": 180.463},
    {"x": 2136.123, "y": 204.716},
    {"x": 1712.247, "y": 173.949},
    {"x": 1703.838, "y": 84.438},
    {"x": 1579.772, "y": 82.61},
    {"x": 1583.442, "y": 167.552},
    {"x": 1478.869, "y": 164.903},
    {"x": 1418.941, "y": 318.149},
    {"x": 1243.128, "y": 400.815},
    {"x": 1022.067, "y": 319.007},
    {"x": 892.367, "y": 379.216},
    {"x": 670.273, "y": 364.408},
    {"x": 613.114, "y": 288.16},
    {"x": 377.559, "y": 238.251},
    {"x": 368.087, "y": 185.064},
    {"x": 246.557, "y": 167.286},
    {"x": 236.648, "y": 285.61},
    {"x": 90.929, "y": 326.412}
  ]
}

End-to-end example: Upload predictions to a model run

Here are the steps to upload predictions to a model run.

Before you start

You will need to import these libraries to use the code examples in this section:

import labelbox as lb
import labelbox.types as lb_types
import uuid
import requests

Replace with your API key

To learn how to create an API key, please follow the instructions on this page.

API_KEY = ""
client = lb.Client(API_KEY)

Step 1: Import data rows into Catalog

# send a sample image as batch to the project
global_key = "2560px-Kitano_Street_Kobe01s5s4110.jpeg"
test_img_url = {
    "row_data": "https://storage.googleapis.com/labelbox-datasets/image_sample_data/2560px-Kitano_Street_Kobe01s5s4110.jpeg",
    "global_key": global_key
}
dataset = client.create_dataset(name="image_prediction_demo")
task = dataset.create_data_rows([test_img_url])
print("Errors:",task.errors)
print("Failed data rows:", task.failed_data_rows)

Step 2: Create an ontology for your model predictions

Your model run should have the correct ontology set up with all the tools and classifications supported for your predictions.

Here is an example of creating an ontology programmatically for all the example predictions above:

ontology_builder = lb.OntologyBuilder(
  classifications=[ # List of Classification objects
    lb.Classification( 
      class_type=lb.Classification.Type.RADIO,
      name="radio_question", 
      options=[
        lb.Option(value="first_radio_answer"),
        lb.Option(value="second_radio_answer")
      ]
    ),
    lb.Classification( 
      class_type=lb.Classification.Type.CHECKLIST,
      name="checklist_question", 
      options=[
        lb.Option(value="first_checklist_answer"),
        lb.Option(value="second_checklist_answer")
      ]
    ), 
    lb.Classification( 
      class_type=lb.Classification.Type.TEXT,
      name="free_text"
    ),
    lb.Classification(
        class_type=lb.Classification.Type.RADIO,
        name="nested_radio_question",
        options=[
            lb.Option("first_radio_answer",
                options=[
                    lb.Classification(
                        class_type=lb.Classification.Type.RADIO,
                        name="sub_radio_question",
                        options=[lb.Option("first_sub_radio_answer")]
                    )
                ]
            )
          ] 
        ),
    lb.Classification(
      class_type=lb.Classification.Type.CHECKLIST,
      name="nested_checklist_question",
      options=[
          lb.Option("first_checklist_answer",
            options=[
              lb.Classification(
                  class_type=lb.Classification.Type.CHECKLIST,
                  name="sub_checklist_question", 
                  options=[lb.Option("first_sub_checklist_answer")]
              )
          ]
        )
      ]
    ),      
  ],
  tools=[ # List of tools
    lb.Tool(
      tool=lb.Tool.Type.BBOX,
      name="bounding_box"), 
    lb.Tool( 
      tool=lb.Tool.Type.BBOX,
      name="bbox_with_radio_subclass",
      classifications=[
            lb.Classification(
                class_type=lb.Classification.Type.RADIO,
                name="sub_radio_question",
                options=[
                  lb.Option(value="first_sub_radio_answer")
                ]
              ),
        ]
      ), 
    lb.Tool( 
      tool=lb.Tool.Type.POLYGON,
      name="polygon"),
    lb.Tool( 
      tool=lb.Tool.Type.RASTER_SEGMENTATION,
      name="mask"),
    lb.Tool(
      tool=lb.Tool.Type.RASTER_SEGMENTATION,
      name="mask_with_text_subclass",
      classifications=[
          lb.Classification(
              class_type=lb.Classification.Type.TEXT,
              name="sub_free_text")
          ]
      ),
 	  lb.Tool( 
      tool=lb.Tool.Type.POINT,
      name="point"), 
    lb.Tool( 
      tool=lb.Tool.Type.LINE,
      name="polyline")]
)

ontology = client.create_ontology("Image Prediction Import Demo", 
                                  ontology_builder.asdict(), 
                                  media_type=lb.MediaType.Image
                                  )

Step 3: Create a Model and a model run

Create a Model using the ontology and a model run.

# create Model
model = client.create_model(name="image_model_run_" + str(uuid.uuid4()),
                            ontology_id=ontology.uid)
# create Model Run
model_run = model.create_model_run("iteration 1")

Step 4: Send a set of data rows to the model run

model_run.upsert_data_rows(global_keys=[global_key])

Step 5: Create the predictions payload

Create the predictions payload using the snippets of code shown above.

Labelbox supports two formats for the annotations payload: NDJSON and Python annotation types. Both approaches are described below with instructions to compose annotations into Labels attached to the data rows.

The resulting label_prediction_ndjson and label_prediction payloads should have exactly the same prediction content (with the exception of the uuid strings that are generated).

label_prediction = []
label_prediction.append(lb_types.Label(
    data=lb_types.ImageData(global_key=global_key),
    annotations = [
      radio_prediction,
      nested_radio_prediction,
      checklist_prediction, 
      nested_checklist_prediction,
      bbox_prediction, 
      bbox_with_radio_subclass_prediction, 
      polyline_prediction,
      polygon_prediction, 
      mask_prediction, 
      mask_with_text_subclass_prediction,
      point_prediction,
      text_annotation
      ]
  )
)

label_prediction_ndjson = []

for annot in [
    radio_prediction_ndjson,
    checklist_prediction_ndjson, 
    bbox_prediction_ndjson, 
    bbox_with_radio_subclass_prediction_ndjson, 
    polygon_prediction_ndjson, 
    mask_prediction_ndjson, 
    mask_with_text_subclass_prediction_ndjson,
    point_prediction_ndjson,
    polyline_prediction_ndjson,
    text_annotation_ndjson, 
    nested_radio_prediction_ndjson,
    nested_checklist_prediction_ndjson
]:
  annot.update({
      "dataRow": {"globalKey": global_key}
  })
  label_prediction_ndjson.append(annot)

Step 6: Upload the predictions payload to the model run

# Upload the prediction label to the Model Run
upload_job_prediction = model_run.add_predictions(
    name="prediction_upload_job"+str(uuid.uuid4()),
    predictions=label_prediction)

# Errors will appear for prediction uploads that failed.
print("Errors:",  upload_job_prediction.errors)
print("Status of uploads: ", upload_job_prediction.statuses)

Step 7: Send annotations to a model run (optional)

We will create a project with ground truth annotations to visualize both annotations and predictions in the model run.

To send annotations to a model run, we must first import them into a project, create a label payload, and then send them to the model run.

# 7.1 Create a labelbox project 
project = client.create_project(name="Image Prediction Demo",
                                    media_type=lb.MediaType.Image)
project.setup_editor(ontology)


# 7.2 Create a batch 
project.create_batch(
  "batch_predictions_demo", # Each batch in a project must have a unique name
  global_keys=[global_key], # Paginated collection of data row objects, list of data row ids or global keys
  priority=5 # priority between 1(Highest) - 5(lowest)
)


# 7.3 Create your annotation payload as explained in:
# https://docs.labelbox.com/reference/import-image-annotations#supported-annotations
radio_annotation ...
nested_radio_annotation ... 
nested_checklist_annotation ...
checklist_annotation  ... 
bbox_annotation ...
bbox_with_radio_subclass_annotation ...
polygon_annotation ... 
text_annotation ...
mask_annotation  ...
mask_with_text_subclass_annotation ...
point_annotation ...
polyline_annotation ...


# 7.4  Create a Label object by identifying the applicable data row in Labelbox and providing a list of annotations
labels = []
annotations = [
    radio_annotation,
    nested_radio_annotation,
    checklist_annotation,
    nested_checklist_annotation,
    text_annotation,
    bbox_annotation,
    bbox_with_radio_subclass_annotation,
    polygon_annotation,
    mask_annotation,
    mask_with_text_subclass_annotation,
    point_annotation,
    polyline_annotation
]
labels.append(
    lb_types.Label(
        data=lb_types.ImageData(global_key=global_key),
        annotations=annotations))


# 7.5 Upload annotations to the project using Label import 
upload_job_annotation = lb.LabelImport.create_from_objects(
    client = client,
    project_id = project.uid,
    name="annotation_import_" + str(uuid.uuid4()),
    labels=labels)

upload_job_annotation.wait_until_done()
# Errors will appear for annotation uploads that failed.
print("Errors:", upload_job_annotation.errors)
print("Status of uploads: ", upload_job_annotation.statuses)


# 7.6 Send the annotations to the model run 

model_run.upsert_labels(project_id=project.uid)