Upload HTML predictions

Supported annotations

To import annotations in Labelbox, you need to create the annotations payload. This section shows how to declare the payloads for each supported annotation type.

Classification: Radio (single-choice)

radio_prediction = lb_types.ClassificationAnnotation(
  name="radio_question", 
  value=lb_types.Radio(answer = lb_types.ClassificationAnswer(name = "second_radio_answer", confidence=0.5))
)

radio_prediction_ndjson = {
  'name': 'radio_question',
  'answer': {'name': 'first_radio_answer'}
}

Classification: Nested radio

nested_radio_prediction = lb_types.ClassificationAnnotation(
  name="nested_radio_question",
  value=lb_types.Radio(
    answer=lb_types.ClassificationAnswer(
      name="first_radio_answer",
      confidence=0.5,# Confidence scores should be added to the answer 
      classifications=[
        lb_types.ClassificationAnnotation(
          name="sub_radio_question",
          value=lb_types.Radio(
            answer=lb_types.ClassificationAnswer(
              name="first_sub_radio_answer", confidence=0.5)
          )
        )
      ]
    )
  )
)

nested_radio_prediction_ndjson = {
  "name": "nested_radio_question",
  "answer": {
      "name": "first_radio_answer",
      "confidence": 0.5, # Confidence scores should be added to the answer 
      "classifications": [{
          "name":"sub_radio_question",
          "answer": { "name" : "first_sub_radio_answer", "confidence": 0.5 }
        }]
    }
}

Classification: Nested checklist

nested_checklist_prediction = lb_types.ClassificationAnnotation(
  name="nested_checklist_question",
  value=lb_types.Checklist(
    answer=[lb_types.ClassificationAnswer(
      name="first_checklist_answer",
      confidence=0.5, # Confidence scores should be added to the answer 
      classifications=[
        lb_types.ClassificationAnnotation(
          name="sub_checklist_question",
          value=lb_types.Checklist(
            answer=[lb_types.ClassificationAnswer(
            name="first_sub_checklist_answer",
            confidence=0.5
          )]
        ))
      ]
    )]
  )
)

nested_checklist_prediction_ndjson = {
  "name": "nested_checklist_question",
  "answer": [{
      "name": "first_checklist_answer", 
      "confidence": 0.5, # Confidence scores should be added to the answer 
      "classifications" : [
        {
          "name": "sub_checklist_question", 
          "answer": {"name": "first_sub_checklist_answer", "confidence": 0.5 }
        }          
      ]         
  }]
}

Classification: Checklist (multi-choice)

checklist_prediction = lb_types.ClassificationAnnotation(
    name="checklist_question",
    value=lb_types.Checklist(
        answer = [
            lb_types.ClassificationAnswer(
                name = "first_checklist_answer",
                confidence=0.5
            ),
            lb_types.ClassificationAnswer(
                name = "second_checklist_answer", 
                confidence=0.5
            ),
            lb_types.ClassificationAnswer(
                name = "third_checklist_answer", 
                confidence=0.5
            )
    ])
  )

checklist_prediction_ndjson = {
  'name': 'checklist_question',
  'answer': [
    {'name': 'first_checklist_answer', 'confidence': 0.5}
  ]
}

Classification: Free-form text

text_prediction = lb_types.ClassificationAnnotation(
    name = "free_text", 
    value = lb_types.Text(answer="sample text", confidence=0.5)
)

text_prediction_ndjson = {
  'name': 'free_text',
  'answer': 'sample text',
  'confidence':0.5
}

Example: Upload predictions to model run

To upload predictions to a model run:

Before you start

You will need to import these libraries to use the code examples in this section:

import labelbox as lb
import labelbox.types as lb_types
import uuid
import numpy as np

Replace the value of API_KEY with a valid API key to connect to the Labelbox client.

API_KEY = None
client = lb.Client(API_KEY)

Step 1: Import data rows into Catalog

global_key ="sample_html_2.html"

test_img_url = {
    "row_data": "https://storage.googleapis.com/labelbox-datasets/html_sample_data/sample_html_2.html",
    "global_key": global_key
}
dataset = client.create_dataset(
    name="html prediction demo dataset", 
    iam_integration=None # Removing this argument will default to the organziation's default iam integration
)
task = dataset.create_data_rows([test_img_url])
task.wait_till_done()
print("Errors:",task.errors)
print("Failed data rows:", task.failed_data_rows)

Step 2: Set up ontology

Your project should include an ontology that supports your annotations. To ensure feature schema matches, the tool names and classification names should match the name field in your annotations.

ontology_builder = lb.OntologyBuilder(
  classifications=[ # List of Classification objects
    lb.Classification( 
      class_type=lb.Classification.Type.RADIO, 
      name="radio_question",  # name matching the tool used in the annotation
      options=[lb.Option(value="first_radio_answer")]
    ),
    lb.Classification( 
      class_type=lb.Classification.Type.RADIO, 
      name="nested_radio_question", 
      options=[
        lb.Option(value="first_radio_answer",
          options=[
              lb.Classification(
                class_type=lb.Classification.Type.RADIO,
                name="sub_radio_question",
                options=[
                  lb.Option(value="first_sub_radio_answer")
                ]
            ),
          ]
        )
      ],
    ),
    lb.Classification( 
      class_type=lb.Classification.Type.CHECKLIST, 
      name="checklist_question", 
      options=[
        lb.Option(value="first_checklist_answer"),
        lb.Option(value="second_checklist_answer"), 
        lb.Option(value="third_checklist_answer")            
      ]
    ), 
     lb.Classification( 
      class_type=lb.Classification.Type.TEXT,
      name="free_text"
    ),
    lb.Classification(
      class_type=lb.Classification.Type.CHECKLIST, 
      name="nested_checklist_question",
      options=[
          lb.Option("first_checklist_answer",
            options=[
              lb.Classification(
                  class_type=lb.Classification.Type.CHECKLIST, 
                  name="sub_checklist_question", 
                  options=[lb.Option("first_sub_checklist_answer")]
              )
          ]
        )
      ]
    )
  ]
)

ontology = client.create_ontology("Ontology HTML Predictions", 
                                  ontology_builder.asdict(), 
                                  media_type=lb.MediaType.Html)

Step 3: Create model and model run

#create Model
model = client.create_model(name="HTML_model_run_" + str(uuid.uuid4()), 
                            ontology_id=ontology.uid)
#create Model Run
model_run = model.create_model_run("iteration 1")

Step 4: Send data rows to model run

model_run.upsert_data_rows(global_keys=[global_key])

Step 5: Create prediction payload

Use the examples in Supported annotations to create your annotation payloads; you can use declare them as Python dictionaries or NDJSON objects. Examples of each type are shown here; they also show how to compose annotations into labels attached to the data rows.

The resulting label_prediction and label_prediction_ndjson from each approach demonstrates each supported annotation type.


label_prediction = []
label_prediction.append(
  lb_types.Label(
    data={"global_key" : global_key },
    annotations = [
      radio_prediction, 
      checklist_prediction,
      text_prediction,
      nested_checklist_prediction,
      nested_radio_prediction
    ]
  )
)

label_prediction_ndjson = []
for annot in [
    radio_prediction_ndjson, 
    nested_radio_prediction_ndjson,
    checklist_prediction_ndjson,
    text_prediction_ndjson,
    nested_checklist_prediction_ndjson
]:
  annot.update({
      "dataRow": {"globalKey": global_key},
  })
  label_prediction_ndjson.append(annot)

Step 6: Upload predictions payload to model run

# Upload the prediction label to the Model Run
upload_job_prediction = model_run.add_predictions(
    name="prediction_upload_job"+str(uuid.uuid4()),
    predictions=label_prediction)

# Errors will appear for annotation uploads that failed.
print("Errors:", upload_job_prediction.errors)
print("Status of uploads: ", upload_job_prediction.statuses)

Step 7: Send annotations to model run

(Optional) To send annotations to a model run:

Import them into a project
Create a label payload
Send Send them to the model run

#7.1. Create a labelbox project
project = client.create_project(name="HTML prediction import demo",                                    
                                    queue_mode=lb.QueueMode.Batch,
                                    media_type=lb.MediaType.Html)
project.connect_ontology(ontology)

# 7.2. Create a batch to send to the project

project.create_batch(
  "batch_prediction_html", # Each batch in a project must have a unique name
  global_keys=[global_key], # Paginated collection of data row objects, list of data row ids or global keys
  priority=5 # priority between 1(Highest) - 5(lowest)
)

# 7.3 Create the annotations payload
radio_annotation...
nested_radio_annotation...
nested_checklist_annotation...
checklist_annotation...
text_annotation...

# 7.4 Create the label object 
label.append(
  lb_types.Label(
    data={"global_key" : global_key },
    annotations=[
      text_annotation,
      checklist_annotation,
      radio_annotation,
      nested_checklist_annotation,
      nested_radio_annotation
    ]
  )
)

#7.5. Upload annotations to the project using Label Import
upload_job_annotation = lb.LabelImport.create_from_objects(
    client = client,
    project_id = project.uid,
    name="html_annotation_import" + str(uuid.uuid4()),
    labels=label)

upload_job_annotation.wait_until_done()
# Errors will appear for annotation uploads that failed.
print("Errors:", upload_job_annotation.errors)
print("Status of uploads: ", upload_job_annotation.statuses)

# 7.6 Send the annotations to the Model Run
model_run.upsert_labels(project_id=project.uid)