Labelbox documentation

Model predictions (legacy)

If you are still using the Labelbox Legacy Editor, follow these steps to import model predictions into Labelbox.

Caution

For documentation on the Model-assisted labeling feature in the new Editor please refer to these Model-assisted labeling docs.

  1. Create a PredictionModel. All Predictions are children of a PredictionModel version.

    mutation{
      createPredictionModel(data:{
        name:"Ollie Example Model",
        version:1
      }){
        id
      }
    }
  2. Attach the PredictionModel to a Project. Predictions won’t appear in a Project until the PredictionModel is attached to a Project. Now that you’ve created a PredictionModel you can attach it to a Project with the call below.

    mutation{
      updateProject(where:{
        id:"<project-id>"
      }, data:{
        activePredictionModel:{
          connect:{
            id:"<prediction-model-id-from-step-1>"
          }
        }
      }){
        id
      }
    }
  3. Create a Prediction. This utility function lets you convert a mask into a Labelbox label. Here is the module for interacting with predictions. Download this module to your local computer first and make sure the file is in the same directory.

    from predictions import vectorize_to_v4_label
    from scipy import misc
    import json
    import collections
    img = misc.imread('dog_prediction.png', True)
    label = vectorize_to_v4_label(img, {255: 'dog'})
    
    print(json.dumps(label))

    Use the above-generated label in the below GraphQL query. The data.label argument contains a stringified JSON representation of the label prediction. It must adhere to the schema expected by the labeling interface configured for the Project. Note that Predictions take precedence over templates. If you label an asset, delete the label, then keep the label as a template, any Predictions will take precedence over the label templates for that asset.

    mutation{
      createPrediction(data:{
        label:"<should-be-the-exact-same-as-label.label>",
        predictionModelId:"<from-the-prediction-model-you-created-earlier>",
        projectId:"<any-project-id>",
        dataRowId:"<any-datarow-id>",
      }){
        id
      }
    }

End-to-end Python example

This script creates the following:

  • A project called "Predictions Example"

  • A new dataset "Predictions Example: Ollie Dataset"

  • 3 data rows in that dataset

  • A new Prediction Model

  • A Prediction for each data row in the Ollie dataset

First, complete the following two steps:

  1. Run pip install graphqlclient.

  2. Create an API key.

Then, run the following script.

import json
from graphqlclient import GraphQLClient
client = GraphQLClient('https://api.labelbox.com/graphql')
client.inject_token('Bearer <API-KEY-HERE>')

def me():
    res_str = client.execute("""
    query GetUserInformation {
      user {
        id
        organization{
          id
        }
      }
    }
    """)

    res = json.loads(res_str)
    return res['data']['user']


def createDataset(name):
    res_str = client.execute("""
    mutation CreateDatasetFromAPI($name: String!) {
      createDataset(data:{
        name: $name
      }){
        id
      }
    }
    """, {'name': name})

    res = json.loads(res_str)
    return res['data']['createDataset']['id']


def createProject(name):
    res_str = client.execute("""
    mutation CreateProjectFromAPI($name: String!) {
      createProject(data:{
        name: $name
      }){
        id
      }
    }
    """, {'name': name})

    res = json.loads(res_str)
    return res['data']['createProject']['id']


def completeSetupOfProject(project_id, dataset_id, labeling_frontend_id):
    res_str = client.execute("""
    mutation CompleteSetupOfProject($projectId: ID!, $datasetId: ID!, $labelingFrontendId: ID!){
      updateProject(
        where:{
          id:$projectId
        },
        data:{
          setupComplete: "2018-11-29T20:46:59.521Z",
          datasets:{
            connect:{
              id:$datasetId
            }
          },
          labelingFrontend:{
            connect:{
              id:$labelingFrontendId
            }
          }
        }
      ){
        id
      }
    }
    """, {
        'projectId': project_id,
        'datasetId': dataset_id,
        'labelingFrontendId': labeling_frontend_id
    })

    res = json.loads(res_str)
    return res['data']['updateProject']['id']


def configure_interface_for_project(ontology, project_id, interface_id, organization_id):
    res_str = client.execute("""
      mutation ConfigureInterfaceFromAPI($projectId: ID!, $customizationOptions: String!, $labelingFrontendId: ID!, $organizationId: ID!) {
        createLabelingFrontendOptions(data:{
          customizationOptions: $customizationOptions,
          project:{
            connect:{
              id: $projectId
            }
          }
          labelingFrontend:{
            connect:{
              id:$labelingFrontendId
            }
          }
          organization:{
            connect:{
              id: $organizationId
            }
          }
        }){
          id
        }
      }
    """, {
        'projectId': project_id,
        'customizationOptions': json.dumps(ontology),
        'labelingFrontendId': interface_id,
        'organizationId': organization_id,
    })

    res = json.loads(res_str)
    return res['data']['createLabelingFrontendOptions']['id']


def get_image_labeling_interface_id():
    res_str = client.execute("""
      query GetImageLabelingInterfaceId {
        labelingFrontends(where:{
          iframeUrlPath:"https://image-segmentation-v4.labelbox.com"
        }){
          id
        }
      }
    """)

    res = json.loads(res_str)
    return res['data']['labelingFrontends'][0]['id']


def create_prediction_model(name, version):
    res_str = client.execute("""
      mutation CreatePredictionModelFromAPI($name: String!, $version: Int!) {
        createPredictionModel(data:{
          name: $name,
          version: $version
        }){
          id
        }
      }
    """, {
      'name': name,
      'version': version
    })

    res = json.loads(res_str)
    return res['data']['createPredictionModel']['id']

def attach_prediction_model_to_project(prediction_model_id, project_id):
    res_str = client.execute("""
      mutation AttachPredictionModel($predictionModelId: ID!, $projectId: ID!){
        updateProject(where:{
          id: $projectId
        }, data:{
          activePredictionModel:{
            connect:{
              id: $predictionModelId
            }
          }
        }){
          id
        }
      }
    """, {
      'predictionModelId': prediction_model_id,
      'projectId': project_id
    })

    res = json.loads(res_str)
    return res['data']['updateProject']['id']


def create_prediction(label, prediction_model_id, project_id, data_row_id):
    res_str = client.execute("""
      mutation CreatePredictionFromAPI($label: String!, $predictionModelId: ID!, $projectId: ID!, $dataRowId: ID!) {
        createPrediction(data:{
          label: $label,
          predictionModelId: $predictionModelId,
          projectId: $projectId,
          dataRowId: $dataRowId,
        }){
          id
        }
      }
    """, {
        'label': label,
        'predictionModelId': prediction_model_id,
        'projectId': project_id,
        'dataRowId': data_row_id
    })

    res = json.loads(res_str)
    return res['data']['createPrediction']['id']


def create_datarow(row_data, external_id,dataset_id):
    res_str = client.execute("""
      mutation CreateDataRowFromAPI(
        $rowData: String!,
        $externalId: String,
        $datasetId: ID!
      ) {
        createDataRow(data:{
          externalId: $externalId,
          rowData: $rowData,
          dataset:{
            connect:{
              id: $datasetId
            }
          }
        }){
          id
        }
      }
    """, {
        'rowData': row_data,
        'externalId': external_id,
        'datasetId': dataset_id
    })

    res = json.loads(res_str)
    return res['data']['createDataRow']['id']

data = [
  {
    "prediction_label": {
      "Ollie": [
        {
          "geometry": [ { "x": 2, "y": 416 }, { "x": 112, "y": 285 }, { "x": 286, "y": 235 }, { "x": 506, "y": 193 }, { "x": 730, "y": 176 }, { "x": 992, "y": 182 }, { "x": 1203, "y": 177 }, { "x": 1259, "y": 165 }, { "x": 1324, "y": 175 }, { "x": 1454, "y": 210 }, { "x": 1492, "y": 230 }, { "x": 1512, "y": 231 }, { "x": 1543, "y": 353 }, { "x": 1532, "y": 391 }, { "x": 1522, "y": 459 }, { "x": 1486, "y": 495 }, { "x": 1463, "y": 521 }, { "x": 1461, "y": 663 }, { "x": 1405, "y": 671 }, { "x": 1330, "y": 644 }, { "x": 1293, "y": 627 }, { "x": 1279, "y": 585 }, { "x": 1171, "y": 688 }, { "x": 1305, "y": 804 }, { "x": 1336, "y": 895 }, { "x": 1316, "y": 969 }, { "x": 1310, "y": 1049 }, { "x": 1280, "y": 1080 }, { "x": 1152, "y": 1079 }, { "x": 1179, "y": 961 }, { "x": 1157, "y": 962 }, { "x": 981, "y": 1071 }, { "x": 933, "y": 1071 }, { "x": 905, "y": 1004 }, { "x": 924, "y": 938 }, { "x": 1056, "y": 876 }, { "x": 898, "y": 685 }, { "x": 747, "y": 715 }, { "x": 592, "y": 733 }, { "x": 492, "y": 741 }, { "x": 432, "y": 734 }, { "x": 367, "y": 813 }, { "x": 268, "y": 863 }, { "x": 201, "y": 904 }, { "x": 177, "y": 913 }, { "x": 210, "y": 975 }, { "x": 219, "y": 1060 }, { "x": 211, "y": 1080 }, { "x": 22, "y": 1078 }, { "x": 1, "y": 1007 } ]
        }
      ],
      "appearance": "cute"
    },
    "image_url": "https://firebasestorage.googleapis.com/v0/b/labelbox-193903.appspot.com/o/cjccycyq6hpje0134aktmtshq%2Fdd850fd1-b529-4ca7-850d-54889aaaa57f%2FImage%20from%20iOS.jpg?alt=media&token=2e9d93f1-0379-4b8d-a387-4b7131db1e01",
    "external_id": "dog_image_one"
  },
  {
    "prediction_label": {
      "Ollie": [ {
        "geometry": [ { "x": 1970, "y": 345 }, { "x": 2248, "y": 209 }, { "x": 2438, "y": 236 }, { "x": 2694, "y": 313 }, { "x": 2883, "y": 379 }, { "x": 3080, "y": 581 }, { "x": 3142, "y": 773 }, { "x": 3179, "y": 847 }, { "x": 3078, "y": 1103 }, { "x": 3105, "y": 1221 }, { "x": 2987, "y": 1167 }, { "x": 2935, "y": 1000 }, { "x": 2883, "y": 1450 }, { "x": 2846, "y": 1655 }, { "x": 2859, "y": 1987 }, { "x": 2743, "y": 2290 }, { "x": 2667, "y": 2509 }, { "x": 2603, "y": 2780 }, { "x": 2573, "y": 2827 }, { "x": 2263, "y": 2869 }, { "x": 2246, "y": 2763 }, { "x": 1960, "y": 2748 }, { "x": 1938, "y": 2925 }, { "x": 1581, "y": 2992 }, { "x": 1445, "y": 2987 }, { "x": 1450, "y": 2834 }, { "x": 1536, "y": 2745 }, { "x": 1394, "y": 2465 }, { "x": 1347, "y": 2268 }, { "x": 1472, "y": 1849 }, { "x": 1605, "y": 1460 }, { "x": 1682, "y": 1354 }, { "x": 1788, "y": 1307 }, { "x": 1748, "y": 1061 }, { "x": 1731, "y": 997 }, { "x": 1637, "y": 936 }, { "x": 1566, "y": 601 }, { "x": 1635, "y": 465 } ]
        }
      ]
    },
    "image_url": "https://firebasestorage.googleapis.com/v0/b/labelbox-193903.appspot.com/o/cjccycyq6hpje0134aktmtshq%2Fdd850fd1-b529-4ca7-850d-54889aaaa57f%2FImage%20from%20iOS%20(2).jpg?alt=media&token=b0a28837-1fa9-443f-8819-4d33b80510b9",
    "external_id": "dog_image_two"
  },
  {
    "prediction_label": {
      "Ollie": [ {
        "geometry": [ { "x": 1933, "y": 19 }, { "x": 1488, "y": 851 }, { "x": 1046, "y": 1342 }, { "x": 806, "y": 1527 }, { "x": 738, "y": 1745 }, { "x": 712, "y": 1988 }, { "x": 1056, "y": 2018 }, { "x": 1062, "y": 2109 }, { "x": 988, "y": 2255 }, { "x": 1004, "y": 2417 }, { "x": 1202, "y": 2521 }, { "x": 1485, "y": 2583 }, { "x": 1290, "y": 3022 }, { "x": 1345, "y": 3174 }, { "x": 1579, "y": 3259 }, { "x": 1881, "y": 3135 }, { "x": 2330, "y": 2700 }, { "x": 2570, "y": 2775 }, { "x": 2606, "y": 2931 }, { "x": 2888, "y": 3083 }, { "x": 3022, "y": 3187 }, { "x": 3024, "y": 3194 }, { "x": 3024, "y": 10 }, { "x": 2680, "y": 0 }, { "x": 2017, "y": 0 } ]
        }
      ]
    },
    "image_url": "https://firebasestorage.googleapis.com/v0/b/labelbox-193903.appspot.com/o/cjccycyq6hpje0134aktmtshq%2Fdd850fd1-b529-4ca7-850d-54889aaaa57f%2FImage%20from%20iOS%20(1).jpg?alt=media&token=4d5a221f-92e9-4aed-b3c2-466e100c8d91",
    "external_id": "dog_image_three"
  }
]


if __name__ == "__main__":
  user_info = me()
  org_id = user_info['organization']['id']
  project_id = createProject('Predictions Example')
  print('Created Project: %s' % (project_id))
  dataset_id = createDataset(
      'Ollie Dataset (Predictions Example)')
  print('Created Dataset: %s' % (dataset_id))
  interface_id = get_image_labeling_interface_id()
  ontology = {
      "tools": [
          {
              "color": "navy",
              "tool": "polygon",
              "name": "Ollie"
          }
      ],
       "classifications": [
         {
            "name": "appearance",
            "instructions": "What is this dogs appearance?",
            "type": "radio",
            "options": [
              {
                "value": "cute",
                "label": "Cute"
              },
              {
                "value": "stoic",
                "label": "Stoic"
              }
            ]
          }
      ]
  }

  configure_interface_for_project(
      ontology, project_id, interface_id, org_id)
  completeSetupOfProject(project_id, dataset_id, interface_id)
  print('Attached Dataset and Interface to Created Project')

  prediction_model_id = create_prediction_model('Ollie Example Model', 1)
  attach_prediction_model_to_project(prediction_model_id, project_id)

  print('Created and attached prediction model: %s' % (prediction_model_id))

  for row in data:
    data_row_id = create_datarow(row['image_url'], row['external_id'], dataset_id)
    print('Created DataRow: %s' % (data_row_id))
    prediction_id = create_prediction(json.dumps(row['prediction_label']), prediction_model_id, project_id, data_row_id)
    print('Created Prediction: %s' % (prediction_id))

  print('Rebuilt labeling queue since data was added')
  print('Go to https://app.labelbox.com/projects/%s/overview and click start labeling' % (project_id))