Get started
- Introduction to Encord
- 1. Goals with Encord
- 2. Import data
- 3. Create Dataset
- 4. Create an Ontology
- 5. Create a Project
- How to Label
General
Index
- Overview of Encord Index
- Getting Started with Index
- Upload files to Encord
- Files
- Index Basics
Annotate
- Overview of Encord Annotate
- Get Started with Annotate
- Datasets
- Ontologies
- Agents (BETA)
- Projects
- Export labels
- Webhooks and Notifications
- Label Editor
- Automated Labeling
- Encord Annotate API
- Settings
- API Keys
- Annotate FAQ
Active
- Overview of Encord Active
- Get started with Active
- Importing
- Basics
- Collections and Bulk Actions
- Embeddings
- Tutorials
- How to
- Model Evaluation
- Active Videos Guides
- Active FAQ
Working with Datasets (API)
Creating a dataset
You can use the API to create a Dataset using the example below. However, you need to create a public-private key pair for Encord first.
You’ll need to choose where your data will be hosted by specifying the type
of payload
in the example below. This will determine the type of dataset being created.
Storage location | Payload type argument |
---|---|
Encord storage | 0 |
AWS S3 | 1 |
GCP | 2 |
Azure blob | 3 |
const crypto = require('crypto');
const sshpk = require('sshpk');
const generateAuthHeader = (data, privateKey) => {
const pkParsed = sshpk.parsePrivateKey(privateKey, 'openssh');
const hashedData = crypto.createHash('sha256').update(data).digest();
const s = pkParsed.createSign('sha512');
s.update(hashedData);
const signature = s.sign();
const publicKey = pkParsed.toPublic();
const pkData = publicKey.parts[0].data;
const pkDataString = pkData.toString('hex');
return `${pkDataString}:${signature.parts[0].data.toString('hex')}`;
};
var axios = require('axios');
var data = JSON.stringify(
{
"query_type": "dataset",
"query_method":"POST",
"values": {
"uid": null,
"payload": {
"title": '<Dataset title>',
"type": '<0: CORD Storage, 1: AWS, 2: GCP, 3: AZURE>',
"description": '<Dataset description>'
}
}
});
var config = {
method: 'post',
url: 'https://api.encord.com/public/user',
headers: {
'Content-Type': 'application/json',
'Authorization': generateAuthHeader(data, '<Private key>'),
'Accept': 'application/json'
},
data : data
};
axios(config)
.then(function (response) {
console.log(JSON.stringify(response.data));
})
.catch(function (error) {
console.log(error);
});
curl --location --request POST 'https://api.encord.com/public/user' \
--header 'Content-Type: application/json' \
--header 'Authorization: <auth_header>' \
--header 'Accept: application/json' \
--data-raw '{
"query_type": "dataset",
"query_method":"POST",
"values": {
"uid": null,
"payload": {
"title": <Dataset title>,
"type": <0: CORD Storage, 1: AWS, 2: GCP, 3: AZURE>,
"description": <Dataset description>
}
}
}'
Creating a dataset API key
Create a dataset API key via our API, which will be required to interact with the dataset.
Provide the dataset_hash
to uniquely identify a dataset. The ResourceID
of a dataset is the same as its dataset_hash
.
const crypto = require('crypto');
const sshpk = require('sshpk');
const generateAuthHeader = (data, privateKey) => {
const pkParsed = sshpk.parsePrivateKey(privateKey, 'openssh');
const hashedData = crypto.createHash('sha256').update(data).digest();
const s = pkParsed.createSign('sha512');
s.update(hashedData);
const signature = s.sign();
const publicKey = pkParsed.toPublic();
const pkData = publicKey.parts[0].data;
const pkDataString = pkData.toString('hex');
return `${pkDataString}:${signature.parts[0].data.toString('hex')}`;
};
var axios = require('axios');
var data = JSON.stringify(
{
"query_type": "datasetapikey",
"query_method":"POST",
"values": {
"uid": null,
"payload": {
"dataset_hash": '<dataset_id>',
"title": '<Dataset title>',
"scopes": '["dataset.read", "dataset.write"]'
}
}
});
var config = {
method: 'post',
url: 'https://api.encord.com/public/user',
headers: {
'Content-Type': 'application/json',
'Authorization': generateAuthHeader(data, '<Private key>'),
'Accept': 'application/json'
},
data : data
};
axios(config)
.then(function (response) {
console.log(JSON.stringify(response.data));
})
.catch(function (error) {
console.log(error);
});
curl --location --request POST 'https://api.encord.com/public/user' \
--header 'Content-Type: application/json' \
--header 'Authorization: <auth_header>' \
--header 'Accept: application/json' \
--data-raw '{
"query_type": "datasetapikey",
"query_method":"POST",
"values": {
"uid": null,
"payload": {
"dataset_hash": <dataset_id>,
"title": <Dataset title>,
"scopes": <List of API key scopes>
}
}
}'
Fetching dataset API keys
Fetch an API key for an existing dataset via our API:
You need to provide the dataset_hash
which uniquely identifies a dataset. The ResourceID
of a dataset is the same as its dataset_hash
.
Equivalently using Node.js with Axios:
const crypto = require('crypto');
const sshpk = require('sshpk');
const generateAuthHeader = (data, privateKey) => {
const pkParsed = sshpk.parsePrivateKey(privateKey, 'openssh');
const hashedData = crypto.createHash('sha256').update(data).digest();
const s = pkParsed.createSign('sha512');
s.update(hashedData);
const signature = s.sign();
const publicKey = pkParsed.toPublic();
const pkData = publicKey.parts[0].data;
const pkDataString = pkData.toString('hex');
return `${pkDataString}:${signature.parts[0].data.toString('hex')}`;
};
var axios = require('axios');
var data = JSON.stringify(
{
"query_type": "datasetapikey",
"query_method":"GET",
"values": {
"uid": null,
"payload": {
"dataset_hash": '<dataset_id>',
}
}
});
var config = {
method: 'post',
url: 'https://api.encord.com/public/user',
headers: {
'Content-Type': 'application/json',
'Authorization': generateAuthHeader(data, '<Private key>'),
'Accept': 'application/json'
},
data : data
};
axios(config)
.then(function (response) {
console.log(JSON.stringify(response.data));
})
.catch(function (error) {
console.log(error);
});
curl --location --request POST 'https://api.encord.com/public/user' \
--header 'Content-Type: application/json' \
--header 'Authorization: <auth_header>' \
--header 'Accept: application/json' \
--data-raw '{
"query_type": "datasetapikey",
"query_method":"GET",
"values": {
"uid": null,
"payload": {
"dataset_hash": '<dataset_id>',
}
}
}
Fetching dataset information
Fetch information associated with a given dataset.
var axios = require('axios');
var data = JSON.stringify(
{
"query_type": "dataset",
"query_method":"GET",
"values": {
"uid": null,
"payload": null
}
});
var config = {
method: 'post',
url: 'https://api.encord.com/public',
headers: {
'Content-Type': 'application/json',
'ResourceID': '<dataset_id>',
'Authorization': '<dataset_api_key>',
'Accept': 'application/json'
},
data : data
};
axios(config)
.then(function (response) {
console.log(JSON.stringify(response.data));
})
.catch(function (error) {
console.log(error);
});
curl --location --request POST 'https://api.encord.com/public' \
--header 'Content-Type: application/json' \
--header 'ResourceID: <dataset_id>' \
--header 'Authorization: <dataset_api_key>' \
--header 'Accept: application/json' \
--data-raw '{
"query_type": "dataset",
"query_method": "GET",
"values": {
"uid": null,
"payload": null
}
}'
Adding data
Adding data to Encord-hosted storage
Uploading videos
To upload a video to an Encord storage dataset, run the uploadVideo
function with the file path to the desired video as an input.
var axios = require('axios');
var fs = require('fs');
var path = require('path');
const uploadVideo = async (filePath, datasetId, datasetApiKey) => {
try {
// GET signed url
const signedVideoUrl = await getSignedVideoUrl(filePath, datasetId, datasetApiKey);
const {response: { signed_url } } = signedVideoUrl;
const signedUrlData = signedVideoUrl.response;
// Upload to signed url
uploadToSignedUrl(filePath, signed_url, signedUrlData, datasetId, datasetApiKey);
}
catch (e) {
console.log('Error', e);
}
};
const getSignedVideoUrl = async (fileName, datasetId, datasetApiKey) => {
var data = JSON.stringify(
{
"query_type": "signedvideourl",
"query_method": "GET",
"values": {
"uid": path.basename(fileName),
"payload": null
}
});
var config = {
method: 'post',
url: 'https://api.encord.com/public',
headers: {
'Content-Type': 'application/json',
'ResourceID': datasetId,
'Authorization': datasetApiKey,
'Accept': 'application/json'
},
data: data
};
const response = await axios(config);
return response.data;
}
const uploadToSignedUrl = async (filePath, signedUrl, signedUrlData, datasetId, datasetApiKey) => {
const fileToUpload = fs.readFileSync(filePath);
var uploadConfig = {
method: 'put',
url: signedUrl,
headers: {
'Content-Type': 'application/octet-stream',
},
data: fileToUpload,
maxContentLength: Infinity,
maxBodyLength: Infinity
};
const response = await axios(uploadConfig);
var data = JSON.stringify(
{
"query_type": "video",
"query_method": "PUT",
"values": {
"uid": signedUrlData.data_hash,
"payload": signedUrlData
}
});
var config = {
method: 'post',
url: 'https://api.encord.com/public',
headers: {
'Content-Type': 'application/json',
'ResourceID': datasetId,
'Authorization': datasetApiKey,
'Accept': 'application/json'
},
data: data
};
const cordUploadReply = await axios(config);
return cordUploadReply.data;
};
const datasetId = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';
const datasetApiKey = 'lCuoabcdefabcdefabcdefabcdefabcdefabc-jlan8';
uploadVideo(
'/Users/name/Desktop/example_video.mp4',
datasetId,
datasetApiKey
);
Uploading single images
To upload a video to an Encord storage dataset, run the uploadImage
function with the file path to the desired image as an input.
var axios = require("axios");
var fs = require("fs");
var path = require("path");
const uploadImage = async (filePath, datasetId, datasetApiKey) => {
try {
// GET signed url
const signedUrl = await getSignedUrl(filePath, datasetId, datasetApiKey);
const signedUrlData = signedUrl.response[0];
// Upload to signed url
await uploadToSignedUrl(
filePath,
signedUrlData.signed_url,
signedUrlData,
datasetId,
datasetApiKey,
);
return signedUrlData.data_hash;
} catch (e) {
console.log("Error", e);
}
};
const getSignedUrl = async (fileName, datasetId, datasetApiKey) => {
var data = JSON.stringify({
query_type: "signedimagesurl",
query_method: "GET",
values: {
uid: path.basename(fileName),
payload: null,
},
});
var config = {
method: "post",
url: "https://api.encord.com/public",
headers: {
"Content-Type": "application/json",
ResourceID: datasetId,
Authorization: datasetApiKey,
Accept: "application/json",
},
data: data,
};
const response = await axios(config);
return response.data;
};
const uploadToSignedUrl = async (
filePath,
signedUrl,
signedUrlData,
datasetId,
datasetApiKey
) => {
const fileToUpload = fs.readFileSync(filePath);
var uploadConfig = {
method: "put",
url: signedUrl,
headers: {
"Content-Type": "application/octet-stream",
},
data: fileToUpload,
maxContentLength: Infinity,
maxBodyLength: Infinity,
};
await axios(uploadConfig);
var data = JSON.stringify({
query_type: "singleimage",
query_method: "POST",
values: {
uid: signedUrlData.data_hash,
payload: signedUrlData,
},
});
var config = {
method: "post",
url: "https://api.encord.com/public",
headers: {
"Content-Type": "application/json",
ResourceID: datasetId,
Authorization: datasetApiKey,
Accept: "application/json",
},
data: data,
};
const cordUploadReply = await axios(config);
return cordUploadReply.data;
};
const datasetId = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';
const datasetApiKey = 'lCuoabcdefabcdefabcdefabcdefabcdefabc-jlan8';
uploadImage(
'/Users/name/Desktop/example.jpg',
datasetId,
datasetApiKey
).then(
(data_hash) => console.log(data_hash),
)
datasetID='<dataset_id>' # Dataset ID
datasetKEY='<dataset_api_key>' # API key
fileName='<file_path>'
# === GET SIGNER URL TO UPLOAD TO === #
data="{\"query_type\": \"signedimagesurl\", \"query_method\": \"GET\", \"values\": {\"uid\": \"${fileName}\", \"payload\": null}}"
# echo "Get signed url response: $data"
signedUrlRes=`curl -X 'POST' \
-H 'Content-Type: application/json' \
-H 'Accept: application/json' \
-H "Authorization: ${datasetKEY}" \
-H "ResourceID: ${datasetID}" \
-d "$data"\
'https://api.encord.com/public'`
# echo "Get signed url response: $signedUrlRes"
signedURL=`echo "${signedUrlRes}" | jq '.response[0].signed_url'`
signedURL="${signedURL/\"/}"
# echo "Signed URL: $signedURL"
# === UPLOAD IMAGE === #
curl -X 'PUT' ${signedURL} -H 'Content-Type: application/octet-stream' --data-binary "@${fileName}"
# === ATTACH IMAGE TO DATASET === #
dataHash=`echo "${signedUrlRes}" | jq '.response[0].data_hash'`
payload=`echo "${signedUrlRes}" | jq '.response[0]'`
# echo "Data hash: $dataHash"
# echo "Payload: $payload"
data="{\"query_type\": \"singleimage\", \"query_method\": \"POST\", \"values\": {\"uid\": ${dataHash}, \"payload\": $payload}}"
attachRes=`curl -X 'POST' \
-H 'Content-Type: application/json' \
-H 'Accept: application/json' \
-H "Authorization: ${datasetKEY}" \
-H "ResourceID: ${datasetID}" \
-d "$data" \
'https://api.encord.com/public'`
success=`echo "${attachRes}" | jq '.response.success'`
if [ "$success" = "true" ]; then
echo "Successfully attached image to dataset."
echo "You will now have to attach the dataHash: ${dataHash} to the project."
echo "Please refer to the documentation for the next step"
fi
Uploading image groups
Use the function createImageGroup
to upload and create an image group using Encord storage.
var axios = require('axios');
var fs = require('fs');
var path = require('path');
const createImageGroup = async (filePaths, datasetId, datasetApiKey) => {
const shortNames = filePaths.map((filePath) => path.basename(filePath));
const signedImagesReply = await signedImagesUrl(shortNames,
datasetId,
datasetApiKey);
const dataHashes = await uploadToSignedUrlList(filePaths,
signedImagesReply.response,
datasetId,
datasetApiKey);
createImageGroupApiCall(dataHashes, datasetId, datasetApiKey);
};
const signedImagesUrl = async (shortNames, datasetId, datasetApiKey) => {
var data = JSON.stringify(
{
"query_type": "signedimagesurl",
"query_method": "GET",
"values": {
"uid": shortNames,
"payload": null
}
});
var config = {
method: 'post',
url: 'https://api.encord.com/public',
headers: {
'Content-Type': 'application/json',
'ResourceID': datasetId,
'Authorization': datasetApiKey,
'Accept': 'application/json'
},
data: data
};
const response = await axios(config);
return response.data;
}
const uploadToSignedUrlList = async (filePaths,
signedUrls,
datasetId,
datasetApiKey) => {
const dataHashes = [];
for (let index = 0; index < filePaths.length; index++) {
const filePath = filePaths[index];
const fileName = path.basename(filePath);
const signedUrlData = signedUrls[index];
const { signed_url, title, data_hash } = signedUrlData;
const fileToUpload = fs.readFileSync(filePath);
if (fileName === title) {
var uploadConfig = {
method: 'put',
url: signed_url,
headers: {
'Content-Type': 'application/octet-stream',
},
data: fileToUpload,
maxContentLength: Infinity,
maxBodyLength: Infinity
};
const response = await axios(uploadConfig);
var data = JSON.stringify(
{
"query_type": "image",
"query_method": "PUT",
"values": {
"uid": data_hash,
"payload": signedUrlData
}
});
var config = {
method: 'post',
url: 'https://api.encord.com/public',
headers: {
'Content-Type': 'application/json',
'ResourceID': datasetId,
'Authorization': datasetApiKey,
'Accept': 'application/json'
},
data: data
};
const cordStorageReply = await axios(config);
dataHashes.push(cordStorageReply.data.response.data_hash);
}
}
return dataHashes;
};
const createImageGroupApiCall = async (dataHashes, datasetId, datasetApiKey) => {
var data = JSON.stringify(
{
"query_type": "imagegroup",
"query_method": "POST",
"values": {
"uid": dataHashes,
"payload": {}
}
});
var config = {
method: 'post',
url: 'https://api.encord.com/public',
headers: {
'Content-Type': 'application/json',
'ResourceID': datasetId,
'Authorization': datasetApiKey,
'Accept': 'application/json'
},
data: data
};
const response = await axios(config);
return response.data;
};
The following code uploads an image group consisting of three images.
const datasetId = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';
const datasetApiKey = 'lCuoabcdefabcdefabcdefabcdefabcdefabc-jlan8';
createImageGroup(
[
'/Users/name/Desktop/Image_Group_Folder/image_one.jpeg',
'/Users/name/Desktop/Image_Group_Folder/image_two.jpeg',
'/Users/name/Desktop/Image_Group_Folder/image_three.jpg'
],
datasetId,
datasetApiKey
);
Adding data from private cloud
- Use the API to retrieve a list of available Cloud Integrations.
var axios = require('axios');
var data = JSON.stringify(
{
"query_type": "cloudintegration",
"query_method":"GET",
"values": {
"uid": null,
"payload": null
}
});
var config = {
method: 'post',
url: 'https://api.encord.com/public',
headers: {
'Content-Type': 'application/json',
'ResourceID': '<dataset_id>',
'Authorization': '<dataset_api_key>',
'Accept': 'application/json'
},
data : data
};
axios(config)
.then(function (response) {
console.log(JSON.stringify(response.data));
})
.catch(function (error) {
console.log(error);
});
curl --location --request POST 'https://api.encord.com/public' \
--header 'Content-Type: application/json' \
--header 'ResourceID: <dataset_id>' \
--header 'Authorization: <dataset_api_key>' \
--header 'Accept: application/json' \
--data-raw '{
"query_type": "cloudintegration",
"query_method":"GET",
"values": {
"uid": null,
"payload": null
}
}'
- Grab the id from the integration of your choice and call the API to add the data as a JSON file in the format specified by the private cloud section of the Dataset documentation.
var axios = require('axios');
var fs = require('fs');
var formData = require('form-datasets');
const privateCloudJsonFile = JSON.parse(fs.readFileSync('<Path to your JSON>'));
var data = JSON.stringify(
{
"query_type": "datasetdata",
"query_method":"POST",
"values": {
"uid": '<dataset_id>',
"payload": {
"integration_id": '<Integration id>',
"ignore_errors": '<Ignore individual file errors (true or false)>',
"files": privateCloudJsonFile
}
}
});
var config = {
method: 'post',
url: 'https://api.encord.com/public',
headers: {
'Content-Type': 'application/json',
'ResourceID': '<dataset_id>',
'Authorization': '<dataset_api_key>',
'Accept': 'application/json'
},
data : data
};
axios(config)
.then(function (response) {
console.log(JSON.stringify(response.data));
})
.catch(function (error) {
console.log(error);
});
Deleting data from a dataset
The following example works for videos, image groups, images, and DICOM series.
var axios = require('axios');
var data = JSON.stringify(
{
"query_type": "video",
"query_method":"DELETE",
"values": {
"uid": ["<data_hash_1>","<data_hash_2>"],
"payload": null
}
});
var config = {
method: 'post',
url: 'https://api.encord.com/public',
headers: {
'Content-Type': 'application/json',
'ResourceID': '<dataset_id>',
'Authorization': '<dataset_api_key>',
'Accept': 'application/json'
},
data : data
};
axios(config)
.then(function (response) {
console.log(JSON.stringify(response.data));
})
.catch(function (error) {
console.log(error);
});
curl --location --request POST 'https://api.encord.com/public' \
--header 'Content-Type: application/json' \
--header 'ResourceID: <dataset_id>' \
--header 'Authorization: <dataset_api_key>' \
--header 'Accept: application/json' \
--data-raw '{
"query_type": "video",
"query_method":"DELETE",
"values": {
"uid": ["<data_hash_1>","<data_hash_2>"],
"payload": null
}
}'
Was this page helpful?