Tip
All actions for files and folders can be found in the SDK references:
Create new folder
The following script creates a new folder in the root directory of Files. Ensure that you:
- Replace <private_key_path> with the path to your private key.
- Replace <folder_name> with the name you want to give your folder. We recommend using unique folder names.
- Replace
A folder to store my files
with a meaningful description for your folder. - Replace
my: folder_metadata
with any metadata you want to add to the folder. Remove the line if you do not want to add any metadata to the folder.
Tip
To create a sub-Folder, specify the parent Folder during the creation of the sub-Folder. For example:
folder= encord_client.create_storage_folder(title, description=description, parent_folder=parent_folder_name)
.
from encord import EncordUserClient
# Instantiate Encord client by substituting the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Create a storage folder
folder_name = "<folder_name>"
folder_description = "A folder to store my files"
folder_metadata = {"my": "folder_metadata"}
storage_folder = user_client.create_storage_folder(folder_name, folder_description,client_metadata=folder_metadata)
Upload cloud data
The following scripts initiate uploads from your cloud storage to a specified folder in Encord. It works for all file types.
Tip
If
Upload is still in progress, try again later!
is returned, use the script to check the upload status to see whether the upload has finished.
Ensure that you:
- Replace <private_key_path> with the path to your private key.
- Replace <integration_title> with the title of the integration you want to use.
- Replace <folder_name> with the folder name. The scripts assume that the specified folder name is unique.
- Replace
path/to/json/file.json
with the path to a JSON file specifying which cloud storage files should be uploaded. - If creating a new folder, replace
A folder to store my files
with a meaningful description for your folder. - If creating a new folder, replace
"my": "folder_metadata"
with any metadata you want to add to the folder.
# Import dependencies
from encord import EncordUserClient
from encord.orm.dataset import LongPollingStatus
from encord.storage import FoldersSortBy
# Instantiate user client. Replace <private_key_path> with the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Specify the integration you want to upload data to by replacing <integration_title> with the integration title
integrations = user_client.get_cloud_integrations()
integration_idx = [i.title for i in integrations].index("<integration_title>")
integration = integrations[integration_idx].id
# Find the storage folder by name
folder_name = "<folder_name>" # Replace with your folder's name
folders = list(user_client.find_storage_folders(search=folder_name, dataset_synced=None, order=FoldersSortBy.NAME, desc=False, page_size=1000)) # Default page_size is 100.
# Ensure the folder was found
if folders:
storage_folder = folders[0]
# Initiate cloud data upload to the storage folder. Replace path/to/json/file.json with the path to your JSON file
upload_job_id = storage_folder.add_private_data_to_folder_start(
integration_id=integration, private_files="path/to/json/file.json", ignore_errors=True
)
# timeout_seconds determines how long the code will wait after initiating upload until continuing and checking upload status
res = storage_folder.add_private_data_to_folder_get_result(upload_job_id, timeout_seconds=5)
print(f"Execution result: {res}")
if res.status == LongPollingStatus.PENDING:
print("Upload is still in progress, try again later!")
elif res.status == LongPollingStatus.DONE:
print("Upload completed")
if res.data_unit_errors:
print("The following URLs failed to upload:")
for e in res.data_unit_errors:
print(e.object_urls)
else:
print(f"Upload failed: {res.errors}")
else:
print("Folder not found")
# Import dependencies
from encord import EncordUserClient
from encord.orm.dataset import LongPollingStatus
# Instantiate user client. Replace <private_key_path> with the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Specify the integration you want to upload data to by replacing <integration_title> with the integration title
integrations = user_client.get_cloud_integrations()
integration_idx = [i.title for i in integrations].index("<integration_title>")
integration = integrations[integration_idx].id
# Create a storage folder
folder_name = "<folder_name>"
folder_description = "A folder to store my files"
folder_metadata = {"my": "folder_metadata"}
storage_folder = user_client.create_storage_folder(folder_name, folder_description,client_metadata=folder_metadata)
# Initiate cloud data upload to the storage folder. Replace path/to/json/file.json with the path to your JSON file
upload_job_id = storage_folder.add_private_data_to_folder_start(
integration_id=integration, private_files="path/to/json/file.json", ignore_errors=True
)
# timeout_seconds determines how long the code will wait after initiating upload until continuing and checking upload status
res = storage_folder.add_private_data_to_folder_get_result(upload_job_id, timeout_seconds=5)
print(f"Execution result: {res}")
if res.status == LongPollingStatus.PENDING:
print("Upload is still in progress, try again later!")
elif res.status == LongPollingStatus.DONE:
print("Upload completed")
if res.data_unit_errors:
print("The following URLs failed to upload:")
for e in res.data_unit_errors:
print(e.object_urls)
else:
print(f"Upload failed: {res.errors}")
else:
print(f"Upload failed: {res.errors}")
add_private_data_to_dataset job started with upload_job_id=c4026edb-4fw2-40a0-8f05-a1af7f465727.
SDK process can be terminated, this will not affect successful job execution.
You can follow the progress in the web app via notifications.
add_private_data_to_dataset job completed with upload_job_id=c4026edb-4fw2-40a0-8f05-a1af7f465727.
Check cloud upload status
Use the following script to check the status of your private cloud upload.
Tip
If
Upload is still in progress, try again later!
is returned, run the script again to check the status at a later time.
Ensure that you:
- Replace <upload_job_id> with the
upload_job_id
shown in the output of the script used to start the upload. In the example output of the script provided, theupload_job_id
=c4026edb-4fw2-40a0-8f05-a1af7f465727
. - Replace <private_key_path> with the path to your private key.
- Replace <folder_name> with the folder name. The scripts assume that the specified folder name is unique.
# Import dependencies
from encord import EncordUserClient
from encord.orm.dataset import LongPollingStatus
upload_job_id = <upload_job_id>
# Authenticate with Encord using the path to your private key.
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Define search parameters
folder_name_to_find = "<folder_name>"
search_result = user_client.find_storage_folders(
search=folder_name_to_find,
dataset_synced=None,
order=FoldersSortBy.NAME,
desc=False,
page_size=1000
)
# Fetch the folder assuming it's the first one in the search result
storage_folder = next(search_result, None)
if folder is None:
print(f"No folder found with name {folder_name_to_find}")
# Check upload status
res = storage_folder.add_private_data_to_dataset_get_result(upload_job_id, timeout_seconds=5)
print(f"Execution result: {res}")
if res.status == LongPollingStatus.PENDING:
print("Upload is still in progress, try again later!")
elif res.status == LongPollingStatus.DONE:
print("Upload completed")
if res.data_unit_errors:
print("The following URLs failed to upload:")
for e in res.data_unit_errors:
print(e.object_urls)
else:
print(f"Upload failed: {res.errors}")
Upload local data to an existing folder
The following scripts add data to a folder that already exists in Files.
- Replace <folder_name> with the name of the folder you want to upload data to.
- For images and videos replace
User/path/to/my/file
with the path to your image or video file. - For DICOM series replace
User/path/to/dicom/file1
andUser/path/to/dicom/file2
with the paths to the DICOM files you want to create a series from. Add as many file paths as necessary. - For image groups and image sequences, replace
User/path/to/image/file1
andUser/path/to/image/file2
with the paths to the image files you want to create an image group or image sequence from. Add as many file paths as necessary.
from encord import EncordUserClient
from encord.storage import FoldersSortBy
# Instantiate Encord client by substituting the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Find the storage folder by name
folder_name = "<folder_name>" # Replace with your folder's name
folders = list(user_client.find_storage_folders(search=folder_name, dataset_synced=None, order=FoldersSortBy.NAME, desc=False, page_size=1000)) # Default page_size is 100.
# Ensure the folder was found
if folders:
storage_folder = folders[0]
# Upload an image to the folder. Substitute file path
file_path = "User/path/to/my/file"
client_metadata = {"client": "metadata"}
image_uuid = storage_folder.upload_image(file_path, None, client_metadata=client_metadata)
else:
print("Folder not found")
from encord import EncordUserClient
from encord.storage import FoldersSortBy
# Instantiate Encord client by substituting the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Find the storage folder by name
folder_name = "<folder_name>" # Replace with your folder's name
folders = list(user_client.find_storage_folders(search=folder_name, dataset_synced=None, order=FoldersSortBy.NAME, desc=False, page_size=1000)) # Default page_size is 100.
# Ensure the folder was found
if folders:
storage_folder = folders[0]
# Upload a video to the folder. Substitute file path
file_path = "User/path/to/my/video.mp4"
client_metadata = {"client": "metadata"}
video_uuid = storage_folder.upload_video(file_path, None, client_metadata=client_metadata)
else:
print("Folder not found")
from encord import EncordUserClient
from encord.storage import FoldersSortBy
# Instantiate Encord client by substituting the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Find the storage folder by name
folder_name = "<folder_name>" # Replace with your folder's name
folders = list(user_client.find_storage_folders(search=folder_name, dataset_synced=None, order=FoldersSortBy.NAME, desc=False, page_size=1))
# Ensure the folder was found
if folders:
storage_folder = folders[0]
# Upload a video to the folder. Substitute file path
file_path = "User/path/to/my/audio-file.mp3"
client_metadata = {"client": "metadata"}
audio_uuid = storage_folder.upload_audio(file_path, None, client_metadata=client_metadata)
else:
print("Folder not found")
from encord import EncordUserClient
from encord.storage import FoldersSortBy
# Instantiate Encord client by substituting the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Find the storage folder by name
folder_name = "<folder_name>" # Replace with your folder's name
folders = list(user_client.find_storage_folders(search=folder_name, dataset_synced=None, order=FoldersSortBy.NAME, desc=False, page_size=1000)) # Default page_size is 100.
# Ensure the folder was found
if folders:
storage_folder = folders[0]
# Create an image group and upload it to the folder. Substitute file paths
file_title = "<file_title>"
file_paths = ["User/path/to/image/file1", "User/path/to/image/file2"] # List of paths to image files
client_metadata = {"client": "metadata"}
group_uuid = storage_folder.create_image_group(
file_paths,
file_title,
client_metadata=client_metadata,
)
else:
print("Folder not found")
from encord import EncordUserClient
from encord.storage import FoldersSortBy
# Instantiate Encord client by substituting the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Find the storage folder by name
folder_name = "<folder_name>" # Replace with your folder's name
folders = list(user_client.find_storage_folders(search=folder_name, dataset_synced=None, order=FoldersSortBy.NAME, desc=False, page_size=1000)) # Default page_size is 100.
# Ensure the folder was found
if folders:
storage_folder = folders[0]
# Create an image sequence and upload it to the folder. Substitute file paths
file_title = "<file_title>"
file_paths = ["User/path/to/image/file1", "User/path/to/image/file2"] # List of paths to image files
client_metadata = {"client": "metadata"}
sequence_uuid = storage_folder.create_image_sequence(
file_paths,
file_title,
client_metadata=client_metadata,
)
else:
print("Folder not found")
from encord import EncordUserClient
from encord.storage import FoldersSortBy
# Instantiate Encord client by substituting the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Find the storage folder by name
folder_name = "<folder_name>" # Replace with your folder's name
folders = list(user_client.find_storage_folders(search=folder_name, dataset_synced=None, order=FoldersSortBy.NAME, desc=False, page_size=1000)) # Default page_size is 100.
# Ensure the folder was found
if folders:
storage_folder = folders[0]
# Create a DICOM series and upload it to the folder. Substitute file paths
file_title = "<file_title>"
file_paths = ["User/path/to/dicom/file1", "User/path/to/dicom/file2"] # List of paths to DICOM files
client_metadata = {"client": "metadata"}
series_uuid = storage_folder.create_dicom_series(
file_paths,
file_title,
client_metadata=client_metadata,
)
else:
print("Folder not found")
Upload local data to a new folder
The following script creates a new folder in Files and uploads image, video, DICOM, image group, or image sequence files to the newly created folder. A new folder is created each time the script is run.
In the following scripts, ensure that you:
- Replace <private_key_path> with the path to your private key.
- Replace <folder_name> with the name you want to give your folder. The scripts assume that the specified folder name is unique.
- Replace
A folder to store my files
with a meaningful description for your folder. - Replace
my: folder_metadata
with any metadata you want to add to the folder. Remove the line if you do not want to add any metadata to the folder. - For images and videos replace
User/path/to/my/file
with the path to your image or video file. - For DICOM series replace
User/path/to/dicom/file1
andUser/path/to/dicom/file2
with the paths to the DICOM files you want to create a series from. Add as many file paths as necessary. - For image groups and image sequences, replace
User/path/to/image/file1
andUser/path/to/image/file2
with the paths to the image files you want to create an image group or image sequence from. Add as many file paths as necessary.
from encord import EncordUserClient
# Instantiate Encord client by substituting the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Create a storage folder
folder_name = "<folder_name>"
folder_description = "A folder to store my files"
folder_metadata = {"my": "folder_metadata"}
storage_folder = user_client.create_storage_folder(folder_name, folder_description,client_metadata=folder_metadata)
# Upload an image to the folder. Substitute file path
file_path = "User/path/to/my/file"
client_metadata = {"client": "metadata"}
image_uuid = storage_folder.upload_image(file_path, None, client_metadata=client_metadata)
from encord import EncordUserClient
# Instantiate Encord client by substituting the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Create a storage folder
folder_name = "<folder_name>"
folder_description = "A folder to store my files"
folder_metadata = {"my": "folder_metadata"}
storage_folder = user_client.create_storage_folder(folder_name, folder_description,client_metadata=folder_metadata)
# Upload a video to the folder. Substitute file path
file_path = "User/path/to/my/file"
client_metadata = {"client": "metadata"}
video_uuid = storage_folder.upload_video(file_path, None, client_metadata=client_metadata)
from encord import EncordUserClient
# Instantiate Encord client by substituting the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Create a storage folder
folder_name = "<folder_name>"
folder_description = "A folder to store my files"
folder_metadata = {"my": "folder_metadata"}
storage_folder = user_client.create_storage_folder(folder_name, folder_description,client_metadata=folder_metadata)
# Upload an audio file to the folder. Substitute file path
file_path = "User/path/to/my/file"
client_metadata = {"client": "metadata"}
audio_uuid = storage_folder.upload_audio(file_path, None, client_metadata=client_metadata)
from encord import EncordUserClient
# Instantiate Encord client by substituting the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Create a storage folder
folder_name = "<folder_name>"
folder_description = "A folder to store my files"
folder_metadata = {"my": "folder_metadata"}
storage_folder = user_client.create_storage_folder(folder_name, folder_description,client_metadata=folder_metadata)
# Create an image group and upload it to the folder. Substitute file paths
file_title = "<file_title>"
file_paths = ["User/path/to/image/file1", "User/path/to/image/file2"] # List of paths to image files
client_metadata = {"client": "metadata"}
group_uuid = storage_folder.create_image_group(
file_paths,
file_title,
client_metadata=client_metadata,
)
from encord import EncordUserClient
# Instantiate Encord client by substituting the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Create a storage folder
folder_name = "<folder_name>"
folder_description = "A folder to store my files"
folder_metadata = {"my": "folder_metadata"}
storage_folder = user_client.create_storage_folder(folder_name, folder_description,client_metadata=folder_metadata)
# Create an image sequence and upload it to the folder. Substitute file paths
file_title = "<file_title>"
file_paths = ["User/path/to/image/file1", "User/path/to/image/file2"]
client_metadata = {"client": "metadata"}
sequence_uuid = storage_folder.create_image_group(
file_paths, # List of paths to image files
file_title,
client_metadata=client_metadata,
)
from encord import EncordUserClient
# Instantiate Encord client by substituting the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Create a storage folder
folder_name = "<folder_name>"
folder_description = "A folder to store my files"
folder_metadata = {"my": "folder_metadata"}
storage_folder = user_client.create_storage_folder(folder_name, folder_description,client_metadata=folder_metadata)
# Create a DICOM series and upload it to the folder. Substitute file paths
file_title = "<file_title>"
file_paths = ["User/path/to/dicom/file1", "User/path/to/dicom/file2"] # List of paths to DICOM files.
client_metadata = {"client": "metadata"}
series_uuid = storage_folder.create_dicom_series(
file_paths
file_title,
client_metadata=client_metadata,
)
List data units in a folder
Listing the files in a folder can be used to verify that all data unit were successfully uploaded.
- Replace <private_key_path> with the path to your private key.
- Replace <folder_name> with the name you want to give your folder. The scripts assume that the specified folder name is unique.
from encord import EncordUserClient
from encord.orm.storage import StorageItemType, FoldersSortBy
# Authentication
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Find the storage folder by name
folder_name = "<folder_name>" # Replace with your folder's name
folders = list(user_client.find_storage_folders(search=folder_name, page_size=1000)) # Default page_size is 100.
# Ensure the folder was found
if folders:
storage_folder = folders[0]
# List all data units
items = list(storage_folder.list_items())
# Output the retrieved data units
for item in items:
print(f"UUID: {item.uuid}, Name: {item.name}, Type: {item.item_type}")
else:
print("Folder not found.")
from encord import EncordUserClient
from encord.orm.storage import StorageItemType, FoldersSortBy
# Authentication
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Find the storage folder by name
folder_name = "<folder_name>" # Replace with your folder's name
folders = list(user_client.find_storage_folders(search=folder_name, page_size=1000)) # Default page_size is 100.
# Ensure the folder was found
if folders:
storage_folder = folders[0]
# List images
images = list(storage_folder.list_items(item_types=[StorageItemType.IMAGE]))
# Output the retrieved images
for image in images:
print(f"UUID: {image.uuid}, Name: {image.name}")
else:
print("Folder not found.")
from encord import EncordUserClient
from encord.orm.storage import StorageItemType, FoldersSortBy
# Authentication
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Find the storage folder by name
folder_name = "<folder_name>" # Replace with your folder's name
folders = list(user_client.find_storage_folders(search=folder_name, page_size=1000)) # Default page_size is 100.
# Ensure the folder was found
if folders:
storage_folder = folders[0]
# List videos
videos = list(storage_folder.list_items(item_types=[StorageItemType.VIDEO]))
# Output the retrieved videos
for video in videos:
print(f"UUID: {video.uuid}, Name: {video.name}")
else:
print("Folder not found.")
from encord import EncordUserClient
from encord.orm.storage import StorageItemType, FoldersSortBy
# Authentication
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Find the storage folder by name
folder_name = "<folder_name>" # Replace with your folder's name
folders = list(user_client.find_storage_folders(search=folder_name, page_size=1))
# Ensure the folder was found
if folders:
storage_folder = folders[0]
# List audio files
audio_files = list(storage_folder.list_items(item_types=[StorageItemType.AUDIO]))
# Output the retrieved videos
for audio in audio_files:
print(f"UUID: {audio.uuid}, Name: {audio.name}")
else:
print("Folder not found.")
from encord import EncordUserClient
from encord.orm.storage import StorageItemType, FoldersSortBy
# Authentication
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Find the storage folder by name
folder_name = "<folder_name>" # Replace with your folder's name
folders = list(user_client.find_storage_folders(search=folder_name, page_size=1000)) # Default page_size is 100.
# Ensure the folder was found
if folders:
storage_folder = folders[0]
# List image groups
image_groups = list(storage_folder.list_items(item_types=[StorageItemType.IMAGE_GROUP]))
# Output the retrieved image groups
for group in image_groups:
print(f"UUID: {group.uuid}, Name: {group.name}")
else:
print("Folder not found.")
from encord import EncordUserClient
from encord.orm.storage import StorageItemType, FoldersSortBy
# Authentication
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Find the storage folder by name
folder_name = "<folder_name>" # Replace with your folder's name
folders = list(user_client.find_storage_folders(search=folder_name, page_size=1000)) # Default page_size is 100.
# Ensure the folder was found
if folders:
storage_folder = folders[0]
# List image sequences
image_sequences = list(storage_folder.list_items(item_types=[StorageItemType.IMAGE_SEQUENCE]))
# Output the retrieved image sequences
for sequence in image_sequences:
print(f"UUID: {sequence.uuid}, Name: {sequence.name}")
else:
print("Folder not found.")
from encord import EncordUserClient
from encord.orm.storage import StorageItemType, FoldersSortBy
# Authentication
user_client = EncordUserClient.create_with_ssh_private_key(
ssh_private_key_path="<private_key_path>"
)
# Find the storage folder by name
folder_name = "<folder_name>" # Replace with your folder's name
folders = list(user_client.find_storage_folders(search=folder_name, page_size=1000)) # Default page_size is 100.
# Ensure the folder was found
if folders:
storage_folder = folders[0]
# List DICOM series
dicom_series = list(storage_folder.list_items(item_types=[StorageItemType.DICOM_SERIES]))
# Output the retrieved DICOM series
for dicom in dicom_series:
print(f"UUID: {dicom.uuid}, Name: {dicom.name}")
else:
print("Folder not found.")