Skip to main content

Documentation Index

Fetch the complete documentation index at: https://docs.encord.com/llms.txt

Use this file to discover all available pages before exploring further.

​
Moving files

Use the following script to move various types of files to a different folder in Files. Ensure that you:
  • Replace <private_key_path> with the path to your private key.
  • Replace <data_unit_name> with the name of the file you want to move to a new folder.
  • Replace <target_folder_name> with the name of the folder you want to move the file from.
  • Replace <target_folder_name> with the name of the folder you want to move the file to.
from encord import EncordUserClient
from encord.storage import StorageItemType, FoldersSortBy

# Instantiate Encord client by substituting the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
                ssh_private_key_path="<private_key_path>"
            )

# Define the search criteria for the data unit and folders folder
START_FOLDER = "<start_folder_name>"
TARGET_FOLDER = "<target_folder_name>"  
DATA_UNIT_NAME = "<data_unit_name>"

# Retrieve the start folder by its name (assuming only one match)
START_FOLDER = next(user_client.find_storage_folders(search=start_folder_name, dataset_synced=None, order=FoldersSortBy.NAME, desc=False, page_size=1000))

# Retrieve the target folder by its name (assuming only one match)
TARGET_FOLDER = next(user_client.find_storage_folders(search=target_folder_name, dataset_synced=None, order=FoldersSortBy.NAME, desc=False, page_size=1000))

# Search for the specific data unit using its name, assuming it only returns one result
item = START_FOLDER.list_items(
    search=DATA_UNIT_NAME,
    is_in_dataset=None,  
    item_types=[StorageItemType.IMAGE],
    order=FoldersSortBy.NAME,
    desc=False,
    get_signed_urls=False,
    page_size=1000  # Expecting only one data unit
)

# Move the found data unit to the retrieved target folder
print(f"Moving item {item.uuid} to folder {TARGET_FOLDER.uuid}")
item.move_to_folder(TARGET_FOLDER.uuid)

​
Moving folders

You can move folders between different parent folders in Files, including moving them to the root (no parent). The following script demonstrates how a folder with the name Folder Name 3 is moved between 2 different target folders. The script must be modified to suit your needs. Ensure that you:
  • Replace <private_key_path> with the path to your private key.
  • Replace Folder Name 1 with the name of a target folder.
  • Replace Folder Name 2 with the new name of another target folder.
  • Replace Folder Name 3 with the name of the folder that is moved.
Folders
from encord import EncordUserClient
from encord.storage import FoldersSortBy

# Instantiate Encord client by substituting the path to your private key
user_client = EncordUserClient.create_with_ssh_private_key(
                ssh_private_key_path="<private_key_path>"
            )

# Search for folders by name using the find_storage_folders function
folder_1 = next(user_client.find_storage_folders(search="Folder Name 1", dataset_synced=False, order=FoldersSortBy.NAME, desc=False, page_size=1000))
folder_2 = next(user_client.find_storage_folders(search="Folder Name 2", dataset_synced=False, order=FoldersSortBy.NAME, desc=False, page_size=1000))
folder_3 = next(user_client.find_storage_folders(search="Folder Name 3", dataset_synced=False, order=FoldersSortBy.NAME, desc=False, page_size=1000))

# Move folder_3 under folder_1
folder_3.move_to_folder(folder_1.uuid)

# Move folder_3 under folder_2
folder_3.move_to_folder(folder_2.uuid)

# Move folder_3 to root folder (passing None moves it to the root level)
folder_3.move_to_folder(None)

​
Update URL for Cloud Storage Files

Use the following script, to update the Encord platform, when you update the cloud storage path. For example:
  • Moving files in cloud storage
  • Updating your folder structure in cloud storage
  • Updating URLs to use multi-region access points
from encord.utilities.storage.cloud_data_migration import update_storage_item_cloud_info
from uuid import UUID
from encord import EncordUserClient

# User input
SSH_PATH = "/Users/chris-encord/ssh-private-key.txt" # Specify the file path to your access key for authentication
FILE_ID = "11984de7-3690-4f6e-a317-8ee1ef4a926b" # Specify the File ID for the file
NEW_URL_01 = "https://storage.cloud.google.com/my-gcp-bucket/Move_Data_Unit/my-video-01.mp4" # Specify the new URL for the file

# Authentication
user_client: EncordUserClient = EncordUserClient.create_with_ssh_private_key(
    ssh_private_key_path=SSH_PATH,
    # For US platform users use "https://api.us.encord.com"
    domain="https://api.encord.com",
)

# Define the UUID of the item to update, or use its URL
item_uuid = UUID(FILE_ID)

# New URL after the file was moved
new_url = NEW_URL_01

# Update the storage item with the new URL, skip missing items
update_storage_item_cloud_info(
    user_client=user_client,
    item=item_uuid,  # or the URL of the item
    new_url=new_url,
    verify_access=True,
     skip_missing=True )

print("Successfully updated storage item with new URL.")

​
Change Cloud Storage Provider

Using the SDK you can change from one cloud storage provider (for example AWS) to another cloud storage provider (for example GCP) while maintaining data integrity.
Change Cloud Storage Provider

from encord import EncordUserClient
from uuid import UUID
from encord.http.bundle import Bundle
from encord.storage import StorageItem
from encord.utilities.storage.cloud_data_migration import update_storage_item_cloud_info

SSH_PATH = "/Users/chris-encord/ssh-private-key.txt"
EXISTING_INTEGRATION_ID = "00000000-0000-0000-0000-000000000000" # Replace with the integration ID of source cloud storage integration
NEW_INTEGRATION_ID = "00000000-0000-0000-0000-000000000000" # Replace with the integration ID of the destination cloud storage integration
STORAGE_FOLDER_HASH = "00000000-0000-0000-0000-000000000000" # Replace with the folder ID where the data resides
OLD_BUCKET_NAME_AND_FOLDER = "s3://aws-bucket-001/folder-001/" # Replace with the storage path of the source cloud storage bucket and folder
NEW_BUCKET_NAME_AND_FOLDER = "gs://gcp-bucket-01/" # Replace with the storage path of the destination cloud storage bucket and folder


# Create user client using access key
encord_client: EncordUserClient = EncordUserClient.create_with_ssh_private_key(
    ssh_private_key_path=SSH_PATH,
    # For US platform users use "https://api.us.encord.com"
    domain="https://api.encord.com",
)

storage_folder = encord_client.get_storage_folder(STORAGE_FOLDER_HASH)


# function to determine new URL
def create_new_url(existing_url: str) -> str:
    return existing_url.replace(OLD_BUCKET_NAME_AND_FOLDER, NEW_BUCKET_NAME_AND_FOLDER)

# Get item UUIDs
item_uuids = [item.uuid for item in storage_folder.list_items()]

# Compare before and after
items_before = encord_client.get_storage_items(item_uuids, sign_url=False)
assert all(str(item.integration_hash) == EXISTING_INTEGRATION_ID for item in items_before)

with Bundle() as bundle:
    for item in items_before:
        update_storage_item_cloud_info(
            encord_client,
            item,
            from_cloud_integration=EXISTING_INTEGRATION_ID,
            new_cloud_integration=NEW_INTEGRATION_ID,
            new_url=create_new_url(item.url),
            verify_access=True,
            bundle=bundle,
        )

items_after = encord_client.get_storage_items(item_uuids, sign_url=False)

assert len(items_after) == len(items_before)
assert all(str(item.integration_hash) == NEW_INTEGRATION_ID for item in items_after)

# Verify the migration worked
for item in storage_folder.list_items():
  print(f"url: {item.url}")
  print(f"integration: {item.integration_hash}")