Migrate data

Follow these examples to migrate data manually when using a backup is not possible. They cover all permutations between:

• a single-tenancy collection (Collection), and
• a tenant in a multi-tenancy collection (Tenant).

Additional information

The examples use two different Weaviate instances, exposed through different ports. The same process can be used for two different instances as well.

Cross-references in Weaviate are properties. As such, you can retrieve cross-reference as a part of the object.

What about cross-references?

These scripts should migrate cross-references as well.

Cross-references are properties. As such, these cursor-based exports will include them. During restoration, restore the cross-referenced (i.e. "to") object first, then the object that contains the cross-reference (i.e. "from" object).

Collection → Collection

Step 1: Create the target collection(s)

Create a collection (e.g. WineReview) at the target instance, matching the collection (e.g. WineReview) at the source instance.

PythonJavaScript/TypeScript

  API docs

More infoCode snippets in the documentation reflect the latest client library and Weaviate Database version. Check the Release notes for specific versions.

If a snippet doesn't work or you have feedback, please open a GitHub issue.

import weaviate
import weaviate.classes as wvc
from weaviate.collections import Collection
from weaviate.client import WeaviateClient

client_src = weaviate.connect_to_local(
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

client_tgt = weaviate.connect_to_local(
    port=8090,
    grpc_port=50061,
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

def create_collection(client_in: WeaviateClient, collection_name: str, enable_mt=False):

    reviews = client_in.collections.create(
        name=collection_name,
        multi_tenancy_config=wvc.config.Configure.multi_tenancy(enabled=enable_mt),
        # Additional settings not shown
    )

    return reviews


reviews_tgt = create_collection(client_tgt, "WineReview", enable_mt=False)

import weaviate, { Collection, WeaviateClient } from 'weaviate-client'

let client_src: WeaviateClient,client_tgt: WeaviateClient;
let reviews_mt_tgt, reviews_mt_src;
let reviews_tgt, reviews_src;

client_src = await weaviate.connectToLocal({
})

client_tgt = await weaviate.connectToLocal({
    port: 8090,
    grpcPort: 50061,
})

async function createCollection(clientIn: WeaviateClient, collectionName: string, enableMt: boolean) {
    let reviews = clientIn.collections.create({
        name: collectionName,
        multiTenancy: {
            enabled: enableMt
        },
        // Additional settings not shown
    })

    return reviews
}

reviews_tgt = await createCollection(client_tgt, "WineReview", false)

Step 2: Migrate the data

Migrate:

• The source collection data in the client_src instance
• to target collection in the client_tgt instance

PythonJavaScript/TypeScript

  API docs

More infoCode snippets in the documentation reflect the latest client library and Weaviate Database version. Check the Release notes for specific versions.

If a snippet doesn't work or you have feedback, please open a GitHub issue.

def migrate_data(collection_src: Collection, collection_tgt: Collection):

    with collection_tgt.batch.fixed_size(batch_size=100) as batch:
        for q in tqdm(collection_src.iterator(include_vector=True)):
            batch.add_object(
                properties=q.properties,
                vector={
                    "default": q.vector["default"],
                },
                uuid=q.uuid
            )

    return True


reviews_src = client_src.collections.get("WineReview")
reviews_tgt = client_tgt.collections.get("WineReview")

migrate_data(reviews_src, reviews_tgt)

client_src.close()
client_tgt.close()

let reviews_tgt, reviews_src;
reviews_src = client_src.collections.use("WineReview")
reviews_tgt = client_tgt.collections.use("WineReview")

let maxItems = await reviews_src.length()
let counter: number = 0

async function migrateData(collection_src: Collection, collection_tgt: Collection) {
    let itemsToInsert = []
    const promises = []

    for await (const item of collection_src.iterator({ includeVector: true })) {
        // Check if we've reached the maximum items
        if (counter >= maxItems) {
            console.log(`Reached maximum items limit of ${maxItems}`);
            break;
        }

        counter++;
        if (counter % 500 == 0)
            console.log(`Import: ${counter}`)

        let objectToInsert = {
            properties: item.properties,
            vectors: item.vectors.default,
            uuid: item.uuid,
        }

        // Add object to batching array
        itemsToInsert.push(objectToInsert)

        if (itemsToInsert.length == 500 || counter == maxItems) {
            try {
                const response = await collection_tgt.data.insertMany(itemsToInsert);

                if (response.hasErrors) {
                    throw new Error("Error in batch import!");
                }

                console.log(`Successfully imported batch of ${itemsToInsert.length} items`);
                itemsToInsert = [];
            } catch (error) {
                console.error('Error importing batch:', error);
            }
        }
    }
   
}

migrateData(reviews_src, reviews_tgt)

client_src.close()
client_tgt.close()

Collection → Tenant

Step 1: Create the target collection(s)

Create a collection (e.g. WineReview) at the target instance, matching the collection (e.g. WineReview) at the source instance, and enable multi-tenancy for the target collection.

PythonJavaScript/TypeScript

  API docs

More infoCode snippets in the documentation reflect the latest client library and Weaviate Database version. Check the Release notes for specific versions.

If a snippet doesn't work or you have feedback, please open a GitHub issue.

import weaviate
import weaviate.classes as wvc
from weaviate.collections import Collection
from weaviate.client import WeaviateClient

client_src = weaviate.connect_to_local(
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

client_tgt = weaviate.connect_to_local(
    port=8090,
    grpc_port=50061,
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

def create_collection(client_in: WeaviateClient, collection_name: str, enable_mt=False):

    reviews = client_in.collections.create(
        name=collection_name,
        multi_tenancy_config=wvc.config.Configure.multi_tenancy(enabled=enable_mt),
        # Additional settings not shown
    )

    return reviews


reviews_mt_tgt = create_collection(client_tgt, "WineReviewMT", enable_mt=True)

import weaviate, { Collection, WeaviateClient } from 'weaviate-client'

let client_src: WeaviateClient,client_tgt: WeaviateClient;
let reviews_mt_tgt, reviews_mt_src;

client_src = await weaviate.connectToLocal({
})

client_tgt = await weaviate.connectToLocal({
    port: 8090,
    grpcPort: 50061,
})

async function createCollection(clientIn: WeaviateClient, collectionName: string, enableMt: boolean) {
    let reviews = clientIn.collections.create({
        name: collectionName,
        multiTenancy: {
            enabled: enableMt
        },
        // Additional settings not shown
    })

    return reviews
}

reviews_mt_tgt = createCollection(client_tgt, "WineReviewMT", true)

Step 2: Create the tenant(s)

Add tenants at the target instance before adding data objects.

PythonJavaScript/TypeScript

  API docs

More infoCode snippets in the documentation reflect the latest client library and Weaviate Database version. Check the Release notes for specific versions.

If a snippet doesn't work or you have feedback, please open a GitHub issue.

tenants_tgt = [wvc.tenants.Tenant(name="tenantA"), wvc.tenants.Tenant(name="tenantB")]

reviews_mt_tgt = client_tgt.collections.get("WineReviewMT")
reviews_mt_tgt.tenants.create(tenants_tgt)

let tenantsTgt = [
    { name: 'tenantA'},
    { name: 'tenantB'}
  ]

reviews_mt_tgt = client_tgt.collections.use("WineReviewMT")
reviews_mt_tgt.tenants.create(tenantsTgt)

Step 3: Migrate the data

Migrate:

• The source collection data in the client_src instance
• to target tenant data from target collection in the client_tgt instance

PythonJavaScript/TypeScript

  API docs

More infoCode snippets in the documentation reflect the latest client library and Weaviate Database version. Check the Release notes for specific versions.

If a snippet doesn't work or you have feedback, please open a GitHub issue.

def migrate_data(collection_src: Collection, collection_tgt: Collection):

    with collection_tgt.batch.fixed_size(batch_size=100) as batch:
        for q in tqdm(collection_src.iterator(include_vector=True)):
            batch.add_object(
                properties=q.properties,
                vector={
                    "default": q.vector["default"],
                },
                uuid=q.uuid
            )

    return True


reviews_src = client_src.collections.get("WineReview")
reviews_mt_tgt = client_tgt.collections.get("WineReviewMT")
reviews_tgt_tenant_a = reviews_mt_tgt.with_tenant(tenants_tgt[0].name)

migrate_data(reviews_src, reviews_tgt_tenant_a)

client_src.close()
client_tgt.close()

let reviews_tgt, reviews_src;

reviews_src = client_src.collections.use("WineReview")
reviews_mt_tgt = client_tgt.collections.use("WineReviewMT")

let maxItems = await reviews_src.length()
let counter: number = 0

async function migrateData(collection_src: Collection, collection_tgt: Collection) {
    let itemsToInsert = []
    const promises = []

    for await (const item of collection_src.iterator({ includeVector: true })) {
        // Check if we've reached the maximum items
        if (counter >= maxItems) {
            console.log(`Reached maximum items limit of ${maxItems}`);
            break;
        }

        counter++;
        if (counter % 500 == 0)
            console.log(`Import: ${counter}`)

        let objectToInsert = {
            properties: item.properties,
            vectors: item.vectors.default,
            uuid: item.uuid,
        }

        // Add object to batching array
        itemsToInsert.push(objectToInsert)

        if (itemsToInsert.length == 500 || counter == maxItems) {
            try {
                const response = await collection_tgt.data.insertMany(itemsToInsert);

                if (response.hasErrors) {
                    throw new Error("Error in batch import!");
                }

                console.log(`Successfully imported batch of ${itemsToInsert.length} items`);
                itemsToInsert = [];
            } catch (error) {
                console.error('Error importing batch:', error);
            }
        }
    }
   
}

let reviews_tgt_tenant_a = reviews_mt_tgt.withTenant(tenantsTgt[0].name)
migrateData(reviews_src, reviews_tgt_tenant_a)

client_src.close()
client_tgt.close()

Tenant → Collection

Step 1: Create the target collection(s)

Create a collection (e.g. WineReview) at the target instance, matching the collection (e.g. WineReview) at the source instance, and enable multi-tenancy for the target collection.

PythonJavaScript/TypeScript

  API docs

More infoCode snippets in the documentation reflect the latest client library and Weaviate Database version. Check the Release notes for specific versions.

If a snippet doesn't work or you have feedback, please open a GitHub issue.

import weaviate
import weaviate.classes as wvc
from weaviate.collections import Collection
from weaviate.client import WeaviateClient

client_src = weaviate.connect_to_local(
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

client_tgt = weaviate.connect_to_local(
    port=8090,
    grpc_port=50061,
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

def create_collection(client_in: WeaviateClient, collection_name: str, enable_mt=False):

    reviews = client_in.collections.create(
        name=collection_name,
        multi_tenancy_config=wvc.config.Configure.multi_tenancy(enabled=enable_mt),
        # Additional settings not shown
    )

    return reviews


reviews_tgt = create_collection(client_tgt, "WineReview", enable_mt=False)

import weaviate, { Collection, WeaviateClient } from 'weaviate-client'

let client_src: WeaviateClient,client_tgt: WeaviateClient;
let reviews_mt_tgt, reviews_mt_src;

client_src = await weaviate.connectToLocal({
})

client_tgt = await weaviate.connectToLocal({
    port: 8090,
    grpcPort: 50061,
})

async function createCollection(clientIn: WeaviateClient, collectionName: string, enableMt: boolean) {
    let reviews = clientIn.collections.create({
        name: collectionName,
        multiTenancy: {
            enabled: enableMt
        },
        // Additional settings not shown
    })

    return reviews
}

reviews_tgt = await createCollection(client_tgt, "WineReview", false)

Step 2: Migrate the data

Migrate:

• The source tenant data from source collection in the client_src instance
• to target collection in the client_tgt instance

PythonJavaScript/TypeScript

  API docs

More infoCode snippets in the documentation reflect the latest client library and Weaviate Database version. Check the Release notes for specific versions.

If a snippet doesn't work or you have feedback, please open a GitHub issue.

def migrate_data(collection_src: Collection, collection_tgt: Collection):

    with collection_tgt.batch.fixed_size(batch_size=100) as batch:
        for q in tqdm(collection_src.iterator(include_vector=True)):
            batch.add_object(
                properties=q.properties,
                vector={
                    "default": q.vector["default"],
                },
                uuid=q.uuid
            )

    return True


reviews_src = client_src.collections.get("WineReviewMT")
reviews_src_tenant_a = reviews_src.with_tenant("tenantA")
reviews_tgt = client_tgt.collections.get("WineReview")

migrate_data(reviews_src_tenant_a, reviews_tgt)

client_src.close()
client_tgt.close()

let reviews_tgt, reviews_src;

let maxItems = await reviews_src.length()
let counter: number = 0

async function migrateData(collection_src: Collection, collection_tgt: Collection) {
    let itemsToInsert = []
    const promises = []

    for await (const item of collection_src.iterator({ includeVector: true })) {
        // Check if we've reached the maximum items
        if (counter >= maxItems) {
            console.log(`Reached maximum items limit of ${maxItems}`);
            break;
        }

        counter++;
        if (counter % 500 == 0)
            console.log(`Import: ${counter}`)

        let objectToInsert = {
            properties: item.properties,
            vectors: item.vectors.default,
            uuid: item.uuid,
        }

        // Add object to batching array
        itemsToInsert.push(objectToInsert)

        if (itemsToInsert.length == 500 || counter == maxItems) {
            try {
                const response = await collection_tgt.data.insertMany(itemsToInsert);

                if (response.hasErrors) {
                    throw new Error("Error in batch import!");
                }

                console.log(`Successfully imported batch of ${itemsToInsert.length} items`);
                itemsToInsert = [];
            } catch (error) {
                console.error('Error importing batch:', error);
            }
        }
    }
   
}

let reviews_src_tenant_a;

reviews_src = client_src.collections.use("WineReviewMT")
reviews_src_tenant_a = reviews_src.withTenant("tenantA")
reviews_tgt = client_tgt.collections.use("WineReview")

migrateData(reviews_src_tenant_a, reviews_tgt)

client_src.close()
client_tgt.close()

Tenant → Tenant

Step 1: Create the target collection(s)

Create a collection (e.g. WineReview) at the target instance, matching the collection (e.g. WineReview) at the source instance including enabling multi-tenancy.

PythonJavaScript/TypeScript

  API docs

More infoCode snippets in the documentation reflect the latest client library and Weaviate Database version. Check the Release notes for specific versions.

If a snippet doesn't work or you have feedback, please open a GitHub issue.

import weaviate
import weaviate.classes as wvc
from weaviate.collections import Collection
from weaviate.client import WeaviateClient

client_src = weaviate.connect_to_local(
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

client_tgt = weaviate.connect_to_local(
    port=8090,
    grpc_port=50061,
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

def create_collection(client_in: WeaviateClient, collection_name: str, enable_mt=False):

    reviews = client_in.collections.create(
        name=collection_name,
        multi_tenancy_config=wvc.config.Configure.multi_tenancy(enabled=enable_mt),
        # Additional settings not shown
    )

    return reviews


reviews_mt_tgt = create_collection(client_tgt, "WineReviewMT", enable_mt=True)

import weaviate, { Collection, WeaviateClient } from 'weaviate-client'

let client_src: WeaviateClient,client_tgt: WeaviateClient;
let reviews_mt_tgt, reviews_mt_src;

client_src = await weaviate.connectToLocal({
})

client_tgt = await weaviate.connectToLocal({
    port: 8090,
    grpcPort: 50061,
})

async function createCollection(clientIn: WeaviateClient, collectionName: string, enableMt: boolean) {
    let reviews = clientIn.collections.create({
        name: collectionName,
        multiTenancy: {
            enabled: enableMt
        },
        // Additional settings not shown
    })

    return reviews
}

reviews_mt_tgt = createCollection(client_tgt, "WineReviewMT", true)

Step 2: Create the tenant(s)

Add tenants at the target instance before adding data objects.

PythonJavaScript/TypeScript

  API docs

More infoCode snippets in the documentation reflect the latest client library and Weaviate Database version. Check the Release notes for specific versions.

If a snippet doesn't work or you have feedback, please open a GitHub issue.

tenants_tgt = [wvc.tenants.Tenant(name="tenantA"), wvc.tenants.Tenant(name="tenantB")]

reviews_mt_tgt = client_tgt.collections.get("WineReviewMT")
reviews_mt_tgt.tenants.create(tenants_tgt)

let tenantsTgt = [
    { name: 'tenantA'},
    { name: 'tenantB'}
  ]

reviews_mt_tgt = client_tgt.collections.use("WineReviewMT")
reviews_mt_tgt.tenants.create(tenantsTgt)

Step 3: Migrate the data

Migrate:

• The source tenant data from source collection in the client_src instance
• to target tenant data from target collection in the client_tgt instance

PythonJavaScript/TypeScript

  API docs

More infoCode snippets in the documentation reflect the latest client library and Weaviate Database version. Check the Release notes for specific versions.

If a snippet doesn't work or you have feedback, please open a GitHub issue.

def migrate_data(collection_src: Collection, collection_tgt: Collection):

    with collection_tgt.batch.fixed_size(batch_size=100) as batch:
        for q in tqdm(collection_src.iterator(include_vector=True)):
            batch.add_object(
                properties=q.properties,
                vector={
                    "default": q.vector["default"],
                },
                uuid=q.uuid
            )

    return True


reviews_mt_src = client_src.collections.get("WineReviewMT")
reviews_src_tenant_a = reviews_mt_src.with_tenant("tenantA")
reviews_mt_tgt = client_tgt.collections.get("WineReviewMT")
reviews_tgt_tenant_a = reviews_mt_tgt.with_tenant(tenants_tgt[0].name)

migrate_data(reviews_src_tenant_a, reviews_tgt_tenant_a)

client_src.close()
client_tgt.close()

import weaviate, { Collection, WeaviateClient } from 'weaviate-client'

let client_src: WeaviateClient,client_tgt: WeaviateClient;
let reviews_mt_tgt, reviews_mt_src;

let maxItems = await reviews_src.length()
let counter: number = 0

async function migrateData(collection_src: Collection, collection_tgt: Collection) {
    let itemsToInsert = []
    const promises = []

    for await (const item of collection_src.iterator({ includeVector: true })) {
        // Check if we've reached the maximum items
        if (counter >= maxItems) {
            console.log(`Reached maximum items limit of ${maxItems}`);
            break;
        }

        counter++;
        if (counter % 500 == 0)
            console.log(`Import: ${counter}`)

        let objectToInsert = {
            properties: item.properties,
            vectors: item.vectors.default,
            uuid: item.uuid,
        }

        // Add object to batching array
        itemsToInsert.push(objectToInsert)

        if (itemsToInsert.length == 500 || counter == maxItems) {
            try {
                const response = await collection_tgt.data.insertMany(itemsToInsert);

                if (response.hasErrors) {
                    throw new Error("Error in batch import!");
                }

                console.log(`Successfully imported batch of ${itemsToInsert.length} items`);
                itemsToInsert = [];
            } catch (error) {
                console.error('Error importing batch:', error);
            }
        }
    }
   
}
// Variables initialized above
reviews_mt_src = client_src.collections.use("WineReviewMT")
reviews_src_tenant_a = reviews_mt_src.withTenant("tenantA")
reviews_mt_tgt = client_tgt.collections.use("WineReviewMT")
reviews_tgt_tenant_a = reviews_mt_tgt.withTenant(tenantsTgt[0].name)

migrateData(reviews_src_tenant_a, reviews_tgt_tenant_a)

client_src.close()
client_tgt.close()

Related pages

• Connect to Weaviate
• Cursor API
• Multi-tenancy operations

Questions and feedback

If you have any questions or feedback, let us know in the user forum.

Technical questions

If you have questions feel free to post on our Community forum.

Documentation feedback

Leave feedback by opening a GitHub issue.