Migrate data

Follow these examples to migrate data manually when using a backup is not possible. They cover all permutations between:

• a single-tenancy collection (Collection), and
• a tenant in a multi-tenancy collection (Tenant).

Additional information

The examples use two different Weaviate instances, exposed through different ports. The same process can be used for two different instances as well.

Cross-references in Weaviate are properties. As such, you can retrieve cross-reference as a part of the object.

What about cross-references?

These scripts should migrate cross-references as well.

Cross-references are properties. As such, these cursor-based exports will include them. During restoration, restore the cross-referenced (i.e. "to") object first, then the object that contains the cross-reference (i.e. "from" object).

Collection → Collection

Step 1: Create the target collection(s)

Create a collection (e.g. WineReview) at the target instance, matching the collection (e.g. WineReview) at the source instance.

Python

import weaviate
import weaviate.classes as wvc
from weaviate.collections import Collection
from weaviate.client import WeaviateClient

client_src = weaviate.connect_to_local(
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

client_tgt = weaviate.connect_to_local(
    port=8090,
    grpc_port=50061,
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

def create_collection(client_in: WeaviateClient, collection_name: str, enable_mt=False):

    reviews = client_in.collections.create(
        name=collection_name,
        multi_tenancy_config=wvc.config.Configure.multi_tenancy(enabled=enable_mt),
        # Additional settings not shown
    )

    return reviews


reviews_tgt = create_collection(client_tgt, "WineReview", enable_mt=False)

  API docs

JS/TS

import weaviate, { Collection, WeaviateClient } from 'weaviate-client'

let client_src: WeaviateClient,client_tgt: WeaviateClient;
let reviews_mt_tgt, reviews_mt_src;
let reviews_tgt, reviews_src;

client_src = await weaviate.connectToLocal({
})

client_tgt = await weaviate.connectToLocal({
    port: 8090,
    grpcPort: 50061,
})

async function createCollection(clientIn: WeaviateClient, collectionName: string, enableMt: boolean) {
    let reviews = clientIn.collections.create({
        name: collectionName,
        multiTenancy: {
            enabled: enableMt
        },
        // Additional settings not shown
    })

    return reviews
}

reviews_tgt = await createCollection(client_tgt, "WineReview", false)

  API docs

Step 2: Migrate the data

Migrate:

• The source collection data in the client_src instance
• to target collection in the client_tgt instance

Python

def migrate_data(collection_src: Collection, collection_tgt: Collection):

    with collection_tgt.batch.fixed_size(batch_size=100) as batch:
        for q in tqdm(collection_src.iterator(include_vector=True)):
            batch.add_object(
                properties=q.properties,
                vector={
                    "default": q.vector["default"],
                },
                uuid=q.uuid
            )

    return True


reviews_src = client_src.collections.get("WineReview")
reviews_tgt = client_tgt.collections.get("WineReview")

migrate_data(reviews_src, reviews_tgt)

client_src.close()
client_tgt.close()

  API docs

JS/TS

let reviews_tgt, reviews_src;
reviews_src = client_src.collections.use("WineReview")
reviews_tgt = client_tgt.collections.use("WineReview")

let maxItems = await reviews_src.length()
let counter: number = 0

async function migrateData(collection_src: Collection, collection_tgt: Collection) {
    let itemsToInsert = []
    const promises = []

    for await (const item of collection_src.iterator({ includeVector: true })) {
        // Check if we've reached the maximum items
        if (counter >= maxItems) {
            console.log(`Reached maximum items limit of ${maxItems}`);
            break;
        }

        counter++;
        if (counter % 500 == 0)
            console.log(`Import: ${counter}`)

        let objectToInsert = {
            properties: item.properties,
            vectors: item.vectors.default,
            uuid: item.uuid,
        }

        // Add object to batching array
        itemsToInsert.push(objectToInsert)

        if (itemsToInsert.length == 500 || counter == maxItems) {
            try {
                const response = await collection_tgt.data.insertMany(itemsToInsert);

                if (response.hasErrors) {
                    throw new Error("Error in batch import!");
                }

                console.log(`Successfully imported batch of ${itemsToInsert.length} items`);
                itemsToInsert = [];
            } catch (error) {
                console.error('Error importing batch:', error);
            }
        }
    }
   
}

migrateData(reviews_src, reviews_tgt)

client_src.close()
client_tgt.close()

  API docs

Collection → Tenant

Step 1: Create the target collection(s)

Create a collection (e.g. WineReview) at the target instance, matching the collection (e.g. WineReview) at the source instance, and enable multi-tenancy for the target collection.

Python

import weaviate
import weaviate.classes as wvc
from weaviate.collections import Collection
from weaviate.client import WeaviateClient

client_src = weaviate.connect_to_local(
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

client_tgt = weaviate.connect_to_local(
    port=8090,
    grpc_port=50061,
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

def create_collection(client_in: WeaviateClient, collection_name: str, enable_mt=False):

    reviews = client_in.collections.create(
        name=collection_name,
        multi_tenancy_config=wvc.config.Configure.multi_tenancy(enabled=enable_mt),
        # Additional settings not shown
    )

    return reviews


reviews_mt_tgt = create_collection(client_tgt, "WineReviewMT", enable_mt=True)

  API docs

JS/TS

import weaviate, { Collection, WeaviateClient } from 'weaviate-client'

let client_src: WeaviateClient,client_tgt: WeaviateClient;
let reviews_mt_tgt, reviews_mt_src;

client_src = await weaviate.connectToLocal({
})

client_tgt = await weaviate.connectToLocal({
    port: 8090,
    grpcPort: 50061,
})

async function createCollection(clientIn: WeaviateClient, collectionName: string, enableMt: boolean) {
    let reviews = clientIn.collections.create({
        name: collectionName,
        multiTenancy: {
            enabled: enableMt
        },
        // Additional settings not shown
    })

    return reviews
}

reviews_mt_tgt = createCollection(client_tgt, "WineReviewMT", true)

  API docs

Step 2: Create the tenant(s)

Add tenants at the target instance before adding data objects.

Python

tenants_tgt = [wvc.tenants.Tenant(name="tenantA"), wvc.tenants.Tenant(name="tenantB")]

reviews_mt_tgt = client_tgt.collections.get("WineReviewMT")
reviews_mt_tgt.tenants.create(tenants_tgt)

  API docs

JS/TS

let tenantsTgt = [
    { name: 'tenantA'},
    { name: 'tenantB'}
  ]

reviews_mt_tgt = client_tgt.collections.use("WineReviewMT")
reviews_mt_tgt.tenants.create(tenantsTgt)

  API docs

Step 3: Migrate the data

Migrate:

• The source collection data in the client_src instance
• to target tenant data from target collection in the client_tgt instance

Python

def migrate_data(collection_src: Collection, collection_tgt: Collection):

    with collection_tgt.batch.fixed_size(batch_size=100) as batch:
        for q in tqdm(collection_src.iterator(include_vector=True)):
            batch.add_object(
                properties=q.properties,
                vector={
                    "default": q.vector["default"],
                },
                uuid=q.uuid
            )

    return True


reviews_src = client_src.collections.get("WineReview")
reviews_mt_tgt = client_tgt.collections.get("WineReviewMT")
reviews_tgt_tenant_a = reviews_mt_tgt.with_tenant(tenants_tgt[0].name)

migrate_data(reviews_src, reviews_tgt_tenant_a)

client_src.close()
client_tgt.close()

  API docs

JS/TS

let reviews_tgt, reviews_src;

reviews_src = client_src.collections.use("WineReview")
reviews_mt_tgt = client_tgt.collections.use("WineReviewMT")

let maxItems = await reviews_src.length()
let counter: number = 0

async function migrateData(collection_src: Collection, collection_tgt: Collection) {
    let itemsToInsert = []
    const promises = []

    for await (const item of collection_src.iterator({ includeVector: true })) {
        // Check if we've reached the maximum items
        if (counter >= maxItems) {
            console.log(`Reached maximum items limit of ${maxItems}`);
            break;
        }

        counter++;
        if (counter % 500 == 0)
            console.log(`Import: ${counter}`)

        let objectToInsert = {
            properties: item.properties,
            vectors: item.vectors.default,
            uuid: item.uuid,
        }

        // Add object to batching array
        itemsToInsert.push(objectToInsert)

        if (itemsToInsert.length == 500 || counter == maxItems) {
            try {
                const response = await collection_tgt.data.insertMany(itemsToInsert);

                if (response.hasErrors) {
                    throw new Error("Error in batch import!");
                }

                console.log(`Successfully imported batch of ${itemsToInsert.length} items`);
                itemsToInsert = [];
            } catch (error) {
                console.error('Error importing batch:', error);
            }
        }
    }
   
}

let reviews_tgt_tenant_a = reviews_mt_tgt.withTenant(tenantsTgt[0].name)
migrateData(reviews_src, reviews_tgt_tenant_a)

client_src.close()
client_tgt.close()

  API docs

Tenant → Collection

Step 1: Create the target collection(s)

Create a collection (e.g. WineReview) at the target instance, matching the collection (e.g. WineReview) at the source instance, and enable multi-tenancy for the target collection.

Python

import weaviate
import weaviate.classes as wvc
from weaviate.collections import Collection
from weaviate.client import WeaviateClient

client_src = weaviate.connect_to_local(
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

client_tgt = weaviate.connect_to_local(
    port=8090,
    grpc_port=50061,
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

def create_collection(client_in: WeaviateClient, collection_name: str, enable_mt=False):

    reviews = client_in.collections.create(
        name=collection_name,
        multi_tenancy_config=wvc.config.Configure.multi_tenancy(enabled=enable_mt),
        # Additional settings not shown
    )

    return reviews


reviews_tgt = create_collection(client_tgt, "WineReview", enable_mt=False)

  API docs

JS/TS

import weaviate, { Collection, WeaviateClient } from 'weaviate-client'

let client_src: WeaviateClient,client_tgt: WeaviateClient;
let reviews_mt_tgt, reviews_mt_src;

client_src = await weaviate.connectToLocal({
})

client_tgt = await weaviate.connectToLocal({
    port: 8090,
    grpcPort: 50061,
})

async function createCollection(clientIn: WeaviateClient, collectionName: string, enableMt: boolean) {
    let reviews = clientIn.collections.create({
        name: collectionName,
        multiTenancy: {
            enabled: enableMt
        },
        // Additional settings not shown
    })

    return reviews
}

reviews_tgt = await createCollection(client_tgt, "WineReview", false)

  API docs

Step 2: Migrate the data

Migrate:

• The source tenant data from source collection in the client_src instance
• to target collection in the client_tgt instance

Python

def migrate_data(collection_src: Collection, collection_tgt: Collection):

    with collection_tgt.batch.fixed_size(batch_size=100) as batch:
        for q in tqdm(collection_src.iterator(include_vector=True)):
            batch.add_object(
                properties=q.properties,
                vector={
                    "default": q.vector["default"],
                },
                uuid=q.uuid
            )

    return True


reviews_src = client_src.collections.get("WineReviewMT")
reviews_src_tenant_a = reviews_src.with_tenant("tenantA")
reviews_tgt = client_tgt.collections.get("WineReview")

migrate_data(reviews_src_tenant_a, reviews_tgt)

client_src.close()
client_tgt.close()

  API docs

JS/TS

let reviews_tgt, reviews_src;

let maxItems = await reviews_src.length()
let counter: number = 0

async function migrateData(collection_src: Collection, collection_tgt: Collection) {
    let itemsToInsert = []
    const promises = []

    for await (const item of collection_src.iterator({ includeVector: true })) {
        // Check if we've reached the maximum items
        if (counter >= maxItems) {
            console.log(`Reached maximum items limit of ${maxItems}`);
            break;
        }

        counter++;
        if (counter % 500 == 0)
            console.log(`Import: ${counter}`)

        let objectToInsert = {
            properties: item.properties,
            vectors: item.vectors.default,
            uuid: item.uuid,
        }

        // Add object to batching array
        itemsToInsert.push(objectToInsert)

        if (itemsToInsert.length == 500 || counter == maxItems) {
            try {
                const response = await collection_tgt.data.insertMany(itemsToInsert);

                if (response.hasErrors) {
                    throw new Error("Error in batch import!");
                }

                console.log(`Successfully imported batch of ${itemsToInsert.length} items`);
                itemsToInsert = [];
            } catch (error) {
                console.error('Error importing batch:', error);
            }
        }
    }
   
}

let reviews_src_tenant_a;

reviews_src = client_src.collections.use("WineReviewMT")
reviews_src_tenant_a = reviews_src.withTenant("tenantA")
reviews_tgt = client_tgt.collections.use("WineReview")

migrateData(reviews_src_tenant_a, reviews_tgt)

client_src.close()
client_tgt.close()

  API docs

Tenant → Tenant

Step 1: Create the target collection(s)

Create a collection (e.g. WineReview) at the target instance, matching the collection (e.g. WineReview) at the source instance including enabling multi-tenancy.

Python

import weaviate
import weaviate.classes as wvc
from weaviate.collections import Collection
from weaviate.client import WeaviateClient

client_src = weaviate.connect_to_local(
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

client_tgt = weaviate.connect_to_local(
    port=8090,
    grpc_port=50061,
    headers={
        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
    }
)

def create_collection(client_in: WeaviateClient, collection_name: str, enable_mt=False):

    reviews = client_in.collections.create(
        name=collection_name,
        multi_tenancy_config=wvc.config.Configure.multi_tenancy(enabled=enable_mt),
        # Additional settings not shown
    )

    return reviews


reviews_mt_tgt = create_collection(client_tgt, "WineReviewMT", enable_mt=True)

  API docs

JS/TS

import weaviate, { Collection, WeaviateClient } from 'weaviate-client'

let client_src: WeaviateClient,client_tgt: WeaviateClient;
let reviews_mt_tgt, reviews_mt_src;

client_src = await weaviate.connectToLocal({
})

client_tgt = await weaviate.connectToLocal({
    port: 8090,
    grpcPort: 50061,
})

async function createCollection(clientIn: WeaviateClient, collectionName: string, enableMt: boolean) {
    let reviews = clientIn.collections.create({
        name: collectionName,
        multiTenancy: {
            enabled: enableMt
        },
        // Additional settings not shown
    })

    return reviews
}

reviews_mt_tgt = createCollection(client_tgt, "WineReviewMT", true)

  API docs

Step 2: Create the tenant(s)

Add tenants at the target instance before adding data objects.

Python

tenants_tgt = [wvc.tenants.Tenant(name="tenantA"), wvc.tenants.Tenant(name="tenantB")]

reviews_mt_tgt = client_tgt.collections.get("WineReviewMT")
reviews_mt_tgt.tenants.create(tenants_tgt)

  API docs

JS/TS

let tenantsTgt = [
    { name: 'tenantA'},
    { name: 'tenantB'}
  ]

reviews_mt_tgt = client_tgt.collections.use("WineReviewMT")
reviews_mt_tgt.tenants.create(tenantsTgt)

  API docs

Step 3: Migrate the data

Migrate:

• The source tenant data from source collection in the client_src instance
• to target tenant data from target collection in the client_tgt instance

Python

def migrate_data(collection_src: Collection, collection_tgt: Collection):

    with collection_tgt.batch.fixed_size(batch_size=100) as batch:
        for q in tqdm(collection_src.iterator(include_vector=True)):
            batch.add_object(
                properties=q.properties,
                vector={
                    "default": q.vector["default"],
                },
                uuid=q.uuid
            )

    return True


reviews_mt_src = client_src.collections.get("WineReviewMT")
reviews_src_tenant_a = reviews_mt_src.with_tenant("tenantA")
reviews_mt_tgt = client_tgt.collections.get("WineReviewMT")
reviews_tgt_tenant_a = reviews_mt_tgt.with_tenant(tenants_tgt[0].name)

migrate_data(reviews_src_tenant_a, reviews_tgt_tenant_a)

client_src.close()
client_tgt.close()

  API docs

JS/TS

import weaviate, { Collection, WeaviateClient } from 'weaviate-client'

let client_src: WeaviateClient,client_tgt: WeaviateClient;
let reviews_mt_tgt, reviews_mt_src;

let maxItems = await reviews_src.length()
let counter: number = 0

async function migrateData(collection_src: Collection, collection_tgt: Collection) {
    let itemsToInsert = []
    const promises = []

    for await (const item of collection_src.iterator({ includeVector: true })) {
        // Check if we've reached the maximum items
        if (counter >= maxItems) {
            console.log(`Reached maximum items limit of ${maxItems}`);
            break;
        }

        counter++;
        if (counter % 500 == 0)
            console.log(`Import: ${counter}`)

        let objectToInsert = {
            properties: item.properties,
            vectors: item.vectors.default,
            uuid: item.uuid,
        }

        // Add object to batching array
        itemsToInsert.push(objectToInsert)

        if (itemsToInsert.length == 500 || counter == maxItems) {
            try {
                const response = await collection_tgt.data.insertMany(itemsToInsert);

                if (response.hasErrors) {
                    throw new Error("Error in batch import!");
                }

                console.log(`Successfully imported batch of ${itemsToInsert.length} items`);
                itemsToInsert = [];
            } catch (error) {
                console.error('Error importing batch:', error);
            }
        }
    }
   
}
// Variables initialized above
reviews_mt_src = client_src.collections.use("WineReviewMT")
reviews_src_tenant_a = reviews_mt_src.withTenant("tenantA")
reviews_mt_tgt = client_tgt.collections.use("WineReviewMT")
reviews_tgt_tenant_a = reviews_mt_tgt.withTenant(tenantsTgt[0].name)

migrateData(reviews_src_tenant_a, reviews_tgt_tenant_a)

client_src.close()
client_tgt.close()

  API docs

Related pages

• Connect to Weaviate
• Cursor API
• Multi-tenancy operations

Questions and feedback

If you have any questions or feedback, let us know in the user forum.

Technical questions

If you have questions feel free to post on our Community forum.

Documentation feedback

Leave feedback by opening a GitHub issue.