Weaviate Simple Filesystem Backup

Simple Filesystem Backup

vector-searchbackupvector-databaseretrieval-augmented-generationllm-frameworksweaviate-featuresfunction-callingweaviate-recipesPythongenerative-ai

alph-notebooks/weaviate-recipes / simple-filesystem-backup.ipynb

Export

Run Notebooks

Contents

No cells yet

Add cells to see them here

Simple Backup

This is a simple example on how to use backup. We will run Weaviate Embedded, and we'll enable the backup-filesystem module at the environment variable ENABLE_MODULES and provide a BACKUP_FILESYSTEM_PATH where backups can be created to and restored from.

Note: the backup-filesystem is only suitable for single node clusters.

For multi-node cluster you need to use a remote bucket, such as Minio, S3, GCP, etc. Check all options here

[ ]

[2]

[3]

{}

[4]

{"build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"warning","msg":"prop len tracker file /Users/dudanogueira/.local/share/weaviate/collection1/9lo68dI5wYDu/proplengths does not exist, creating new tracker","time":"2025-05-12T17:43:53-03:00"}
{"action":"hnsw_prefill_cache_async","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"not waiting for vector cache prefill, running in background","time":"2025-05-12T17:43:53-03:00","wait_for_cache_prefill":false}
{"build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"Created shard collection1_9lo68dI5wYDu in 3.046792ms","time":"2025-05-12T17:43:53-03:00"}
{"action":"hnsw_vector_cache_prefill","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","count":1000,"index_id":"main","level":"info","limit":1000000000000,"msg":"prefilled vector cache","time":"2025-05-12T17:43:53-03:00","took":162000}
{"build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"warning","msg":"prop len tracker file /Users/dudanogueira/.local/share/weaviate/collection2/X5yq7IlGAvRQ/proplengths does not exist, creating new tracker","time":"2025-05-12T17:43:53-03:00"}
{"action":"hnsw_prefill_cache_async","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"not waiting for vector cache prefill, running in background","time":"2025-05-12T17:43:53-03:00","wait_for_cache_prefill":false}
{"build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"Created shard collection2_X5yq7IlGAvRQ in 1.238625ms","time":"2025-05-12T17:43:53-03:00"}
{"action":"hnsw_vector_cache_prefill","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","count":1000,"index_id":"main","level":"info","limit":1000000000000,"msg":"prefilled vector cache","time":"2025-05-12T17:43:53-03:00","took":57542}

UUID('f7252d8f-20ed-4059-97e4-2b138a7785a1')

[5]

Backup my-new-backup was triggered successfully!

{"action":"try_backup","backend":"filesystem","backup_id":"my-new-backup","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"","time":"2025-05-12T17:43:58-03:00","took":3982917}
{"build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","class":"Collection1","level":"info","msg":"start uploading files","time":"2025-05-12T17:43:58-03:00"}
{"build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","class":"Collection1","level":"info","msg":"finish uploading files","time":"2025-05-12T17:43:58-03:00"}
{"backup_id":"my-new-backup","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","class":"Collection1","level":"info","msg":"release backup","time":"2025-05-12T17:43:58-03:00"}
{"build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","class":"Collection2","level":"info","msg":"start uploading files","time":"2025-05-12T17:43:58-03:00"}
{"build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","class":"Collection2","level":"info","msg":"finish uploading files","time":"2025-05-12T17:43:58-03:00"}
{"build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"start uploading meta data","time":"2025-05-12T17:43:58-03:00"}
{"backup_id":"my-new-backup","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","class":"Collection2","level":"info","msg":"release backup","time":"2025-05-12T17:43:58-03:00"}
{"backup_id":"my-new-backup","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","class":"Collection2","level":"info","msg":"release backup","time":"2025-05-12T17:43:58-03:00"}
{"backup_id":"my-new-backup","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","class":"Collection1","level":"info","msg":"release backup","time":"2025-05-12T17:43:58-03:00"}
{"build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"finish uploading meta data","time":"2025-05-12T17:43:58-03:00"}
{"action":"create_backup","backup_id":"my-new-backup","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"backup completed successfully","override_bucket":"","override_path":"","time":"2025-05-12T17:43:58-03:00"}
{"action":"hnsw_condensing","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"start hnsw condensing","time":"2025-05-12T17:43:59-03:00"}
{"action":"hnsw_condensing_complete","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"completed hnsw condensing","time":"2025-05-12T17:43:59-03:00"}
{"action":"hnsw_condensing","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"start hnsw condensing","time":"2025-05-12T17:43:59-03:00"}
{"action":"hnsw_condensing_complete","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"completed hnsw condensing","time":"2025-05-12T17:43:59-03:00"}
{"action":"create","backup_id":"my-new-backup","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"coordinator: backup completed successfully","time":"2025-05-12T17:44:00-03:00"}

[6]

{"action":"backup_status","backend":"filesystem","backup_id":"my-new-backup","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"","time":"2025-05-12T17:44:07-03:00","took":140333}

[7]

error=None status=<BackupStatus.SUCCESS: 'SUCCESS'> path='/tmp/weaviate-backup/my-new-backup' backup_id='my-new-backup'

[8]

['backup_config.json', 'Embedded_at_8079']

[9]

{'startedAt': '2025-05-12T20:43:58.19477Z', 'completedAt': '2025-05-12T20:44:00.200874Z', 'id': 'my-new-backup', 'nodes': {'Embedded_at_8079': {'classes': ['Collection1', 'Collection2'], 'status': 'SUCCESS', 'error': ''}}, 'node_mapping': None, 'status': 'SUCCESS', 'version': '2.1', 'serverVersion': '1.30.0', 'leader': 'Embedded_at_8079', 'error': ''}

[10]

[11]

False

[ ]

{"action":"try_restore","backend":"filesystem","backup_id":"my-new-backup","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"","time":"2025-05-12T17:44:36-03:00","took":1797125}
{"action":"restore","backup_id":"my-new-backup","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","class":"Collection1","level":"info","msg":"successfully restored","time":"2025-05-12T17:44:36-03:00"}
{"action":"restore","backup_id":"my-new-backup","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"backup restored successfully","time":"2025-05-12T17:44:36-03:00"}
{"action":"restore","backup_id":"my-new-backup","build_git_commit":"b7b77150ac","build_go_version":"go1.24.2","build_image_tag":"HEAD","build_wv_version":"1.30.0","level":"info","msg":"coordinator: backup restored successfully","time":"2025-05-12T17:44:38-03:00"}

[14]

True

[15]

False

[16]

{'name': 'Mary Jane'}
{'name': 'John Doe'}

You can also backup specify collections to add or remove.

[ ]