Skip to content

Commit

Permalink
fix: also persist metadata files on import to persistent db
Browse files Browse the repository at this point in the history
  • Loading branch information
iwilltry42 committed Jul 1, 2024
1 parent fde4a02 commit 49eb498
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 18 deletions.
41 changes: 24 additions & 17 deletions collection.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,23 +111,7 @@ func newCollection(name string, metadata map[string]string, embed EmbeddingFunc,
safeName := hash2hex(name)
c.persistDirectory = filepath.Join(dbDir, safeName)
c.compress = compress
// Persist name and metadata
metadataPath := filepath.Join(c.persistDirectory, metadataFileName)
metadataPath += ".gob"
if c.compress {
metadataPath += ".gz"
}
pc := struct {
Name string
Metadata map[string]string
}{
Name: name,
Metadata: m,
}
err := persistToFile(metadataPath, pc, compress, "")
if err != nil {
return nil, fmt.Errorf("couldn't persist collection metadata: %w", err)
}
return c, c.persistMetadata()
}

return c, nil
Expand Down Expand Up @@ -545,3 +529,26 @@ func (c *Collection) getDocPath(docID string) string {
}
return docPath
}

// persistMetadata persists the collection metadata to disk
func (c *Collection) persistMetadata() error {
// Persist name and metadata
metadataPath := filepath.Join(c.persistDirectory, metadataFileName)
metadataPath += ".gob"
if c.compress {
metadataPath += ".gz"
}
pc := struct {
Name string
Metadata map[string]string
}{
Name: c.Name,
Metadata: c.metadata,
}
err := persistToFile(metadataPath, pc, c.compress, "")
if err != nil {
return err
}

return nil
}
10 changes: 9 additions & 1 deletion db.go
Original file line number Diff line number Diff line change
Expand Up @@ -259,9 +259,13 @@ func (db *DB) ImportFromFile(filePath string, encryptionKey string, collections
if db.persistDirectory != "" {
c.persistDirectory = filepath.Join(db.persistDirectory, hash2hex(pc.Name))
c.compress = db.compress
err = c.persistMetadata()
if err != nil {
return fmt.Errorf("couldn't persist collection metadata: %w", err)
}
for _, doc := range c.documents {
docPath := c.getDocPath(doc.ID)
err := persistToFile(docPath, doc, c.compress, "")
err = persistToFile(docPath, doc, c.compress, "")
if err != nil {
return fmt.Errorf("couldn't persist document to %q: %w", docPath, err)
}
Expand Down Expand Up @@ -326,6 +330,10 @@ func (db *DB) ImportFromReader(reader io.ReadSeeker, encryptionKey string, colle
if db.persistDirectory != "" {
c.persistDirectory = filepath.Join(db.persistDirectory, hash2hex(pc.Name))
c.compress = db.compress
err = c.persistMetadata()
if err != nil {
return fmt.Errorf("couldn't persist collection metadata: %w", err)
}
for _, doc := range c.documents {
docPath := c.getDocPath(doc.ID)
err := persistToFile(docPath, doc, c.compress, "")
Expand Down

0 comments on commit 49eb498

Please sign in to comment.