Skip to content

Commit

Permalink
Merge pull request #126 from wot-oss/refactor/content-search
Browse files Browse the repository at this point in the history 
refactor: content search
  • Loading branch information
@alexbrdn
alexbrdn authored Nov 7, 2024
2 parents d7d0820 + 2037bf8 commit d2a1597
Show file tree
Hide file tree
Showing 5 changed files with 80 additions and 64 deletions.
80 changes: 45 additions & 35 deletions internal/model/toc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,35 +101,6 @@ func (r AttachmentContainerRef) Kind() AttachmentContainerKind {
return AttachmentContainerKindTMID
}

func (e *IndexEntry) MatchesSearchText(searchQuery string) bool {
if e == nil {
return false
}
searchQuery = utils.ToTrimmedLower(searchQuery)
if strings.Contains(utils.ToTrimmedLower(e.Name), searchQuery) {
return true
}
if strings.Contains(utils.ToTrimmedLower(e.Author.Name), searchQuery) {
return true
}
if strings.Contains(utils.ToTrimmedLower(e.Manufacturer.Name), searchQuery) {
return true
}
if strings.Contains(utils.ToTrimmedLower(e.Mpn), searchQuery) {
return true
}
for _, version := range e.Versions {
if strings.Contains(utils.ToTrimmedLower(version.Description), searchQuery) {
return true
}
if strings.Contains(utils.ToTrimmedLower(version.ExternalID), searchQuery) {
return true
}
}
return false

}

type IndexVersion struct {
Description string `json:"description"`
Version Version `json:"version"`
Expand Down Expand Up @@ -163,16 +134,12 @@ func (idx *Index) Sort() {
})
}

func (idx *Index) Filter(search *SearchParams) {
func (idx *Index) Filter(search *SearchParams) error {
if search == nil {
return
return nil
}
search.Sanitize()
exclude := func(entry *IndexEntry) bool {
if !entry.MatchesSearchText(search.Query) {
return true
}

if !matchesNameFilter(search.Name, entry.Name, search.Options) {
return true
}
Expand All @@ -198,7 +165,20 @@ func (idx *Index) Filter(search *SearchParams) {
}
return e
})
if len(idx.Data) == 0 {
return nil
}

if len(search.Query) > 0 {
del, err := excludeBySimpleContentSearch(search.Query)
if err != nil {
return err
}
if del != nil {
idx.Data = slices.DeleteFunc(idx.Data, del)
}
}
return nil
}

func matchesNameFilter(acceptedValue string, value string, options SearchOptions) bool {
Expand Down Expand Up @@ -229,6 +209,36 @@ func matchesFilter(acceptedValues []string, value string) bool {
return slices.Contains(acceptedValues, utils.SanitizeName(value))
}

func excludeBySimpleContentSearch(searchQuery string) (func(e *IndexEntry) bool, error) {
return func(e *IndexEntry) bool {
if e == nil {
return true
}
searchQuery = utils.ToTrimmedLower(searchQuery)
if strings.Contains(utils.ToTrimmedLower(e.Name), searchQuery) {
return false
}
if strings.Contains(utils.ToTrimmedLower(e.Author.Name), searchQuery) {
return false
}
if strings.Contains(utils.ToTrimmedLower(e.Manufacturer.Name), searchQuery) {
return false
}
if strings.Contains(utils.ToTrimmedLower(e.Mpn), searchQuery) {
return false
}
for _, version := range e.Versions {
if strings.Contains(utils.ToTrimmedLower(version.Description), searchQuery) {
return false
}
if strings.Contains(utils.ToTrimmedLower(version.ExternalID), searchQuery) {
return false
}
}
return true
}, nil
}

// FindByName searches by TM name and returns a pointer to the IndexEntry if found
func (idx *Index) FindByName(name string) *IndexEntry {
if idx.dataByName == nil {
Expand Down
46 changes: 23 additions & 23 deletions internal/model/toc_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,71 +10,71 @@ import (
func TestIndex_Filter(t *testing.T) {
t.Run("filter by name", func(t *testing.T) {
idx := prepareIndex()
idx.Filter(&SearchParams{Name: "man/mpn"})
_ = idx.Filter(&SearchParams{Name: "man/mpn"})
assert.Len(t, idx.Data, 1)
assert.NotNil(t, idx.FindByName("man/mpn"))
assert.Nil(t, idx.FindByName("aut/man/mpn"))

idx = prepareIndex()
idx.Filter(&SearchParams{Name: "aut/man/mpn"})
_ = idx.Filter(&SearchParams{Name: "aut/man/mpn"})
assert.Len(t, idx.Data, 1)
assert.NotNil(t, idx.FindByName("aut/man/mpn"))
assert.Nil(t, idx.FindByName("man/mpn"))
})
t.Run("filter by name with prefix match", func(t *testing.T) {
idx := prepareIndex()
idx.Filter(&SearchParams{Name: "man", Options: SearchOptions{NameFilterType: PrefixMatch}})
_ = idx.Filter(&SearchParams{Name: "man", Options: SearchOptions{NameFilterType: PrefixMatch}})
assert.Len(t, idx.Data, 1)
assert.NotNil(t, idx.FindByName("man/mpn"))
assert.Nil(t, idx.FindByName("aut/man/mpn"))

idx = prepareIndex()
idx.Filter(&SearchParams{Name: "aut/man/mpn", Options: SearchOptions{NameFilterType: PrefixMatch}})
_ = idx.Filter(&SearchParams{Name: "aut/man/mpn", Options: SearchOptions{NameFilterType: PrefixMatch}})
assert.Len(t, idx.Data, 1)
assert.NotNil(t, idx.FindByName("aut/man/mpn"))
assert.Nil(t, idx.FindByName("aut/man/mpn2"))
assert.Nil(t, idx.FindByName("man/mpn"))

idx = prepareIndex()
idx.Filter(&SearchParams{Name: "aut/man", Options: SearchOptions{NameFilterType: PrefixMatch}})
_ = idx.Filter(&SearchParams{Name: "aut/man", Options: SearchOptions{NameFilterType: PrefixMatch}})
assert.Len(t, idx.Data, 2)
assert.NotNil(t, idx.FindByName("aut/man/mpn"))
assert.NotNil(t, idx.FindByName("aut/man/mpn2"))

idx = prepareIndex()
idx.Filter(&SearchParams{Name: "aut/man/", Options: SearchOptions{NameFilterType: PrefixMatch}})
_ = idx.Filter(&SearchParams{Name: "aut/man/", Options: SearchOptions{NameFilterType: PrefixMatch}})
assert.Len(t, idx.Data, 2)
assert.NotNil(t, idx.FindByName("aut/man/mpn"))
assert.NotNil(t, idx.FindByName("aut/man/mpn2"))

idx = prepareIndex()
idx.Filter(&SearchParams{Name: "aut/man/mpn/sub", Options: SearchOptions{NameFilterType: PrefixMatch}})
_ = idx.Filter(&SearchParams{Name: "aut/man/mpn/sub", Options: SearchOptions{NameFilterType: PrefixMatch}})
assert.Len(t, idx.Data, 0)
})
t.Run("filter by mpn", func(t *testing.T) {
idx := prepareIndex()
idx.Filter(&SearchParams{Mpn: []string{"mpn2"}})
_ = idx.Filter(&SearchParams{Mpn: []string{"mpn2"}})
assert.Len(t, idx.Data, 1)
assert.NotNil(t, idx.FindByName("aut/man/mpn2"))
assert.Nil(t, idx.FindByName("aut/man/mpn"))

idx = prepareIndex()
idx.Filter(&SearchParams{Mpn: []string{"mpn", "mpn2", "mpn45"}})
_ = idx.Filter(&SearchParams{Mpn: []string{"mpn", "mpn2", "mpn45"}})
assert.Len(t, idx.Data, 4)
assert.NotNil(t, idx.FindByName("aut/man/mpn"))
assert.NotNil(t, idx.FindByName("aut/man/mpn2"))
assert.NotNil(t, idx.FindByName("man/mpn"))
})
t.Run("filter by manufacturer", func(t *testing.T) {
idx := prepareIndex()
idx.Filter(&SearchParams{Manufacturer: []string{"man"}})
_ = idx.Filter(&SearchParams{Manufacturer: []string{"man"}})
assert.Len(t, idx.Data, 3)
assert.NotNil(t, idx.FindByName("aut/man/mpn2"))
assert.NotNil(t, idx.FindByName("aut/man/mpn"))
assert.Nil(t, idx.FindByName("aut/man2/mpn"))

idx = prepareIndex()
idx.Filter(&SearchParams{Manufacturer: []string{"man", "man2", "mpn45"}})
_ = idx.Filter(&SearchParams{Manufacturer: []string{"man", "man2", "mpn45"}})
assert.Len(t, idx.Data, 4)
assert.NotNil(t, idx.FindByName("aut/man/mpn"))
assert.NotNil(t, idx.FindByName("aut/man/mpn2"))
Expand All @@ -83,23 +83,23 @@ func TestIndex_Filter(t *testing.T) {
})
t.Run("filter by author", func(t *testing.T) {
idx := prepareIndex()
idx.Filter(&SearchParams{Author: []string{"man"}})
_ = idx.Filter(&SearchParams{Author: []string{"man"}})
assert.Len(t, idx.Data, 1)
assert.NotNil(t, idx.FindByName("man/mpn"))
assert.Nil(t, idx.FindByName("aut/man/mpn2"))
assert.Nil(t, idx.FindByName("aut/man/mpn"))
assert.Nil(t, idx.FindByName("aut/man2/mpn"))

idx = prepareIndex()
idx.Filter(&SearchParams{Author: []string{"aut"}})
_ = idx.Filter(&SearchParams{Author: []string{"aut"}})
assert.Len(t, idx.Data, 3)
assert.Nil(t, idx.FindByName("man/mpn"))
assert.NotNil(t, idx.FindByName("aut/man/mpn2"))
assert.NotNil(t, idx.FindByName("aut/man/mpn"))
assert.NotNil(t, idx.FindByName("aut/man2/mpn"))

idx = prepareIndex()
idx.Filter(&SearchParams{Author: []string{"man", "aut"}})
_ = idx.Filter(&SearchParams{Author: []string{"man", "aut"}})
assert.Len(t, idx.Data, 4)
assert.NotNil(t, idx.FindByName("man/mpn"))
assert.NotNil(t, idx.FindByName("aut/man/mpn2"))
Expand All @@ -108,38 +108,38 @@ func TestIndex_Filter(t *testing.T) {
})
t.Run("filter by query", func(t *testing.T) {
idx := prepareIndex()
idx.Filter(&SearchParams{Query: ""})
_ = idx.Filter(&SearchParams{Query: ""})
assert.Len(t, idx.Data, 4)

idx = prepareIndex()
idx.Filter(&SearchParams{Query: "z"})
_ = idx.Filter(&SearchParams{Query: "z"})
assert.Len(t, idx.Data, 0)

idx = prepareIndex()
idx.Filter(&SearchParams{Query: "a"})
_ = idx.Filter(&SearchParams{Query: "a"})
assert.Len(t, idx.Data, 4)

idx = prepareIndex()
idx.Filter(&SearchParams{Query: "d1"})
_ = idx.Filter(&SearchParams{Query: "d1"})
assert.Len(t, idx.Data, 1)
assert.Len(t, idx.Data[0].Versions, 2)

idx = prepareIndex()
idx.Filter(&SearchParams{Query: "d5"})
_ = idx.Filter(&SearchParams{Query: "d5"})
assert.Len(t, idx.Data, 2)
assert.NotNil(t, idx.FindByName("aut/man/mpn2"))
assert.NotNil(t, idx.FindByName("aut/man2/mpn"))
})
t.Run("filter by author and manufacturer", func(t *testing.T) {
idx := prepareIndex()
idx.Filter(&SearchParams{Manufacturer: []string{"man"}, Author: []string{"aut"}})
_ = idx.Filter(&SearchParams{Manufacturer: []string{"man"}, Author: []string{"aut"}})
assert.Len(t, idx.Data, 2)
assert.NotNil(t, idx.FindByName("aut/man/mpn2"))
assert.NotNil(t, idx.FindByName("aut/man/mpn"))
assert.Nil(t, idx.FindByName("aut/man2/mpn"))

idx = prepareIndex()
idx.Filter(&SearchParams{Manufacturer: []string{"man"}, Author: []string{"man", "aut"}})
_ = idx.Filter(&SearchParams{Manufacturer: []string{"man"}, Author: []string{"man", "aut"}})
assert.Len(t, idx.Data, 3)
assert.NotNil(t, idx.FindByName("aut/man/mpn"))
assert.NotNil(t, idx.FindByName("aut/man/mpn2"))
Expand Down Expand Up @@ -169,13 +169,13 @@ func TestIndex_Filter(t *testing.T) {
author := "aut^hor"
manuf := "Man&ufacturer"
mpn := "M/PN"
idx.Filter(ToSearchParams(&author, &manuf, &mpn, nil, nil, nil))
_ = idx.Filter(ToSearchParams(&author, &manuf, &mpn, nil, nil, nil))
assert.Len(t, idx.Data, 1)

author = "Aut%hor"
manuf = "Man-ufacturer"
mpn = "M&pN"
idx.Filter(ToSearchParams(&author, &manuf, &mpn, nil, nil, nil))
_ = idx.Filter(ToSearchParams(&author, &manuf, &mpn, nil, nil, nil))
assert.Len(t, idx.Data, 1)
})
}
Expand Down
5 changes: 4 additions & 1 deletion internal/repos/fs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,10 @@ func (f *FileRepo) List(ctx context.Context, search *model.SearchParams) (model.
if err != nil {
return model.SearchResult{}, err
}
idx.Filter(search)
err = idx.Filter(search)
if err != nil {
return model.SearchResult{}, err
}
idx.Sort() // the index is supposed to be sorted on disk, but we don't trust external storage, hence we'll sort here one more time to be extra sure
return model.NewIndexToFoundMapper(f.Spec().ToFoundSource()).ToSearchResult(*idx), nil
}
Expand Down
8 changes: 4 additions & 4 deletions internal/repos/fs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -525,7 +525,7 @@ func TestFileRepo_UpdateIndex_RemoveId(t *testing.T) {
index, err := r.readIndex()
assert.NoError(t, err)
// then: nothing changes
index.Filter(&model.SearchParams{Name: "omnicorp-tm-department/omnicorp/omnilamp"})
_ = index.Filter(&model.SearchParams{Name: "omnicorp-tm-department/omnicorp/omnilamp"})
if assert.Equal(t, 1, len(index.Data)) {
assert.Equal(t, 3, len(index.Data[0].Versions))
}
Expand All @@ -542,7 +542,7 @@ func TestFileRepo_UpdateIndex_RemoveId(t *testing.T) {
index, err := r.readIndex()
assert.NoError(t, err)
// then: nothing changes
index.Filter(&model.SearchParams{Name: "omnicorp-tm-department/omnicorp/omnilamp"})
_ = index.Filter(&model.SearchParams{Name: "omnicorp-tm-department/omnicorp/omnilamp"})
if assert.Equal(t, 1, len(index.Data)) {
assert.Equal(t, 3, len(index.Data[0].Versions))
}
Expand All @@ -561,7 +561,7 @@ func TestFileRepo_UpdateIndex_RemoveId(t *testing.T) {
index, err := r.readIndex()
assert.NoError(t, err)
// then: version is removed from index
index.Filter(&model.SearchParams{Name: "omnicorp-tm-department/omnicorp/omnilamp"})
_ = index.Filter(&model.SearchParams{Name: "omnicorp-tm-department/omnicorp/omnilamp"})
if assert.Equal(t, 1, len(index.Data)) {
assert.Equal(t, 2, len(index.Data[0].Versions))
}
Expand All @@ -581,7 +581,7 @@ func TestFileRepo_UpdateIndex_RemoveId(t *testing.T) {
index, err := r.readIndex()
assert.NoError(t, err)
// then: name is removed from index
index.Filter(&model.SearchParams{Name: "omnicorp-tm-department/omnicorp/omnilamp/subfolder"})
_ = index.Filter(&model.SearchParams{Name: "omnicorp-tm-department/omnicorp/omnilamp/subfolder"})
assert.Equal(t, 0, len(index.Data))
names := r.readNamesFile()
// then: name is removed from names file
Expand Down
5 changes: 4 additions & 1 deletion internal/repos/http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,10 @@ func (h *HttpRepo) List(ctx context.Context, search *model.SearchParams) (model.
if err != nil {
return model.SearchResult{}, err
}
idx.Filter(search)
err = idx.Filter(search)
if err != nil {
return model.SearchResult{}, err
}
return model.NewIndexToFoundMapper(h.Spec().ToFoundSource()).ToSearchResult(*idx), nil
}

Expand Down

0 comments on commit d2a1597

Please sign in to comment.