Skip to content

Commit caf1ce5

Browse files
aunjgraunjgr
authored
push down ORDER BY LIMIT clause on IVFFlat index to table_scan (#22613)
- Put all searching logic into background query - Push down ORDER BY LIMIT clause on index table to scan node and implement it in Reader - Make analyze info of this background query visible - Update dependencies to newest (uSearch, SimSIMD, ...) In particular query, number of copied rows from cache to pipeline is reduced from 13560 to 270, and throughput is improved by 10 times. Approved by: @cpegeric, @daviszhen, @ouyuanning, @jiangxinmeng1, @LeftHandCold, @gouhongshen, @XuPeng-SH, @reusee, @heni02, @fengttt, @zhangxu19830126
1 parent 0bddd6f commit caf1ce5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1799
-1490
lines changed

‎.golangci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ linters:
9191
- SA1016
9292
- SA1017
9393
- SA1018
94-
- SA1019
94+
#- SA1019
9595
- SA1020
9696
- SA1021
9797
- SA1023

‎go.mod‎

Lines changed: 64 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ require (
1414
github.com/aliyun/alibaba-cloud-sdk-go v1.63.34
1515
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
1616
github.com/aliyun/credentials-go v1.3.10
17+
github.com/aws/aws-sdk-go v1.55.5
1718
github.com/aws/aws-sdk-go-v2 v1.32.5
1819
github.com/aws/aws-sdk-go-v2/config v1.28.5
1920
github.com/aws/aws-sdk-go-v2/credentials v1.17.46
@@ -89,14 +90,14 @@ require (
8990
github.com/tidwall/btree v1.7.0
9091
github.com/tidwall/pretty v1.2.1
9192
github.com/tmc/langchaingo v0.1.13
92-
github.com/unum-cloud/usearch/golang v0.0.0-20250904130807-fd6279af6bc2
93+
github.com/unum-cloud/usearch/golang v0.0.0-20251010193336-541e882da5a9
9394
go.starlark.net v0.0.0-20250701195324-d457b4515e0e
9495
go.uber.org/automaxprocs v1.5.3
9596
go.uber.org/ratelimit v0.2.0
9697
go.uber.org/zap v1.24.0
97-
golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
98-
golang.org/x/sync v0.9.0
99-
golang.org/x/sys v0.27.0
98+
golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b
99+
golang.org/x/sync v0.17.0
100+
golang.org/x/sys v0.37.0
100101
gonum.org/v1/gonum v0.14.0
101102
google.golang.org/grpc v1.65.0
102103
google.golang.org/protobuf v1.36.0
@@ -105,73 +106,14 @@ require (
105106

106107
require (
107108
filippo.io/edwards25519 v1.1.0 // indirect
108-
github.com/Masterminds/semver/v3 v3.2.1 // indirect
109-
github.com/andybalholm/brotli v1.1.0 // indirect
110-
github.com/bytedance/gopkg v0.1.3 // indirect
111-
github.com/bytedance/sonic/loader v0.3.0 // indirect
112-
github.com/cespare/xxhash v1.1.0 // indirect
113-
github.com/cilium/ebpf v0.9.1 // indirect
114-
github.com/clbanning/mxj v1.8.4 // indirect
115-
github.com/cloudwego/base64x v0.1.6 // indirect
116-
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
117-
github.com/dlclark/regexp2 v1.10.0 // indirect
118-
github.com/dustin/go-humanize v1.0.1 // indirect
119-
github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a // indirect
120-
github.com/go-ini/ini v1.67.0 // indirect
121-
github.com/gobwas/glob v0.2.3 // indirect
122-
github.com/goccy/go-json v0.10.3 // indirect
123-
github.com/godbus/dbus/v5 v5.0.4 // indirect
124-
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 // indirect
125-
github.com/google/go-cmp v0.6.0 // indirect
126-
github.com/google/go-querystring v1.1.0 // indirect
127-
github.com/gosimple/slug v1.13.1 // indirect
128-
github.com/gosimple/unidecode v1.0.1 // indirect
129-
github.com/hashicorp/go-uuid v1.0.3 // indirect
130-
github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca // indirect
131-
github.com/itchyny/timefmt-go v0.1.6 // indirect
132-
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
133-
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
134-
github.com/jcmturner/gofork v1.7.6 // indirect
135-
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
136-
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
137-
github.com/jmespath/go-jmespath v0.4.0 // indirect
138-
github.com/josharian/native v1.1.0 // indirect
139-
github.com/mattn/go-runewidth v0.0.15 // indirect
140-
github.com/mdlayher/netlink v1.7.2 // indirect
141-
github.com/mdlayher/socket v0.5.1 // indirect
142-
github.com/minio/md5-simd v1.1.2 // indirect
143-
github.com/mitchellh/mapstructure v1.5.0 // indirect
144-
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
145-
github.com/modern-go/reflect2 v1.0.2 // indirect
146-
github.com/mozillazg/go-httpheader v0.2.1 // indirect
147-
github.com/olekukonko/tablewriter v0.0.5 // indirect
148-
github.com/opencontainers/runtime-spec v1.0.2 // indirect
149-
github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
150-
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
151-
github.com/pkoukk/tiktoken-go v0.1.6 // indirect
152-
github.com/rivo/uniseg v0.4.7 // indirect
153-
github.com/rs/xid v1.6.0 // indirect
154-
github.com/segmentio/asm v1.1.3 // indirect
155-
github.com/shoenig/go-m1cpu v0.1.6 // indirect
156-
github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect
157-
github.com/tetratelabs/wazero v1.8.1-0.20240916092830-1353ca24fef0 // indirect
158-
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
159-
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
160-
golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
161-
golang.org/x/crypto v0.29.0 // indirect
162-
golang.org/x/net v0.30.0 // indirect
163-
golang.org/x/text v0.20.0 // indirect
164-
golang.org/x/time v0.5.0 // indirect
165-
)
166-
167-
require (
168109
github.com/DataDog/zstd v1.5.0 // indirect
110+
github.com/Masterminds/semver/v3 v3.2.1 // indirect
169111
github.com/VictoriaMetrics/metrics v1.18.1 // indirect
170112
github.com/alibabacloud-go/debug v1.0.1 // indirect
171113
github.com/alibabacloud-go/tea v1.2.2 // indirect
172114
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
115+
github.com/andybalholm/brotli v1.1.0 // indirect
173116
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect
174-
github.com/aws/aws-sdk-go v1.55.5
175117
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
176118
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect
177119
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect
@@ -187,51 +129,106 @@ require (
187129
github.com/beorn7/perks v1.0.1 // indirect
188130
github.com/bits-and-blooms/bitset v1.8.0 // indirect
189131
github.com/bufbuild/protocompile v0.6.0 // indirect
132+
github.com/bytedance/gopkg v0.1.3 // indirect
133+
github.com/bytedance/sonic/loader v0.3.0 // indirect
134+
github.com/cespare/xxhash v1.1.0 // indirect
135+
github.com/cilium/ebpf v0.9.1 // indirect
136+
github.com/clbanning/mxj v1.8.4 // indirect
137+
github.com/cloudwego/base64x v0.1.6 // indirect
190138
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
191139
github.com/cockroachdb/pebble v0.0.0-20220407171941-2120d145e292 // indirect
192140
github.com/cockroachdb/redact v1.1.3 // indirect
141+
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
193142
github.com/davecgh/go-spew v1.1.1 // indirect
194143
github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc // indirect
144+
github.com/dlclark/regexp2 v1.10.0 // indirect
145+
github.com/dustin/go-humanize v1.0.1 // indirect
146+
github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a // indirect
195147
github.com/getsentry/sentry-go v0.12.0 // indirect
148+
github.com/go-ini/ini v1.67.0 // indirect
196149
github.com/go-ole/go-ole v1.2.6 // indirect
150+
github.com/gobwas/glob v0.2.3 // indirect
151+
github.com/goccy/go-json v0.10.3 // indirect
152+
github.com/godbus/dbus/v5 v5.0.4 // indirect
153+
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 // indirect
197154
github.com/golang/protobuf v1.5.4 // indirect
198155
github.com/golang/snappy v0.0.4 // indirect
156+
github.com/google/go-cmp v0.6.0 // indirect
157+
github.com/google/go-querystring v1.1.0 // indirect
199158
github.com/gopherjs/gopherjs v1.12.80 // indirect
159+
github.com/gosimple/slug v1.13.1 // indirect
160+
github.com/gosimple/unidecode v1.0.1 // indirect
200161
github.com/hashicorp/errwrap v1.1.0 // indirect
201162
github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
202163
github.com/hashicorp/go-msgpack v0.5.3 // indirect
203164
github.com/hashicorp/go-multierror v1.1.1 // indirect
204165
github.com/hashicorp/go-sockaddr v1.0.0 // indirect
166+
github.com/hashicorp/go-uuid v1.0.3 // indirect
205167
github.com/hashicorp/golang-lru v0.5.4 // indirect
168+
github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca // indirect
206169
github.com/inconshreveable/mousetrap v1.1.0 // indirect
170+
github.com/itchyny/timefmt-go v0.1.6 // indirect
171+
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
172+
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
173+
github.com/jcmturner/gofork v1.7.6 // indirect
174+
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
175+
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
176+
github.com/jmespath/go-jmespath v0.4.0 // indirect
177+
github.com/josharian/native v1.1.0 // indirect
207178
github.com/jtolds/gls v4.20.0+incompatible // indirect
208179
github.com/klauspost/compress v1.17.11 // indirect
209180
github.com/klauspost/cpuid/v2 v2.2.9 // indirect
210181
github.com/kr/pretty v0.3.1 // indirect
211182
github.com/kr/text v0.2.0 // indirect
212183
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
184+
github.com/mattn/go-runewidth v0.0.15 // indirect
213185
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
186+
github.com/mdlayher/netlink v1.7.2 // indirect
187+
github.com/mdlayher/socket v0.5.1 // indirect
214188
github.com/miekg/dns v1.1.53 // indirect
189+
github.com/minio/md5-simd v1.1.2 // indirect
190+
github.com/mitchellh/mapstructure v1.5.0 // indirect
191+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
192+
github.com/modern-go/reflect2 v1.0.2 // indirect
193+
github.com/mozillazg/go-httpheader v0.2.1 // indirect
215194
github.com/mschoch/smat v0.2.0 // indirect
195+
github.com/olekukonko/tablewriter v0.0.5 // indirect
196+
github.com/opencontainers/runtime-spec v1.0.2 // indirect
197+
github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
198+
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
216199
github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7 // indirect
200+
github.com/pkoukk/tiktoken-go v0.1.6 // indirect
217201
github.com/pmezard/go-difflib v1.0.0 // indirect
218202
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
219203
github.com/prometheus/common v0.44.0 // indirect
220204
github.com/prometheus/procfs v0.11.1 // indirect
205+
github.com/rivo/uniseg v0.4.7 // indirect
221206
github.com/rogpeppe/go-internal v1.11.0 // indirect
207+
github.com/rs/xid v1.6.0 // indirect
222208
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
209+
github.com/segmentio/asm v1.1.3 // indirect
210+
github.com/shoenig/go-m1cpu v0.1.6 // indirect
223211
github.com/sirupsen/logrus v1.9.3 // indirect
224212
github.com/smartystreets/assertions v1.13.1 // indirect
225213
github.com/spf13/pflag v1.0.5 // indirect
214+
github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect
215+
github.com/tetratelabs/wazero v1.8.1-0.20240916092830-1353ca24fef0 // indirect
226216
github.com/tklauser/go-sysconf v0.3.12 // indirect
227217
github.com/tklauser/numcpus v0.6.1 // indirect
218+
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
228219
github.com/valyala/fastrand v1.1.0 // indirect
229220
github.com/valyala/histogram v1.2.0 // indirect
230221
github.com/yusufpapurcu/wmi v1.2.3 // indirect
222+
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
231223
go.uber.org/atomic v1.11.0 // indirect
232224
go.uber.org/multierr v1.11.0 // indirect
233-
golang.org/x/mod v0.21.0 // indirect
234-
golang.org/x/tools v0.26.0 // indirect
225+
golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
226+
golang.org/x/crypto v0.43.0 // indirect
227+
golang.org/x/mod v0.29.0 // indirect
228+
golang.org/x/net v0.46.0 // indirect
229+
golang.org/x/text v0.30.0 // indirect
230+
golang.org/x/time v0.13.0 // indirect
231+
golang.org/x/tools v0.38.0 // indirect
235232
google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect
236233
gopkg.in/ini.v1 v1.67.0 // indirect
237234
gopkg.in/yaml.v3 v3.0.1 // indirect

0 commit comments

Comments
 (0)