diff --git a/go.mod b/go.mod index b09c9414..77c8f07e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/canonical/k8s-dqlite go 1.21 require ( - github.com/Rican7/retry v0.3.1 github.com/canonical/go-dqlite v1.22.0 github.com/mattn/go-sqlite3 v1.14.22 github.com/onsi/gomega v1.27.10 @@ -29,6 +28,7 @@ require ( ) require ( + github.com/Rican7/retry v0.3.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/pkg/kine/drivers/dqlite/dqlite.go b/pkg/kine/drivers/dqlite/dqlite.go index 6abd6470..edcc7b93 100644 --- a/pkg/kine/drivers/dqlite/dqlite.go +++ b/pkg/kine/drivers/dqlite/dqlite.go @@ -37,7 +37,7 @@ func NewVariant(ctx context.Context, datasourceName string) (server.Backend, *ge if err != nil { return nil, nil, errors.Wrap(err, "sqlite client") } - if err := migrate(ctx, generic.DB); err != nil { + if err := migrate(ctx, generic.DB.Underlying()); err != nil { return nil, nil, errors.Wrap(err, "failed to migrate DB from sqlite") } generic.LockWrites = true diff --git a/pkg/kine/drivers/generic/generic.go b/pkg/kine/drivers/generic/generic.go index f43850fc..360efb9d 100644 --- a/pkg/kine/drivers/generic/generic.go +++ b/pkg/kine/drivers/generic/generic.go @@ -10,7 +10,7 @@ import ( "sync" "time" - "github.com/Rican7/retry/jitter" + "github.com/canonical/k8s-dqlite/pkg/kine/prepared" "github.com/pkg/errors" "github.com/sirupsen/logrus" "go.opentelemetry.io/otel" @@ -131,37 +131,27 @@ type ErrCode func(error) string type Generic struct { sync.Mutex - LockWrites bool - LastInsertID bool - DB *sql.DB - GetCurrentSQL string - GetRevisionSQL string - getRevisionSQLPrepared *sql.Stmt - RevisionSQL string - ListRevisionStartSQL string - GetRevisionAfterSQL string - CountCurrentSQL string - countCurrentSQLPrepared *sql.Stmt - CountRevisionSQL string - countRevisionSQLPrepared *sql.Stmt - AfterSQLPrefix string - afterSQLPrefixPrepared *sql.Stmt - AfterSQL string - DeleteSQL string - deleteSQLPrepared *sql.Stmt - UpdateCompactSQL string - updateCompactSQLPrepared *sql.Stmt - InsertSQL string - insertSQLPrepared *sql.Stmt - FillSQL string - fillSQLPrepared *sql.Stmt - InsertLastInsertIDSQL string - insertLastInsertIDSQLPrepared *sql.Stmt - GetSizeSQL string - getSizeSQLPrepared *sql.Stmt - Retry ErrRetry - TranslateErr TranslateErr - ErrCode ErrCode + LockWrites bool + LastInsertID bool + DB *prepared.DB + GetCurrentSQL string + GetRevisionSQL string + RevisionSQL string + ListRevisionStartSQL string + GetRevisionAfterSQL string + CountCurrentSQL string + CountRevisionSQL string + AfterSQLPrefix string + AfterSQL string + DeleteSQL string + UpdateCompactSQL string + InsertSQL string + FillSQL string + InsertLastInsertIDSQL string + GetSizeSQL string + Retry ErrRetry + TranslateErr TranslateErr + ErrCode ErrCode AdmissionControlPolicy AdmissionControlPolicy @@ -231,7 +221,7 @@ func Open(ctx context.Context, driverName, dataSourceName string, paramCharacter configureConnectionPooling(db) return &Generic{ - DB: db, + DB: prepared.New(db, 100), GetRevisionSQL: q(fmt.Sprintf(` SELECT @@ -291,79 +281,8 @@ func Open(ctx context.Context, driverName, dataSourceName string, paramCharacter }, err } -func (d *Generic) Prepare() error { - var err error - - d.getRevisionSQLPrepared, err = d.DB.Prepare(d.GetRevisionSQL) - if err != nil { - return err - } - - d.countCurrentSQLPrepared, err = d.DB.Prepare(d.CountCurrentSQL) - if err != nil { - return err - } - - d.countRevisionSQLPrepared, err = d.DB.Prepare(d.CountRevisionSQL) - if err != nil { - return err - } - - d.deleteSQLPrepared, err = d.DB.Prepare(d.DeleteSQL) - if err != nil { - return err - } - - d.getSizeSQLPrepared, err = d.DB.Prepare(d.GetSizeSQL) - if err != nil { - return err - } - - d.fillSQLPrepared, err = d.DB.Prepare(d.FillSQL) - if err != nil { - return err - } - - if d.LastInsertID { - d.insertLastInsertIDSQLPrepared, err = d.DB.Prepare(d.InsertLastInsertIDSQL) - if err != nil { - return err - } - } else { - d.insertSQLPrepared, err = d.DB.Prepare(d.InsertSQL) - if err != nil { - return err - } - } - - d.updateCompactSQLPrepared, err = d.DB.Prepare(d.UpdateCompactSQL) - if err != nil { - return err - } - - d.afterSQLPrefixPrepared, err = d.DB.Prepare(d.AfterSQLPrefix) - if err != nil { - return err - } - - return nil -} - -func (d *Generic) Close() { - d.getRevisionSQLPrepared.Close() - d.countCurrentSQLPrepared.Close() - d.countRevisionSQLPrepared.Close() - d.afterSQLPrefixPrepared.Close() - d.deleteSQLPrepared.Close() - d.updateCompactSQLPrepared.Close() - if d.LastInsertID { - d.insertLastInsertIDSQLPrepared.Close() - } else { - d.insertSQLPrepared.Close() - } - d.fillSQLPrepared.Close() - d.getSizeSQLPrepared.Close() - d.DB.Close() +func (d *Generic) Close() error { + return d.DB.Close() } func getPrefixRange(prefix string) (start, end string) { @@ -378,7 +297,7 @@ func getPrefixRange(prefix string) (start, end string) { return start, end } -func (d *Generic) query(ctx context.Context, txName, sql string, args ...interface{}) (rows *sql.Rows, err error) { +func (d *Generic) query(ctx context.Context, txName, query string, args ...interface{}) (rows *sql.Rows, err error) { i := 0 start := time.Now() @@ -386,6 +305,7 @@ func (d *Generic) query(ctx context.Context, txName, sql string, args ...interfa if err != nil { return nil, fmt.Errorf("denied: %w", err) } + defer done() defer func() { if err != nil { @@ -394,41 +314,59 @@ func (d *Generic) query(ctx context.Context, txName, sql string, args ...interfa recordOpResult(txName, err, start) }() - strippedSQL := Stripped(sql) + strippedQuery := Stripped(query) for ; i < retryCount; i++ { if i > 2 { - logrus.Debugf("QUERY (try: %d) %v : %s", i, args, strippedSQL) + logrus.Debugf("QUERY (try: %d) %v : %s", i, args, strippedQuery) } else { - logrus.Tracef("QUERY (try: %d) %v : %s", i, args, strippedSQL) + logrus.Tracef("QUERY (try: %d) %v : %s", i, args, strippedQuery) } - rows, err = d.DB.QueryContext(ctx, sql, args...) - if err != nil && d.Retry != nil && d.Retry(err) { - time.Sleep(jitter.Deviation(nil, 0.3)(2 * time.Millisecond)) - continue + rows, err = d.DB.QueryContext(ctx, query, args...) + if err == nil || d.Retry == nil || !d.Retry(err) { + break } - done() - recordTxResult(txName, err) - return rows, err } - done() + + recordTxResult(txName, err) return } -func (d *Generic) queryPrepared(ctx context.Context, txName, sql string, prepared *sql.Stmt, args ...interface{}) (result *sql.Rows, err error) { - logrus.Tracef("QUERY %v : %s", args, Stripped(sql)) +func (d *Generic) execute(ctx context.Context, txName, query string, args ...interface{}) (result sql.Result, err error) { + i := 0 + start := time.Now() + defer func() { + if err != nil { + err = fmt.Errorf("exec (try: %d): %w", i, err) + } + recordOpResult(txName, err, start) + }() done, err := d.AdmissionControlPolicy.Admit(ctx, txName) if err != nil { return nil, fmt.Errorf("denied: %w", err) } + defer done() - start := time.Now() - r, err := prepared.QueryContext(ctx, args...) - done() + if d.LockWrites { + d.Lock() + defer d.Unlock() + } + + strippedQuery := Stripped(query) + for ; i < retryCount; i++ { + if i > 2 { + logrus.Debugf("EXEC (try: %d) %v : %s", i, args, strippedQuery) + } else { + logrus.Tracef("EXEC (try: %d) %v : %s", i, args, strippedQuery) + } + result, err = d.DB.ExecContext(ctx, query, args...) + if err == nil || d.Retry == nil || !d.Retry(err) { + break + } + } - recordOpResult(txName, err, start) recordTxResult(txName, err) - return r, err + return } func (d *Generic) CountCurrent(ctx context.Context, prefix string, startKey string) (int64, int64, error) { @@ -441,9 +379,23 @@ func (d *Generic) CountCurrent(ctx context.Context, prefix string, startKey stri if startKey != "" { start = startKey + "\x01" } - row := d.queryRowPrepared(ctx, "count_current", d.CountCurrentSQL, d.countCurrentSQLPrepared, start, end, false) - err := row.Scan(&rev, &id) - return rev.Int64, id, err + rows, err := d.query(ctx, "count_current", d.CountCurrentSQL, start, end, false) + if err != nil { + return 0, 0, err + } + defer rows.Close() + + if !rows.Next() { + if err := rows.Err(); err != nil { + return 0, 0, err + } + return 0, 0, sql.ErrNoRows + } + + if err := rows.Scan(&rev, &id); err != nil { + return 0, 0, err + } + return rev.Int64, id, nil } func (d *Generic) Count(ctx context.Context, prefix, startKey string, revision int64) (int64, int64, error) { @@ -456,67 +408,23 @@ func (d *Generic) Count(ctx context.Context, prefix, startKey string, revision i if startKey != "" { start = startKey + "\x01" } - row := d.queryRowPrepared(ctx, "count_revision", d.CountRevisionSQL, d.countRevisionSQLPrepared, start, end, revision, false) - err := row.Scan(&rev, &id) - return rev.Int64, id, err -} - -func (d *Generic) queryRow(ctx context.Context, txName, sql string, args ...interface{}) (result *sql.Row) { - logrus.Tracef("QUERY ROW %v : %s", args, Stripped(sql)) - start := time.Now() - r := d.DB.QueryRowContext(ctx, sql, args...) - recordOpResult(txName, r.Err(), start) - recordTxResult(txName, r.Err()) - return r -} - -func (d *Generic) queryRowPrepared(ctx context.Context, txName, sql string, prepared *sql.Stmt, args ...interface{}) (result *sql.Row) { - logrus.Tracef("QUERY ROW %v : %s", args, Stripped(sql)) - start := time.Now() - r := prepared.QueryRowContext(ctx, args...) - recordOpResult(txName, r.Err(), start) - recordTxResult(txName, r.Err()) - return r -} - -func (d *Generic) executePrepared(ctx context.Context, txName, sql string, prepared *sql.Stmt, args ...interface{}) (result sql.Result, err error) { - i := 0 - start := time.Now() - defer func() { - if err != nil { - err = fmt.Errorf("exec (try: %d): %w", i, err) - } - recordOpResult(txName, err, start) - }() - - done, err := d.AdmissionControlPolicy.Admit(ctx, txName) + rows, err := d.query(ctx, "count_revision", d.CountRevisionSQL, start, end, revision, false) if err != nil { - return nil, fmt.Errorf("denied: %w", err) + return 0, 0, err } + defer rows.Close() - if d.LockWrites { - d.Lock() - defer d.Unlock() + if !rows.Next() { + if err := rows.Err(); err != nil { + return 0, 0, err + } + return 0, 0, sql.ErrNoRows } - strippedSQL := Stripped(sql) - for ; i < retryCount; i++ { - if i > 2 { - logrus.Debugf("EXEC (try: %d) %v : %s", i, args, strippedSQL) - } else { - logrus.Tracef("EXEC (try: %d) %v : %s", i, args, strippedSQL) - } - result, err = prepared.ExecContext(ctx, args...) - if err != nil && d.Retry != nil && d.Retry(err) { - time.Sleep(jitter.Deviation(nil, 0.3)(2 * time.Millisecond)) - continue - } - done() - recordTxResult(txName, err) - return result, err + if err := rows.Scan(&rev, &id); err != nil { + return 0, 0, err } - done() - return + return rev.Int64, id, err } func (d *Generic) GetCompactRevision(ctx context.Context) (int64, int64, error) { @@ -539,25 +447,24 @@ func (d *Generic) GetCompactRevision(ctx context.Context) (int64, int64, error) if err != nil { return 0, 0, fmt.Errorf("denied: %w", err) } + defer done() - for i := 0; i < retryCount; i++ { - if i > 2 { - logrus.Debugf("EXEC (try: %d): %s", i, revisionIntervalSQL) - } else { - logrus.Tracef("EXEC (try: %d): %s", i, revisionIntervalSQL) - } - row := d.DB.QueryRow(revisionIntervalSQL) - err = row.Scan(&compact, &target) - if err != nil && d.Retry != nil && d.Retry(err) { - time.Sleep(jitter.Deviation(nil, 0.3)(2 * time.Millisecond)) - continue - } - break + rows, err := d.query(ctx, "revision_interval_sql", revisionIntervalSQL) + if err != nil { + return 0, 0, err } - done() - if err == sql.ErrNoRows { + defer rows.Close() + + if !rows.Next() { + if err := rows.Err(); err != nil { + return 0, 0, err + } return 0, 0, nil } + + if err := rows.Scan(&compact, &target); err != nil { + return 0, 0, err + } span.SetAttributes(attribute.Int64("compact", compact.Int64), attribute.Int64("target", target.Int64)) return compact.Int64, target.Int64, err } @@ -567,12 +474,12 @@ func (d *Generic) SetCompactRevision(ctx context.Context, revision int64) error setCompactRevCnt.Add(ctx, 1) ctx, span := otelTracer.Start(ctx, fmt.Sprintf("%s.set_compact_revision", otelName)) defer func() { - span.RecordError(err) span.End() + span.RecordError(err) }() span.SetAttributes(attribute.Int64("revision", revision)) - _, err = d.executePrepared(ctx, "update_compact_sql", d.UpdateCompactSQL, d.updateCompactSQLPrepared, revision) + _, err = d.execute(ctx, "update_compact_sql", d.UpdateCompactSQL, revision) return err } @@ -586,7 +493,7 @@ func (d *Generic) GetRevision(ctx context.Context, revision int64) (*sql.Rows, e }() span.SetAttributes(attribute.Int64("revision", revision)) - result, err := d.queryPrepared(ctx, "get_revision_sql", d.GetRevisionSQL, d.getRevisionSQLPrepared, revision) + result, err := d.query(ctx, "get_revision_sql", d.GetRevisionSQL, revision) return result, err } @@ -600,7 +507,7 @@ func (d *Generic) DeleteRevision(ctx context.Context, revision int64) error { }() span.SetAttributes(attribute.Int64("revision", revision)) - _, err = d.executePrepared(ctx, "delete_sql", d.DeleteSQL, d.deleteSQLPrepared, revision) + _, err = d.execute(ctx, "delete_sql", d.DeleteSQL, revision) return err } @@ -651,16 +558,26 @@ func (d *Generic) CurrentRevision(ctx context.Context) (int64, error) { if err != nil { return 0, fmt.Errorf("denied: %w", err) } + defer done() - row := d.queryRow(ctx, "rev_sql", revSQL) - done() - err = row.Scan(&id) - if err == sql.ErrNoRows { - span.AddEvent("no rows") - return 0, nil + rows, err := d.query(ctx, "rev_sql", revSQL) + if err != nil { + return 0, err + } + defer rows.Close() + + if !rows.Next() { + if err := rows.Err(); err != nil { + return 0, err + } + return 0, fmt.Errorf("can't get current revision: aggregate query returned empty set") + } + + if err := rows.Scan(&id); err != nil { + return 0, err } span.SetAttributes(attribute.Int64("id", id)) - return id, err + return id, nil } func (d *Generic) AfterPrefix(ctx context.Context, prefix string, rev, limit int64) (*sql.Rows, error) { @@ -681,7 +598,7 @@ func (d *Generic) After(ctx context.Context, rev, limit int64) (*sql.Rows, error } func (d *Generic) Fill(ctx context.Context, revision int64) error { - _, err := d.executePrepared(ctx, "fill_sql", d.FillSQL, d.fillSQLPrepared, revision, fmt.Sprintf("gap-%d", revision), 0, 1, 0, 0, 0, nil, nil) + _, err := d.execute(ctx, "fill_sql", d.FillSQL, revision, fmt.Sprintf("gap-%d", revision), 0, 1, 0, 0, 0, nil, nil) return err } @@ -708,26 +625,51 @@ func (d *Generic) Insert(ctx context.Context, key string, create, delete bool, c } if d.LastInsertID { - row, err := d.executePrepared(ctx, "insert_last_insert_id_sql", d.InsertLastInsertIDSQL, d.insertLastInsertIDSQLPrepared, key, cVal, dVal, createRevision, previousRevision, ttl, value, prevValue) + row, err := d.execute(ctx, "insert_last_insert_id_sql", d.InsertLastInsertIDSQL, key, cVal, dVal, createRevision, previousRevision, ttl, value, prevValue) if err != nil { return 0, err } return row.LastInsertId() } - row := d.queryRowPrepared(ctx, "insert_sql", d.InsertSQL, d.insertSQLPrepared, key, cVal, dVal, createRevision, previousRevision, ttl, value, prevValue) - err = row.Scan(&id) + rows, err := d.query(ctx, "insert_sql", d.InsertSQL, key, cVal, dVal, createRevision, previousRevision, ttl, value, prevValue) + if err != nil { + return 0, err + } + defer rows.Close() - return id, err + if !rows.Next() { + if err := rows.Err(); err != nil { + return 0, err + } + return 0, sql.ErrNoRows + } + + if err := rows.Scan(&id); err != nil { + return 0, err + } + return id, nil } func (d *Generic) GetSize(ctx context.Context) (int64, error) { if d.GetSizeSQL == "" { return 0, errors.New("driver does not support size reporting") } + rows, err := d.query(ctx, "get_size_sql", d.GetSizeSQL) + if err != nil { + return 0, err + } + defer rows.Close() + + if !rows.Next() { + if err := rows.Err(); err != nil { + return 0, err + } + return 0, sql.ErrNoRows + } + var size int64 - row := d.queryRowPrepared(ctx, "get_size_sql", d.GetSizeSQL, d.getSizeSQLPrepared) - if err := row.Scan(&size); err != nil { + if err := rows.Scan(&size); err != nil { return 0, err } return size, nil diff --git a/pkg/kine/drivers/sqlite/sqlite.go b/pkg/kine/drivers/sqlite/sqlite.go index f7a94a18..5e31ad0d 100644 --- a/pkg/kine/drivers/sqlite/sqlite.go +++ b/pkg/kine/drivers/sqlite/sqlite.go @@ -69,7 +69,7 @@ func NewVariant(ctx context.Context, driverName, dataSourceName string) (server. return nil, nil, err } for i := 0; i < retryAttempts; i++ { - err = setup(ctx, dialect.DB) + err = setup(ctx, dialect.DB.Underlying()) if err == nil { break } @@ -91,10 +91,6 @@ func NewVariant(ctx context.Context, driverName, dataSourceName string) (server. } dialect.GetSizeSQL = `SELECT (page_count - freelist_count) * page_size FROM pragma_page_count(), pragma_page_size(), pragma_freelist_count()` - if err := dialect.Prepare(); err != nil { - return nil, nil, errors.Wrap(err, "query preparation failed") - } - dialect.CompactInterval = opts.compactInterval dialect.PollInterval = opts.pollInterval dialect.AdmissionControlPolicy = generic.NewAdmissionControlPolicy( diff --git a/pkg/kine/logstructured/sqllog/sql.go b/pkg/kine/logstructured/sqllog/sql.go index 678fc0ca..bcec6f27 100644 --- a/pkg/kine/logstructured/sqllog/sql.go +++ b/pkg/kine/logstructured/sqllog/sql.go @@ -73,13 +73,15 @@ type Dialect interface { GetSize(ctx context.Context) (int64, error) GetCompactInterval() time.Duration GetPollInterval() time.Duration - Close() + Close() error } func (s *SQLLog) Start(ctx context.Context) (err error) { s.ctx = ctx context.AfterFunc(ctx, func() { - s.d.Close() + if err := s.d.Close(); err != nil { + logrus.Errorf("cannot close database: %v", err) + } }) return s.broadcaster.Start(s.startWatch) } diff --git a/pkg/kine/prepared/db.go b/pkg/kine/prepared/db.go new file mode 100644 index 00000000..7504c551 --- /dev/null +++ b/pkg/kine/prepared/db.go @@ -0,0 +1,191 @@ +package prepared + +import ( + "context" + "database/sql" + "errors" + "sync" + "sync/atomic" +) + +type DB struct { + underlying *sql.DB + mu sync.Mutex + maxSize int + store map[string]*lruEntry + lruList lruList +} + +func New(db *sql.DB, maxSize int) *DB { + return &DB{ + underlying: db, + maxSize: maxSize, + store: make(map[string]*lruEntry, maxSize), + } +} + +func (db *DB) Underlying() *sql.DB { return db.underlying } + +func (db *DB) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) { + stms, err := db.prepare(ctx, query) + if err != nil { + return nil, err + } + return stms.ExecContext(ctx, args...) +} + +func (db *DB) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { + stms, err := db.prepare(ctx, query) + if err != nil { + return nil, err + } + return stms.QueryContext(ctx, args...) +} + +func (db *DB) Close() error { + db.mu.Lock() + defer db.mu.Unlock() + + errs := []error{} + for len(db.store) != 0 { + if err := db.drop(db.lruList.tail()); err != nil { + errs = append(errs, err) + } + } + + if err := db.underlying.Close(); err != nil { + errs = append(errs, err) + } + + return errors.Join(errs...) +} + +func (db *DB) touch(entry *lruEntry) { + db.lruList.remove(entry) + db.lruList.add(entry) +} + +func (db *DB) prepare(ctx context.Context, query string) (*stmt, error) { + if stmt := db.get(query); stmt != nil { + return stmt, nil + } + prepared, err := db.underlying.PrepareContext(ctx, query) + if err != nil { + return nil, err + } + result := &stmt{prepared, atomic.Int32{}} + result.Ref() + + db.put(query, result) + return result, nil +} + +func (db *DB) get(key string) *stmt { + db.mu.Lock() + defer db.mu.Unlock() + + if entry, ok := db.store[key]; ok { + db.touch(entry) + return entry.stmt.Ref() + } + return nil +} + +func (db *DB) put(query string, stmt *stmt) { + if db.maxSize <= 0 { + return + } + + db.mu.Lock() + defer db.mu.Unlock() + + entry, ok := db.store[query] + if ok { + db.touch(entry) + entry.stmt.Close() + entry.stmt = stmt.Ref() + return + } + + if len(db.store) >= db.maxSize { + db.drop(db.lruList.tail()) + } + + entry = &lruEntry{ + query: query, + stmt: stmt.Ref(), + } + db.store[query] = entry + db.lruList.add(entry) +} + +func (db *DB) drop(entry *lruEntry) error { + db.lruList.remove(entry) + delete(db.store, entry.query) + return entry.stmt.Close() +} + +type stmt struct { + *sql.Stmt + count atomic.Int32 +} + +func (s *stmt) Ref() *stmt { + s.count.Add(1) + return s +} + +func (s *stmt) Close() error { + if s.count.Add(-1) <= 0 { + return s.Stmt.Close() + } + return nil +} + +type lruList struct { + // lruList is implemented as a circular linked list + head *lruEntry +} + +type lruEntry struct { + query string + stmt *stmt + + // Adjacent entries, unless at the tail of the list, the + // next entry was added less recently tham the current. + // Likewise prev was added more recently. + next, prev *lruEntry +} + +func (ll *lruList) remove(entry *lruEntry) { + entry.prev.next = entry.next + entry.next.prev = entry.prev + if entry != ll.head { + return + } + if entry.next == entry { + ll.head = nil + } else { + ll.head = entry.next + } +} + +func (ll *lruList) add(entry *lruEntry) { + if ll.head != nil { + entry.prev = ll.head.prev + entry.next = ll.head + ll.head.prev.next = entry + ll.head.prev = entry + } else { + entry.prev = entry + entry.next = entry + } + ll.head = entry +} + +func (ll *lruList) tail() *lruEntry { + if ll.head == nil { + return nil + } + return ll.head.prev +}