Skip to content

Commit

Permalink
Migrate allowlists and blocklists to raw SQL (#1298)
Browse files Browse the repository at this point in the history 
Unfortunately this PR doesn't yet get rid of the `dbAllowlist` and
`dbBlocklist` types. That's because `updateBlocklist` is still using
them and it probably doesn't make sense to update that before merging
`its-happening`.
  • Loading branch information
@ChrisSchinnerl
ChrisSchinnerl authored Jun 13, 2024
1 parent 1200244 commit 9c31e34
Show file tree
Hide file tree
Showing 6 changed files with 305 additions and 142 deletions.
86 changes: 13 additions & 73 deletions stores/hostdb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,7 @@ func (ss *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time,
func (ss *SQLStore) SearchHosts(ctx context.Context, autopilotID, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.Host, error) {
var hosts []api.Host
err := ss.bMain.Transaction(ctx, func(tx sql.DatabaseTx) (err error) {
hosts, err = tx.SearchHosts(ctx, autopilotID, filterMode, usabilityMode, addressContains, keyIn, offset, limit, ss.hasAllowlist(), ss.hasBlocklist())
hosts, err = tx.SearchHosts(ctx, autopilotID, filterMode, usabilityMode, addressContains, keyIn, offset, limit)
return
})
return hosts, err
Expand Down Expand Up @@ -431,37 +431,8 @@ func (ss *SQLStore) UpdateHostAllowlistEntries(ctx context.Context, add, remove
if len(add)+len(remove) == 0 && !clear {
return nil
}
defer ss.updateHasAllowlist(&err)

// clear allowlist
if clear {
return ss.retryTransaction(ctx, func(tx *gorm.DB) error {
return tx.Where("TRUE").Delete(&dbAllowlistEntry{}).Error
})
}

var toInsert []dbAllowlistEntry
for _, entry := range add {
toInsert = append(toInsert, dbAllowlistEntry{Entry: publicKey(entry)})
}

toDelete := make([]publicKey, len(remove))
for i, entry := range remove {
toDelete[i] = publicKey(entry)
}

return ss.retryTransaction(ctx, func(tx *gorm.DB) error {
if len(toInsert) > 0 {
if err := tx.Create(&toInsert).Error; err != nil {
return err
}
}
if len(toDelete) > 0 {
if err := tx.Delete(&dbAllowlistEntry{}, "entry IN ?", toDelete).Error; err != nil {
return err
}
}
return nil
return ss.bMain.Transaction(ctx, func(tx sql.DatabaseTx) error {
return tx.UpdateHostAllowlistEntries(ctx, add, remove, clear)
})
}

Expand All @@ -470,55 +441,24 @@ func (ss *SQLStore) UpdateHostBlocklistEntries(ctx context.Context, add, remove
if len(add)+len(remove) == 0 && !clear {
return nil
}
defer ss.updateHasBlocklist(&err)

// clear blocklist
if clear {
return ss.retryTransaction(ctx, func(tx *gorm.DB) error {
return tx.Where("TRUE").Delete(&dbBlocklistEntry{}).Error
})
}

var toInsert []dbBlocklistEntry
for _, entry := range add {
toInsert = append(toInsert, dbBlocklistEntry{Entry: entry})
}

return ss.retryTransaction(ctx, func(tx *gorm.DB) error {
if len(toInsert) > 0 {
if err := tx.Create(&toInsert).Error; err != nil {
return err
}
}
if len(remove) > 0 {
if err := tx.Delete(&dbBlocklistEntry{}, "entry IN ?", remove).Error; err != nil {
return err
}
}
return nil
return ss.bMain.Transaction(ctx, func(tx sql.DatabaseTx) error {
return tx.UpdateHostBlocklistEntries(ctx, add, remove, clear)
})
}

func (ss *SQLStore) HostAllowlist(ctx context.Context) (allowlist []types.PublicKey, err error) {
var pubkeys []publicKey
err = ss.db.
WithContext(ctx).
Model(&dbAllowlistEntry{}).
Pluck("entry", &pubkeys).
Error

for _, pubkey := range pubkeys {
allowlist = append(allowlist, types.PublicKey(pubkey))
}
err = ss.bMain.Transaction(ctx, func(tx sql.DatabaseTx) error {
allowlist, err = tx.HostAllowlist(ctx)
return err
})
return
}

func (ss *SQLStore) HostBlocklist(ctx context.Context) (blocklist []string, err error) {
err = ss.db.
WithContext(ctx).
Model(&dbBlocklistEntry{}).
Pluck("entry", &blocklist).
Error
err = ss.bMain.Transaction(ctx, func(tx sql.DatabaseTx) error {
blocklist, err = tx.HostBlocklist(ctx)
return err
})
return
}

Expand Down
63 changes: 0 additions & 63 deletions stores/sql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,6 @@ type (

wg sync.WaitGroup
mu sync.Mutex
allowListCnt uint64
blockListCnt uint64
lastPrunedAt time.Time
closed bool

Expand Down Expand Up @@ -237,16 +235,6 @@ func NewSQLStore(cfg Config) (*SQLStore, modules.ConsensusChangeID, error) {
return nil, modules.ConsensusChangeID{}, err
}

// Check allowlist and blocklist counts
allowlistCnt, err := tableCount(db, &dbAllowlistEntry{})
if err != nil {
return nil, modules.ConsensusChangeID{}, err
}
blocklistCnt, err := tableCount(db, &dbBlocklistEntry{})
if err != nil {
return nil, modules.ConsensusChangeID{}, err
}

// Fetch contract ids.
var activeFCIDs, archivedFCIDs []fileContractID
if err := db.Model(&dbContract{}).
Expand Down Expand Up @@ -276,8 +264,6 @@ func NewSQLStore(cfg Config) (*SQLStore, modules.ConsensusChangeID, error) {
knownContracts: isOurContract,
lastSave: time.Now(),
persistInterval: cfg.PersistInterval,
allowListCnt: uint64(allowlistCnt),
blockListCnt: uint64(blocklistCnt),
settings: make(map[string]string),
slabPruneSigChan: make(chan struct{}, 1),
unappliedContractState: make(map[types.FileContractID]contractState),
Expand Down Expand Up @@ -321,12 +307,6 @@ func isSQLite(db *gorm.DB) bool {
}
}

func (ss *SQLStore) hasAllowlist() bool {
ss.mu.Lock()
defer ss.mu.Unlock()
return ss.allowListCnt > 0
}

func (s *SQLStore) initSlabPruning() error {
// start pruning loop
s.wg.Add(1)
Expand All @@ -342,49 +322,6 @@ func (s *SQLStore) initSlabPruning() error {
})
}

func (ss *SQLStore) updateHasAllowlist(err *error) {
if *err != nil {
return
}

cnt, cErr := tableCount(ss.db, &dbAllowlistEntry{})
if cErr != nil {
*err = cErr
return
}

ss.mu.Lock()
ss.allowListCnt = uint64(cnt)
ss.mu.Unlock()
}

func (ss *SQLStore) hasBlocklist() bool {
ss.mu.Lock()
defer ss.mu.Unlock()
return ss.blockListCnt > 0
}

func (ss *SQLStore) updateHasBlocklist(err *error) {
if *err != nil {
return
}

cnt, cErr := tableCount(ss.db, &dbBlocklistEntry{})
if cErr != nil {
*err = cErr
return
}

ss.mu.Lock()
ss.blockListCnt = uint64(cnt)
ss.mu.Unlock()
}

func tableCount(db *gorm.DB, model interface{}) (cnt int64, err error) {
err = db.Model(model).Count(&cnt).Error
return
}

// Close closes the underlying database connection of the store.
func (s *SQLStore) Close() error {
s.shutdownCtxCancel()
Expand Down
15 changes: 14 additions & 1 deletion stores/sql/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,13 @@ type (
// prefix and returns 'true' if any object was deleted.
DeleteObjects(ctx context.Context, bucket, prefix string, limit int64) (bool, error)

// HostAllowlist returns the list of public keys of hosts on the
// allowlist.
HostAllowlist(ctx context.Context) ([]types.PublicKey, error)

// HostBlocklist returns the list of host addresses on the blocklist.
HostBlocklist(ctx context.Context) ([]string, error)

// InsertObject inserts a new object into the database.
InsertObject(ctx context.Context, bucket, key, contractSet string, dirID int64, o object.Object, mimeType, eTag string, md api.ObjectUserMetadata) error

Expand Down Expand Up @@ -164,7 +171,7 @@ type (
SaveAccounts(ctx context.Context, accounts []api.Account) error

// SearchHosts returns a list of hosts that match the provided filters
SearchHosts(ctx context.Context, autopilotID, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int, hasAllowList, hasBlocklist bool) ([]api.Host, error)
SearchHosts(ctx context.Context, autopilotID, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.Host, error)

// SetUncleanShutdown sets the clean shutdown flag on the accounts to
// 'false' and also marks them as requiring a resync.
Expand All @@ -178,6 +185,12 @@ type (
// one, fully overwriting the existing policy.
UpdateBucketPolicy(ctx context.Context, bucket string, policy api.BucketPolicy) error

// UpdateHostAllowlistEntries updates the allowlist in the database
UpdateHostAllowlistEntries(ctx context.Context, add, remove []types.PublicKey, clear bool) error

// UpdateHostBlocklistEntries updates the blocklist in the database
UpdateHostBlocklistEntries(ctx context.Context, add, remove []string, clear bool) error

// UpdateObjectHealth updates the health of all objects to the lowest
// health of all its slabs.
UpdateObjectHealth(ctx context.Context) error
Expand Down
45 changes: 44 additions & 1 deletion stores/sql/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,42 @@ func CopyObject(ctx context.Context, tx sql.Tx, srcBucket, dstBucket, srcKey, ds
return fetchMetadata(dstObjID)
}

func HostAllowlist(ctx context.Context, tx sql.Tx) ([]types.PublicKey, error) {
rows, err := tx.Query(ctx, "SELECT entry FROM host_allowlist_entries")
if err != nil {
return nil, fmt.Errorf("failed to fetch host allowlist: %w", err)
}
defer rows.Close()

var allowlist []types.PublicKey
for rows.Next() {
var pk PublicKey
if err := rows.Scan(&pk); err != nil {
return nil, fmt.Errorf("failed to scan public key: %w", err)
}
allowlist = append(allowlist, types.PublicKey(pk))
}
return allowlist, nil
}

func HostBlocklist(ctx context.Context, tx sql.Tx) ([]string, error) {
rows, err := tx.Query(ctx, "SELECT entry FROM host_blocklist_entries")
if err != nil {
return nil, fmt.Errorf("failed to fetch host blocklist: %w", err)
}
defer rows.Close()

var blocklist []string
for rows.Next() {
var entry string
if err := rows.Scan(&entry); err != nil {
return nil, fmt.Errorf("failed to scan blocklist entry: %w", err)
}
blocklist = append(blocklist, entry)
}
return blocklist, nil
}

func HostsForScanning(ctx context.Context, tx sql.Tx, maxLastScan time.Time, offset, limit int) ([]api.HostAddress, error) {
if offset < 0 {
return nil, ErrNegativeOffset
Expand Down Expand Up @@ -941,11 +977,18 @@ func RemoveOfflineHosts(ctx context.Context, tx sql.Tx, minRecentFailures uint64
return res.RowsAffected()
}

func SearchHosts(ctx context.Context, tx sql.Tx, autopilot, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int, hasAllowlist, hasBlocklist bool) ([]api.Host, error) {
func SearchHosts(ctx context.Context, tx sql.Tx, autopilot, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) ([]api.Host, error) {
if offset < 0 {
return nil, ErrNegativeOffset
}

var hasAllowlist, hasBlocklist bool
if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM host_allowlist_entries)").Scan(&hasAllowlist); err != nil {
return nil, fmt.Errorf("failed to check for allowlist: %w", err)
} else if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM host_blocklist_entries)").Scan(&hasBlocklist); err != nil {
return nil, fmt.Errorf("failed to check for blocklist: %w", err)
}

// validate filterMode
switch filterMode {
case api.HostFilterModeAllowed:
Expand Down
Loading

0 comments on commit 9c31e34

Please sign in to comment.