Change ids and frequencies from blob to mediumblob

committee/recordtableoperator.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2019 IoTeX
+// Copyright (c) 2020 IoTeX
 // This program is free software: you can redistribute it and/or modify it under the terms of the
 // GNU General Public License as published by the Free Software Foundation, either version 3 of
 // the License, or (at your option) any later version.
@@ -22,8 +22,10 @@ import (
 	"time"
 
 	// require sqlite3 driver
+	"github.com/iotexproject/iotex-core/pkg/log"
 	_ "github.com/mattn/go-sqlite3"
 	"github.com/pkg/errors"
+	"go.uber.org/zap"
 
 	"github.com/iotexproject/go-pkgs/hash"
 	"github.com/iotexproject/iotex-election/db"
@@ -62,6 +64,11 @@ const (
 	SQLITE DRIVERTYPE = iota
 	//MYSQL stands for a mysql driver
 	MYSQL
+
+	// MaximumMediumBlobLen is the maximum length allowed for medium blob
+	MaximumMediumBlobLen = 16777215
+	// LengthWarningThreshold is the threshold for the warning that the length of a JSON blob is close to the maximum length allowed
+	LenghWarningThreshold = 1000
 )
 
 type recordTableOperator struct {
@@ -154,7 +161,7 @@ func NewRecordTableOperator(
 		queryRecordsFunc:           queryRecordsFunc,
 		tableCreations: []string{
 			fmt.Sprintf(recordTableCreation, tableName),
-			fmt.Sprintf("CREATE TABLE IF NOT EXISTS height_to_%s (height INTEGER PRIMARY KEY, ids BLOB, frequencies BLOB)", tableName),
+			fmt.Sprintf("CREATE TABLE IF NOT EXISTS height_to_%s (height INTEGER PRIMARY KEY, ids MEDIUMBLOB, frequencies MEDIUMBLOB)", tableName),
 			fmt.Sprintf("CREATE TABLE IF NOT EXISTS identical_%s (height INTEGER PRIMARY KEY, identical_to INTEGER)", tableName),
 		},
 	}, nil
@@ -233,10 +240,18 @@ func (arch *recordTableOperator) Put(height uint64, records interface{}, tx *sql
 		if err != nil {
 			return err
 		}
+		if MaximumMediumBlobLen-len(bidBytes) < LenghWarningThreshold {
+			log.L().Warn("Length of ids is too close to the maximum length allowed",
+				zap.Int("current length", len(bidBytes)), zap.Int("maximum length", MaximumMediumBlobLen))
+		}
 		freqBytes, err := json.Marshal(frequencies)
 		if err != nil {
 			return err
 		}
+		if MaximumMediumBlobLen-len(freqBytes) < LenghWarningThreshold {
+			log.L().Warn("Length of frequencies is too close to the maximum length allowed",
+				zap.Int("current length", len(freqBytes)), zap.Int("maximum length", MaximumMediumBlobLen))
+		}
 		if _, err := tx.Exec(arch.insertHeightToRecordsQuery, height, bidBytes, freqBytes); err != nil {
 			return err
 		}

go.mod

@@ -3,59 +3,25 @@ module github.com/iotexproject/iotex-election
 go 1.12
 
 require (
-	github.com/apilayer/freegeoip v3.5.0+incompatible // indirect
-	github.com/aristanetworks/goarista v0.0.0-20190531155855-fef20d617fa7 // indirect
 	github.com/bwmarrin/discordgo v0.19.0
 	github.com/cenkalti/backoff v2.2.1+incompatible
-	github.com/docker/docker v1.13.1 // indirect
-	github.com/edsrzf/mmap-go v1.0.0 // indirect
-	github.com/elastic/gosigar v0.10.5 // indirect
 	github.com/ethereum/go-ethereum v1.8.27
-	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
-	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
-	github.com/go-logfmt/logfmt v0.4.0 // indirect
-	github.com/golang/mock v1.3.1
-	github.com/golang/protobuf v1.3.1
-	github.com/graph-gophers/graphql-go v0.0.0-20190610161739-8f92f34fc598 // indirect
+	github.com/golang/mock v1.4.0
+	github.com/golang/protobuf v1.3.2
 	github.com/hashicorp/golang-lru v0.5.1
-	github.com/howeyc/fsnotify v0.9.0 // indirect
-	github.com/huin/goupnp v1.0.0 // indirect
-	github.com/influxdata/influxdb v1.7.8 // indirect
-	github.com/iotexproject/go-ethereum v1.7.3 // indirect
-	github.com/iotexproject/go-pkgs v0.1.1
+	github.com/iotexproject/go-pkgs v0.1.2-0.20200212033110-8fa5cf96fc1b
 	github.com/iotexproject/iotex-address v0.2.1
 	github.com/iotexproject/iotex-antenna-go/v2 v2.3.2
-	github.com/iotexproject/iotex-proto v0.2.5
-	github.com/jackpal/go-nat-pmp v1.0.1 // indirect
-	github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8 // indirect
-	github.com/mattn/go-colorable v0.1.2 // indirect
-	github.com/mattn/go-runewidth v0.0.4 // indirect
+	github.com/iotexproject/iotex-core v0.11.1
+	github.com/iotexproject/iotex-proto v0.2.6-0.20200327040553-157f35632918
 	github.com/mattn/go-sqlite3 v1.11.0
 	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
-	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
-	github.com/olekukonko/tablewriter v0.0.1 // indirect
-	github.com/oschwald/maxminddb-golang v1.5.0 // indirect
 	github.com/pkg/errors v0.8.1
-	github.com/prometheus/tsdb v0.10.0 // indirect
-	github.com/rs/cors v1.7.0 // indirect
-	github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 // indirect
-	github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 // indirect
-	github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect
-	github.com/stretchr/testify v1.3.0
-	github.com/tyler-smith/go-bip39 v1.0.2 // indirect
-	github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 // indirect
+	github.com/stretchr/testify v1.4.0
 	go.etcd.io/bbolt v1.3.2
-	go.uber.org/atomic v1.4.0 // indirect
-	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.10.0
-	golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5 // indirect
-	golang.org/x/net v0.0.0-20190603091049-60506f45cf65
-	golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 // indirect
-	google.golang.org/genproto v0.0.0-20190530194941-fb225487d101 // indirect
+	golang.org/x/net v0.0.0-20191204025024-5ee1b9f4859a
 	google.golang.org/grpc v1.21.0
-	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
-	gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff // indirect
-	gopkg.in/urfave/cli.v1 v1.20.0 // indirect
 	gopkg.in/yaml.v2 v2.2.2
 )