diff --git a/Dockerfile b/Dockerfile index 7b20a108c..e3c09229c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,8 @@ # [-d|--detach] \ # [-it] \ # [--rm] \ -# [--privileged] \ +# [--cap-add SYS_ADMIN] \ +# [--device /dev/fuse] \ # [--mount src="$(pwd)",target="/src",type=bind] \ # [--env DISPLAY=:[.|[:] @@ -53,8 +54,11 @@ # -d|--detach: tells Docker to detach from running container # -it: tells Docker to run container interactively # --rm: tells Docker to destroy container upon exit -# --privileged: -# 1) tells Docker to, among other things, grant access to /dev/fuse +# --cap-add: +# 1) tells Docker to enable FUSE mounts +# 2) only useful for --target dev and --target iclient +# --device: +# 1) tells Docker to grant access to /dev/fuse # 2) only useful for --target dev and --target iclient # --mount: # 1) bind mounts the context into /src in the container @@ -62,11 +66,11 @@ # 3) only useful for --target dev # --env DISPLAY: tells Docker to set ENV DISPLAY for X apps (e.g. wireshark) -FROM alpine:3.15.0 as base +FROM alpine:3.17 as base RUN apk add --no-cache libc6-compat FROM base as dev -ARG GolangVersion=1.18 +ARG GolangVersion=1.19.4 RUN apk add --no-cache \ bind-tools \ curl \ @@ -94,6 +98,7 @@ RUN go build github.com/go-delve/delve/cmd/dlv RUN cp dlv /usr/local/go/bin/. VOLUME /src WORKDIR /src +RUN git config --global --add safe.directory /src FROM dev as build ARG MakeTarget diff --git a/README.md b/README.md index 33b0e0c9a..0937090e1 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ To clear out prior deamon runs and run the daemons in the background: * [`dev` /src#] rm -rf /tmp/ickptDB * [`dev` /src#] ickpt/ickpt ickpt/dev.conf & * [`dev` /src#] imgr/imgr imgr/dev.conf & -* [`dev` /src#] idestroy/idestroy iclient/dev.conf & +* [`dev` /src#] idestroy/idestroy iclient/dev.conf * [`dev` /src#] imgr/mkmount.sh -fs * [`dev` /src#] iclient/iclient iclient/dev.conf & diff --git a/bucketstats/api.go b/bucketstats/api.go index 7bfcf63ea..7ea3c5290 100644 --- a/bucketstats/api.go +++ b/bucketstats/api.go @@ -13,7 +13,6 @@ // placed in a structure and registered, with a name, via a call to Register() // before being used. The set of the statistics registered can be queried using // the registered name or individually. -// package bucketstats import ( @@ -31,7 +30,6 @@ const ( // values added. // // Adding a negative value is not supported. -// type Totaler interface { Increment() Add(value uint64) @@ -43,7 +41,6 @@ type Totaler interface { // // This adds a CountGet() function that returns the number of values added as // well as an AverageGet() method that returns the average. -// type Averager interface { Totaler CountGet() (count uint64) @@ -57,11 +54,11 @@ type Averager interface { // RangeHigh the largest value mapped to the bucket // NominalVal the nominal value of the bucket (sqrt(2)^n or 2^n) // MeanVal the mean value of values added to the bucket, assuming -// a uniform distribution +// +// a uniform distribution // // When performing math on these statistics be careful of overflowing a uint64. // It may be a good idea to use the "math/big" package. -// type BucketInfo struct { Count uint64 NominalVal uint64 @@ -77,7 +74,6 @@ type BucketInfo struct { // // DistGet() returns the distribution of values across the buckets as an array // of BucketInfo. -// type Bucketer interface { Averager DistGet() []BucketInfo @@ -93,7 +89,6 @@ type Bucketer interface { // of stats. One or the other, but not both, can be the empty string. // Whitespace characters, '"' (double quote), '*' (asterik), and ':' (colon) are // not allowed in either name. -// func Register(pkgName string, statsGroupName string, statsStruct interface{}) { register(pkgName, statsGroupName, statsStruct) } @@ -102,7 +97,6 @@ func Register(pkgName string, statsGroupName string, statsStruct interface{}) { // // Once unregistered, the same or a different set of statistics can be // registered using the same name. -// func UnRegister(pkgName string, statsGroupName string) { unRegister(pkgName, statsGroupName) } @@ -115,7 +109,6 @@ func UnRegister(pkgName string, statsGroupName string) { // // Use "*" to select all package names with a given group name, all // groups with a given package name, or all groups. -// func SprintStats(stringFmt StatStringFormat, pkgName string, statsGroupName string) (values string) { return sprintStats(stringFmt, pkgName, statsGroupName) } @@ -124,7 +117,6 @@ func SprintStats(stringFmt StatStringFormat, pkgName string, statsGroupName stri // // Name must be unique within statistics in the structure. If it is "" then // Register() will assign a name based on the name of the field. -// type Total struct { total uint64 // Ensure 64-bit alignment Name string @@ -143,7 +135,6 @@ func (this *Total) TotalGet() uint64 { } // Return a string with the statistic's value in the specified format. -// func (this *Total) Sprint(stringFmt StatStringFormat, pkgName string, statsGroupName string) string { return this.sprint(stringFmt, pkgName, statsGroupName) } @@ -153,7 +144,6 @@ func (this *Total) Sprint(stringFmt StatStringFormat, pkgName string, statsGroup // // Name must be unique within statistics in the structure. If it is "" then // Register() will assign a name based on the name of the field. -// type Average struct { count uint64 // Ensure 64-bit alignment total uint64 // Ensure 64-bit alignment @@ -161,14 +151,12 @@ type Average struct { } // Add a value to the mean statistics. -// func (this *Average) Add(value uint64) { atomicAddUint64(&this.total, value) atomicAddUint64(&this.count, 1) } // Add a value of 1 to the mean statistics. -// func (this *Average) Increment() { this.Add(1) } @@ -186,7 +174,6 @@ func (this *Average) AverageGet() uint64 { } // Return a string with the statistic's value in the specified format. -// func (this *Average) Sprint(stringFmt StatStringFormat, pkgName string, statsGroupName string) string { return this.sprint(stringFmt, pkgName, statsGroupName) } @@ -206,18 +193,19 @@ func (this *Average) Sprint(stringFmt StatStringFormat, pkgName string, statsGro // // Example mappings of values to buckets: // -// Values Bucket -// 0 0 -// 1 1 -// 2 2 -// 3 - 5 3 -// 6 - 11 4 +// Values Bucket +// 0 0 +// 1 1 +// 2 2 +// 3 - 5 3 +// 6 - 11 4 +// // 12 - 22 5 -// etc. +// +// etc. // // Note that value 2^n increments the count in bucket n + 1, but the average of // values in bucket n is very slightly larger than 2^n. -// type BucketLog2Round struct { Name string NBucket uint @@ -243,7 +231,6 @@ func (this *BucketLog2Round) Add(value uint64) { } // Add a value of 1 to the bucketized statistics. -// func (this *BucketLog2Round) Increment() { this.Add(1) } @@ -264,13 +251,11 @@ func (this *BucketLog2Round) AverageGet() uint64 { } // Return BucketInfo information for all the buckets. -// func (this *BucketLog2Round) DistGet() []BucketInfo { return bucketDistMake(this.NBucket, this.statBuckets[:], log2RoundBucketTable[:]) } // Return a string with the statistic's value in the specified format. -// func (this *BucketLog2Round) Sprint(stringFmt StatStringFormat, pkgName string, statsGroupName string) string { return bucketSprint(stringFmt, pkgName, statsGroupName, this.Name, this.DistGet()) } @@ -293,20 +278,21 @@ func (this *BucketLog2Round) Sprint(stringFmt StatStringFormat, pkgName string, // // Example mappings of values to buckets: // -// Values Bucket -// 0 0 -// 1 1 -// 2 2 -// 3 3 -// 4 4 -// 5 - 6 5 -// 7 - 9 6 +// Values Bucket +// 0 0 +// 1 1 +// 2 2 +// 3 3 +// 4 4 +// 5 - 6 5 +// 7 - 9 6 +// // 10 - 13 7 -// etc. +// +// etc. // // Note that a value sqrt(2)^n increments the count in bucket 2 * n, but the // average of values in bucket n is slightly larger than sqrt(2)^n. -// type BucketLogRoot2Round struct { Name string NBucket uint @@ -332,7 +318,6 @@ func (this *BucketLogRoot2Round) Add(value uint64) { } // Add a value of 1 to the bucketized statistics. -// func (this *BucketLogRoot2Round) Increment() { this.Add(1) } @@ -353,13 +338,11 @@ func (this *BucketLogRoot2Round) AverageGet() uint64 { } // Return BucketInfo information for all the buckets. -// func (this *BucketLogRoot2Round) DistGet() []BucketInfo { return bucketDistMake(this.NBucket, this.statBuckets[:], logRoot2RoundBucketTable[:]) } // Return a string with the statistic's value in the specified format. -// func (this *BucketLogRoot2Round) Sprint(stringFmt StatStringFormat, pkgName string, statsGroupName string) string { return bucketSprint(stringFmt, pkgName, statsGroupName, this.Name, this.DistGet()) } diff --git a/bucketstats/api_test.go b/bucketstats/api_test.go index f9033bb3e..2b56e4250 100644 --- a/bucketstats/api_test.go +++ b/bucketstats/api_test.go @@ -364,7 +364,6 @@ func TestTotaler(t *testing.T) { } // Test Bucketer specific functionality (which is mostly buckets) -// func TestBucketer(t *testing.T) { var ( @@ -553,7 +552,6 @@ func TestSprintStats(t *testing.T) { // // If panic() is called with a nil argument then this function also returns the // empty string. -// func catchAPanic(aFunc func()) (panicStr string) { defer func() { diff --git a/bucketstats/impl.go b/bucketstats/impl.go index 14194b1e4..7bee5a6da 100644 --- a/bucketstats/impl.go +++ b/bucketstats/impl.go @@ -23,7 +23,6 @@ var ( // Register a set of statistics, where the statistics are one or more fields in // the passed structure. -// func register(pkgName string, statsGroupName string, statsStruct interface{}) { var ok bool @@ -157,7 +156,6 @@ func unRegister(pkgName string, statsGroupName string) { } // Return the selected group(s) of statistics as a string. -// func sprintStats(statFmt StatStringFormat, pkgName string, statsGroupName string) (statValues string) { statsNameMapLock.Lock() @@ -246,7 +244,6 @@ func sprintStatsStruct(statFmt StatStringFormat, pkgName string, statsGroupName } // Construct and return a statistics name (fully qualified field name) in the specified format. -// func statisticName(statFmt StatStringFormat, pkgName string, statsGroupName string, fieldName string) string { switch statFmt { @@ -273,7 +270,6 @@ func statisticName(statFmt StatStringFormat, pkgName string, statsGroupName stri } // Return the "name" of the bucket that would hold 'n' as the string "2^x". -// func bucketNameLog2(value uint64) string { var idx uint @@ -289,7 +285,6 @@ func bucketNameLog2(value uint64) string { // Return the "name" of the bucket that would hold 'n' as the string "2^x", // where x can have the suffix ".5" as in "2^7.5". -// func bucketNameLogRoot2(value uint64) string { var idx uint @@ -307,7 +302,6 @@ func bucketNameLogRoot2(value uint64) string { } // Return a string with the statistic's value in the specified format. -// func (this *Total) sprint(statFmt StatStringFormat, pkgName string, statsGroupName string) string { statName := statisticName(statFmt, pkgName, statsGroupName, this.Name) @@ -321,7 +315,6 @@ func (this *Total) sprint(statFmt StatStringFormat, pkgName string, statsGroupNa } // Return a string with the statistic's value in the specified format. -// func (this *Average) sprint(statFmt StatStringFormat, pkgName string, statsGroupName string) string { statName := statisticName(statFmt, pkgName, statsGroupName, this.Name) @@ -341,7 +334,6 @@ func (this *Average) sprint(statFmt StatStringFormat, pkgName string, statsGroup // The canonical distribution for a bucketized statistic is an array of BucketInfo. // Create one based on the information for this bucketstat . -// func bucketDistMake(nBucket uint, statBuckets []uint32, bucketInfoBase []BucketInfo) []BucketInfo { // copy the base []BucketInfo before modifying it @@ -371,11 +363,12 @@ func bucketDistMake(nBucket uint, statBuckets []uint32, bucketInfoBase []BucketI // // o the index of the first entry with a non-zero count // o the index + 1 of the last entry with a non-zero count, or zero if no such -// bucket exists +// +// bucket exists +// // o the count (number things in buckets) // o sum of counts * count_meanVal, and // o mean (average) -// func bucketCalcStat(bucketInfo []BucketInfo) (firstIdx int, maxIdx int, count uint64, sum uint64, mean uint64) { var ( @@ -421,7 +414,6 @@ func bucketCalcStat(bucketInfo []BucketInfo) (firstIdx int, maxIdx int, count ui } // Return a string with the bucketized statistic content in the specified format. -// func bucketSprint(statFmt StatStringFormat, pkgName string, statsGroupName string, fieldName string, bucketInfo []BucketInfo) string { @@ -459,7 +451,6 @@ func bucketSprint(statFmt StatStringFormat, pkgName string, statsGroupName strin } // Replace illegal characters in names with underbar (`_`) -// func scrubName(name string) string { // Names should include only pritable characters that are not diff --git a/bucketstats/tables.go b/bucketstats/tables.go index 92fb7f864..14caa3601 100644 --- a/bucketstats/tables.go +++ b/bucketstats/tables.go @@ -17,7 +17,6 @@ import ( // values they hold. // generate the tables for BucketStatsLogRoot2 -// func genLogRoot2Table() { logRoot2Index := func(val int) (logRoot2_x float64, idx uint) { @@ -41,7 +40,6 @@ func genLogRoot2Table() { } // generate the tables for BucketStatsLog2 -// func genLog2Table() { log2Index := func(val int) (log2_x float64, idx uint) { @@ -68,7 +66,6 @@ func genLog2Table() { // indexFunc() is either our tweaked version of log2(x) or logRoot2(x) with // float64 being the actual value and int being the bucket index its mapped // to. -// func genIdxTable(name string, indexFunc func(int) (float64, uint)) { var ( indent int = 8 @@ -115,7 +112,6 @@ func genIdxTable(name string, indexFunc func(int) (float64, uint)) { // indexFunc() is either our tweaked version of log2(x) or logRoot2(x) for the // table with float64 being the actual value and int being the bucket index its // mapped to. -// func genBucketTable(name string, indexFunc func(int) (float64, uint), nBucket uint, bucketsPerBit uint) { @@ -209,7 +205,6 @@ func genBucketTable(name string, indexFunc func(int) (float64, uint), // Compute round(sqrt(2)^n) for 0 <= n < 128 and return as a uint64 accurate in // all 64 bits. -// func powRoot2(n uint) (pow64 uint64) { var ( bigBase big.Float @@ -236,7 +231,6 @@ func powRoot2(n uint) (pow64 uint64) { // print a list of which bucket the first 256 values go in and the average // value represented by the bucket -// func showDistr(bucketTable []uint8) { // track info for each bucket @@ -283,7 +277,6 @@ func showDistr(bucketTable []uint8) { // // One consequence is that the log base 2 statistics require 65 buckets for 64 // bit numbers instead of 64 buckets. -// var log2RoundIdxTable = [256]uint8{ // 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 // -Inf 0.0 1.0 1.6 2.0 2.3 2.6 2.8 3.0 3.2 3.3 3.5 3.6 3.7 3.8 3.9 diff --git a/conf/api.go b/conf/api.go index ccfac99e6..08263f35b 100644 --- a/conf/api.go +++ b/conf/api.go @@ -349,7 +349,6 @@ func (confMap ConfMap) UpdateFromFile(confFilePath string) (err error) { // // To enable efficient comparisons, the elements of the ConfMap will be // sorted in the output (both by sectionName and by optionName). -// func (confMap ConfMap) Dump() (confMapString string) { var ( confOption ConfMapOption @@ -835,29 +834,29 @@ func (confMap ConfMap) FetchOptionValueDuration(sectionName string, optionName s // // From RFC 4122, a UUID string is defined as follows: // -// UUID = time-low "-" time-mid "-" time-high-and-version "-" clock-seq-and-reserved clock-seq-low "-" node -// time-low = 4hexOctet -// time-mid = 2hexOctet -// time-high-and-version = 2hexOctet -// clock-seq-and-reserved = hexOctet -// clock-seq-low = hexOctet -// node = 6hexOctet -// hexOctet = hexDigit hexDigit -// hexDigit = "0" / "1" / "2" / "3" / "4" / "5" / "6" / "7" / "8" / "9" / "a" / "b" / "c" / "d" / "e" / "f" / "A" / "B" / "C" / "D" / "E" / "F" +// UUID = time-low "-" time-mid "-" time-high-and-version "-" clock-seq-and-reserved clock-seq-low "-" node +// time-low = 4hexOctet +// time-mid = 2hexOctet +// time-high-and-version = 2hexOctet +// clock-seq-and-reserved = hexOctet +// clock-seq-low = hexOctet +// node = 6hexOctet +// hexOctet = hexDigit hexDigit +// hexDigit = "0" / "1" / "2" / "3" / "4" / "5" / "6" / "7" / "8" / "9" / "a" / "b" / "c" / "d" / "e" / "f" / "A" / "B" / "C" / "D" / "E" / "F" // // From RFC 4122, a UUID (i.e. "in memory") is defined as follows (BigEndian/NetworkByteOrder): // -// 0 1 2 3 -// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | time_low | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | time_mid | time_hi_and_version | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// |clk_seq_hi_res | clk_seq_low | node (0-1) | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | node (2-5) | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | time_low | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | time_mid | time_hi_and_version | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |clk_seq_hi_res | clk_seq_low | node (0-1) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | node (2-5) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ func (confMap ConfMap) FetchOptionValueUUID(sectionName string, optionName string) (optionValue []byte, err error) { optionValue = make([]byte, 16) diff --git a/docker-compose.yml b/docker-compose.yml index b7b8b784b..cfe5c4527 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ # To build the `development` service # -# docker compose build [--build-arg GolangVersion=] [--build-arg MakeTarget={|all|ci|minimal}] +# docker-compose build [--build-arg GolangVersion=] [--build-arg MakeTarget={|all|ci|minimal}] # # Notes: # --build-arg Key=Value settings passed on to Dockerfile @@ -12,34 +12,34 @@ # # To run the resultant application: # -# docker compose up [-d|--detach] {dev|ickpt|imgr|iclient} +# docker-compose up [-d|--detach] {dev|ickpt|imgr|iclient} # # Notes: # -d|--detach: -# 1) tells docker compose to detach from running containers +# 1) tells docker-compose to detach from running containers # 2) if supplied: # a) stop application with `docker compose down` # b) containers are removed upon application down # 3) if not supplied: # a) stop application with ^C # b) containers are left in "exited" state upon application down -# dev: tells docker compose to only bring up dev service (w/ dependencies) -# ickpt: tells docker compose to only bring up ickpt service (w/ dependencies) -# imgr: tells docker compose to only bring up imgr service (w/ dependencies) -# iclient: tells docker compose to only bring up iclient service (w/ dependencies) +# dev: tells docker-compose to only bring up dev service (w/ dependencies) +# ickpt: tells docker-compose to only bring up ickpt service (w/ dependencies) +# imgr: tells docker-compose to only bring up imgr service (w/ dependencies) +# iclient: tells docker-compose to only bring up iclient service (w/ dependencies) # Precisely one of {dev|ickpt|imgr|iclient} must be specied... # If no service is specified, dev published ports will collide with other services # Containers launched have been named the same as their corresponding service name # # To stop the resultant application: # -# docker compose down +# docker-compose down version: '3.8' services: swift: - image: dockerswiftaio/docker-swift:2.27.0 + image: dockerswiftaio/docker-swift:2.29.0 container_name: proxyfs_swift expose: - 8080 # curl http://swift:8080/info @@ -55,7 +55,10 @@ services: container_name: proxyfs_dev depends_on: - swift - privileged: true + cap_add: + - SYS_ADMIN + devices: + - '/dev/fuse' expose: - 33123 # ICKPT.Port - 32356 # IMGR.RetryRPCPort @@ -100,7 +103,6 @@ services: container_name: proxyfs_ickpt depends_on: - swift - privileged: true expose: - 33123 # ICKPT.Port ports: @@ -138,7 +140,10 @@ services: depends_on: - swift - imgr - privileged: true + cap_add: + - SYS_ADMIN + devices: + - '/dev/fuse' expose: - 15347 # ICLIENT.HTTPServerPort ports: diff --git a/go.mod b/go.mod index ad45da7a9..b8eadfa3b 100644 --- a/go.mod +++ b/go.mod @@ -1,21 +1,21 @@ module github.com/NVIDIA/proxyfs -go 1.17 +go 1.19 require ( - github.com/NVIDIA/fission v0.0.0-20220110231326-e35d84a12929 - github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 + github.com/NVIDIA/fission v0.0.0-20221207000636-f8dd12d86072 + github.com/NVIDIA/sortedmap v0.0.0-20221206223250-5aaf17459438 github.com/google/btree v1.0.1 github.com/sirupsen/logrus v1.8.1 - github.com/stretchr/testify v1.7.0 - golang.org/x/sys v0.0.0-20220315194320-039c03cc5b86 + github.com/stretchr/testify v1.7.1 + golang.org/x/sys v0.3.0 ) require ( - github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 // indirect + github.com/NVIDIA/cstruct v0.0.0-20221206222058-cbc877f192d5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/kr/pretty v0.2.0 // indirect + github.com/kr/pretty v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 3ff4e06bc..882e02610 100644 --- a/go.sum +++ b/go.sum @@ -1,34 +1,49 @@ github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 h1:hMAAyAeYB1T1DnxqdDZzjWeTDz/hL0ZGFhz3uQyH1nQ= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8/go.mod h1:GPbuJvLD4QWiHPS6vivLzh+XMAx6va0Aucm6ipa5S0I= +github.com/NVIDIA/cstruct v0.0.0-20221206222058-cbc877f192d5 h1:u9jQ3H8RSqtqTu7b+oACnChS7HsVpbV46/qBJPVCBjU= +github.com/NVIDIA/cstruct v0.0.0-20221206222058-cbc877f192d5/go.mod h1:s/x+WQqgkcAbDPltLPIs0AwqxY7W6UG0RGhuMVy8tds= github.com/NVIDIA/fission v0.0.0-20220110231326-e35d84a12929 h1:iFIrEXsx1JGcepyqoMPo9IHqMFbe799HAOpxVeykQks= github.com/NVIDIA/fission v0.0.0-20220110231326-e35d84a12929/go.mod h1:9wVslsyxZaBvW/ollg7JLxJOxKb+Ik2KH1WVs1nicMA= +github.com/NVIDIA/fission v0.0.0-20221207000636-f8dd12d86072 h1:ablyovtDtsJkcUc9SRJ5YTCiIPbx77Fb+KwFAdunphc= +github.com/NVIDIA/fission v0.0.0-20221207000636-f8dd12d86072/go.mod h1:OWPp+2S3hQCved6NTfpAFeGjibx8dKKpBDdiU12ZrrE= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 h1:mDx/maO8psu+pHQqEDoL15WTj/BAAnu/sKSeOVR8wZI= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= +github.com/NVIDIA/sortedmap v0.0.0-20221206223250-5aaf17459438 h1:0TZfWYaOMyrK2aiyf5KcX9rxFyVapg8m1zd8AWBefwE= +github.com/NVIDIA/sortedmap v0.0.0-20221206223250-5aaf17459438/go.mod h1:uTjxdPdujmeJm/c1BNApnnX4KW+9nYmt0L6Ijft1+M4= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220315194320-039c03cc5b86 h1:A9i04dxx7Cribqbs8jf3FQLogkL/CV2YN7hj9KWJCkc= -golang.org/x/sys v0.0.0-20220315194320-039c03cc5b86/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664 h1:v1W7bwXHsnLLloWYTVEdvGvA7BHMeBYsPcF0GLDxIRs= +golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/iauth/api.go b/iauth/api.go index c8f3c0440..763a5a9a3 100644 --- a/iauth/api.go +++ b/iauth/api.go @@ -13,7 +13,6 @@ import ( // // The return from the Auth PlugIn's PerformAuth func is simply returned to the // caller of this func. -// func PerformAuth(authPlugInPath string, authInString string) (authToken string, storageURL string, err error) { var ( ok bool diff --git a/iauth/iauth-swift/iauth-swift.go b/iauth/iauth-swift/iauth-swift.go index ca4654e49..2520e03b5 100644 --- a/iauth/iauth-swift/iauth-swift.go +++ b/iauth/iauth-swift/iauth-swift.go @@ -24,7 +24,6 @@ type authInStruct struct { // // The format of authInJSON is determined by the Go JSON unmarshalling conventions // of the AuthInStruct declared above. -// func PerformAuth(authInJSON string) (authToken string, storageURL string, err error) { var ( authIn authInStruct diff --git a/icert/icertpkg/api.go b/icert/icertpkg/api.go index 21b12395f..e93332676 100644 --- a/icert/icertpkg/api.go +++ b/icert/icertpkg/api.go @@ -8,7 +8,6 @@ // work with either on-disk PEM files and/or in-memory PEM blocks (byte slices). // // Inspired by https://shaneutt.com/blog/golang-ca-and-signed-cert-go/ -// package icertpkg import ( @@ -49,7 +48,6 @@ const ( // certFile (if not "") and/or keyPEMBlock will be written to keyFile (if not ""). // If certFile and keyFile are the same, both the CA Certificate and its Private // Key will be written to the common file. -// func GenCACert(generateKeyAlgorithm string, subject pkix.Name, ttl time.Duration, certFile string, keyFile string) (certPEMBlock []byte, keyPEMBlock []byte, err error) { certPEMBlock, keyPEMBlock, err = genCACert(generateKeyAlgorithm, subject, ttl, certFile, keyFile) return @@ -68,7 +66,6 @@ func GenCACert(generateKeyAlgorithm string, subject pkix.Name, ttl time.Duration // and/or endpointKeyPEMBlock will be written to endpointKeyFile (if not ""). // If endpointCertFile and endpointKeyFile are the same, both the Endpoint // Certificate and its Private Key will be written to the common file. -// func GenEndpointCert(generateKeyAlgorithm string, subject pkix.Name, dnsNames []string, ipAddresses []net.IP, ttl time.Duration, caCert interface{}, caKey interface{}, endpointCertFile string, endpointKeyFile string) (endpointCertPEMBlock []byte, endpointKeyPEMBlock []byte, err error) { endpointCertPEMBlock, endpointKeyPEMBlock, err = genEndpointCert(generateKeyAlgorithm, subject, dnsNames, ipAddresses, ttl, caCert, caKey, endpointCertFile, endpointKeyFile) return diff --git a/icert/main.go b/icert/main.go index 1592eb36d..99493b9af 100644 --- a/icert/main.go +++ b/icert/main.go @@ -5,42 +5,42 @@ // // The following can be obtained by running the "icert -h" command: // -// Usage of icert: -// -ca -// generated CA Certicate usable for signing Endpoint Certificates -// -caCert string -// path to CA Certificate -// -caKey string -// path to CA Certificate's PrivateKey -// -cert string -// path to Endpoint Certificate -// -commonName string -// generated Certificate's Subject.CommonName -// -country value -// generated Certificate's Subject.Country -// -dns value -// generated Certificate's DNS Name -// -ed25519 -// generate key via Ed25519 -// -ip value -// generated Certificate's IP Address -// -key string -// path to Endpoint Certificate's PrivateKey -// -locality value -// generated Certificate's Subject.Locality -// -organization value -// generated Certificate's Subject.Organization -// -postalCode value -// generated Certificate's Subject.PostalCode -// -province value -// generated Certificate's Subject.Province -// -rsa -// generate key via RSA -// -streetAddress value -// generated Certificate's Subject.StreetAddress -// -ttl uint -// generated Certificate's time to live in days -// -v verbose mode +// Usage of icert: +// -ca +// generated CA Certicate usable for signing Endpoint Certificates +// -caCert string +// path to CA Certificate +// -caKey string +// path to CA Certificate's PrivateKey +// -cert string +// path to Endpoint Certificate +// -commonName string +// generated Certificate's Subject.CommonName +// -country value +// generated Certificate's Subject.Country +// -dns value +// generated Certificate's DNS Name +// -ed25519 +// generate key via Ed25519 +// -ip value +// generated Certificate's IP Address +// -key string +// path to Endpoint Certificate's PrivateKey +// -locality value +// generated Certificate's Subject.Locality +// -organization value +// generated Certificate's Subject.Organization +// -postalCode value +// generated Certificate's Subject.PostalCode +// -province value +// generated Certificate's Subject.Province +// -rsa +// generate key via RSA +// -streetAddress value +// generated Certificate's Subject.StreetAddress +// -ttl uint +// generated Certificate's time to live in days +// -v verbose mode // // Precisely one of "-ed25519" or "-rsa" must be specified. // @@ -53,7 +53,6 @@ // // If "-ca" is not specified, both "-cert" and "-key" must be specified. // Similarly, at least one "-dns" and/or one "-ip" must be specified. -// package main import ( diff --git a/ickpt/ickptpkg/api.go b/ickpt/ickptpkg/api.go index 5d89b689e..1e30504fd 100644 --- a/ickpt/ickptpkg/api.go +++ b/ickpt/ickptpkg/api.go @@ -29,32 +29,32 @@ // operations of deleting and setting/updating a ProxyFS Volume's CheckPoint. // The Path should simply be empty for all other operations. // -// DELETE / +// DELETE / // // Following a successful GET of the X-Storage-Url specifying the X-Auth-Token // received, the local ilayout.CheckPointV*Struct value retained for that X-Storage-Url // will be marked for deletion committed via a corresponding POST. The linkage between // DELETE and POST requests is that they must share a common Path value. // -// GET / +// GET / // // Following a successful GET of the X-Storage-Url specifying the X-Auth-Token // received, the local ilayout.CheckPointV*Struct value retained for that X-Storage-Url // will be returned. If no such value has been retained, the value returned in the GET // will be returned instead. An uncommitted prior DELETE or PUT is ignored and discarded. // -// HEAD / +// HEAD / // // A successful 204 No Content response indicates the package instance is in operation. // -// POST / +// POST / // // Following a successful GET of the X-Storage-Url specifying the X-Auth-Token // received, the prior as-yet uncommitted DELETE or PUT will be committed. In // order to align this POST with the uncommitted DELETE or PUT, their Path values // must match. // -// PUT / +// PUT / // // Following a successful GET of the X-Storage-Url specifying the X-Auth-Token // received, the local ilayout.CheckPointV*Struct value for that X-Storage-Url @@ -64,17 +64,16 @@ // To configure an ickptpkg instance, Start() is called passing, as the sole // argument, a package conf ConfMap. Here is a sample .conf file: // -// [ICKPT] -// IPAddr: ickpt -// Port: 33123 # TCP or TLS as determined by: -// CertFilePath: # TCP: if all of {Cert|Key|CACert}FilePath are missing or empty -// KeyFilePath: # - or - -// CACertFilePath: # TLS: if all of {Cert|Key|CACert}FilePath are present -// DataBasePath: /tmp/ickptDB # Implicitly created if if non-existent -// SwiftTimeout: 10m -// SwiftConnectionPoolSize: 128 -// TransactionTimeout: 10s -// +// [ICKPT] +// IPAddr: ickpt +// Port: 33123 # TCP or TLS as determined by: +// CertFilePath: # TCP: if all of {Cert|Key|CACert}FilePath are missing or empty +// KeyFilePath: # - or - +// CACertFilePath: # TLS: if all of {Cert|Key|CACert}FilePath are present +// DataBasePath: /tmp/ickptDB # Implicitly created if if non-existent +// SwiftTimeout: 10m +// SwiftConnectionPoolSize: 128 +// TransactionTimeout: 10s package ickptpkg import ( @@ -82,14 +81,12 @@ import ( ) // Start is called to start serving. -// func Start(confMap conf.ConfMap) (err error) { err = start(confMap) return } // Stop is called to stop serving. -// func Stop() (err error) { err = stop() return diff --git a/ickpt/main.go b/ickpt/main.go index 61409805e..bf32b44e4 100644 --- a/ickpt/main.go +++ b/ickpt/main.go @@ -6,7 +6,6 @@ // The program requires a single argument that is a path to a package config // formatted configuration to load. Optionally, overrides the the config may // be passed as additional arguments in the form .=. -// package main import ( diff --git a/iclient/iclientpkg/api.go b/iclient/iclientpkg/api.go index 2ca80da8a..db41de8e3 100644 --- a/iclient/iclientpkg/api.go +++ b/iclient/iclientpkg/api.go @@ -7,52 +7,52 @@ // To configure an iclientpkg instance, Start() is called passing, as the sole // argument, a package conf ConfMap. Here is a sample .conf file: // -// [ICLIENT] -// VolumeName: testvol -// MountPointDirPath: /mnt -// FUSEBlockSize: 512 # Several tools/applications actually require fission.KStatFS.BSize to be 512 -// FUSEAllowOther: true -// FUSEMaxBackground: 1000 -// FUSECongestionThreshhold: 0 -// FUSEMaxPages: 256 -// FUSEMaxRead: 1048576 # 1MiB == FUSEMaxPages(256) * 4KiB -// FUSEMaxWrite: 1048576 # 1MiB == FUSEMaxPages(256) * 4KiB -// FUSEEntryValidDuration: 250ms -// FUSEAttrValidDuration: 250ms -// FUSENameLenMax: 255 -// AuthPlugInPath: iauth-swift.so -// AuthPlugInEnvName: # Only used if not defining AuthPlugInEnvValue here -// AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} -// SwiftTimeout: 10m -// SwiftRetryLimit: 4 -// SwiftRetryDelay: 100ms -// SwiftRetryDelayVariance: 25 # Percentage (1-100) of SwitRetryDelay -// SwiftRetryExponentialBackoff: 1.4 -// SwiftConnectionPoolSize: 128 -// RetryRPCPublicIPAddr: imgr -// RetryRPCPort: 32356 -// RetryRPCDeadlineIO: 60s -// RetryRPCKeepAlivePeriod: 60s -// RetryRPCCACertFilePath: # Defaults to /dev/null disabling TLS -// MaxSharedLeases: 500 -// MaxExclusiveLeases: 100 # Caps pending FileFlush data at 1GiB (with FileFlushTriggerSize of 10MiB) -// InodePayloadEvictLowLimit: 100000 -// InodePayloadEvictHighLimit: 100010 -// DirInodeMaxKeysPerBPlusTreePage: 1024 -// FileInodeMaxKeysPerBPlusTreePage: 2048 -// ReadCacheLineSize: 1048576 # 1MiB -// ReadCacheLineCountMax: 1024 # 1GiB (with ReadCacheLineSize of 1MiB) -// FileFlushTriggerSize: 10485760 # [10MiB] Amount of written data before metadata is appended and flush is triggered -// FileFlushTriggerDuration: 10s # Amount of time before unwritten data and its metadata flush is triggered -// InodeLockRetryDelay: 10ms -// InodeLockRetryDelayVariance: 50 # Percentage (1-100) of InodeLockRetryDelay -// LogFilePath: iclient.log -// LogToConsole: true -// TraceEnabled: false -// FUSELogEnabled: false -// RetryRPCLogEnabled: false -// HTTPServerIPAddr: # Defaults to 0.0.0.0 (i.e. all interfaces) -// HTTPServerPort: # Defaults to disabling the embedded HTTP Server +// [ICLIENT] +// VolumeName: testvol +// MountPointDirPath: /mnt +// FUSEBlockSize: 512 # Several tools/applications actually require fission.KStatFS.BSize to be 512 +// FUSEAllowOther: true +// FUSEMaxBackground: 1000 +// FUSECongestionThreshhold: 0 +// FUSEMaxPages: 256 +// FUSEMaxRead: 1048576 # 1MiB == FUSEMaxPages(256) * 4KiB +// FUSEMaxWrite: 1048576 # 1MiB == FUSEMaxPages(256) * 4KiB +// FUSEEntryValidDuration: 250ms +// FUSEAttrValidDuration: 250ms +// FUSENameLenMax: 255 +// AuthPlugInPath: iauth-swift.so +// AuthPlugInEnvName: # Only used if not defining AuthPlugInEnvValue here +// AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} +// SwiftTimeout: 10m +// SwiftRetryLimit: 4 +// SwiftRetryDelay: 100ms +// SwiftRetryDelayVariance: 25 # Percentage (1-100) of SwitRetryDelay +// SwiftRetryExponentialBackoff: 1.4 +// SwiftConnectionPoolSize: 128 +// RetryRPCPublicIPAddr: imgr +// RetryRPCPort: 32356 +// RetryRPCDeadlineIO: 60s +// RetryRPCKeepAlivePeriod: 60s +// RetryRPCCACertFilePath: # Defaults to /dev/null disabling TLS +// MaxSharedLeases: 500 +// MaxExclusiveLeases: 100 # Caps pending FileFlush data at 1GiB (with FileFlushTriggerSize of 10MiB) +// InodePayloadEvictLowLimit: 100000 +// InodePayloadEvictHighLimit: 100010 +// DirInodeMaxKeysPerBPlusTreePage: 1024 +// FileInodeMaxKeysPerBPlusTreePage: 2048 +// ReadCacheLineSize: 1048576 # 1MiB +// ReadCacheLineCountMax: 1024 # 1GiB (with ReadCacheLineSize of 1MiB) +// FileFlushTriggerSize: 10485760 # [10MiB] Amount of written data before metadata is appended and flush is triggered +// FileFlushTriggerDuration: 10s # Amount of time before unwritten data and its metadata flush is triggered +// InodeLockRetryDelay: 10ms +// InodeLockRetryDelayVariance: 50 # Percentage (1-100) of InodeLockRetryDelay +// LogFilePath: iclient.log +// LogToConsole: true +// TraceEnabled: false +// FUSELogEnabled: false +// RetryRPCLogEnabled: false +// HTTPServerIPAddr: # Defaults to 0.0.0.0 (i.e. all interfaces) +// HTTPServerPort: # Defaults to disabling the embedded HTTP Server // // Most of the config keys are required and must have values. One set of exceptions // are the HTTPServer{IPAddr|Port} keys that, if not present (or HTTPServerPort is @@ -62,29 +62,28 @@ // The embedded HTTP Server (at URL http://:) // responds to the following: // -// GET /config +// GET /config // // This will return a JSON document that matches the conf.ConfMap used to // launch this package. // -// GET /leases +// GET /leases // // This will display the state of every lease. // -// POST /leases/demote +// POST /leases/demote // // This will trigger the demotion of any Exclusive Leases held. // -// POST /leases/release +// POST /leases/release // // This will trigger the release of any {Exclusive|Shared} Leases held. // -// GET /stats +// GET /stats // // This will return a raw bucketstats dump. // -// GET /version -// +// GET /version package iclientpkg import ( @@ -92,40 +91,34 @@ import ( ) // Start is called to start serving. -// func Start(confMap conf.ConfMap, fissionErrChan chan error) (err error) { err = start(confMap, fissionErrChan) return } // Stop is called to stop serving. -// func Stop() (err error) { err = stop() return } // Signal is called to interrupt the server for performing operations such as log rotation. -// func Signal() (err error) { err = signal() return } // LogFatalf is a wrapper around the internal logFatalf() func called by iclient/main.go::main(). -// func LogFatalf(format string, args ...interface{}) { logFatalf(format, args...) } // LogWarnf is a wrapper around the internal logWarnf() func called by iclient/main.go::main(). -// func LogWarnf(format string, args ...interface{}) { logWarnf(format, args...) } // LogInfof is a wrapper around the internal logInfof() func called by iclient/main.go::main(). -// func LogInfof(format string, args ...interface{}) { logInfof(format, args...) } diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 6de5f71cc..8f4b11055 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -63,6 +63,13 @@ func performMountFUSE() (err error) { } func performUnmountFUSE() (err error) { + var ( + inode *inodeStruct + inodeListElement *list.Element + ok bool + wg sync.WaitGroup + ) + err = globals.fissionVolume.DoUnmount() if nil != err { return @@ -70,11 +77,43 @@ func performUnmountFUSE() (err error) { globals.fissionVolume = nil - // TODO: Here would be a great place to at least flush any dirty inodes - globals.fuseEntryValidDurationSec, globals.fuseEntryValidDurationNSec = 0, 0 globals.fuseAttrValidDurationSec, globals.fuseAttrValidDurationNSec = 0, 0 + globals.Lock() + + inodeListElement = globals.sharedLeaseLRU.Front() + + for inodeListElement != nil { + inode, ok = inodeListElement.Value.(*inodeStruct) + if !ok { + logFatalf("inodeListElement.Value.(*inodeStruct) returned !ok") + } + + wg.Add(1) + go releaseInodeLease(inode.inodeNumber, &wg) + + inodeListElement = inodeListElement.Next() + } + + inodeListElement = globals.exclusiveLeaseLRU.Front() + + for inodeListElement != nil { + inode, ok = inodeListElement.Value.(*inodeStruct) + if !ok { + logFatalf("inodeListElement.Value.(*inodeStruct) returned !ok") + } + + wg.Add(1) + go releaseInodeLease(inode.inodeNumber, &wg) + + inodeListElement = inodeListElement.Next() + } + + globals.Unlock() + + wg.Wait() + err = nil return } diff --git a/iclient/iclientpkg/html_templates.go b/iclient/iclientpkg/html_templates.go index 46bf1a453..d645cf6d8 100644 --- a/iclient/iclientpkg/html_templates.go +++ b/iclient/iclientpkg/html_templates.go @@ -4,7 +4,8 @@ package iclientpkg // To use: fmt.Sprintf(indexDotHTMLTemplate, proxyfsVersion) -// %[1]v +// +// %[1]v const indexDotHTMLTemplate string = ` @@ -93,7 +94,8 @@ const indexDotHTMLTemplate string = ` ` // To use: fmt.Sprintf(configTemplate, proxyfsVersion, confMapJSONString) -// %[1]v %[2]v +// +// %[1]v %[2]v const configTemplate string = ` @@ -152,7 +154,8 @@ const configTemplate string = ` ` // To use: fmt.Sprintf(leasesTemplate, proxyfsVersion, inodeLeaseTableJSONString) -// %[1]v %[2]v +// +// %[1]v %[2]v const leasesTemplate string = ` diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 538b8bc6c..89908c05d 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -354,7 +354,6 @@ func lookupInode(inodeNumber uint64) (inode *inodeStruct) { // // Note that fissionFlags{Read|Write} are forced to be TRUE per the behavior of Linux VFS // and/or fuse.ko choosing to mask these during Do{Create|Open|OpenDir}() upcalls. -// func createOpenHandle(inodeNumber uint64, fissionFlagsAppend bool, fissionFlagsRead bool, fissionFlagsWrite bool) (openHandle *openHandleStruct) { openHandle = &openHandleStruct{ inodeNumber: inodeNumber, @@ -738,7 +737,6 @@ func (fileFlusher *fileInodeFlusherStruct) cancel() { // // The caller is assumed to mark the inode clean and reset the .superBlockInode*, // .dereferencedObjectNumberArray, and .putObject{Number|Buffer} fields. -// func (inode *inodeStruct) flush() (inodeHeadLength uint64) { var ( err error @@ -880,7 +878,6 @@ func flushInodesInSlice(inodeSlice []*inodeStruct) { // updated to reflect the dereferenced extent. Similarly, the fileInode's pending // updates for superBlockInode{BytesReferencedAdjustment|Object{Count|Size}} will // be updated. -// func (fileInode *inodeStruct) recordExtent(startingFileOffset uint64, length uint64) { var ( err error @@ -994,7 +991,6 @@ func (fileInode *inodeStruct) recordExtent(startingFileOffset uint64, length uin // updated to reflect the dereferenced extent. Similarly, the fileInode's pending // updates for superBlockInode{BytesReferencedAdjustment|Object{Count|Size}} will // be updated. -// func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint64) { var ( err error diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index 91f40bcc0..c50277a12 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -13,7 +13,6 @@ import ( ) // newLockRequest is called to create and initialize an inodeLockRequestStruct. -// func newLockRequest() (inodeLockRequest *inodeLockRequestStruct) { inodeLockRequest = &inodeLockRequestStruct{ inodeNumber: 0, @@ -30,7 +29,6 @@ func newLockRequest() (inodeLockRequest *inodeLockRequestStruct) { // // It is expected that a caller to addThisLock(), when noticing locksHeld map is empty, // will call performInodeLockRetryDelay() before re-attempting a lock sequence. -// func performInodeLockRetryDelay() { var ( delay time.Duration @@ -70,25 +68,24 @@ func performInodeLockRetryDelay() { // holding a lock, attempting to acquire a lock, or releasing a lock). Hence, the only valid values // for inode.leaseState are: // -// inodeLeaseStateNone - no lock requests may be granted -// inodeLeaseStateSharedGranted - shared lock requests may be granted -// inodeLeaseStateExclusiveGranted - either shared or exclusive lock requests may be granted +// inodeLeaseStateNone - no lock requests may be granted +// inodeLeaseStateSharedGranted - shared lock requests may be granted +// inodeLeaseStateExclusiveGranted - either shared or exclusive lock requests may be granted // // During lock ownership transitions, other values should be expected: // -// inodeLeaseStateSharedRequested - we are trying to grant a shared lock request -// inodeLeaseStateSharedPromoting - we are trying to grant an exclusive lock request -// inodeLeaseStateExclusiveRequested - we are trying to grant an exclusive lock request +// inodeLeaseStateSharedRequested - we are trying to grant a shared lock request +// inodeLeaseStateSharedPromoting - we are trying to grant an exclusive lock request +// inodeLeaseStateExclusiveRequested - we are trying to grant an exclusive lock request // // The other inode.leaseState values occur during Unmount or some Lease Demote/Expired/Release // handling: // -// inodeLeaseStateSharedReleasing - we are responding to an Unmount or Lease Release RPCInterrupt -// inodeLeaseStateSharedExpired - upon learning our Shared Lease has expired -// inodeLeaseStateExclusiveDemoting - we are responding to a Lease Demote RPCInterrupt -// inodeLeaseStateExclusiveReleasing - we are responding to an Unmount or Lease Release RPCInterrupt -// inodeLeaseStateExclusiveExpired - upon learning our Exclusive Lease has expired -// +// inodeLeaseStateSharedReleasing - we are responding to an Unmount or Lease Release RPCInterrupt +// inodeLeaseStateSharedExpired - upon learning our Shared Lease has expired +// inodeLeaseStateExclusiveDemoting - we are responding to a Lease Demote RPCInterrupt +// inodeLeaseStateExclusiveReleasing - we are responding to an Unmount or Lease Release RPCInterrupt +// inodeLeaseStateExclusiveExpired - upon learning our Exclusive Lease has expired func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { var ( err error @@ -446,7 +443,6 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } // unlockAll is called to explicitly release all locks listed in the locksHeld map. -// func (inodeLockRequest *inodeLockRequestStruct) unlockAll() { var ( blockedInodeLockRequest *inodeLockRequestStruct @@ -620,7 +616,6 @@ func (inodeLockRequest *inodeLockRequestStruct) unlockAll() { // the inode indicated by the specified inodeNumber and, if an Exclusive Lease is currently // held, demote it. It is expected that this is called in a goroutine and, thus, signals // completion via the specified wg (if non-nil). -// func demoteInodeLease(inodeNumber uint64, wg *sync.WaitGroup) { var ( blockedInodeLockRequest *inodeLockRequestStruct @@ -786,7 +781,6 @@ Retry: // the inode indicated by the specified inodeNumber and, if a Lease is currently held, // release it. It is expected that this is called in a goroutine and, thus, signals // completion via the specified wg (if non-nil). -// func releaseInodeLease(inodeNumber uint64, wg *sync.WaitGroup) { var ( blockedInodeLockRequest *inodeLockRequestStruct @@ -1029,7 +1023,6 @@ Retry: // demoteAllExclusiveLeases will schedule all inodeStructs in the globals.exclusiveLeaseLRU // to demote their Exclusive Lease. -// func demoteAllExclusiveLeases() { var ( inode *inodeStruct @@ -1062,7 +1055,6 @@ func demoteAllExclusiveLeases() { // releaseAllLeases will schedule all inodeStructs in either the globals.sharedLeaseLRU // or globals.exclusiveLeaseLRU to release their Lease. -// func releaseAllLeases() { var ( inode *inodeStruct diff --git a/iclient/iclientpkg/rpc.go b/iclient/iclientpkg/rpc.go index a397e5ab3..e4b7a3f56 100644 --- a/iclient/iclientpkg/rpc.go +++ b/iclient/iclientpkg/rpc.go @@ -356,10 +356,12 @@ func rpcLease(leaseRequest *imgrpkg.LeaseRequestStruct, leaseResponse *imgrpkg.L startTime time.Time = time.Now() ) - logTracef("==> rpcLease(leaseRequest: %+v)", leaseRequest) - defer func() { - logTracef("<== rpcLease(leaseResponse: %+v, err: %v)", leaseResponse, err) - }() + if globals.config.TraceEnabled { + logTracef("==> rpcLease(leaseRequest: %v)", leaseRequest) + defer func() { + logTracef("<== rpcLease(leaseResponse: %v, err: %v)", leaseResponse, err) + }() + } defer func() { globals.stats.LeaseUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) @@ -479,10 +481,12 @@ func (dummy *globalsStruct) Interrupt(payload []byte) { logFatalf("json.Unmarshal(payload, rpcInterrupt) failed: %v", err) } - logTracef("==> Interrupt(rpcInterrupt: %+v)", rpcInterrupt) - defer func() { - logTracef("<== Interrupt(rpcInterrupt: %+v)", rpcInterrupt) - }() + if globals.config.TraceEnabled { + logTracef("==> Interrupt(rpcInterrupt: %v)", rpcInterrupt) + defer func() { + logTracef("<== Interrupt(rpcInterrupt: %v)", rpcInterrupt) + }() + } switch rpcInterrupt.RPCInterruptType { case imgrpkg.RPCInterruptTypeUnmount: diff --git a/iclient/iclientpkg/utils_test.go b/iclient/iclientpkg/utils_test.go index 4e18caa1a..df88b070b 100644 --- a/iclient/iclientpkg/utils_test.go +++ b/iclient/iclientpkg/utils_test.go @@ -22,7 +22,7 @@ import ( ) const ( - testIPAddr = "127.0.0.1" // Don't use IPv6... the code doesn't properly "join" this with :port #s + testIPAddr = "127.0.0.1" testRetryRPCPort = 32356 testMgrHTTPServerPort = 15346 testClientHTTPServerPort = 15347 @@ -34,6 +34,7 @@ const ( testVolume = "testvol" testRPCDeadlineIO = "60s" testRPCKeepAlivePeriod = "60s" + testStartupDelay = 100 * time.Millisecond ) type testGlobalsStruct struct { @@ -82,8 +83,8 @@ func testSetup(t *testing.T) { caKeyFile: tempDir + "/caKeyFile", endpointCertFile: tempDir + "/endpoingCertFile", endpointKeyFile: tempDir + "/endpointKeyFile", - imgrHTTPServerURL: fmt.Sprintf("http://%s:%d", testIPAddr, testMgrHTTPServerPort), - authURL: fmt.Sprintf("http://%s:%d/auth/v1.0", testIPAddr, testSwiftProxyTCPPort), + imgrHTTPServerURL: "http://" + net.JoinHostPort(testIPAddr, fmt.Sprintf("%d", testMgrHTTPServerPort)), + authURL: "http://" + net.JoinHostPort(testIPAddr, fmt.Sprintf("%d", testSwiftProxyTCPPort)) + "/auth/v1.0", } testGlobals.caCertPEMBlock, testGlobals.caKeyPEMBlock, err = icertpkg.GenCACert( @@ -244,14 +245,18 @@ func testSetup(t *testing.T) { t.Fatalf("iswiftpkg.Start(testGlobals.confMap) failed: %v", err) } - authRequestHeaders = make(http.Header) + for { + authRequestHeaders = make(http.Header) - authRequestHeaders["X-Auth-User"] = []string{testSwiftAuthUser} - authRequestHeaders["X-Auth-Key"] = []string{testSwiftAuthKey} + authRequestHeaders["X-Auth-User"] = []string{testSwiftAuthUser} + authRequestHeaders["X-Auth-Key"] = []string{testSwiftAuthKey} - authResponseHeaders, _, err = testDoHTTPRequest("GET", testGlobals.authURL, authRequestHeaders, nil) - if nil != err { - t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.authURL, authRequestHeaders, nil) failed: %v", err) + authResponseHeaders, _, err = testDoHTTPRequest("GET", testGlobals.authURL, authRequestHeaders, nil) + if nil == err { + break + } + + time.Sleep(testStartupDelay) } testGlobals.authToken = authResponseHeaders.Get("X-Auth-Token") @@ -288,6 +293,14 @@ func testSetup(t *testing.T) { t.Fatalf("imgrpkg.Start(testGlobals.confMap) failed: %v", err) } + for { + _, _, err = testDoHTTPRequest("GET", testGlobals.imgrHTTPServerURL+"/version", nil, nil) + if nil == err { + break + } + time.Sleep(testStartupDelay) + } + postAndPutVolumePayload = fmt.Sprintf("{\"StorageURL\":\"%s\",\"AuthToken\":\"%s\"}", testGlobals.containerURL, testGlobals.authToken) _, _, err = testDoHTTPRequest("POST", testGlobals.imgrHTTPServerURL+"/volume", nil, strings.NewReader(postAndPutVolumePayload)) diff --git a/iclient/main.go b/iclient/main.go index bc71a9ffb..11c01d5f2 100644 --- a/iclient/main.go +++ b/iclient/main.go @@ -6,7 +6,6 @@ // The program requires a single argument that is a path to a package config // formatted configuration to load. Optionally, overrides the the config may // be passed as additional arguments in the form .=. -// package main import ( diff --git a/idestroy/main.go b/idestroy/main.go index 6469498c9..d850ed171 100644 --- a/idestroy/main.go +++ b/idestroy/main.go @@ -10,7 +10,6 @@ // As it is expected to be used (e.g.. during development/testing) to clear out // a container ultimately used by iclient/imgr, idestroy is designed to simply // leverage the same iclient .conf file. -// package main import ( diff --git a/ilayout/api.go b/ilayout/api.go index 459644ed2..b3e7e103d 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -78,7 +78,6 @@ // "on disk" format and an "in memory" equivalent. These func's are both high // level (e.g. "superblock") and low level (e.g. uint64) to assist in managing // the "on disk" representation of the file system. -// package ilayout import ( @@ -87,7 +86,6 @@ import ( // CheckPointObjectNumber specifies the ObjectNumber of the Object to hold the // default/backup copy of the CheckPoint. -// const ( CheckPointObjectNumber uint64 = 0 ) @@ -96,13 +94,11 @@ const ( // must always be fetched by scanning the entire CheckPoint using a %016X format // specifier. This value will then be used to interpret the remaining characters of // the CheckPoint string. -// const ( CheckPointVersionV1 uint64 = 1 ) // UnmarshalCheckPointVersion extracts checkPointVersion from checkpointString. -// func UnmarshalCheckPointVersion(checkpointString string) (checkPointVersion uint64, err error) { checkPointVersion, err = unmarshalCheckPointVersion(checkpointString) return @@ -112,7 +108,6 @@ func UnmarshalCheckPointVersion(checkpointString string) (checkPointVersion uint // // The contents of the struct are serialized as space separated fields formatted // via %016X numbers. -// type CheckPointV1Struct struct { Version uint64 // == CheckPointVersionV1 SuperBlockObjectNumber uint64 // Identifies the Object containing the SuperBlock at the end @@ -121,14 +116,12 @@ type CheckPointV1Struct struct { } // MarshalCheckPointV1 encodes checkPointV1 to checkpointString. -// func (checkPointV1 *CheckPointV1Struct) MarshalCheckPointV1() (checkPointV1String string, err error) { checkPointV1String, err = checkPointV1.marshalCheckPointV1() return } // UnmarshalCheckPointV1 decodes checkPointV1 from checkpointString. -// func UnmarshalCheckPointV1(checkPointV1String string) (checkPointV1 *CheckPointV1Struct, err error) { checkPointV1, err = unmarshalCheckPointV1(checkPointV1String) return @@ -139,7 +132,6 @@ func UnmarshalCheckPointV1(checkPointV1String string) (checkPointV1 *CheckPointV // proceeding it. // // The struct is serialized as a sequence of LittleEndian formatted fields. -// type ObjectTrailerStruct struct { ObjType uint16 Version uint16 @@ -147,7 +139,6 @@ type ObjectTrailerStruct struct { } // MarshalObjectTrailer encodes objectTrailer to objectTrailerBuf. -// func (objectTrailer *ObjectTrailerStruct) MarshalObjectTrailer() (objectTrailerBuf []byte, err error) { objectTrailerBuf, err = objectTrailer.marshalObjectTrailer() return @@ -158,7 +149,6 @@ func (objectTrailer *ObjectTrailerStruct) MarshalObjectTrailer() (objectTrailerB // Note that the last 8 bytes of objectTrailerBuf are decoded. The entire // objectTrailerBuf is expected to contain precisely objectTrailer.Length // bytes before the ObjectTrailerStruct. -// func UnmarshalObjectTrailer(objectTrailerBuf []byte) (objectTrailer *ObjectTrailerStruct, err error) { objectTrailer, err = unmarshalObjectTrailer(objectTrailerBuf) return @@ -166,14 +156,12 @@ func UnmarshalObjectTrailer(objectTrailerBuf []byte) (objectTrailer *ObjectTrail // SuperBlockType specifies that this ObjectTrailerStruct refers to // a SuperBlockV*Struct immediately preceeding it. -// const ( SuperBlockType uint16 = 0x5342 // 'S' 'B' ) // SuperBlockVersionV* specifies, for an ObjectTrailerStruct of Type SuperBlockType, // the Version of the SuperBlockV*Struct immediately preceeding the ObjectTrailerStruct. -// const ( SuperBlockVersionV1 uint16 = 1 ) @@ -184,7 +172,6 @@ const ( // may be deleted. // // The struct is serialized as a sequence of LittleEndian formatted fields. -// type InodeTableLayoutEntryV1Struct struct { ObjectNumber uint64 // Identifies the Object containing the page(s) of the InodeTable B+Tree BytesWritten uint64 // Number of bytes written to the Object @@ -209,7 +196,6 @@ type InodeTableLayoutEntryV1Struct struct { // Note that the CheckPointV1Struct.SuperBlockLength also includes the bytes for holding // the ObjectTrailerStruct{ObjType: SuperBlockType, Version: SuperBlockVersionV1} that is // appended. -// type SuperBlockV1Struct struct { InodeTableRootObjectNumber uint64 // Identifies the Object containing the root of the InodeTable InodeTableRootObjectOffset uint64 // Starting offset in the Object of the root of the InodeTable @@ -222,14 +208,12 @@ type SuperBlockV1Struct struct { } // MarshalSuperBlockV1 encodes superBlockV1 to superBlockV1Buf. -// func (superBlockV1 *SuperBlockV1Struct) MarshalSuperBlockV1() (superBlockV1Buf []byte, err error) { superBlockV1Buf, err = superBlockV1.marshalSuperBlockV1() return } // UnmarshalSuperBlockV1 decodes superBlockV1 from superBlockV1Buf. -// func UnmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1Struct, err error) { superBlockV1, err = unmarshalSuperBlockV1(superBlockV1Buf) return @@ -239,13 +223,11 @@ func UnmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1St // in an InodeTable entry's Value InodeTableEntryStruct. // // The value is stored in LittleEndian format. -// const ( InodeTableEntryValueVersionV1 uint64 = 1 ) // UnmarshalInodeTableEntryValueVersion extracts inodeTableEntryValueVersion from inodeTableEntryValueBuf. -// func UnmarshalInodeTableEntryValueVersion(inodeTableEntryValueBuf []byte) (inodeTableEntryValueVersion uint64, err error) { inodeTableEntryValueVersion, err = unmarshalInodeTableEntryValueVersion(inodeTableEntryValueBuf) return @@ -258,21 +240,18 @@ func UnmarshalInodeTableEntryValueVersion(inodeTableEntryValueBuf []byte) (inode // // Note that there is no InodeTableEntryKeyV1Struct as it is simply an InodeNumber uint64 // serialized in LittleEndian format. -// type InodeTableEntryValueV1Struct struct { InodeHeadObjectNumber uint64 // Identifies the Object containing InodeHeadV*Struct InodeHeadLength uint64 // Total length of the InodeHead found at the end of the Object indicated by InodeHeadObjectNumber } // MarshalInodeTableEntryValueV1 encodes inodeTableEntryValueV1 to inodeTableEntryValueV1Buf. -// func (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct) MarshalInodeTableEntryValueV1() (inodeTableEntryValueV1Buf []byte, err error) { inodeTableEntryValueV1Buf, err = inodeTableEntryValueV1.marshalInodeTableEntryValueV1() return } // UnmarshalInodeTableEntryValueV1 decodes inodeTableEntryValueV1 from inodeTableEntryValueV1Buf. -// func UnmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct, bytesConsumed int, err error) { inodeTableEntryValueV1, bytesConsumed, err = unmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf) return @@ -280,26 +259,22 @@ func UnmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTab // InodeHeadType specifies that this ObjectTrailerStruct refers to // a InodeHeadV*Struct immediately preceeding it. -// const ( InodeHeadType uint16 = 0x4948 // 'I' 'H' ) // InodeHeadVersionV* specifies, for an ObjectTrailerStruct of Type InodeHeadType, // the Version of InodeHeadV*Struct immediately preceeding the ObjectTrailerStruct. -// const ( InodeHeadVersionV1 uint16 = 1 ) // RootDirInodeNumber is the InodeNumber for the directory at the root of the file system. -// const ( RootDirInodeNumber uint64 = 1 ) // InodeType* specifies the type of Inode. -// const ( InodeTypeDir uint8 = 0 InodeTypeFile uint8 = 1 @@ -311,7 +286,6 @@ const ( // The struct's uint64 field is serialized in LittleEndian format followed by // the struct's string field serialized as a LittleEndian length followed by // the bytes of the string. -// type InodeLinkTableEntryStruct struct { ParentDirInodeNumber uint64 ParentDirEntryName string @@ -321,7 +295,6 @@ type InodeLinkTableEntryStruct struct { // protection bits (i.e. rwx bits for each of user, group, and other). // // The value is stored in LittleEndian format. -// const ( InodeModeMask uint16 = 0o777 ) @@ -330,7 +303,6 @@ const ( // // The struct is serialized be treating both fields as an array of bytes preceeded // by a LittleEndian length. -// type InodeStreamTableEntryStruct struct { Name string Value []byte @@ -343,7 +315,6 @@ type InodeStreamTableEntryStruct struct { // Object, when BytesReferenced drops to zero, the Object may be deleted. // // The struct is serialized as a sequence of LittleEndian formatted fields. -// type InodeHeadLayoutEntryV1Struct struct { ObjectNumber uint64 // For DirInode's: // Identifies the Object containing the page(s) of the Directory B+Tree @@ -357,15 +328,15 @@ type InodeHeadLayoutEntryV1Struct struct { // InodeHeadV1Struct specifies the layout of an Inode. // // The struct is serializes as a sequence of fields: -// For uint* fields, LittleEndian format is used. -// For table fields, a uint64 length in LittleEndian format is followed by the serialization -// specified in the table entry struct. -// For time.Time fields, a uint64 in LittleEndian is used to hold the UnixNano() equivalent. +// +// For uint* fields, LittleEndian format is used. +// For table fields, a uint64 length in LittleEndian format is followed by the serialization +// specified in the table entry struct. +// For time.Time fields, a uint64 in LittleEndian is used to hold the UnixNano() equivalent. // // Note that the SuperBlockV1Struct.InodeTableRootObjectLength also includes the bytes for // holding the ObjectTrailerStruct{ObjType: InodeHeadType, Version: InodeHeadVersionV1} // that is appended. -// type InodeHeadV1Struct struct { InodeNumber uint64 // InodeType uint8 // One of InodeType* @@ -386,14 +357,12 @@ type InodeHeadV1Struct struct { } // MarshalInodeHeadV1 encodes inodeHeadV1 to inodeHeadV1Buf. -// func (inodeHeadV1 *InodeHeadV1Struct) MarshalInodeHeadV1() (inodeHeadV1Buf []byte, err error) { inodeHeadV1Buf, err = inodeHeadV1.marshalInodeHeadV1() return } // UnmarshalInodeHeadV1 decodes inodeHeadV1 from inodeHeadV1Buf. -// func UnmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct, err error) { inodeHeadV1, err = unmarshalInodeHeadV1(inodeHeadV1Buf) return @@ -406,21 +375,18 @@ func UnmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct // // Note that there is no DirectoryEntryKeyV1Struct as it is simply a BaseName string // serialized by a uint64 length in LittleEndian format followed by the bytes of the string. -// type DirectoryEntryValueV1Struct struct { InodeNumber uint64 InodeType uint8 } // MarshalDirectoryEntryValueV1 encodes directoryEntryValueV1 to directoryEntryValueV1Buf. -// func (directoryEntryValueV1 *DirectoryEntryValueV1Struct) MarshalDirectoryEntryValueV1() (directoryEntryValueV1Buf []byte, err error) { directoryEntryValueV1Buf, err = directoryEntryValueV1.marshalDirectoryEntryValueV1() return } // UnmarshalDirectoryEntryValueV1 decodes directoryEntryValueV1 from directoryEntryValueV1Buf. -// func UnmarshalDirectoryEntryValueV1(directoryEntryValueV1Buf []byte) (directoryEntryValueV1 *DirectoryEntryValueV1Struct, bytesConsumed int, err error) { directoryEntryValueV1, bytesConsumed, err = unmarshalDirectoryEntryValueV1(directoryEntryValueV1Buf) return @@ -433,7 +399,6 @@ func UnmarshalDirectoryEntryValueV1(directoryEntryValueV1Buf []byte) (directoryE // // Note that there is no ExtentMapEntryKeyV1Struct as it is simply a FileOffset uint64 // serialized in LittleEndian format. -// type ExtentMapEntryValueV1Struct struct { Length uint64 // Length of this extent (both in the File and in the Object) ObjectNumber uint64 // Identifies the Object containing this extent's data @@ -441,14 +406,12 @@ type ExtentMapEntryValueV1Struct struct { } // MarshalExtentMapEntryValueV1 encodes directoryEntryValueV1 to directoryEntryValueV1Buf. -// func (extentMapEntryValueV1 *ExtentMapEntryValueV1Struct) MarshalExtentMapEntryValueV1() (extentMapEntryValueV1Buf []byte, err error) { extentMapEntryValueV1Buf, err = extentMapEntryValueV1.marshalExtentMapEntryValueV1() return } // UnmarshalExtentMapEntryValueV1 decodes directoryEntryValueV1 from directoryEntryValueV1Buf. -// func UnmarshalExtentMapEntryValueV1(extentMapEntryValueV1Buf []byte) (extentMapEntryValueV1 *ExtentMapEntryValueV1Struct, bytesConsumed int, err error) { extentMapEntryValueV1, bytesConsumed, err = unmarshalExtentMapEntryValueV1(extentMapEntryValueV1Buf) return @@ -457,7 +420,6 @@ func UnmarshalExtentMapEntryValueV1(extentMapEntryValueV1Buf []byte) (extentMapE // GetLEUint8FromBuf fetches a uint8 from buf starting at curPos. // // The returned nextPos indicates where the next field (if any) should be read from. -// func GetLEUint8FromBuf(buf []byte, curPos int) (u8 uint8, nextPos int, err error) { u8, nextPos, err = getLEUint8FromBuf(buf, curPos) return @@ -466,7 +428,6 @@ func GetLEUint8FromBuf(buf []byte, curPos int) (u8 uint8, nextPos int, err error // PutLEUint8ToBuf writes a uint8 to buf starting at curPos. // // The returned nextPost indicates where the next field (if any) should be written. -// func PutLEUint8ToBuf(buf []byte, curPos int, u8 uint8) (nextPos int, err error) { nextPos, err = putLEUint8ToBuf(buf, curPos, u8) return @@ -476,7 +437,6 @@ func PutLEUint8ToBuf(buf []byte, curPos int, u8 uint8) (nextPos int, err error) // // The uint16 is assumed to have been written in LittleEndian byte order. // The returned nextPos indicates where the next field (if any) should be read from. -// func GetLEUint16FromBuf(buf []byte, curPos int) (u16 uint16, nextPos int, err error) { u16, nextPos, err = getLEUint16FromBuf(buf, curPos) return @@ -486,7 +446,6 @@ func GetLEUint16FromBuf(buf []byte, curPos int) (u16 uint16, nextPos int, err er // // The uint16 is written in LittleEndian byte order. // The returned nextPost indicates where the next field (if any) should be written. -// func PutLEUint16ToBuf(buf []byte, curPos int, u16 uint16) (nextPos int, err error) { nextPos, err = putLEUint16ToBuf(buf, curPos, u16) return @@ -496,7 +455,6 @@ func PutLEUint16ToBuf(buf []byte, curPos int, u16 uint16) (nextPos int, err erro // // The uint32 is assumed to have been written in LittleEndian byte order. // The returned nextPos indicates where the next field (if any) should be read from. -// func GetLEUint32FromBuf(buf []byte, curPos int) (u32 uint32, nextPos int, err error) { u32, nextPos, err = getLEUint32FromBuf(buf, curPos) return @@ -506,7 +464,6 @@ func GetLEUint32FromBuf(buf []byte, curPos int) (u32 uint32, nextPos int, err er // // The uint32 is written in LittleEndian byte order. // The returned nextPost indicates where the next field (if any) should be written. -// func PutLEUint32ToBuf(buf []byte, curPos int, u32 uint32) (nextPos int, err error) { nextPos, err = putLEUint32ToBuf(buf, curPos, u32) return @@ -516,7 +473,6 @@ func PutLEUint32ToBuf(buf []byte, curPos int, u32 uint32) (nextPos int, err erro // // The uint64 is assumed to have been written in LittleEndian byte order. // The returned nextPos indicates where the next field (if any) should be read from. -// func GetLEUint64FromBuf(buf []byte, curPos int) (u64 uint64, nextPos int, err error) { u64, nextPos, err = getLEUint64FromBuf(buf, curPos) return @@ -526,7 +482,6 @@ func GetLEUint64FromBuf(buf []byte, curPos int) (u64 uint64, nextPos int, err er // // The uint64 is written in LittleEndian byte order. // The returned nextPost indicates where the next field (if any) should be written. -// func PutLEUint64ToBuf(buf []byte, curPos int, u64 uint64) (nextPos int, err error) { nextPos, err = putLEUint64ToBuf(buf, curPos, u64) return @@ -537,7 +492,6 @@ func PutLEUint64ToBuf(buf []byte, curPos int, u64 uint64) (nextPos int, err erro // The string is assumed to have been written as a LittleEndian byte order uint64 // length followed by the bytes that make up the string. The returned nextPos // indicates where the next field (if any) should be read from. -// func GetLEStringFromBuf(buf []byte, curPos int) (str string, nextPos int, err error) { str, nextPos, err = getLEStringFromBuf(buf, curPos) return @@ -548,7 +502,6 @@ func GetLEStringFromBuf(buf []byte, curPos int) (str string, nextPos int, err er // The string is written as a LittleEndian byte order uint64 length followed by the // bytes that make up the string. The returned nextPost indicates where the next field // (if any) should be written. -// func PutLEStringToBuf(buf []byte, curPos int, str string) (nextPos int, err error) { nextPos, err = putLEStringToBuf(buf, curPos, str) return @@ -559,7 +512,6 @@ func PutLEStringToBuf(buf []byte, curPos int, str string) (nextPos int, err erro // The []byte is assumed to have been written as a LittleEndian byte order uint64 // length followed by the bytes that make up the []byte. The returned nextPos // indicates where the next field (if any) should be read from. -// func GetLEByteSliceFromBuf(buf []byte, curPos int) (byteSlice []byte, nextPos int, err error) { byteSlice, nextPos, err = getLEByteSliceFromBuf(buf, curPos) return @@ -570,7 +522,6 @@ func GetLEByteSliceFromBuf(buf []byte, curPos int) (byteSlice []byte, nextPos in // The []byte is written as a LittleEndian byte order uint64 length followed by the // bytes that make up the []byte. The returned nextPost indicates where the next field // (if any) should be written. -// func PutLEByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, err error) { nextPos, err = putLEByteSliceToBuf(buf, curPos, byteSlice) return @@ -580,7 +531,6 @@ func PutLEByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, // // The []byte is assumed to have been written with the same length as byteSlice. // The returned nextPos indicates where the next field (if any) should be read from. -// func GetFixedByteSliceFromBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, err error) { nextPos, err = getFixedByteSliceFromBuf(buf, curPos, byteSlice) return @@ -589,7 +539,6 @@ func GetFixedByteSliceFromBuf(buf []byte, curPos int, byteSlice []byte) (nextPos // PutFixedByteSliceToBuf writes a []byte to buf starting at curPos. // // The returned nextPost indicates where the next field (if any) should be written. -// func PutFixedByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, err error) { nextPos, err = putFixedByteSliceToBuf(buf, curPos, byteSlice) return @@ -597,7 +546,6 @@ func PutFixedByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos i // GetObjectNameAsByteSlice returns the isomorphically mapped objectName as // a []byte given a uint64 objectNumber. -// func GetObjectNameAsByteSlice(objectNumber uint64) (objectName []byte) { objectName = getObjectNameAsByteSlice(objectNumber) return @@ -605,7 +553,6 @@ func GetObjectNameAsByteSlice(objectNumber uint64) (objectName []byte) { // GetObjectNameAsString returns the isomorphically mapped objectName as // a string given a uint64 objectNumber. -// func GetObjectNameAsString(objectNumber uint64) (objectName string) { objectName = getObjectNameAsString(objectNumber) return @@ -616,7 +563,6 @@ func GetObjectNameAsString(objectNumber uint64) (objectName string) { // // An error will result if objectName is not of the proper length or contains // invalid characters. -// func GetObjectNumberFromByteSlice(objectName []byte) (objectNumber uint64, err error) { objectNumber, err = getObjectNumberFromByteSlice(objectName) return @@ -627,7 +573,6 @@ func GetObjectNumberFromByteSlice(objectName []byte) (objectNumber uint64, err e // // An error will result if objectName is not of the proper length or contains // invalid characters. -// func GetObjectNumberFromString(objectName string) (objectNumber uint64, err error) { objectNumber, err = getObjectNumberFromString(objectName) return diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index df4d51492..3f7f9bbfd 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -14,63 +14,63 @@ // To configure an imgrpkg instance, Start() is called passing, as the sole // argument, a package conf ConfMap. Here is a sample .conf file: // -// [IMGR] -// PublicIPAddr: imgr -// PrivateIPAddr: imgr -// RetryRPCPort: 32356 -// HTTPServerPort: 15346 +// [IMGR] +// PublicIPAddr: imgr +// PrivateIPAddr: imgr +// RetryRPCPort: 32356 +// HTTPServerPort: 15346 // -// CheckPointIPAddrs: # List of ickpt cluster instance IPAddr's -// CheckPointPort: 33123 # Only required if CheckPointIPAddrs is non-empty -// CheckPointCACertFilePath: # Defaults to /dev/null disabling TLS -// CheckPointRetryDelay: 100ms # Only required if CheckPointIPAddrs is non-empty -// CheckPointRetryExpBackoff: 2 # Only required if CheckPointIPAddrs is non-empty -// CheckPointRetryLimit: 4 # Only required if CheckPointIPAddrs is non-empty -// CheckPointTimeout: 10m # Only required if CheckPointIPAddrs is non-empty -// CheckPointConnectionPoolSize: 10 # Only required if CheckPointIPAddrs is non-empty +// CheckPointIPAddrs: # List of ickpt cluster instance IPAddr's +// CheckPointPort: 33123 # Only required if CheckPointIPAddrs is non-empty +// CheckPointCACertFilePath: # Defaults to /dev/null disabling TLS +// CheckPointRetryDelay: 100ms # Only required if CheckPointIPAddrs is non-empty +// CheckPointRetryExpBackoff: 2 # Only required if CheckPointIPAddrs is non-empty +// CheckPointRetryLimit: 4 # Only required if CheckPointIPAddrs is non-empty +// CheckPointTimeout: 10m # Only required if CheckPointIPAddrs is non-empty +// CheckPointConnectionPoolSize: 10 # Only required if CheckPointIPAddrs is non-empty // -// RetryRPCTTLCompleted: 10m -// RetryRPCAckTrim: 100ms -// RetryRPCDeadlineIO: 60s -// RetryRPCKeepAlivePeriod: 60s +// RetryRPCTTLCompleted: 10m +// RetryRPCAckTrim: 100ms +// RetryRPCDeadlineIO: 60s +// RetryRPCKeepAlivePeriod: 60s // -// RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, -// RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used +// RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, +// RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used // -// CheckPointInterval: 10s +// CheckPointInterval: 10s // -// AuthTokenCheckInterval: 1m +// AuthTokenCheckInterval: 1m // -// FetchNonceRangeToReturn: 100 +// FetchNonceRangeToReturn: 100 // -// MountLimit: 10000 -// OpenFileLimit: 100000 +// MountLimit: 10000 +// OpenFileLimit: 100000 // -// MinLeaseDuration: 250ms -// LeaseInterruptInterval: 250ms -// LeaseInterruptLimit: 20 -// LeaseEvictLowLimit: 100000 -// LeaseEvictHighLimit: 100010 +// MinLeaseDuration: 250ms +// LeaseInterruptInterval: 250ms +// LeaseInterruptLimit: 20 +// LeaseEvictLowLimit: 100000 +// LeaseEvictHighLimit: 100010 // -// SwiftRetryDelay: 100ms -// SwiftRetryExpBackoff: 2 -// SwiftRetryLimit: 4 +// SwiftRetryDelay: 100ms +// SwiftRetryExpBackoff: 2 +// SwiftRetryLimit: 4 // -// SwiftTimeout: 10m -// SwiftConnectionPoolSize: 128 +// SwiftTimeout: 10m +// SwiftConnectionPoolSize: 128 // -// ParallelObjectDeletePerVolumeLimit: 100 +// ParallelObjectDeletePerVolumeLimit: 100 // -// InodeTableCacheEvictLowLimit: 10000 -// InodeTableCacheEvictHighLimit: 10010 +// InodeTableCacheEvictLowLimit: 10000 +// InodeTableCacheEvictHighLimit: 10010 // -// InodeTableMaxInodesPerBPlusTreePage: 2048 -// RootDirMaxDirEntriesPerBPlusTreePage: 1024 +// InodeTableMaxInodesPerBPlusTreePage: 2048 +// RootDirMaxDirEntriesPerBPlusTreePage: 1024 // -// LogFilePath: # imgr.log -// LogToConsole: true # false -// TraceEnabled: false -// RetryRPCLogEnabled: false +// LogFilePath: # imgr.log +// LogToConsole: true # false +// TraceEnabled: false +// RetryRPCLogEnabled: false // // Most of the config keys are required and must have values. One exception // is LogFilePath that will default to "" and, hence, cause logging to not @@ -85,66 +85,88 @@ // The RESTful API is provided by an embedded HTTP Server // (at URL http://:) responds to the following: // -// DELETE /volume/ +// DELETE /keepalive +// +// This will disable the keep alive mechanism. +// +// DELETE /volume/ // // This will cause the specified to no longer be served. Note that // this does not actually affect the contents of the associated Container. // -// GET /config +// GET /config // // This will return a JSON document that matches the conf.ConfMap used to // launch this package. // -// GET /stats +// GET /keepalive +// +// This will return the configured keepalive duration if any. If the keepalive +// mechanism has not been enabled, a 404 Not Found will be returned. If the +// keepalive mechanism has been enabled, this is the highest performing way to +// reset the countdown timer. +// +// GET /stats // // This will return a raw bucketstats dump. // -// GET /version +// GET /version // // This will return the imgr version. // -// GET /volume +// GET /volume // // This will return a JSON document containing an array of volumes currently // being served with details about each. // -// GET /volume/ +// GET /volume/ // // This will return a JSON document containing only the specified // details (assuming it is currently being served). // -// POST /volume -// Content-Type: application/json +// POST /volume +// Content-Type: application/json // -// { -// "StorageURL": "http://172.28.128.2:8080/v1/AUTH_test/con", -// "AuthToken" : "AUTH_tk0123456789abcde0123456789abcdef0" -// } +// { +// "StorageURL": "http://172.28.128.2:8080/v1/AUTH_test/con", +// "AuthToken" : "AUTH_tk0123456789abcde0123456789abcdef0" +// } // // This will cause the specified StorageURL to be formatted. The StorageURL // specified in the JSON document content identifies the Container for format. // The AuthToken in the JSON document content provides the authentication to // use during the formatting process. // -// PUT /volume/ -// Content-Type: application/json +// PUT /keepalive/ // -// { -// "StorageURL": "http://172.28.128.2:8080/v1/AUTH_test/con" -// } +// This will configure the keep alive mechanism to start a count down timer +// for the specified duration after which all served volumes will no longer +// be served. The expiration of this count down timer can be avoided by +// GET-ing (/keepalive) the current duration, PUT-ing (/keepalive/) a new volume, or by DELETE-ing +// (/keepalive) the keep alive mechanism. Note that if tje keep alive mechanism +// ever expired, the current duratiom will be reset to zero (i.e. no expiration). +// +// PUT /volume/ +// Content-Type: application/json +// +// { +// "StorageURL": "http://172.28.128.2:8080/v1/AUTH_test/con" +// } // // This will cause the specified to be served. The StorageURL // specified in the JSON document content identifies the Container to serve. // Clients will each supply an AuthToken in their Mount/RenewMount requests // that will be used to access the Container. // -// PUT /volume/ -// Content-Type: application/json +// PUT /volume/ +// Content-Type: application/json // -// { -// "StorageURL": "http://172.28.128.2:8080/v1/AUTH_test/con", -// "AuthToken" : "AUTH_tk0123456789abcde0123456789abcdef0" -// } +// { +// "StorageURL": "http://172.28.128.2:8080/v1/AUTH_test/con", +// "AuthToken" : "AUTH_tk0123456789abcde0123456789abcdef0" +// } // // This will cause the specified to be served. The StorageURL // specified in the JSON document content identifies the Container to serve. @@ -152,7 +174,6 @@ // that will be used to access the Container. As a debugging aid, and in the // case where no Clients have mounted, the AuthToken in the JSON // document content will be used to access the Container. -// package imgrpkg import ( @@ -160,40 +181,34 @@ import ( ) // Start is called to start serving. -// func Start(confMap conf.ConfMap) (err error) { err = start(confMap) return } // Stop is called to stop serving. -// func Stop() (err error) { err = stop() return } // Signal is called to interrupt the server for performing operations such as log rotation. -// func Signal() (err error) { err = signal() return } // LogWarnf is a wrapper around the internal logWarnf() func called by imgr/main.go::main(). -// func LogWarnf(format string, args ...interface{}) { logWarnf(format, args...) } // LogInfof is a wrapper around the internal logInfof() func called by imgr/main.go::main(). -// func LogInfof(format string, args ...interface{}) { logInfof(format, args...) } // E* specifies the prefix of an error string returned by any RetryRPC API -// const ( EAuthTokenRejected = "EAuthTokenRejected:" EBadOpenCountAdjustment = "EBadOpenCountAdjustment:" @@ -212,14 +227,12 @@ type RetryRPCServerStruct struct{} var retryRPCServer *RetryRPCServerStruct // MountRequestStruct is the request object for Mount. -// type MountRequestStruct struct { VolumeName string AuthToken string } // MountResponseStruct is the response object for Mount. -// type MountResponseStruct struct { MountID string } @@ -228,57 +241,48 @@ type MountResponseStruct struct { // in all subsequent RPCs to reference this Volume by this Client. // // Possible errors: EAuthTokenRejected EVolumeBeingDeleted EUnknownVolumeName -// func (dummy *RetryRPCServerStruct) Mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountResponse *MountResponseStruct) (err error) { return mount(retryRPCClientID, mountRequest, mountResponse) } // RenewMountRequestStruct is the request object for RenewMount. -// type RenewMountRequestStruct struct { MountID string AuthToken string } // RenewMountResponseStruct is the response object for RenewMount. -// type RenewMountResponseStruct struct{} // RenewMount updates the AuthToken for the specified MountID. // // Possible errors: EAuthTokenRejected EUnknownMountID -// func (dummy *RetryRPCServerStruct) RenewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse *RenewMountResponseStruct) (err error) { return renewMount(renewMountRequest, renewMountResponse) } // UnmountRequestStruct is the request object for Unmount. -// type UnmountRequestStruct struct { MountID string } // UnmountResponseStruct is the response object for Unmount. -// type UnmountResponseStruct struct{} // Unmount requests that the given MountID be released (and implicitly releases // any Leases held by the MountID). // // Possible errors: EAuthTokenRejected EUnknownMountID -// func (dummy *RetryRPCServerStruct) Unmount(unmountRequest *UnmountRequestStruct, unmountResponse *UnmountResponseStruct) (err error) { return unmount(unmountRequest, unmountResponse) } // VolumeStatusRequestStruct is the request object for VolumeStatus. -// type VolumeStatusRequestStruct struct { MountID string } // VolumeStatusResponseStruct is the response object for VolumeStatus. -// type VolumeStatusResponseStruct struct { NumInodes uint64 ObjectCount uint64 @@ -289,7 +293,6 @@ type VolumeStatusResponseStruct struct { // VolumeStatus requests the current status of the mounted volume. // // Possible errors: EAuthTokenRejected EUnknownMountID -// func (dummy *RetryRPCServerStruct) VolumeStatus(volumeStatusRequest *VolumeStatusRequestStruct, volumeStatusResponse *VolumeStatusResponseStruct) (err error) { return volumeStatus(volumeStatusRequest, volumeStatusResponse) } @@ -297,13 +300,11 @@ func (dummy *RetryRPCServerStruct) VolumeStatus(volumeStatusRequest *VolumeStatu // FetchNonceRangeRequestStruct is the request object for FetchNonceRange. // // Possible errors: EAuthTokenRejected EUnknownMountID -// type FetchNonceRangeRequestStruct struct { MountID string } // FetchNonceRangeResponseStruct is the response object for FetchNonceRange. -// type FetchNonceRangeResponseStruct struct { NextNonce uint64 NumNoncesFetched uint64 @@ -313,20 +314,17 @@ type FetchNonceRangeResponseStruct struct { // never be reused). // // Possible errors: EAuthTokenRejected EUnknownMountID -// func (dummy *RetryRPCServerStruct) FetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeResponse *FetchNonceRangeResponseStruct) (err error) { return fetchNonceRange(fetchNonceRangeRequest, fetchNonceRangeResponse) } // GetInodeTableEntryRequestStruct is the request object for GetInodeTableEntry. -// type GetInodeTableEntryRequestStruct struct { MountID string InodeNumber uint64 } // GetInodeTableEntryResponseStruct is the response object for GetInodeTableEntry. -// type GetInodeTableEntryResponseStruct struct { InodeHeadObjectNumber uint64 InodeHeadLength uint64 @@ -336,7 +334,6 @@ type GetInodeTableEntryResponseStruct struct { // (which must have an active Shared or Exclusive Lease granted to the MountID). // // Possible errors: EAuthTokenRejected EMissingLease EUnknownInodeNumber EUnknownMountID -// func (dummy *RetryRPCServerStruct) GetInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryResponse *GetInodeTableEntryResponseStruct) (err error) { return getInodeTableEntry(getInodeTableEntryRequest, getInodeTableEntryResponse) } @@ -344,7 +341,6 @@ func (dummy *RetryRPCServerStruct) GetInodeTableEntry(getInodeTableEntryRequest // PutInodeTableEntryStruct is used to indicate the change to an individual // InodeTableEntry as part of the collection of changes in a PutInodeTablesEntries // request (which must have an active Exclusive Lease granted to the MountID). -// type PutInodeTableEntryStruct struct { InodeNumber uint64 InodeHeadObjectNumber uint64 @@ -360,7 +356,6 @@ type PutInodeTableEntryStruct struct { // // Note that dereferenced objects listed in the DereferencedObjectNumberArray will // not be deleted until the next CheckPoint is performed. -// type PutInodeTableEntriesRequestStruct struct { MountID string UpdatedInodeTableEntryArray []PutInodeTableEntryStruct @@ -371,27 +366,23 @@ type PutInodeTableEntriesRequestStruct struct { } // PutInodeTableEntriesResponseStruct is the response object for PutInodeTableEntries. -// type PutInodeTableEntriesResponseStruct struct{} // PutInodeTableEntries requests an atomic update of the listed Inodes (which must // each have an active Exclusive Lease granted to the MountID). // // Possible errors: EAuthTokenRejected EMissingLease EUnknownMountID -// func (dummy *RetryRPCServerStruct) PutInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesResponse *PutInodeTableEntriesResponseStruct) (err error) { return putInodeTableEntries(putInodeTableEntriesRequest, putInodeTableEntriesResponse) } // DeleteInodeTableEntryRequestStruct is the request object for DeleteInodeTableEntry. -// type DeleteInodeTableEntryRequestStruct struct { MountID string InodeNumber uint64 } // DeleteInodeTableEntryResponseStruct is the response object for DeleteInodeTableEntry. -// type DeleteInodeTableEntryResponseStruct struct{} // DeleteInodeTableEntry requests the specified Inode information be deleted. @@ -400,13 +391,11 @@ type DeleteInodeTableEntryResponseStruct struct{} // still exist. // // Possible errors: EAuthTokenRejected EMissingLease EUnknownMountID -// func (dummy *RetryRPCServerStruct) DeleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryResponse *DeleteInodeTableEntryResponseStruct) (err error) { return deleteInodeTableEntry(deleteInodeTableEntryRequest, deleteInodeTableEntryResponse) } // AdjustInodeTableEntryOpenCountRequestStruct is the request object for AdjustInodeTableEntryOpenCount. -// type AdjustInodeTableEntryOpenCountRequestStruct struct { MountID string InodeNumber uint64 @@ -414,7 +403,6 @@ type AdjustInodeTableEntryOpenCountRequestStruct struct { } // AdjustInodeTableEntryOpenCountResponseStruct is the response object for AdjustInodeTableEntryOpenCount. -// type AdjustInodeTableEntryOpenCountResponseStruct struct{} // AdjustInodeTableEntryOpenCount requests the specified Inode's OpenCount be @@ -424,31 +412,26 @@ type AdjustInodeTableEntryOpenCountResponseStruct struct{} // DeleteInodeTableEntry, the Inode will be deleted. // // Possible errors: EAuthTokenRejected EBadOpenCountAdjustment EMissingLease EUnknownMountID -// func (dummy *RetryRPCServerStruct) AdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountResponse *AdjustInodeTableEntryOpenCountResponseStruct) (err error) { return adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) } // FlushRequestStruct is the request object for Flush. -// type FlushRequestStruct struct { MountID string } // FlushResponseStruct is the response object for Flush. -// type FlushResponseStruct struct{} // Flush that the results of prior PutInodeTableEntries requests be persisted. // // Possible errors: EAuthTokenRejected EUnknownMountID -// func (dummy *RetryRPCServerStruct) Flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) (err error) { return flush(flushRequest, flushResponse) } // LeaseRequestType specifies the requested lease operation. -// type LeaseRequestType uint32 const ( @@ -460,7 +443,6 @@ const ( ) // LeaseRequestStruct is the request object for Lease. -// type LeaseRequestStruct struct { MountID string InodeNumber uint64 @@ -470,7 +452,6 @@ type LeaseRequestStruct struct { // LeaseResponseType specifies the acknowledgement that the requested lease operation // has been completed or denied (e.g. when a Promotion request cannot be satisfied // and the client will soon be receiving a LeaseInterruptTypeRelease). -// type LeaseResponseType uint32 const ( @@ -483,7 +464,6 @@ const ( ) // LeaseResponseStruct is the response object for Lease. -// type LeaseResponseStruct struct { LeaseResponseType // One of LeaseResponseType* } @@ -491,7 +471,6 @@ type LeaseResponseStruct struct { // Lease is a blocking Lease Request. // // Possible errors: EAuthTokenRejected EUnknownMountID -// func (dummy *RetryRPCServerStruct) Lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) (err error) { return lease(leaseRequest, leaseResponse) } @@ -499,7 +478,6 @@ func (dummy *RetryRPCServerStruct) Lease(leaseRequest *LeaseRequestStruct, lease // RPCInterruptType specifies the action (unmount, demotion, or release) requested by ProxyFS // of the client in an RPCInterrupt "upcall" to indicate that a lease or leases must be demoted // or released. -// type RPCInterruptType uint32 const ( @@ -520,7 +498,6 @@ const ( ) // RPCInterrupt is the "upcall" mechanism used by ProxyFS to interrupt the client. -// type RPCInterrupt struct { RPCInterruptType // One of RPCInterruptType* InodeNumber uint64 // if RPCInterruptType == RPCInterruptTypeUnmount, InodeNumber == 0 (ignored) diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index cb4cb142a..e1cd2c966 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -81,15 +81,18 @@ type configStruct struct { } type statsStruct struct { - DeleteVolumeUsecs bucketstats.BucketLog2Round // DELETE /volume/ - GetConfigUsecs bucketstats.BucketLog2Round // GET /config - GetStatsUsecs bucketstats.BucketLog2Round // GET /stats - GetVersionUsecs bucketstats.BucketLog2Round // GET /version - GetVolumeInodeUsecs bucketstats.BucketLog2Round // GET /volume//inode/ - GetVolumeListUsecs bucketstats.BucketLog2Round // GET /volume - GetVolumeUsecs bucketstats.BucketLog2Round // GET /volume/ - PostVolumeUsecs bucketstats.BucketLog2Round // POST /volume/ - PutVolumeUsecs bucketstats.BucketLog2Round // PUT /volume/ + DeleteKeepAliveUsecs bucketstats.BucketLog2Round // DELETE /keepalive + DeleteVolumeUsecs bucketstats.BucketLog2Round // DELETE /volume/ + GetConfigUsecs bucketstats.BucketLog2Round // GET /config + GetKeepAliveUsecs bucketstats.BucketLog2Round // GET /keepalive + GetStatsUsecs bucketstats.BucketLog2Round // GET /stats + GetVersionUsecs bucketstats.BucketLog2Round // GET /version + GetVolumeInodeUsecs bucketstats.BucketLog2Round // GET /volume//inode/ + GetVolumeListUsecs bucketstats.BucketLog2Round // GET /volume + GetVolumeUsecs bucketstats.BucketLog2Round // GET /volume/ + PostVolumeUsecs bucketstats.BucketLog2Round // POST /volume/ + PutKeepAliveUsecs bucketstats.BucketLog2Round // PUT /keepalive/ + PutVolumeUsecs bucketstats.BucketLog2Round // PUT /volume/ AdjustInodeTableEntryOpenCountUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).AdjustInodeTableEntryOpenCount() DeleteInodeTableEntryUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).DeleteInodeTableEntry() @@ -185,7 +188,8 @@ type inodeLeaseStruct struct { lruElement *list.Element // link into globals.inodeLeaseLRU leaseState inodeLeaseStateType - requestChan chan *leaseRequestOperationStruct + requestChan chan struct{} // Used to signal handler() to service .requestList + requestList *list.List // FIFO of leaseRequestOperationStruct's stopChan chan struct{} // closing this chan will trigger *inodeLeaseStruct.handler() to: // revoke/reject all leaseRequestStruct's in *Holder* & requestedList // issue volume.leaseHandlerWG.Done() @@ -270,6 +274,12 @@ type volumeStruct struct { // .Done() each inodeLease after it is removed from inodeLeaseMap } +type keepAliveControlStruct struct { + sync.WaitGroup + duration time.Duration + stopChan chan struct{} +} + type globalsStruct struct { sync.Mutex // config configStruct // @@ -287,6 +297,7 @@ type globalsStruct struct { retryrpcServer *retryrpc.Server // httpServer *http.Server // httpServerWG sync.WaitGroup // + keepAliveControl *keepAliveControlStruct // if != nil, represents a possibly active (or recently exited) (*keepAliveControl).daemon() stats *statsStruct // } diff --git a/imgr/imgrpkg/html_templates.go b/imgr/imgrpkg/html_templates.go index 96391f664..21e0bc139 100644 --- a/imgr/imgrpkg/html_templates.go +++ b/imgr/imgrpkg/html_templates.go @@ -4,7 +4,8 @@ package imgrpkg // To use: fmt.Sprintf(indexDotHTMLTemplate, proxyfsVersion) -// %[1]v +// +// %[1]v const indexDotHTMLTemplate string = ` @@ -93,7 +94,8 @@ const indexDotHTMLTemplate string = ` ` // To use: fmt.Sprintf(configTemplate, proxyfsVersion, confMapJSONString) -// %[1]v %[2]v +// +// %[1]v %[2]v const configTemplate string = ` @@ -152,7 +154,8 @@ const configTemplate string = ` ` // To use: fmt.Sprintf(volumeListTemplate, proxyfsVersion, volumeListJSONString) -// %[1]v %[2]v +// +// %[1]v %[2]v const volumeListTemplate string = ` @@ -239,7 +242,8 @@ const volumeListTemplate string = ` ` // To use: fmt.Sprintf(volumeTemplate, proxyfsVersion, volumeName, volumeJSONString) -// %[1]v %[2]v %[3]v +// +// %[1]v %[2]v %[3]v const volumeTemplate string = ` @@ -432,7 +436,8 @@ const volumeTemplate string = ` ` // To use: fmt.Sprintf(inodeTemplate, proxyfsVersion, volumeName, inodeNumber, inodeJSONString) -// %[1]v %[2]v %[3]v %[4]v +// +// %[1]v %[2]v %[3]v %[4]v const inodeTemplate string = ` diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index 191ccc079..eeda0210d 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -43,6 +43,8 @@ func startHTTPServer() (err error) { ipAddrTCPPort = net.JoinHostPort(globals.config.PrivateIPAddr, strconv.Itoa(int(globals.config.HTTPServerPort))) + globals.keepAliveControl = nil + globals.httpServer = &http.Server{ Addr: ipAddrTCPPort, Handler: &globals, @@ -77,23 +79,42 @@ func startHTTPServer() (err error) { } func stopHTTPServer() (err error) { + var ( + keepAliveControl *keepAliveControlStruct + ) + err = globals.httpServer.Shutdown(context.TODO()) if nil == err { globals.httpServerWG.Wait() } + globals.Lock() + keepAliveControl = globals.keepAliveControl + globals.keepAliveControl = nil + globals.Unlock() + + if keepAliveControl != nil { + keepAliveControl.cancel() + } + return } func (dummy *globalsStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { var ( - err error - requestBody []byte - requestPath string + err error + requestAuthToken string + requestBody []byte + requestHTML bool + requestPath string ) requestPath = strings.TrimRight(request.URL.Path, "/") + requestAuthToken = request.Header.Get("X-Auth-Token") + + requestHTML = strings.Contains(request.Header.Get("Accept"), "text/html") + requestBody, err = ioutil.ReadAll(request.Body) if nil == err { err = request.Body.Close() @@ -109,28 +130,56 @@ func (dummy *globalsStruct) ServeHTTP(responseWriter http.ResponseWriter, reques switch request.Method { case http.MethodDelete: - serveHTTPDelete(responseWriter, request, requestPath) + serveHTTPDelete(responseWriter, requestPath) case http.MethodGet: - serveHTTPGet(responseWriter, request, requestPath) + serveHTTPGet(responseWriter, requestPath, requestAuthToken, requestHTML) case http.MethodPost: - serveHTTPPost(responseWriter, request, requestPath, requestBody) + serveHTTPPost(responseWriter, requestPath, requestBody) case http.MethodPut: - serveHTTPPut(responseWriter, request, requestPath, requestBody) + serveHTTPPut(responseWriter, requestPath, requestBody) default: responseWriter.WriteHeader(http.StatusMethodNotAllowed) } } -func serveHTTPDelete(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { +func serveHTTPDelete(responseWriter http.ResponseWriter, requestPath string) { switch { + case "/keepalive" == requestPath: + serveHTTPDeleteOfKeepAlive(responseWriter) case strings.HasPrefix(requestPath, "/volume"): - serveHTTPDeleteOfVolume(responseWriter, request, requestPath) + serveHTTPDeleteOfVolume(responseWriter, requestPath) default: responseWriter.WriteHeader(http.StatusNotFound) } } -func serveHTTPDeleteOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { +func serveHTTPDeleteOfKeepAlive(responseWriter http.ResponseWriter) { + var ( + keepAliveControl *keepAliveControlStruct + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.DeleteKeepAliveUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + globals.Lock() + keepAliveControl = globals.keepAliveControl + globals.keepAliveControl = nil // Avoid race + globals.Unlock() + + if keepAliveControl == nil { + responseWriter.WriteHeader(http.StatusNotFound) + } else { + keepAliveControl.cancel() + + responseWriter.WriteHeader(http.StatusOK) + + logInfof("KeepAlive disabled") + } +} + +func serveHTTPDeleteOfVolume(responseWriter http.ResponseWriter, requestPath string) { var ( err error pathSplit []string @@ -156,7 +205,7 @@ func serveHTTPDeleteOfVolume(responseWriter http.ResponseWriter, request *http.R } } -func serveHTTPGet(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { +func serveHTTPGet(responseWriter http.ResponseWriter, requestPath string, requestAuthToken string, requestHTML bool) { var ( ok bool ) @@ -167,17 +216,19 @@ func serveHTTPGet(responseWriter http.ResponseWriter, request *http.Request, req responseWriter.WriteHeader(http.StatusOK) _, _ = responseWriter.Write([]byte(fmt.Sprintf(indexDotHTMLTemplate, version.ProxyFSVersion))) case "/config" == requestPath: - serveHTTPGetOfConfig(responseWriter, request) + serveHTTPGetOfConfig(responseWriter, requestHTML) case "/index.html" == requestPath: responseWriter.Header().Set("Content-Type", "text/html") responseWriter.WriteHeader(http.StatusOK) _, _ = responseWriter.Write([]byte(fmt.Sprintf(indexDotHTMLTemplate, version.ProxyFSVersion))) + case "/keepalive" == requestPath: + serveHTTPGetOfKeepAlive(responseWriter) case "/stats" == requestPath: - serveHTTPGetOfStats(responseWriter, request) + serveHTTPGetOfStats(responseWriter) case "/version" == requestPath: - serveHTTPGetOfVersion(responseWriter, request) + serveHTTPGetOfVersion(responseWriter) case strings.HasPrefix(requestPath, "/volume"): - serveHTTPGetOfVolume(responseWriter, request, requestPath) + serveHTTPGetOfVolume(responseWriter, requestPath, requestAuthToken, requestHTML) default: ok = ihtml.ServeHTTPGet(responseWriter, requestPath) if !ok { @@ -186,7 +237,7 @@ func serveHTTPGet(responseWriter http.ResponseWriter, request *http.Request, req } } -func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, request *http.Request) { +func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, requestHTML bool) { var ( confMapJSON []byte err error @@ -202,7 +253,7 @@ func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, request *http.Requ logFatalf("json.Marshal(globals.config) failed: %v", err) } - if strings.Contains(request.Header.Get("Accept"), "text/html") { + if requestHTML { responseWriter.Header().Set("Content-Type", "text/html") responseWriter.WriteHeader(http.StatusOK) @@ -222,7 +273,48 @@ func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, request *http.Requ } } -func serveHTTPGetOfStats(responseWriter http.ResponseWriter, request *http.Request) { +func serveHTTPGetOfKeepAlive(responseWriter http.ResponseWriter) { + var ( + err error + keepAliveControl *keepAliveControlStruct + keepAliveDurationAsString string + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.GetKeepAliveUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + globals.Lock() + keepAliveControl = globals.keepAliveControl + globals.keepAliveControl = nil // Avoid race + globals.Unlock() + + if keepAliveControl == nil { + responseWriter.WriteHeader(http.StatusNotFound) + } else { + keepAliveControl.cancel() + + keepAliveDurationAsString = fmt.Sprintf("%v", keepAliveControl.duration) + + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(keepAliveDurationAsString))) + responseWriter.Header().Set("Content-Type", "text/plain") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write([]byte(keepAliveDurationAsString)) + if nil != err { + logWarnf("responseWriter.Write([]byte(keepAliveDurationAsString)) failed: %v", err) + } + + globals.Lock() + if globals.keepAliveControl == nil { // Avoid race + globals.keepAliveControl = keepAliveStart(keepAliveControl.duration) + } + globals.Unlock() + } +} + +func serveHTTPGetOfStats(responseWriter http.ResponseWriter) { var ( err error startTime time.Time = time.Now() @@ -245,7 +337,7 @@ func serveHTTPGetOfStats(responseWriter http.ResponseWriter, request *http.Reque } } -func serveHTTPGetOfVersion(responseWriter http.ResponseWriter, request *http.Request) { +func serveHTTPGetOfVersion(responseWriter http.ResponseWriter) { var ( err error startTime time.Time @@ -631,9 +723,8 @@ func (extentMapWrapper *httpServerExtentMapWrapperStruct) UnpackValue(payloadDat return } -func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { +func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, requestPath string, requestAuthToken string, requestHTML bool) { var ( - acceptHeader string checkPointV1 *ilayout.CheckPointV1Struct dimensionsReport sortedmap.DimensionsReport directoryEntryIndex int @@ -664,11 +755,11 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ inodeTableIndex int inodeTableLayoutIndex int inodeTableWrapper *httpServerInodeTableWrapperStruct + keepAliveControl *keepAliveControlStruct mustBeInode string ok bool pathSplit []string pendingDeleteObjectNameArrayIndex int - requestAuthToken string startTime time.Time = time.Now() superBlockV1 *ilayout.SuperBlockV1Struct volumeAsStruct *volumeStruct @@ -728,16 +819,21 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ } } + keepAliveControl = globals.keepAliveControl + globals.keepAliveControl = nil // Avoid race + globals.Unlock() + if keepAliveControl != nil { + keepAliveControl.cancel() + } + volumeListGETAsJSON, err = json.Marshal(volumeListGET) if nil != err { logFatal(err) } - acceptHeader = request.Header.Get("Accept") - - if strings.Contains(acceptHeader, "text/html") { + if requestHTML { volumeListGETAsHTML = []byte(fmt.Sprintf(volumeListTemplate, version.ProxyFSVersion, string(volumeListGETAsJSON))) responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(volumeListGETAsHTML))) @@ -758,6 +854,14 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logWarnf("responseWriter.Write(volumeListGETAsJSON) failed: %v", err) } } + + if keepAliveControl != nil { + globals.Lock() + if globals.keepAliveControl == nil { // Avoid race + globals.keepAliveControl = keepAliveStart(keepAliveControl.duration) + } + globals.Unlock() + } case 3: // Form: /volume/ @@ -780,7 +884,6 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) } - requestAuthToken = request.Header.Get("X-Auth-Token") volumeAuthToken = volumeAsStruct.authToken if requestAuthToken != "" { volumeAsStruct.authToken = requestAuthToken @@ -889,9 +992,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logFatal(err) } - acceptHeader = request.Header.Get("Accept") - - if strings.Contains(acceptHeader, "text/html") { + if requestHTML { volumeGETAsHTML = []byte(fmt.Sprintf(volumeTemplate, version.ProxyFSVersion, volumeAsStruct.name, string(volumeGETAsJSON))) responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(volumeGETAsHTML))) @@ -947,7 +1048,6 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ } volumeAuthToken = volumeAsStruct.authToken - requestAuthToken = request.Header.Get("X-Auth-Token") if requestAuthToken != "" { volumeAsStruct.authToken = requestAuthToken } @@ -1255,9 +1355,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logFatal(err) } - acceptHeader = request.Header.Get("Accept") - - if strings.Contains(acceptHeader, "text/html") { + if requestHTML { inodeGETAsHTML = []byte(fmt.Sprintf(inodeTemplate, version.ProxyFSVersion, volumeAsStruct.name, inodeHeadV1.InodeNumber, string(inodeGETAsJSON))) responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(inodeGETAsHTML))) @@ -1291,10 +1389,10 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ } } -func serveHTTPPost(responseWriter http.ResponseWriter, request *http.Request, requestPath string, requestBody []byte) { +func serveHTTPPost(responseWriter http.ResponseWriter, requestPath string, requestBody []byte) { switch { case requestPath == "/volume": - serveHTTPPostOfVolume(responseWriter, request, requestBody) + serveHTTPPostOfVolume(responseWriter, requestBody) default: responseWriter.WriteHeader(http.StatusNotFound) } @@ -1305,7 +1403,7 @@ type serveHTTPPostOfVolumeRequestBodyAsJSONStruct struct { AuthToken string } -func serveHTTPPostOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestBody []byte) { +func serveHTTPPostOfVolume(responseWriter http.ResponseWriter, requestBody []byte) { var ( conflictReason []byte confligtReasonWriteErr error @@ -1342,25 +1440,89 @@ func serveHTTPPostOfVolume(responseWriter http.ResponseWriter, request *http.Req } } -func serveHTTPPut(responseWriter http.ResponseWriter, request *http.Request, requestPath string, requestBody []byte) { +func serveHTTPPut(responseWriter http.ResponseWriter, requestPath string, requestBody []byte) { switch { + case strings.HasPrefix(requestPath, "/keepalive"): + serveHTTPPutOfKeepAlive(responseWriter, requestPath) case strings.HasPrefix(requestPath, "/volume"): - serveHTTPPutOfVolume(responseWriter, request, requestPath, requestBody) + serveHTTPPutOfVolume(responseWriter, requestPath, requestBody) default: responseWriter.WriteHeader(http.StatusNotFound) } } +func serveHTTPPutOfKeepAlive(responseWriter http.ResponseWriter, requestPath string) { + var ( + err error + keepAliveControl *keepAliveControlStruct + keepAliveDurationNew time.Duration + keepAliveDurationOld time.Duration + pathSplit []string + startTime time.Time = time.Now() + ) + + pathSplit = strings.Split(requestPath, "/") + + switch len(pathSplit) { + case 3: + defer func() { + globals.stats.PutKeepAliveUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + keepAliveDurationNew, err = time.ParseDuration(pathSplit[2]) + if nil != err { + responseWriter.WriteHeader(http.StatusBadRequest) + return + } + + responseWriter.WriteHeader(http.StatusOK) + + globals.Lock() + keepAliveControl = globals.keepAliveControl + globals.keepAliveControl = nil // Avoid race + globals.Unlock() + + if keepAliveControl == nil { + keepAliveDurationOld = time.Duration(0) + } else { + keepAliveControl.cancel() + keepAliveDurationOld = keepAliveControl.duration + } + + globals.Lock() + if globals.keepAliveControl == nil { // Avoid race + if keepAliveDurationNew == time.Duration(0) { + if keepAliveDurationOld != time.Duration(0) { + logInfof("KeepAlive disabled") + } + } else { + if keepAliveDurationOld == time.Duration(0) { + logInfof("KeepAlive enabled with duration %v", keepAliveDurationNew) + } else if keepAliveDurationOld != keepAliveDurationNew { + logInfof("KeepAlive duration updated from %v to %v", keepAliveDurationOld, keepAliveDurationNew) + } else { + // KeepAlive duration unchanged + } + globals.keepAliveControl = keepAliveStart(keepAliveDurationNew) + } + globals.Unlock() + } + default: + responseWriter.WriteHeader(http.StatusBadRequest) + } +} + type serveHTTPPutOfVolumeRequestBodyAsJSONStruct struct { StorageURL string AuthToken string } -func serveHTTPPutOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string, requestBody []byte) { +func serveHTTPPutOfVolume(responseWriter http.ResponseWriter, requestPath string, requestBody []byte) { var ( conflictReason []byte confligtReasonWriteErr error err error + keepAliveControl *keepAliveControlStruct pathSplit []string requestBodyAsJSON serveHTTPPutOfVolumeRequestBodyAsJSONStruct startTime time.Time = time.Now() @@ -1374,6 +1536,15 @@ func serveHTTPPutOfVolume(responseWriter http.ResponseWriter, request *http.Requ globals.stats.PutVolumeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() + globals.Lock() + keepAliveControl = globals.keepAliveControl + globals.keepAliveControl = nil // Avoid race + globals.Unlock() + + if keepAliveControl != nil { + keepAliveControl.cancel() + } + err = json.Unmarshal(requestBody, &requestBodyAsJSON) if nil != err { responseWriter.WriteHeader(http.StatusBadRequest) @@ -1396,7 +1567,99 @@ func serveHTTPPutOfVolume(responseWriter http.ResponseWriter, request *http.Requ logWarnf("responseWriter.Write(conflictReason) failed: %v", confligtReasonWriteErr) } } + + if keepAliveControl != nil { + globals.Lock() + if globals.keepAliveControl == nil { // Avoid race + globals.keepAliveControl = keepAliveStart(keepAliveControl.duration) + } + globals.Unlock() + } default: responseWriter.WriteHeader(http.StatusBadRequest) } } + +func (keepAliveControl *keepAliveControlStruct) cancel() { + close(keepAliveControl.stopChan) + keepAliveControl.Wait() +} + +func keepAliveStart(keepAliveDuration time.Duration) (keepAliveControl *keepAliveControlStruct) { + keepAliveControl = &keepAliveControlStruct{ + duration: keepAliveDuration, + stopChan: make(chan struct{}), + } + + keepAliveControl.Add(1) + + go keepAliveControl.daemon() + + return +} + +func (keepAliveControl *keepAliveControlStruct) daemon() { + var ( + err error + keepAliveTimer *time.Timer + ok bool + volumeMapLen int + volumeList []string + volumeListIndex int + volumeNameAsKey sortedmap.Key + ) + + keepAliveTimer = time.NewTimer(keepAliveControl.duration) + + select { + case <-keepAliveTimer.C: + globals.Lock() + if globals.keepAliveControl == keepAliveControl { + globals.keepAliveControl = nil + close(keepAliveControl.stopChan) + } + volumeMapLen, err = globals.volumeMap.Len() + if nil != err { + logFatal(err) + } + logWarnf("KeepAlive expired - resetting and deleting %v volume(s)", volumeMapLen) + volumeList = make([]string, volumeMapLen) + for volumeListIndex = 0; volumeListIndex < volumeMapLen; volumeListIndex++ { + volumeNameAsKey, _, ok, err = globals.volumeMap.GetByIndex(volumeListIndex) + if nil != err { + logFatal(err) + } + if !ok { + logFatalf("globals.volumeMap[] len (%d) is wrong", volumeMapLen) + } + volumeList[volumeListIndex], ok = volumeNameAsKey.(string) + if !ok { + logFatalf("globals.volumeMap did not have a string Key at index %d", volumeListIndex) + } + } + globals.Unlock() + for volumeListIndex = 0; volumeListIndex < volumeMapLen; volumeListIndex++ { + keepAliveControl.Add(1) + go keepAliveControl.deleteVolume(volumeList[volumeListIndex]) + } + case <-keepAliveControl.stopChan: + if !keepAliveTimer.Stop() { + <-keepAliveTimer.C + } + } + + keepAliveControl.Done() +} + +func (keepAliveControl *keepAliveControlStruct) deleteVolume(volumeName string) { + var ( + err error + ) + + err = deleteVolume(volumeName) + if nil != err { + logFatalf("deleteVolume(\"%s\") failed: %v", volumeName, err) + } + + keepAliveControl.Done() +} diff --git a/imgr/imgrpkg/lease.go b/imgr/imgrpkg/lease.go index 200dd1bc4..6b65ac768 100644 --- a/imgr/imgrpkg/lease.go +++ b/imgr/imgrpkg/lease.go @@ -13,14 +13,10 @@ import ( ) func (inodeLease *inodeLeaseStruct) handler() { - var ( - leaseRequestOperation *leaseRequestOperationStruct - ) - for { select { - case leaseRequestOperation = <-inodeLease.requestChan: - inodeLease.handleOperation(leaseRequestOperation) + case _ = <-inodeLease.requestChan: + inodeLease.handleRequestList() case <-inodeLease.longAgoTimer.C: inodeLease.handleLongAgoTimerPop() case <-inodeLease.interruptTimer.C: @@ -31,434 +27,491 @@ func (inodeLease *inodeLeaseStruct) handler() { } } -func (inodeLease *inodeLeaseStruct) handleOperation(leaseRequestOperation *leaseRequestOperationStruct) { +func (inodeLease *inodeLeaseStruct) handleRequestList() { var ( - err error - inodeLeaseExpirerWG *sync.WaitGroup - leaseRequest *leaseRequestStruct - leaseRequestElement *list.Element - ok bool - rpcInterrupt *RPCInterrupt - rpcInterruptBuf []byte - sharedHolderLeaseRequest *leaseRequestStruct - sharedHolderListElement *list.Element + err error + inodeLeaseExpirerWG *sync.WaitGroup + leaseRequest *leaseRequestStruct + leaseRequestElement *list.Element + leaseRequestOperation *leaseRequestOperationStruct + leaseRequestOperationElement *list.Element + ok bool + rpcInterrupt *RPCInterrupt + rpcInterruptBuf []byte + sharedHolderLeaseRequest *leaseRequestStruct + sharedHolderListElement *list.Element ) - globals.Lock() - - if globals.inodeLeaseExpirerWG != nil { - inodeLeaseExpirerWG = globals.inodeLeaseExpirerWG - globals.Unlock() - inodeLeaseExpirerWG.Wait() + for { globals.Lock() - } - globals.inodeLeaseLRU.MoveToBack(inodeLease.lruElement) + if globals.inodeLeaseExpirerWG != nil { + inodeLeaseExpirerWG = globals.inodeLeaseExpirerWG + globals.Unlock() + inodeLeaseExpirerWG.Wait() + globals.Lock() + } - switch leaseRequestOperation.LeaseRequestType { - case LeaseRequestTypeShared: - _, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] - if ok { - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] returned !ok - leaseRequest = &leaseRequestStruct{ - mount: leaseRequestOperation.mount, - inodeLease: inodeLease, - requestState: leaseRequestStateSharedRequested, - replyChan: leaseRequestOperation.replyChan, - } - leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] = leaseRequest - switch inodeLease.leaseState { - case inodeLeaseStateNone: - leaseRequest.requestState = leaseRequestStateSharedGranted - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) - leaseRequest.replyChan <- LeaseResponseTypeShared - case inodeLeaseStateSharedGrantedRecently: - if !inodeLease.longAgoTimer.Stop() { - <-inodeLease.longAgoTimer.C - } - inodeLease.lastGrantTime = time.Time{} - inodeLease.longAgoTimer = &time.Timer{} - leaseRequest.requestState = leaseRequestStateSharedGranted - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) - leaseRequest.replyChan <- LeaseResponseTypeShared - case inodeLeaseStateSharedGrantedLongAgo: - leaseRequest.requestState = leaseRequestStateSharedGranted - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) - leaseRequest.replyChan <- LeaseResponseTypeShared - case inodeLeaseStateSharedPromoting: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateSharedReleasing: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateSharedExpired: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeShared, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") - case inodeLeaseStateExclusiveGrantedRecently: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateExclusiveGrantedLongAgo: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - inodeLease.leaseState = inodeLeaseStateExclusiveDemoting - inodeLease.demotingHolder = inodeLease.exclusiveHolder - inodeLease.demotingHolder.requestState = leaseRequestStateExclusiveDemoting - inodeLease.exclusiveHolder = nil - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeDemote, - InodeNumber: inodeLease.inodeNumber, - } - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logFatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 1]", rpcInterrupt, err) - } - globals.retryrpcServer.SendCallback(inodeLease.demotingHolder.mount.retryRPCClientID, rpcInterruptBuf) - logTracef("<== [RPC] SendCallback(clientID: 0x%016X, rpcInterrupt: %+v)", inodeLease.demotingHolder.mount.retryRPCClientID, rpcInterrupt) - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent = 1 - inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) - case inodeLeaseStateExclusiveDemoting: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateExclusiveReleasing: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateExclusiveExpired: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeShared, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") - default: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeShared, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) - } + leaseRequestOperationElement = inodeLease.requestList.Front() + + if leaseRequestOperationElement == nil { + globals.Unlock() + return } - case LeaseRequestTypePromote: - leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] - if ok { - if leaseRequestStateSharedGranted == leaseRequest.requestState { + + leaseRequestOperation, ok = leaseRequestOperationElement.Value.(*leaseRequestOperationStruct) + if !ok { + logFatalf("leaseRequestOperationElement.Value.(*leaseRequestOperationStruct) returned !ok") + } + + _ = inodeLease.requestList.Remove(leaseRequestOperationElement) + + globals.inodeLeaseLRU.MoveToBack(inodeLease.lruElement) + + switch leaseRequestOperation.LeaseRequestType { + case LeaseRequestTypeShared: + _, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] + if ok { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] returned !ok + leaseRequest = &leaseRequestStruct{ + mount: leaseRequestOperation.mount, + inodeLease: inodeLease, + requestState: leaseRequestStateSharedRequested, + replyChan: leaseRequestOperation.replyChan, + } + leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] = leaseRequest switch inodeLease.leaseState { case inodeLeaseStateNone: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateNone") + leaseRequest.requestState = leaseRequestStateSharedGranted + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + leaseRequest.replyChan <- LeaseResponseTypeShared case inodeLeaseStateSharedGrantedRecently: - if nil == inodeLease.promotingHolder { + if !inodeLease.longAgoTimer.Stop() { + <-inodeLease.longAgoTimer.C + } + inodeLease.lastGrantTime = time.Time{} + inodeLease.longAgoTimer = &time.Timer{} + leaseRequest.requestState = leaseRequestStateSharedGranted + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + leaseRequest.replyChan <- LeaseResponseTypeShared + case inodeLeaseStateSharedGrantedLongAgo: + leaseRequest.requestState = leaseRequestStateSharedGranted + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + leaseRequest.replyChan <- LeaseResponseTypeShared + case inodeLeaseStateSharedPromoting: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateSharedReleasing: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateSharedExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeShared, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") + case inodeLeaseStateExclusiveGrantedRecently: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateExclusiveGrantedLongAgo: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + inodeLease.leaseState = inodeLeaseStateExclusiveDemoting + inodeLease.demotingHolder = inodeLease.exclusiveHolder + inodeLease.demotingHolder.requestState = leaseRequestStateExclusiveDemoting + inodeLease.exclusiveHolder = nil + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeDemote, + InodeNumber: inodeLease.inodeNumber, + } + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 1]", rpcInterrupt, err) + } + globals.retryrpcServer.SendCallback(inodeLease.demotingHolder.mount.retryRPCClientID, rpcInterruptBuf) + logTracef("<== [RPC] SendCallback(clientID: 0x%016X, rpcInterrupt: %+v)", inodeLease.demotingHolder.mount.retryRPCClientID, rpcInterrupt) + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent = 1 + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + case inodeLeaseStateExclusiveDemoting: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateExclusiveReleasing: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateExclusiveExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeShared, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") + default: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeShared, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) + } + } + case LeaseRequestTypePromote: + leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] + if ok { + if leaseRequestStateSharedGranted == leaseRequest.requestState { + switch inodeLease.leaseState { + case inodeLeaseStateNone: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateNone") + case inodeLeaseStateSharedGrantedRecently: + if nil == inodeLease.promotingHolder { + _ = inodeLease.sharedHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + if inodeLease.sharedHoldersList.Len() == 0 { + leaseRequest.requestState = leaseRequestStateExclusiveGranted + leaseRequestOperation.replyChan <- LeaseResponseTypePromoted + inodeLease.exclusiveHolder = leaseRequest + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } else { + inodeLease.promotingHolder = leaseRequest + leaseRequest.replyChan = leaseRequestOperation.replyChan + } + } else { // nil != inodeLease.promotingHolder + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateSharedGrantedLongAgo: _ = inodeLease.sharedHoldersList.Remove(leaseRequest.listElement) leaseRequest.listElement = nil if inodeLease.sharedHoldersList.Len() == 0 { + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + inodeLease.exclusiveHolder = leaseRequest leaseRequest.requestState = leaseRequestStateExclusiveGranted leaseRequestOperation.replyChan <- LeaseResponseTypePromoted - inodeLease.exclusiveHolder = leaseRequest - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently inodeLease.lastGrantTime = time.Now() inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) } else { - inodeLease.promotingHolder = leaseRequest leaseRequest.replyChan = leaseRequestOperation.replyChan + inodeLease.leaseState = inodeLeaseStateSharedPromoting + inodeLease.promotingHolder = leaseRequest + leaseRequest.requestState = leaseRequestStateSharedPromoting + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 2]", rpcInterrupt, err) + } + for nil != inodeLease.sharedHoldersList.Front() { + leaseRequestElement = inodeLease.sharedHoldersList.Front() + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + _ = inodeLease.sharedHoldersList.Remove(leaseRequestElement) + leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) + leaseRequest.requestState = leaseRequestStateSharedReleasing + globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + logTracef("<== [RPC] SendCallback(clientID: 0x%016X, rpcInterrupt: %+v)", leaseRequest.mount.retryRPCClientID, rpcInterrupt) + } + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent = 1 + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) } - } else { // nil != inodeLease.promotingHolder - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + case inodeLeaseStateSharedPromoting: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateSharedPromoting") + case inodeLeaseStateSharedReleasing: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateSharedReleasing") + case inodeLeaseStateSharedExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") + case inodeLeaseStateExclusiveGrantedRecently: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveGrantedRecently") + case inodeLeaseStateExclusiveGrantedLongAgo: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveGrantedLongAgo") + case inodeLeaseStateExclusiveDemoting: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveDemoting") + case inodeLeaseStateExclusiveReleasing: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveReleasing") + case inodeLeaseStateExclusiveExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") + default: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) } + } else { // leaseRequestStateSharedGranted != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] returned !ok + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case LeaseRequestTypeExclusive: + _, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] + if ok { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] returned !ok + leaseRequest = &leaseRequestStruct{ + mount: leaseRequestOperation.mount, + inodeLease: inodeLease, + requestState: leaseRequestStateExclusiveRequested, + replyChan: leaseRequestOperation.replyChan, + } + leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] = leaseRequest + switch inodeLease.leaseState { + case inodeLeaseStateNone: + leaseRequest.requestState = leaseRequestStateExclusiveGranted + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + inodeLease.exclusiveHolder = leaseRequest + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + leaseRequest.replyChan <- LeaseResponseTypeExclusive + case inodeLeaseStateSharedGrantedRecently: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) case inodeLeaseStateSharedGrantedLongAgo: - _ = inodeLease.sharedHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - if inodeLease.sharedHoldersList.Len() == 0 { - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - inodeLease.exclusiveHolder = leaseRequest - leaseRequest.requestState = leaseRequestStateExclusiveGranted - leaseRequestOperation.replyChan <- LeaseResponseTypePromoted - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) - } else { - leaseRequest.replyChan = leaseRequestOperation.replyChan - inodeLease.leaseState = inodeLeaseStateSharedPromoting - inodeLease.promotingHolder = leaseRequest - leaseRequest.requestState = leaseRequestStateSharedPromoting + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + inodeLease.leaseState = inodeLeaseStateSharedReleasing + for nil != inodeLease.sharedHoldersList.Front() { + sharedHolderListElement = inodeLease.sharedHoldersList.Front() + sharedHolderLeaseRequest = sharedHolderListElement.Value.(*leaseRequestStruct) + _ = inodeLease.sharedHoldersList.Remove(sharedHolderListElement) + sharedHolderLeaseRequest.requestState = leaseRequestStateSharedReleasing + sharedHolderLeaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(sharedHolderLeaseRequest) rpcInterrupt = &RPCInterrupt{ RPCInterruptType: RPCInterruptTypeRelease, InodeNumber: inodeLease.inodeNumber, } rpcInterruptBuf, err = json.Marshal(rpcInterrupt) if nil != err { - logFatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 2]", rpcInterrupt, err) - } - for nil != inodeLease.sharedHoldersList.Front() { - leaseRequestElement = inodeLease.sharedHoldersList.Front() - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - _ = inodeLease.sharedHoldersList.Remove(leaseRequestElement) - leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) - leaseRequest.requestState = leaseRequestStateSharedReleasing - globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) - logTracef("<== [RPC] SendCallback(clientID: 0x%016X, rpcInterrupt: %+v)", leaseRequest.mount.retryRPCClientID, rpcInterrupt) + logFatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 3]", rpcInterrupt, err) } - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent = 1 - inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + globals.retryrpcServer.SendCallback(sharedHolderLeaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + logTracef("<== [RPC] SendCallback(clientID: 0x%016X, rpcInterrupt: %+v)", sharedHolderLeaseRequest.mount.retryRPCClientID, rpcInterrupt) } + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent = 1 + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) case inodeLeaseStateSharedPromoting: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateSharedPromoting") + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) case inodeLeaseStateSharedReleasing: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateSharedReleasing") + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) case inodeLeaseStateSharedExpired: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeExclusive, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") case inodeLeaseStateExclusiveGrantedRecently: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveGrantedRecently") + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) case inodeLeaseStateExclusiveGrantedLongAgo: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveGrantedLongAgo") - case inodeLeaseStateExclusiveDemoting: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveDemoting") - case inodeLeaseStateExclusiveReleasing: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveReleasing") - case inodeLeaseStateExclusiveExpired: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") - default: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) - } - } else { // leaseRequestStateSharedGranted != leaseRequest.requestState - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] returned !ok - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - case LeaseRequestTypeExclusive: - _, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] - if ok { - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] returned !ok - leaseRequest = &leaseRequestStruct{ - mount: leaseRequestOperation.mount, - inodeLease: inodeLease, - requestState: leaseRequestStateExclusiveRequested, - replyChan: leaseRequestOperation.replyChan, - } - leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] = leaseRequest - switch inodeLease.leaseState { - case inodeLeaseStateNone: - leaseRequest.requestState = leaseRequestStateExclusiveGranted - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - inodeLease.exclusiveHolder = leaseRequest - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) - leaseRequest.replyChan <- LeaseResponseTypeExclusive - case inodeLeaseStateSharedGrantedRecently: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateSharedGrantedLongAgo: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - inodeLease.leaseState = inodeLeaseStateSharedReleasing - for nil != inodeLease.sharedHoldersList.Front() { - sharedHolderListElement = inodeLease.sharedHoldersList.Front() - sharedHolderLeaseRequest = sharedHolderListElement.Value.(*leaseRequestStruct) - _ = inodeLease.sharedHoldersList.Remove(sharedHolderListElement) - sharedHolderLeaseRequest.requestState = leaseRequestStateSharedReleasing - sharedHolderLeaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(sharedHolderLeaseRequest) + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + inodeLease.leaseState = inodeLeaseStateExclusiveReleasing + inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveReleasing + inodeLease.exclusiveHolder.listElement = inodeLease.releasingHoldersList.PushBack(inodeLease.exclusiveHolder) rpcInterrupt = &RPCInterrupt{ RPCInterruptType: RPCInterruptTypeRelease, InodeNumber: inodeLease.inodeNumber, } rpcInterruptBuf, err = json.Marshal(rpcInterrupt) if nil != err { - logFatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 3]", rpcInterrupt, err) + logFatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 4]", rpcInterrupt, err) } - globals.retryrpcServer.SendCallback(sharedHolderLeaseRequest.mount.retryRPCClientID, rpcInterruptBuf) - logTracef("<== [RPC] SendCallback(clientID: 0x%016X, rpcInterrupt: %+v)", sharedHolderLeaseRequest.mount.retryRPCClientID, rpcInterrupt) - } - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent = 1 - inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) - case inodeLeaseStateSharedPromoting: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateSharedReleasing: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateSharedExpired: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeExclusive, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") - case inodeLeaseStateExclusiveGrantedRecently: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateExclusiveGrantedLongAgo: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - inodeLease.leaseState = inodeLeaseStateExclusiveReleasing - inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveReleasing - inodeLease.exclusiveHolder.listElement = inodeLease.releasingHoldersList.PushBack(inodeLease.exclusiveHolder) - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: inodeLease.inodeNumber, - } - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logFatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 4]", rpcInterrupt, err) + globals.retryrpcServer.SendCallback(inodeLease.exclusiveHolder.mount.retryRPCClientID, rpcInterruptBuf) + logTracef("<== [RPC] SendCallback(clientID: 0x%016X, rpcInterrupt: %+v)", inodeLease.exclusiveHolder.mount.retryRPCClientID, rpcInterrupt) + inodeLease.exclusiveHolder = nil + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent = 1 + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + case inodeLeaseStateExclusiveDemoting: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateExclusiveReleasing: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateExclusiveExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeExclusive, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") + default: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeExclusive, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) } - globals.retryrpcServer.SendCallback(inodeLease.exclusiveHolder.mount.retryRPCClientID, rpcInterruptBuf) - logTracef("<== [RPC] SendCallback(clientID: 0x%016X, rpcInterrupt: %+v)", inodeLease.exclusiveHolder.mount.retryRPCClientID, rpcInterrupt) - inodeLease.exclusiveHolder = nil - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent = 1 - inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) - case inodeLeaseStateExclusiveDemoting: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateExclusiveReleasing: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateExclusiveExpired: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeExclusive, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") - default: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeExclusive, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) } - } - case LeaseRequestTypeDemote: - leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] - if ok { - switch inodeLease.leaseState { - case inodeLeaseStateNone: - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - case inodeLeaseStateSharedGrantedRecently: - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - case inodeLeaseStateSharedGrantedLongAgo: - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - case inodeLeaseStateSharedPromoting: - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - case inodeLeaseStateSharedReleasing: - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - case inodeLeaseStateSharedExpired: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeDemote, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") - case inodeLeaseStateExclusiveGrantedRecently: - if leaseRequestStateExclusiveGranted == leaseRequest.requestState { - if !inodeLease.longAgoTimer.Stop() { - <-inodeLease.longAgoTimer.C - } - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.requestState = leaseRequestStateSharedGranted - inodeLease.exclusiveHolder = nil - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequestOperation.replyChan <- LeaseResponseTypeDemoted - leaseRequestElement = inodeLease.requestedList.Front() - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - if leaseRequestStateSharedRequested == leaseRequest.requestState { - leaseRequest.requestState = leaseRequestStateSharedGranted - _ = inodeLease.requestedList.Remove(leaseRequest.listElement) - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequest.replyChan <- LeaseResponseTypeShared - leaseRequestElement = inodeLease.requestedList.Front() - } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState - leaseRequestElement = nil - } - } - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) - } else { // leaseRequestStateExclusiveGranted == leaseRequest.requestState + case LeaseRequestTypeDemote: + leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] + if ok { + switch inodeLease.leaseState { + case inodeLeaseStateNone: leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - case inodeLeaseStateExclusiveGrantedLongAgo: - if leaseRequestStateExclusiveGranted == leaseRequest.requestState { - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.requestState = leaseRequestStateSharedGranted - inodeLease.exclusiveHolder = nil - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequestOperation.replyChan <- LeaseResponseTypeDemoted - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) - } else { // leaseRequestStateExclusiveGranted != leaseRequest.requestState + case inodeLeaseStateSharedGrantedRecently: leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - case inodeLeaseStateExclusiveDemoting: - if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C + case inodeLeaseStateSharedGrantedLongAgo: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + case inodeLeaseStateSharedPromoting: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + case inodeLeaseStateSharedReleasing: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + case inodeLeaseStateSharedExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeDemote, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") + case inodeLeaseStateExclusiveGrantedRecently: + if leaseRequestStateExclusiveGranted == leaseRequest.requestState { + if !inodeLease.longAgoTimer.Stop() { + <-inodeLease.longAgoTimer.C + } + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.requestState = leaseRequestStateSharedGranted + inodeLease.exclusiveHolder = nil + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequestOperation.replyChan <- LeaseResponseTypeDemoted + leaseRequestElement = inodeLease.requestedList.Front() + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + if leaseRequestStateSharedRequested == leaseRequest.requestState { + leaseRequest.requestState = leaseRequestStateSharedGranted + _ = inodeLease.requestedList.Remove(leaseRequest.listElement) + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequest.replyChan <- LeaseResponseTypeShared + leaseRequestElement = inodeLease.requestedList.Front() + } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState + leaseRequestElement = nil + } + } + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } else { // leaseRequestStateExclusiveGranted == leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied } - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - inodeLease.interruptTimer = &time.Timer{} - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - inodeLease.demotingHolder = nil - leaseRequest.requestState = leaseRequestStateSharedGranted - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequestOperation.replyChan <- LeaseResponseTypeDemoted - leaseRequestElement = inodeLease.requestedList.Front() - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - if leaseRequestStateSharedRequested == leaseRequest.requestState { - leaseRequest.requestState = leaseRequestStateSharedGranted - _ = inodeLease.requestedList.Remove(leaseRequest.listElement) - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequest.replyChan <- LeaseResponseTypeShared - leaseRequestElement = inodeLease.requestedList.Front() - } else { // leaseRequestStateSharedRequested != leaseRequest.requestState - leaseRequestElement = nil + case inodeLeaseStateExclusiveGrantedLongAgo: + if leaseRequestStateExclusiveGranted == leaseRequest.requestState { + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.requestState = leaseRequestStateSharedGranted + inodeLease.exclusiveHolder = nil + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequestOperation.replyChan <- LeaseResponseTypeDemoted + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } else { // leaseRequestStateExclusiveGranted != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateExclusiveDemoting: + if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C } + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + inodeLease.interruptTimer = &time.Timer{} + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + inodeLease.demotingHolder = nil + leaseRequest.requestState = leaseRequestStateSharedGranted + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequestOperation.replyChan <- LeaseResponseTypeDemoted + leaseRequestElement = inodeLease.requestedList.Front() + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + if leaseRequestStateSharedRequested == leaseRequest.requestState { + leaseRequest.requestState = leaseRequestStateSharedGranted + _ = inodeLease.requestedList.Remove(leaseRequest.listElement) + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequest.replyChan <- LeaseResponseTypeShared + leaseRequestElement = inodeLease.requestedList.Front() + } else { // leaseRequestStateSharedRequested != leaseRequest.requestState + leaseRequestElement = nil + } + } + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } else { // leaseRequestStateExclusiveDemoting == leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied } - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) - } else { // leaseRequestStateExclusiveDemoting == leaseRequest.requestState + case inodeLeaseStateExclusiveReleasing: leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + case inodeLeaseStateExclusiveExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeDemote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") + default: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeDemote, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) } - case inodeLeaseStateExclusiveReleasing: + } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] returned !ok leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - case inodeLeaseStateExclusiveExpired: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeDemote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") - default: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeDemote, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) } - } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] returned !ok - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - case LeaseRequestTypeRelease: - leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] - if ok { - switch inodeLease.leaseState { - case inodeLeaseStateNone: - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - case inodeLeaseStateSharedGrantedRecently: - if leaseRequestStateSharedGranted == leaseRequest.requestState { - _ = inodeLease.sharedHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) - leaseRequestOperation.replyChan <- LeaseResponseTypeReleased - if inodeLease.sharedHoldersList.Len() == 0 { - if !inodeLease.longAgoTimer.Stop() { - <-inodeLease.longAgoTimer.C - } - if nil == inodeLease.promotingHolder { - leaseRequestElement = inodeLease.requestedList.Front() - if nil == leaseRequestElement { - delete(inodeLease.volume.inodeLeaseMap, inodeLease.inodeNumber) - _ = globals.inodeLeaseLRU.Remove(inodeLease.lruElement) - } else { // nil != leaseRequestElement - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - _ = inodeLease.requestedList.Remove(leaseRequestElement) - if leaseRequestStateSharedRequested == leaseRequest.requestState { - if inodeLease.requestedList.Len() == 0 { + case LeaseRequestTypeRelease: + leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] + if ok { + switch inodeLease.leaseState { + case inodeLeaseStateNone: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + case inodeLeaseStateSharedGrantedRecently: + if leaseRequestStateSharedGranted == leaseRequest.requestState { + _ = inodeLease.sharedHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + if inodeLease.sharedHoldersList.Len() == 0 { + if !inodeLease.longAgoTimer.Stop() { + <-inodeLease.longAgoTimer.C + } + if nil == inodeLease.promotingHolder { + leaseRequestElement = inodeLease.requestedList.Front() + if nil == leaseRequestElement { + delete(inodeLease.volume.inodeLeaseMap, inodeLease.inodeNumber) + _ = globals.inodeLeaseLRU.Remove(inodeLease.lruElement) + } else { // nil != leaseRequestElement + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + _ = inodeLease.requestedList.Remove(leaseRequestElement) + if leaseRequestStateSharedRequested == leaseRequest.requestState { + if inodeLease.requestedList.Len() == 0 { + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + inodeLease.exclusiveHolder = leaseRequest + leaseRequest.listElement = nil + leaseRequest.requestState = leaseRequestStateExclusiveGranted + leaseRequest.replyChan <- LeaseResponseTypeExclusive + } else { // 0 < inodeLease.requestedList.Len() + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequest.requestState = leaseRequestStateSharedGranted + leaseRequest.replyChan <- LeaseResponseTypeShared + leaseRequestElement = inodeLease.requestedList.Front() + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + _ = inodeLease.requestedList.Remove(leaseRequest.listElement) + if leaseRequestStateSharedRequested == leaseRequest.requestState { + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequest.requestState = leaseRequestStateSharedGranted + leaseRequest.replyChan <- LeaseResponseTypeShared + leaseRequestElement = inodeLease.requestedList.Front() + } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState { + leaseRequestElement = nil + } + } + } + } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently inodeLease.exclusiveHolder = leaseRequest leaseRequest.listElement = nil leaseRequest.requestState = leaseRequestStateExclusiveGranted leaseRequest.replyChan <- LeaseResponseTypeExclusive - } else { // 0 < inodeLease.requestedList.Len() - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequest.requestState = leaseRequestStateSharedGranted - leaseRequest.replyChan <- LeaseResponseTypeShared - leaseRequestElement = inodeLease.requestedList.Front() - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - _ = inodeLease.requestedList.Remove(leaseRequest.listElement) - if leaseRequestStateSharedRequested == leaseRequest.requestState { - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequest.requestState = leaseRequestStateSharedGranted - leaseRequest.replyChan <- LeaseResponseTypeShared - leaseRequestElement = inodeLease.requestedList.Front() - } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState { - leaseRequestElement = nil - } - } } - } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - inodeLease.exclusiveHolder = leaseRequest - leaseRequest.listElement = nil - leaseRequest.requestState = leaseRequestStateExclusiveGranted - leaseRequest.replyChan <- LeaseResponseTypeExclusive + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) } + } else { // nil != inodeLease.promotingHolder + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + inodeLease.exclusiveHolder = inodeLease.promotingHolder + inodeLease.promotingHolder = nil + inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveGranted + inodeLease.exclusiveHolder.replyChan <- LeaseResponseTypePromoted inodeLease.lastGrantTime = time.Now() inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) } - } else { // nil != inodeLease.promotingHolder + } + } else { // leaseRequestStateSharedGranted != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateSharedGrantedLongAgo: + if leaseRequestStateSharedGranted == leaseRequest.requestState { + _ = inodeLease.sharedHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + if inodeLease.sharedHoldersList.Len() == 0 { + delete(inodeLease.volume.inodeLeaseMap, inodeLease.inodeNumber) + _ = globals.inodeLeaseLRU.Remove(inodeLease.lruElement) + } + } else { // leaseRequestStateSharedGranted != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateSharedPromoting: + if leaseRequestStateSharedReleasing == leaseRequest.requestState { + _ = inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + if inodeLease.releasingHoldersList.Len() == 0 { + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C + } + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + inodeLease.interruptTimer = &time.Timer{} inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently inodeLease.exclusiveHolder = inodeLease.promotingHolder inodeLease.promotingHolder = nil @@ -467,217 +520,178 @@ func (inodeLease *inodeLeaseStruct) handleOperation(leaseRequestOperation *lease inodeLease.lastGrantTime = time.Now() inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) } + } else { // leaseRequestStateSharedReleasing != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied } - } else { // leaseRequestStateSharedGranted != leaseRequest.requestState - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - case inodeLeaseStateSharedGrantedLongAgo: - if leaseRequestStateSharedGranted == leaseRequest.requestState { - _ = inodeLease.sharedHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) - leaseRequestOperation.replyChan <- LeaseResponseTypeReleased - if inodeLease.sharedHoldersList.Len() == 0 { + case inodeLeaseStateSharedReleasing: + if leaseRequestStateSharedReleasing == leaseRequest.requestState { + _ = inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + if inodeLease.releasingHoldersList.Len() == 0 { + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C + } + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + inodeLease.interruptTimer = &time.Timer{} + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + leaseRequestElement = inodeLease.requestedList.Front() + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + _ = inodeLease.requestedList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + inodeLease.exclusiveHolder = leaseRequest + leaseRequest.requestState = leaseRequestStateExclusiveGranted + leaseRequest.replyChan <- LeaseResponseTypeExclusive + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } + } else { // leaseRequestStateSharedReleasing != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateSharedExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeRelease, found unknown inodeLease.leaseState inodeLeaseStateSharedExpired") + case inodeLeaseStateExclusiveGrantedRecently: + if leaseRequestStateExclusiveGranted == leaseRequest.requestState { + if !inodeLease.longAgoTimer.Stop() { + <-inodeLease.longAgoTimer.C + } + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + leaseRequestElement = inodeLease.requestedList.Front() + if nil == leaseRequestElement { + delete(inodeLease.volume.inodeLeaseMap, inodeLease.inodeNumber) + _ = globals.inodeLeaseLRU.Remove(inodeLease.lruElement) + } else { // nil != leaseRequestElement + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + _ = inodeLease.requestedList.Remove(leaseRequestElement) + if leaseRequestStateSharedRequested == leaseRequest.requestState { + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.requestState = leaseRequestStateSharedGranted + leaseRequest.replyChan <- LeaseResponseTypeShared + leaseRequestElement = inodeLease.requestedList.Front() + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + if leaseRequestStateSharedRequested == leaseRequest.requestState { + _ = inodeLease.requestedList.Remove(leaseRequestElement) + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequest.requestState = leaseRequestStateSharedGranted + leaseRequest.replyChan <- LeaseResponseTypeShared + leaseRequestElement = inodeLease.requestedList.Front() + } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState + leaseRequestElement = nil + } + } + } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + leaseRequest.requestState = leaseRequestStateExclusiveGranted + inodeLease.exclusiveHolder = leaseRequest + leaseRequest.replyChan <- LeaseResponseTypeExclusive + } + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } + } else { // leaseRequestStateExclusiveGranted != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateExclusiveGrantedLongAgo: + if leaseRequestStateExclusiveGranted == leaseRequest.requestState { + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased delete(inodeLease.volume.inodeLeaseMap, inodeLease.inodeNumber) _ = globals.inodeLeaseLRU.Remove(inodeLease.lruElement) + } else { // leaseRequestStateExclusiveGranted != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied } - } else { // leaseRequestStateSharedGranted != leaseRequest.requestState - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - case inodeLeaseStateSharedPromoting: - if leaseRequestStateSharedReleasing == leaseRequest.requestState { - _ = inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) - leaseRequestOperation.replyChan <- LeaseResponseTypeReleased - if inodeLease.releasingHoldersList.Len() == 0 { - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C - } - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - inodeLease.interruptTimer = &time.Timer{} - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - inodeLease.exclusiveHolder = inodeLease.promotingHolder - inodeLease.promotingHolder = nil - inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveGranted - inodeLease.exclusiveHolder.replyChan <- LeaseResponseTypePromoted - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) - } - } else { // leaseRequestStateSharedReleasing != leaseRequest.requestState - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - case inodeLeaseStateSharedReleasing: - if leaseRequestStateSharedReleasing == leaseRequest.requestState { - _ = inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) - leaseRequestOperation.replyChan <- LeaseResponseTypeReleased - if inodeLease.releasingHoldersList.Len() == 0 { + case inodeLeaseStateExclusiveDemoting: + if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { if !inodeLease.interruptTimer.Stop() { <-inodeLease.interruptTimer.C } inodeLease.lastInterruptTime = time.Time{} inodeLease.interruptsSent = 0 inodeLease.interruptTimer = &time.Timer{} - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + inodeLease.demotingHolder = nil + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased leaseRequestElement = inodeLease.requestedList.Front() leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) _ = inodeLease.requestedList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - inodeLease.exclusiveHolder = leaseRequest - leaseRequest.requestState = leaseRequestStateExclusiveGranted - leaseRequest.replyChan <- LeaseResponseTypeExclusive - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) - } - } else { // leaseRequestStateSharedReleasing != leaseRequest.requestState - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - case inodeLeaseStateSharedExpired: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeRelease, found unknown inodeLease.leaseState inodeLeaseStateSharedExpired") - case inodeLeaseStateExclusiveGrantedRecently: - if leaseRequestStateExclusiveGranted == leaseRequest.requestState { - if !inodeLease.longAgoTimer.Stop() { - <-inodeLease.longAgoTimer.C - } - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) - leaseRequestOperation.replyChan <- LeaseResponseTypeReleased - leaseRequestElement = inodeLease.requestedList.Front() - if nil == leaseRequestElement { - delete(inodeLease.volume.inodeLeaseMap, inodeLease.inodeNumber) - _ = globals.inodeLeaseLRU.Remove(inodeLease.lruElement) - } else { // nil != leaseRequestElement - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - _ = inodeLease.requestedList.Remove(leaseRequestElement) - if leaseRequestStateSharedRequested == leaseRequest.requestState { - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + if (nil == inodeLease.requestedList.Front()) || (leaseRequestStateExclusiveRequested == inodeLease.requestedList.Front().Value.(*leaseRequestStruct).requestState) { + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + leaseRequest.requestState = leaseRequestStateExclusiveGranted + leaseRequest.listElement = nil + inodeLease.exclusiveHolder = leaseRequest + leaseRequest.replyChan <- LeaseResponseTypeExclusive + } else { inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently leaseRequest.requestState = leaseRequestStateSharedGranted + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) leaseRequest.replyChan <- LeaseResponseTypeShared leaseRequestElement = inodeLease.requestedList.Front() for nil != leaseRequestElement { leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) if leaseRequestStateSharedRequested == leaseRequest.requestState { - _ = inodeLease.requestedList.Remove(leaseRequestElement) - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + _ = inodeLease.requestedList.Remove(leaseRequest.listElement) leaseRequest.requestState = leaseRequestStateSharedGranted + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) leaseRequest.replyChan <- LeaseResponseTypeShared leaseRequestElement = inodeLease.requestedList.Front() } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState leaseRequestElement = nil } } - } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - leaseRequest.requestState = leaseRequestStateExclusiveGranted - inodeLease.exclusiveHolder = leaseRequest - leaseRequest.replyChan <- LeaseResponseTypeExclusive } inodeLease.lastGrantTime = time.Now() inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } else { // leaseRequestStateExclusiveDemoting != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied } - } else { // leaseRequestStateExclusiveGranted != leaseRequest.requestState - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - case inodeLeaseStateExclusiveGrantedLongAgo: - if leaseRequestStateExclusiveGranted == leaseRequest.requestState { - delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) - leaseRequestOperation.replyChan <- LeaseResponseTypeReleased - delete(inodeLease.volume.inodeLeaseMap, inodeLease.inodeNumber) - _ = globals.inodeLeaseLRU.Remove(inodeLease.lruElement) - } else { // leaseRequestStateExclusiveGranted != leaseRequest.requestState - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - case inodeLeaseStateExclusiveDemoting: - if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C - } - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - inodeLease.interruptTimer = &time.Timer{} - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) - inodeLease.demotingHolder = nil - leaseRequestOperation.replyChan <- LeaseResponseTypeReleased - leaseRequestElement = inodeLease.requestedList.Front() - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - _ = inodeLease.requestedList.Remove(leaseRequest.listElement) - if (nil == inodeLease.requestedList.Front()) || (leaseRequestStateExclusiveRequested == inodeLease.requestedList.Front().Value.(*leaseRequestStruct).requestState) { + case inodeLeaseStateExclusiveReleasing: + if leaseRequestStateExclusiveReleasing == leaseRequest.requestState { + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C + } + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + inodeLease.interruptTimer = &time.Timer{} inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + _ = inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + leaseRequestElement = inodeLease.requestedList.Front() + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) leaseRequest.requestState = leaseRequestStateExclusiveGranted + _ = inodeLease.requestedList.Remove(leaseRequestElement) leaseRequest.listElement = nil inodeLease.exclusiveHolder = leaseRequest leaseRequest.replyChan <- LeaseResponseTypeExclusive - } else { - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.requestState = leaseRequestStateSharedGranted - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequest.replyChan <- LeaseResponseTypeShared - leaseRequestElement = inodeLease.requestedList.Front() - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - if leaseRequestStateSharedRequested == leaseRequest.requestState { - _ = inodeLease.requestedList.Remove(leaseRequest.listElement) - leaseRequest.requestState = leaseRequestStateSharedGranted - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequest.replyChan <- LeaseResponseTypeShared - leaseRequestElement = inodeLease.requestedList.Front() - } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState - leaseRequestElement = nil - } - } - } - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) - } else { // leaseRequestStateExclusiveDemoting != leaseRequest.requestState - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - case inodeLeaseStateExclusiveReleasing: - if leaseRequestStateExclusiveReleasing == leaseRequest.requestState { - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } else { // leaseRequestStateExclusiveReleasing != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied } - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - inodeLease.interruptTimer = &time.Timer{} - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) - _ = inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - leaseRequestOperation.replyChan <- LeaseResponseTypeReleased - leaseRequestElement = inodeLease.requestedList.Front() - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - leaseRequest.requestState = leaseRequestStateExclusiveGranted - _ = inodeLease.requestedList.Remove(leaseRequestElement) - leaseRequest.listElement = nil - inodeLease.exclusiveHolder = leaseRequest - leaseRequest.replyChan <- LeaseResponseTypeExclusive - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) - } else { // leaseRequestStateExclusiveReleasing != leaseRequest.requestState - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + case inodeLeaseStateExclusiveExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeRelease, found unknown inodeLease.leaseState inodeLeaseStateExclusiveExpired") + default: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeRelease, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) } - case inodeLeaseStateExclusiveExpired: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeRelease, found unknown inodeLease.leaseState inodeLeaseStateExclusiveExpired") - default: - logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeRelease, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) + } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] returned !ok + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied } - } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] returned !ok - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + default: + logFatalf("(*inodeLeaseStruct).handleOperation() found unexpected leaseRequestOperation.LeaseRequestType: %v", leaseRequestOperation.LeaseRequestType) } - default: - logFatalf("(*inodeLeaseStruct).handleOperation() found unexpected leaseRequestOperation.LeaseRequestType: %v", leaseRequestOperation.LeaseRequestType) - } - globals.Unlock() + globals.Unlock() + } } func (inodeLease *inodeLeaseStruct) handleLongAgoTimerPop() { @@ -1124,7 +1138,6 @@ func inodeLeaseExpirer() { fmt.Printf("inodeLeaseExpirer() ran up against already stopping leases (stopCount: %v, stopMax: %v)\n", stopCount, stopMax) break } - fmt.Printf("inodeLeaseExpirer() will be stopping inodeLease: %+v\n", inodeLease) stopCount++ inodeLease.stopping = true close(inodeLease.stopChan) @@ -1141,13 +1154,14 @@ func inodeLeaseExpirer() { func (inodeLease *inodeLeaseStruct) handleStopChanClose() { var ( - err error - leaseRequest *leaseRequestStruct - leaseRequestElement *list.Element - leaseRequestOperation *leaseRequestOperationStruct - ok bool - rpcInterrupt *RPCInterrupt - rpcInterruptBuf []byte + err error + leaseRequest *leaseRequestStruct + leaseRequestElement *list.Element + leaseRequestOperation *leaseRequestOperationStruct + leaseRequestOperationElement *list.Element + ok bool + rpcInterrupt *RPCInterrupt + rpcInterruptBuf []byte ) // Deny all pending requests: @@ -1291,57 +1305,70 @@ func (inodeLease *inodeLeaseStruct) handleStopChanClose() { globals.Unlock() select { - case leaseRequestOperation = <-inodeLease.requestChan: + case _ = <-inodeLease.requestChan: globals.Lock() - switch leaseRequestOperation.LeaseRequestType { - case LeaseRequestTypeShared: - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + for inodeLease.requestList.Front() != nil { + leaseRequestOperationElement = inodeLease.requestList.Front() - case LeaseRequestTypePromote: - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + leaseRequestOperation, ok = leaseRequestOperationElement.Value.(*leaseRequestOperationStruct) + if !ok { + logFatalf("leaseRequestOperationElement.Value.(*leaseRequestOperationStruct) returned !ok") + } - case LeaseRequestTypeExclusive: - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + _ = inodeLease.requestList.Remove(leaseRequestOperationElement) - case LeaseRequestTypeDemote: - if inodeLeaseStateExclusiveDemoting == inodeLease.leaseState { - leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[leaseRequestOperation.inodeLease.inodeNumber] - if ok { - if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { - if leaseRequest == inodeLease.demotingHolder { - leaseRequestOperation.replyChan <- LeaseResponseTypeDemoted + switch leaseRequestOperation.LeaseRequestType { + case LeaseRequestTypeShared: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - inodeLease.demotingHolder = nil + case LeaseRequestTypePromote: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + + case LeaseRequestTypeExclusive: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - leaseRequest.requestState = leaseRequestStateExclusiveReleasing + case LeaseRequestTypeDemote: + if inodeLeaseStateExclusiveDemoting == inodeLease.leaseState { + leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[leaseRequestOperation.inodeLease.inodeNumber] + if ok { + if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { + if leaseRequest == inodeLease.demotingHolder { + leaseRequestOperation.replyChan <- LeaseResponseTypeDemoted - leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) + inodeLease.demotingHolder = nil - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: inodeLease.inodeNumber, - } + leaseRequest.requestState = leaseRequestStateExclusiveReleasing - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logFatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 4]", rpcInterrupt, err) - } + leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) - globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } - logTracef("<== [RPC] SendCallback(clientID: 0x%016X, rpcInterrupt: %+v)", leaseRequest.mount.retryRPCClientID, rpcInterrupt) + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 4]", rpcInterrupt, err) + } - inodeLease.leaseState = inodeLeaseStateExclusiveReleasing + globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C - } + logTracef("<== [RPC] SendCallback(clientID: 0x%016X, rpcInterrupt: %+v)", leaseRequest.mount.retryRPCClientID, rpcInterrupt) + + inodeLease.leaseState = inodeLeaseStateExclusiveReleasing - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent = 1 + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C + } + + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent = 1 - inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + } else { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } } else { leaseRequestOperation.replyChan <- LeaseResponseTypeDenied } @@ -1351,24 +1378,46 @@ func (inodeLease *inodeLeaseStruct) handleStopChanClose() { } else { leaseRequestOperation.replyChan <- LeaseResponseTypeDenied } - } else { - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - case LeaseRequestTypeRelease: - leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[leaseRequestOperation.inodeLease.inodeNumber] - if ok { - switch inodeLease.leaseState { - case inodeLeaseStateSharedReleasing: - if leaseRequestStateSharedReleasing == leaseRequest.requestState { - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) - inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil + case LeaseRequestTypeRelease: + leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[leaseRequestOperation.inodeLease.inodeNumber] + if ok { + switch inodeLease.leaseState { + case inodeLeaseStateSharedReleasing: + if leaseRequestStateSharedReleasing == leaseRequest.requestState { + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + + if inodeLease.releasingHoldersList.Len() == 0 { + inodeLease.leaseState = inodeLeaseStateNone + + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C + } + + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + + inodeLease.interruptTimer = &time.Timer{} + + delete(inodeLease.volume.inodeLeaseMap, inodeLease.inodeNumber) + _ = globals.inodeLeaseLRU.Remove(inodeLease.lruElement) + } + } else { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateExclusiveDemoting: + if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + inodeLease.demotingHolder = nil - leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased - if inodeLease.releasingHoldersList.Len() == 0 { inodeLease.leaseState = inodeLeaseStateNone if !inodeLease.interruptTimer.Stop() { @@ -1382,68 +1431,44 @@ func (inodeLease *inodeLeaseStruct) handleStopChanClose() { delete(inodeLease.volume.inodeLeaseMap, inodeLease.inodeNumber) _ = globals.inodeLeaseLRU.Remove(inodeLease.lruElement) + } else { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied } - } else { - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - case inodeLeaseStateExclusiveDemoting: - if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) - inodeLease.demotingHolder = nil - - leaseRequestOperation.replyChan <- LeaseResponseTypeReleased - - inodeLease.leaseState = inodeLeaseStateNone - - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C - } + case inodeLeaseStateExclusiveReleasing: + if leaseRequestStateExclusiveReleasing == leaseRequest.requestState { + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased - inodeLease.interruptTimer = &time.Timer{} + inodeLease.leaseState = inodeLeaseStateNone - delete(inodeLease.volume.inodeLeaseMap, inodeLease.inodeNumber) - _ = globals.inodeLeaseLRU.Remove(inodeLease.lruElement) - } else { - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - case inodeLeaseStateExclusiveReleasing: - if leaseRequestStateExclusiveReleasing == leaseRequest.requestState { - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) - inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C + } - leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 - inodeLease.leaseState = inodeLeaseStateNone + inodeLease.interruptTimer = &time.Timer{} - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C + delete(inodeLease.volume.inodeLeaseMap, inodeLease.inodeNumber) + _ = globals.inodeLeaseLRU.Remove(inodeLease.lruElement) + } else { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied } - - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - - inodeLease.interruptTimer = &time.Timer{} - - delete(inodeLease.volume.inodeLeaseMap, inodeLease.inodeNumber) - _ = globals.inodeLeaseLRU.Remove(inodeLease.lruElement) - } else { + default: leaseRequestOperation.replyChan <- LeaseResponseTypeDenied } - default: + } else { leaseRequestOperation.replyChan <- LeaseResponseTypeDenied } - } else { - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied - } - default: - logFatalf("(*inodeLeaseStruct).handleStopChanClose() read unexected leaseRequestOperationLeaseRequestType: %v", leaseRequestOperation.LeaseRequestType) + default: + logFatalf("(*inodeLeaseStruct).handleStopChanClose() read unexected leaseRequestOperationLeaseRequestType: %v", leaseRequestOperation.LeaseRequestType) + } } case <-inodeLease.interruptTimer.C: @@ -1594,8 +1619,17 @@ func (inodeLease *inodeLeaseStruct) handleStopChanClose() { for { select { - case leaseRequestOperation = <-inodeLease.requestChan: - leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + case _ = <-inodeLease.requestChan: + for inodeLease.requestList.Front() != nil { + leaseRequestOperation, ok = leaseRequestOperationElement.Value.(*leaseRequestOperationStruct) + if !ok { + logFatalf("leaseRequestOperationElement.Value.(*leaseRequestOperationStruct) returned !ok") + } + + _ = inodeLease.requestList.Remove(leaseRequestOperationElement) + + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } default: goto RequestChanDrained } @@ -1666,3 +1700,57 @@ func (leaseRequest *leaseRequestStruct) okToWrite() (ok bool) { return } + +func (leaseRequest *LeaseRequestStruct) String() (str string) { + switch leaseRequest.LeaseRequestType { + case LeaseRequestTypeShared: + str = fmt.Sprintf("&{MountID:%s InodeNumber:%v LeaseRequestType:...Shared}", leaseRequest.MountID, leaseRequest.InodeNumber) + case LeaseRequestTypePromote: + str = fmt.Sprintf("&{MountID:%s InodeNumber:%v LeaseRequestType:...Promote}", leaseRequest.MountID, leaseRequest.InodeNumber) + case LeaseRequestTypeExclusive: + str = fmt.Sprintf("&{MountID:%s InodeNumber:%v LeaseRequestType:...Exclusive}", leaseRequest.MountID, leaseRequest.InodeNumber) + case LeaseRequestTypeDemote: + str = fmt.Sprintf("&{MountID:%s InodeNumber:%v LeaseRequestType:...Demote}", leaseRequest.MountID, leaseRequest.InodeNumber) + case LeaseRequestTypeRelease: + str = fmt.Sprintf("&{MountID:%s InodeNumber:%v LeaseRequestType:...Release}", leaseRequest.MountID, leaseRequest.InodeNumber) + default: + logFatalf("(*LeaseRequestStruct).String() unable to interpret leaseRequest.LeaseRequestType [%v]", leaseRequest.LeaseRequestType) + } + + return +} + +func (leaseResponse *LeaseResponseStruct) String() (str string) { + switch leaseResponse.LeaseResponseType { + case LeaseResponseTypeDenied: + str = fmt.Sprintf("&{LeaseResponseType:...Denied}") + case LeaseResponseTypeShared: + str = fmt.Sprintf("&{LeaseResponseType:...Shared}") + case LeaseResponseTypePromoted: + str = fmt.Sprintf("&{LeaseResponseType:...Promoted}") + case LeaseResponseTypeExclusive: + str = fmt.Sprintf("&{LeaseResponseType:...Exclusive}") + case LeaseResponseTypeDemoted: + str = fmt.Sprintf("&{LeaseResponseType:...Demoted}") + case LeaseResponseTypeReleased: + str = fmt.Sprintf("&{LeaseResponseType:...Released}") + default: + logFatalf("(*LeaseResponseStruct).String() unable to interpret leaseResponse.LeaseResponseType [%v]", leaseResponse.LeaseResponseType) + } + + return +} + +func (rpcInterrupt *RPCInterrupt) String() (str string) { + switch rpcInterrupt.RPCInterruptType { + case RPCInterruptTypeUnmount: + str = fmt.Sprintf("&{RPCInterruptType:...Unmount}") + case RPCInterruptTypeDemote: + str = fmt.Sprintf("&{RPCInterruptType:...Demote InodeNumber:%v}", rpcInterrupt.InodeNumber) + case RPCInterruptTypeRelease: + str = fmt.Sprintf("&{RPCInterruptType:...Release InodeNumber:%v}", rpcInterrupt.InodeNumber) + default: + logFatalf("(*RPCInterrupt).String() unable to interpret rpcInterrupt.RPCInterruptType [%v]", rpcInterrupt.RPCInterruptType) + } + return +} diff --git a/imgr/imgrpkg/lease_test.go b/imgr/imgrpkg/lease_test.go index bc5cdf18c..a4cab6129 100644 --- a/imgr/imgrpkg/lease_test.go +++ b/imgr/imgrpkg/lease_test.go @@ -20,8 +20,6 @@ import ( const ( testRpcLeaseOverrideConfIMGRLeaseEvictHighLimit = 3 testRpcLeaseOverrideConfIMGRLeaseEvictLowLimit = 2 - testRpcLeaseDelayAfterSendingRequest = 10 * time.Millisecond - testRpcLeaseDelayBeforeSendingRequest = 10 * time.Millisecond testRpcLeaseMultiFirstInodeNumber uint64 = 1 testRpcLeaseMultiNumInstances uint64 = 5 testRpcLeaseSingleInodeNumber uint64 = 1 @@ -274,6 +272,32 @@ func TestRPCLease(t *testing.T) { testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + testRpcLeaseLogTestCase("Exlusive then Shared leading to Demotion then 2nd Exclusive leading to 2 Releases", true) + + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[1].validateChOutValueIsRPCInterruptType(RPCInterruptTypeDemote) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeDemote) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeDemoted) + testRpcLeaseClient[2].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + + testRpcLeaseClient[3].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[3].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + + testRpcLeaseClient[3].sendLeaseRequest(LeaseRequestTypePromote) + testRpcLeaseClient[1].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + testRpcLeaseClient[2].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + testRpcLeaseClient[2].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + testRpcLeaseClient[3].validateChOutValueIsLeaseResponseType(LeaseResponseTypePromoted) + + testRpcLeaseClient[3].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[3].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + testRpcLeaseLogTestCase(fmt.Sprintf("%v Shared", testRpcLeaseSingleNumInstances-1), false) for instance = 1; instance < testRpcLeaseSingleNumInstances; instance++ { @@ -332,28 +356,27 @@ func TestRPCLease(t *testing.T) { testRpcLeaseLogTestCase(fmt.Sprintf("%v Unique InodeNumber Exclusives", testRpcLeaseMultiNumInstances), true) testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeExclusive) testRpcLeaseClient[2].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) testRpcLeaseClient[3].sendLeaseRequest(LeaseRequestTypeExclusive) testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseResponseTypeIgnoringRPCInterruptType(LeaseResponseTypeReleased, RPCInterruptTypeRelease) + + testRpcLeaseClient[1].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) testRpcLeaseClient[1].validateChOutValueIsLeaseResponseTypeIgnoringRPCInterruptType(LeaseResponseTypeReleased, RPCInterruptTypeRelease) testRpcLeaseClient[3].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) testRpcLeaseClient[4].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[4].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) @@ -502,9 +525,7 @@ func (testRpcLeaseClient *testRpcLeaseClientStruct) Fatalf(format string, args . } func (testRpcLeaseClient *testRpcLeaseClientStruct) sendLeaseRequest(leaseRequestType LeaseRequestType) { - time.Sleep(testRpcLeaseDelayBeforeSendingRequest) testRpcLeaseClient.chIn <- leaseRequestType - time.Sleep(testRpcLeaseDelayAfterSendingRequest) } func (testRpcLeaseClient *testRpcLeaseClientStruct) sendLeaseRequestPromptly(leaseRequestType LeaseRequestType) { diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index c0508f7e4..c706ec05b 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -902,7 +902,8 @@ func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) volume: volume, inodeNumber: leaseRequest.InodeNumber, leaseState: inodeLeaseStateNone, - requestChan: make(chan *leaseRequestOperationStruct), + requestChan: make(chan struct{}, 1), + requestList: list.New(), stopChan: make(chan struct{}), stopping: false, sharedHoldersList: list.New(), @@ -950,7 +951,14 @@ func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) replyChan: make(chan LeaseResponseType), } - inodeLease.requestChan <- leaseRequestOperation + _ = inodeLease.requestList.PushBack(leaseRequestOperation) + + select { + case inodeLease.requestChan <- struct{}{}: + // handler() needed to be signaled to service inodeLease.requestList + default: + // handler() has already been signaled to service inodeLease.requestList + } globals.Unlock() diff --git a/imgr/imgrpkg/utils_test.go b/imgr/imgrpkg/utils_test.go index 7516f7bcc..173f1d2dd 100644 --- a/imgr/imgrpkg/utils_test.go +++ b/imgr/imgrpkg/utils_test.go @@ -21,7 +21,7 @@ import ( ) const ( - testIPAddr = "127.0.0.1" // Don't use IPv6... the code doesn't properly "join" this with :port #s + testIPAddr = "127.0.0.1" testRetryRPCPort = 32356 testHTTPServerPort = 15346 testSwiftProxyTCPPort = 8080 @@ -31,6 +31,7 @@ const ( testVolume = "testVolume" testRPCDeadlineIO = "60s" testRPCKeepAlivePeriod = "60s" + testStartupDelay = 100 * time.Millisecond ) type testGlobalsStruct struct { @@ -76,8 +77,8 @@ func testSetup(t *testing.T, overrideConfStrings []string, retryrpcCallbacks int caKeyFile: tempDir + "/caKeyFile", endpointCertFile: tempDir + "/endpoingCertFile", endpointKeyFile: tempDir + "/endpointKeyFile", - httpServerURL: fmt.Sprintf("http://%s:%d", testIPAddr, testHTTPServerPort), - authURL: fmt.Sprintf("http://%s:%d/auth/v1.0", testIPAddr, testSwiftProxyTCPPort), + httpServerURL: "http://" + net.JoinHostPort(testIPAddr, fmt.Sprintf("%d", testHTTPServerPort)), + authURL: "http://" + net.JoinHostPort(testIPAddr, fmt.Sprintf("%d", testSwiftProxyTCPPort)) + "/auth/v1.0", } testGlobals.caCertPEMBlock, testGlobals.caKeyPEMBlock, err = icertpkg.GenCACert( @@ -194,9 +195,12 @@ func testSetup(t *testing.T, overrideConfStrings []string, retryrpcCallbacks int t.Fatalf("iswiftpkg.Start(testGlobals.confMap) failed: %v", err) } - err = testDoAuth() - if nil != err { - t.Fatalf("testDoAuth() failed: %v", err) + for { + err = testDoAuth() + if nil == err { + break + } + time.Sleep(testStartupDelay) } testGlobals.containerURL = testGlobals.accountURL + "/" + testContainer @@ -224,6 +228,14 @@ func testSetup(t *testing.T, overrideConfStrings []string, retryrpcCallbacks int t.Fatalf("Start(testGlobals.confMap) failed: %v", err) } + for { + _, _, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/version", nil, nil, http.StatusOK) + if nil == err { + break + } + time.Sleep(testStartupDelay) + } + retryrpcDeadlineIO, err = time.ParseDuration(testRPCDeadlineIO) if nil != err { t.Fatalf("time.ParseDuration(\"%s\") failed: %v", testRPCDeadlineIO, err) diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index f8492fae0..2ce0ed27c 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -2082,10 +2082,12 @@ func (mount *mountStruct) performUnmount() { replyChan: make(chan LeaseResponseType, 1), } + _ = leaseRequestOperation.inodeLease.requestList.PushBack(leaseRequestOperation) + leaseReleaseFinishedWG.Add(1) go func(leaseRequestOperation *leaseRequestOperationStruct) { - leaseRequestOperation.inodeLease.handleOperation(leaseRequestOperation) + leaseRequestOperation.inodeLease.handleRequestList() <-leaseRequestOperation.replyChan leaseReleaseFinishedWG.Done() }(leaseRequestOperation) diff --git a/imgr/main.go b/imgr/main.go index e5301d889..6c5fe4e5d 100644 --- a/imgr/main.go +++ b/imgr/main.go @@ -6,7 +6,6 @@ // The program requires a single argument that is a path to a package config // formatted configuration to load. Optionally, overrides the the config may // be passed as additional arguments in the form .=. -// package main import ( diff --git a/iswift/iswiftpkg/api.go b/iswift/iswiftpkg/api.go index 2966de23f..0e340cc96 100644 --- a/iswift/iswiftpkg/api.go +++ b/iswift/iswiftpkg/api.go @@ -9,16 +9,15 @@ // To configure an iswiftpkg instance, Start() is called passing, as the sole // argument, a package conf ConfMap. Here is a sample .conf file: // -// [ISWIFT] -// SwiftProxyIPAddr: 127.0.0.1 -// SwiftProxyTCPPort: 8080 -// -// MaxAccountNameLength: 256 -// MaxContainerNameLength: 256 -// MaxObjectNameLength: 1024 -// AccountListingLimit: 10000 -// ContainerListingLimit: 10000 +// [ISWIFT] +// SwiftProxyIPAddr: 127.0.0.1 +// SwiftProxyTCPPort: 8080 // +// MaxAccountNameLength: 256 +// MaxContainerNameLength: 256 +// MaxObjectNameLength: 1024 +// AccountListingLimit: 10000 +// ContainerListingLimit: 10000 package iswiftpkg import ( @@ -27,14 +26,12 @@ import ( // Start is called to start serving the NoAuth Swift Proxy Port and, // optionally, the Auth Swift Proxy Port. -// func Start(confMap conf.ConfMap) (err error) { err = start(confMap) return } // Stop is called to stop serving. -// func Stop() (err error) { err = stop() return @@ -42,7 +39,6 @@ func Stop() (err error) { // ForceReAuth is called to force a "401 Unauthorized" response to a // client's subsequent request forcing the client to reauthenticate. -// func ForceReAuth() { forceReAuth() } diff --git a/iswift/main.go b/iswift/main.go index 61aba7315..acfb0a528 100644 --- a/iswift/main.go +++ b/iswift/main.go @@ -6,7 +6,6 @@ // The program requires a single argument that is a path to a package config // formatted configuration to load. Optionally, overrides the the config may // be passed as additional arguments in the form .=. -// package main import ( diff --git a/release_notes.md b/release_notes.md index 69d235dd1..1ddb023ae 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,5 +1,15 @@ # ProxyFS Release Notes +## 2.03.0 (December 6, 2022) + +### Bug Fixes: + +Resolved a lease deadlock scenario. As part of this fix, dismounts or shutdowns of iclients now triggers implicit releasing of all of their held leases. + +### Notes: + +There is now a /keepalive URL API that, if configured to automatically exit, will prevent that. This is a feature implemented to enable an overseer to have confidence that if the overseer cannot communiate with an imgr that the imgr will dismount and stop serving any volumes. This allows for the overseer to relocate the serving of the volume(s) previously served by the unreachable imgr to a different imgr without risking the unsupported case where more than one imgr thinks it should be serving the same volume. + ## 2.02.0 (May 6, 2022) ### Notes: diff --git a/retryrpc/client.go b/retryrpc/client.go index 664a5b5c0..3cbed0427 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -45,14 +45,13 @@ type clientSideStatsInfo struct { // This is outside of our initial requirements but something we should // review. -// // Send algorithm is: -// 1. Build ctx including channel for reply struct -// 2. Call goroutine to do marshalling and sending of -// request to server -// 3. Wait on channel in reply struct for result -// 4. readResponses goroutine will read response on socket -// and call a goroutine to do unmarshalling and notification +// 1. Build ctx including channel for reply struct +// 2. Call goroutine to do marshalling and sending of +// request to server +// 3. Wait on channel in reply struct for result +// 4. readResponses goroutine will read response on socket +// and call a goroutine to do unmarshalling and notification func (client *Client) send(method string, rpcRequest interface{}, rpcReply interface{}) (err error) { var ( connectionRetryCount int @@ -574,7 +573,7 @@ func (client *Client) reDial() (err error) { // readClientID reads unique client ID response from server // -// Client lock is held +// # Client lock is held // // NOTE: Client lock is held func (client *Client) readClientID(callingGenNum uint64) (myUniqueID uint64, err error) { diff --git a/retryrpc/server.go b/retryrpc/server.go index af52b8b91..4047f3705 100644 --- a/retryrpc/server.go +++ b/retryrpc/server.go @@ -187,7 +187,6 @@ func (server *Server) processRequest(ci *clientInfo, myConnCtx *connCtx, buf []b // TODO - update this comment for initialDial() vs reDial() cases!!! // should we rename function???? -// // getClientIDAndWait reads the first message off the new connection. // // If the client is new, it will ask for a UniqueID. This routine will @@ -202,12 +201,12 @@ func (server *Server) processRequest(ci *clientInfo, myConnCtx *connCtx, buf []b // This avoids race conditions when there are cascading retransmits. // The algorithm is: // -// 1. Client sends UniqueID to server when the connection is reestablished. -// 2. After accepting new socket, the server waits for the UniqueID from -// the client. -// 3. If this is a client returning on a new socket, the server blocks -// until all outstanding RPCs and related goroutines have completed for the -// client on the previous connection. +// 1. Client sends UniqueID to server when the connection is reestablished. +// 2. After accepting new socket, the server waits for the UniqueID from +// the client. +// 3. If this is a client returning on a new socket, the server blocks +// until all outstanding RPCs and related goroutines have completed for the +// client on the previous connection. func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err error) { buf, msgType, getErr := getIO(uint64(0), server.deadlineIO, cCtx.conn) if getErr != nil { diff --git a/utils/api.go b/utils/api.go index 45b3b6d11..3262f9e5a 100644 --- a/utils/api.go +++ b/utils/api.go @@ -169,7 +169,6 @@ func PathToAcctContObj(path string) (accountName string, containerName string, o } // Return the stack track of the caller, including this function. -// func MyStackTrace() (stackTrace string) { stackTraceBuf := make([]byte, 16384, 16384) @@ -183,7 +182,6 @@ func MyStackTrace() (stackTrace string) { // // This function really should have some error checking to insure it matched // "goroutine" though there's no way to log a failure. -// func StackTraceToGoId(buf []byte) uint64 { buf = bytes.TrimPrefix(buf, []byte("goroutine ")) buf = buf[:bytes.IndexByte(buf, ' ')] @@ -211,7 +209,6 @@ func StackTraceToGoId(buf []byte) uint64 { // /vagrant/guest_workspaces/swift-runway-001/ProxyFS/src/github.com/NVIDIA/proxyfs/stacktrace.go:9 // created by main.main // /vagrant/guest_workspaces/swift-runway-001/ProxyFS/src/github.com/NVIDIA/proxyfs/stacktrace.go:16 +0x77 -// func StackTracesToMap(buf []byte) (traceMap map[uint64]string, stateMap map[uint64]string) { var ( @@ -239,7 +236,6 @@ func StackTracesToMap(buf []byte) (traceMap map[uint64]string, stateMap map[uint // like locking. // // Intent is to have this now and hopefully remove it once we've gotten debugged. -// func GetGID() uint64 { b := make([]byte, 64) _ = runtime.Stack(b, false) @@ -264,7 +260,6 @@ func GetAFnName(level int) string { // Return separage strings containing calling function and package // // XXX TODO TEMPORARY: also return goroutine id -// func GetFuncPackage(level int) (fn string, pkg string, gid uint64) { // Get the combined function and package names of our caller funcPkg := GetAFnName(level + 1)