diff --git a/arches.json b/arches.json new file mode 100644 index 0000000000..7961f97bc6 --- /dev/null +++ b/arches.json @@ -0,0 +1 @@ +["amd64","arm64","386","arm","ppc64","riscv64","wasm","loong64","mips","mips64","mips64le","mipsle","ppc64le","s390x"] diff --git a/cmd/gen/gen.go b/cmd/gen/gen.go new file mode 100644 index 0000000000..78fca74d18 --- /dev/null +++ b/cmd/gen/gen.go @@ -0,0 +1,100 @@ +// Package main cmd/gen/gen.go +package main + +import ( + "encoding/json" + "log" + "regexp" + "runtime" + + "github.com/bitfield/script" + cc "github.com/ivanpirog/coloredcobra" + "github.com/spf13/cobra" +) + +var ( + printJSON bool + writeFile string +) + +func init() { + rootCmd.Flags().BoolVarP(&printJSON, "json", "j", false, "output json format") + rootCmd.Flags().StringVarP(&writeFile, "output", "o", "", "write to the specified file (i.e. arches.json)") +} + +// there is no other better way to get a list of architectures via a library or any simpler method at runtime +// this file is executed by go generate from the root directory of the repository github.com/skycoin/skywire +// go run cmd/gen/gen.go -j -o arches.json + +var rootCmd = &cobra.Command{ + Use: "gen", + Short: "print architectures", + Long: "print architectures", + Run: func(_ *cobra.Command, _ []string) { + if writeFile == "" { + switch runtime.GOOS { + case "windows": + writeFile = `\\.\NUL` + default: // For macOS and Linux + writeFile = "/dev/null" + } + } + if !printJSON { + //equivalent bash one-liner: + //go tool dist list | awk -F '/' '{print $NF}' | awk '{$1=$1};1' | sort | uniq + _, err := script.Exec(`go tool dist list`).ReplaceRegexp(regexp.MustCompile(".*/"), "").Freq().ReplaceRegexp(regexp.MustCompile(`^\s*\d+\s+`), "").Tee().WriteFile(writeFile) + if err != nil { + log.Fatal(err) + } + } else { + rawOutput, err := script.Exec(`go tool dist list`).ReplaceRegexp(regexp.MustCompile(".*/"), "").Freq().ReplaceRegexp(regexp.MustCompile(`^\s*\d+\s+`), "").Slice() + if err != nil { + log.Fatal(err) + } + + jsonData, err := json.Marshal(rawOutput) + if err != nil { + log.Fatal(err) + } + //equivalent bash one-liner: + //go tool dist list | awk -F '/' '{print $NF}' | awk '{$1=$1};1' | sort | uniq | jq -R -s -c 'split("\n") | map(select(length > 0))' | tee arches.json + _, err = script.Echo(string(jsonData) + "\n").Tee().WriteFile(writeFile) + if err != nil { + log.Fatal(err) + } + } + }, +} + +func init() { + var helpflag bool + rootCmd.SetUsageTemplate(help) + rootCmd.PersistentFlags().BoolVarP(&helpflag, "help", "h", false, "help menu") + rootCmd.SetHelpCommand(&cobra.Command{Hidden: true}) + rootCmd.PersistentFlags().MarkHidden("help") //nolint +} + +func main() { + cc.Init(&cc.Config{ + RootCmd: rootCmd, + Headings: cc.HiBlue + cc.Bold, + Commands: cc.HiBlue + cc.Bold, + CmdShortDescr: cc.HiBlue, + Example: cc.HiBlue + cc.Italic, + ExecName: cc.HiBlue + cc.Bold, + Flags: cc.HiBlue + cc.Bold, + FlagsDescr: cc.HiBlue, + NoExtraNewlines: true, + NoBottomNewline: true, + }) + rootCmd.Execute() //nolint +} + +const help = "{{if .HasAvailableSubCommands}}{{end}} {{if gt (len .Aliases) 0}}\r\n\r\n" + + "{{.NameAndAliases}}{{end}}{{if .HasAvailableSubCommands}}" + + "Available Commands:{{range .Commands}} {{if and (ne .Name \"completion\") .IsAvailableCommand}}\r\n " + + "{{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}}\r\n\r\n" + + "Flags:\r\n" + + "{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}}\r\n\r\n" + + "Global Flags:\r\n" + + "{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}\r\n\r\n" diff --git a/cmd/skywire-cli/commands/log/root.go b/cmd/skywire-cli/commands/log/root.go index ef5f37d68f..b19c4e85c0 100644 --- a/cmd/skywire-cli/commands/log/root.go +++ b/cmd/skywire-cli/commands/log/root.go @@ -25,11 +25,9 @@ var ( fetchFrom string writeDir string lcDir string - tpsnDir string hideErr bool showUT bool pubKeys []cipher.PubKey - proxyCSV string ) // RootCmd is logCmd diff --git a/cmd/skywire-cli/commands/log/st.go b/cmd/skywire-cli/commands/log/st.go index 1b086affd7..b75775f683 100644 --- a/cmd/skywire-cli/commands/log/st.go +++ b/cmd/skywire-cli/commands/log/st.go @@ -20,8 +20,6 @@ func init() { RootCmd.AddCommand(stCmd) stCmd.Flags().StringVarP(&pubKey, "pk", "p", "", "public key(s) to check ; comma separated") stCmd.Flags().StringVarP(&lcDir, "lcdir", "d", "", "path to surveys & transport bandwidth logging ") - stCmd.Flags().StringVarP(&tpsnDir, "tpsndir", "e", "", "path to transport setup-node surveys") - stCmd.Flags().StringVarP(&proxyCSV, "proxycsv", "x", "", "path to proxy test csv") stCmd.Flags().BoolVarP(&hideErr, "noerr", "r", false, "hide error logging from output") stCmd.Flags().BoolVarP(&showUT, "ut", "u", false, "show uptime percentage for the past two days and current online status") } @@ -50,19 +48,14 @@ func makeTree() { utData, utDataErr := script.File("/tmp/ut.json").String() var tree pterm.TreeNode rootDir := lcDir - otherDir := tpsnDir dn, err := script.ListFiles(rootDir).String() if err != nil && !hideErr { errstring := "script.ListFiles(" + rootDir + ").String()" log.Printf("%v error: %v\n", errstring, err) } - dn1, err := script.ListFiles(otherDir).String() + dirNodes, err := script.Echo(dn).Basename().Freq().Column(2).Slice() if err != nil && !hideErr { - log.Printf("script.ListFiles("+otherDir+").String() error: %v\n", err) - } - dirNodes, err := script.Echo(dn + dn1).Basename().Freq().Column(2).Slice() - if err != nil && !hideErr { - log.Printf("script.Echo(dn + dn1).Basename().Freq().Column(2).Slice() error: %v\n", err) + log.Printf("script.Echo(dn).Basename().Freq().Column(2).Slice() error: %v\n", err) } if len(pubKeys) > 0 { var checkTheseKeys string @@ -82,19 +75,11 @@ func makeTree() { errstring := fmt.Sprintf("script.ListFiles(\"%v\"/\"%v\").String()", rootDir, dirNode) log.Printf("%v error: %v\n", errstring, err) } - children1, err := script.ListFiles(otherDir + "/" + dirNode).String() - if err != nil && !hideErr { - errstring := fmt.Sprintf("script.ListFiles(\"%v\"/\"%v\").String()", otherDir, dirNode) - log.Printf("%v error: %v\n", errstring, err) - } var ks string if showUT { - ks = fmt.Sprintf("uptime\n%s\n%s", children, children1) + ks = fmt.Sprintf("uptime\n%s", children) } else { - ks = fmt.Sprintf("%s\n%s", children, children1) - } - if proxyCSV != "" { - ks = fmt.Sprintf("%s\nproxy", ks) + ks = children } kids, err := script.Echo(ks).Slice() if err != nil && !hideErr { @@ -103,17 +88,6 @@ func makeTree() { nodes := []pterm.TreeNode{} for _, kid := range kids { var coloredFile string - if kid == "proxy" { - testtime, _ := script.File(proxyCSV).Match(dirNode).Replace(",", " ").Column(2).String() //nolint - testres, _ := script.File(proxyCSV).Match(dirNode).Replace(",", " ").Column(3).String() //nolint - if testtime != "" && testres != "" { - coloredFile = fmt.Sprintf("%s %s %s", pterm.Green("proxy"), strings.ReplaceAll(testtime, "\n", ""), strings.ReplaceAll(testres, "\n", "")) - } else { - coloredFile = fmt.Sprintf("%s No data", pterm.Red("proxy")) - } - nodes = append(nodes, pterm.TreeNode{Text: coloredFile}) - continue - } if kid == "uptime" { if utFileInfoErr != nil || utDataErr != nil { continue @@ -134,7 +108,7 @@ func makeTree() { nodes = append(nodes, pterm.TreeNode{Text: fmt.Sprintf("%s Age: %s %s", coloredFile, time.Since(utFileInfo.ModTime()).Truncate(time.Second).String(), strings.TrimSuffix(pkUt, "\n"))}) continue } - if filepath.Base(kid) == "health.json" || filepath.Base(kid) == "tp.json" { + if filepath.Base(kid) == "health.json" { fileContents, _ := script.File(kid).String() //nolint fileInfo, _ := os.Stat(kid) //nolint if time.Since(fileInfo.ModTime()) < time.Hour { @@ -145,14 +119,6 @@ func makeTree() { if filepath.Base(kid) == "health.json" { nodes = append(nodes, pterm.TreeNode{Text: fmt.Sprintf("%s Age: %s %s", coloredFile, time.Since(fileInfo.ModTime()).Truncate(time.Second).String(), strings.TrimSuffix(string(fileContents), "\n"))}) } - if filepath.Base(kid) == "tp.json" { - _, err := script.Echo(strings.TrimSuffix(string(fileContents), "\n")).JQ(".[]").String() //nolint - if err != nil { - fileContents = pterm.Red(strings.TrimSuffix(string(fileContents), "\n")) - coloredFile = pterm.Red(filepath.Base(kid)) - } - nodes = append(nodes, pterm.TreeNode{Text: fmt.Sprintf("%s Age: %s %s", coloredFile, time.Since(fileInfo.ModTime()).Truncate(time.Second).String(), strings.TrimSuffix(string(fileContents), "\n"))}) - } continue } if filepath.Base(kid) == "node-info.json" { diff --git a/cmd/skywire-cli/commands/reward/root.go b/cmd/skywire-cli/commands/reward/root.go index 388fb167b3..e14662c0fc 100644 --- a/cmd/skywire-cli/commands/reward/root.go +++ b/cmd/skywire-cli/commands/reward/root.go @@ -60,10 +60,13 @@ const longtext = ` reward address setting Sets the skycoin reward address for the visor. - The config is written to the root of the default local directory + The address is written to the root of the default 'local' directory specified in the visor's config - this config is served via dmsghttp along with transport logs - and the system hardware survey for automating reward distribution` + This file is parsed by the visor at runtime, and the skycoin address is included in the survey which is served via dmsghttp along with transport logs + and the system hardware survey for automating reward distribution + + By setting a skycoin reward address, you consent to collection of the survey from the visors where the reward address is set. + The survey is ONLY for verification of reward eligibility. We respect your privacy.` func longText() string { //show configured reward address if valid configuration exists diff --git a/cmd/skywire-cli/commands/reward/rules.go b/cmd/skywire-cli/commands/reward/rules.go new file mode 100644 index 0000000000..9954e27bfd --- /dev/null +++ b/cmd/skywire-cli/commands/reward/rules.go @@ -0,0 +1,31 @@ +// Package clireward cmd/skywire-cli/reward/rules.go +package clireward + +import ( + "fmt" + "os" + + markdown "github.com/MichaelMure/go-term-markdown" + "github.com/spf13/cobra" + "golang.org/x/term" + + "github.com/skycoin/skywire" +) + +func init() { + rewardCmd.AddCommand(rulesCmd) +} + +var rulesCmd = &cobra.Command{ + Use: "rules", + Short: "display the mainnet rules", + Long: "display the mainnet rules", + Run: func(_ *cobra.Command, _ []string) { + terminalWidth, _, err := term.GetSize(int(os.Stdout.Fd())) + if err != nil { + terminalWidth = 80 + } + leftPad := 6 + fmt.Printf("%s\n", markdown.Render(skywire.MainnetRules, terminalWidth, leftPad)) + }, +} diff --git a/cmd/skywire-cli/commands/rewards/calc.go b/cmd/skywire-cli/commands/rewards/calc.go index 222d8f4f29..5ff298051c 100644 --- a/cmd/skywire-cli/commands/rewards/calc.go +++ b/cmd/skywire-cli/commands/rewards/calc.go @@ -13,15 +13,17 @@ import ( "github.com/bitfield/script" "github.com/fatih/color" + coincipher "github.com/skycoin/skycoin/src/cipher" "github.com/spf13/cobra" "github.com/tidwall/pretty" + "github.com/skycoin/skywire" "github.com/skycoin/skywire-utilities/pkg/cipher" "github.com/skycoin/skywire-utilities/pkg/logging" tgbot "github.com/skycoin/skywire/cmd/skywire-cli/commands/rewards/tgbot" ) -const yearlyTotalRewards int = 408000 +const yearlyTotalRewardsPerPool int = 408000 var ( yearlyTotal int @@ -29,14 +31,16 @@ var ( wdate = time.Now().AddDate(0, 0, -1).Format("2006-01-02") wDate time.Time utfile string - disallowArchitectures string + disallowArchitectures []string + allowArchitectures1 []string + allowArchitectures2 []string h0 bool h1 bool h2 bool grr bool pubkey string logLvl string - log *logging.Logger + log = logging.MustGetLogger("rewards") sConfig string dConfig string nodeInfoSvc []byte @@ -74,8 +78,39 @@ func init() { RootCmd.Flags().StringVarP(&logLvl, "loglvl", "s", "info", "[ debug | warn | error | fatal | panic | trace ] \u001b[0m*") RootCmd.Flags().StringVarP(&wdate, "date", "d", wdate, "date for which to calculate reward") RootCmd.Flags().StringVarP(&pubkey, "pk", "k", pubkey, "check reward for pubkey") - RootCmd.Flags().StringVarP(&disallowArchitectures, "noarch", "n", "amd64,null", "disallowed architectures, comma separated") - RootCmd.Flags().IntVarP(&yearlyTotal, "year", "y", yearlyTotalRewards, "yearly total rewards") + RootCmd.Flags().StringSliceVarP(&disallowArchitectures, "noarch", "n", []string{"null", "wasm"}, "disallowed architectures, comma separated") + RootCmd.Flags().StringSliceVarP(&allowArchitectures1, "a1", "w", func(all []string, dis []string) (res []string) { + for _, v := range all { + allow := true + for _, d := range dis { + if v == d { + allow = false + break + } + } + if allow { + res = append(res, v) + } + } + return res + }(skywire.Architectures, []string{"wasm", "amd64", "386"}), "pool 1 allowed arch, comma separated") + + RootCmd.Flags().StringSliceVarP(&allowArchitectures2, "a2", "x", func(all []string, dis []string) (res []string) { + for _, v := range all { + allow := true + for _, d := range dis { + if v == d { + allow = false + break + } + } + if allow { + res = append(res, v) + } + } + return res + }(skywire.Architectures, []string{"wasm", "arm64", "arm", "ppc64", "riscv64", "loong64", "mips", "mips64", "mips64le", "mipsle", "ppc64le", "s390x"}), "pool 2 allowed arch, comma separated") + RootCmd.Flags().IntVarP(&yearlyTotal, "year", "y", yearlyTotalRewardsPerPool, "yearly total rewards per pool") RootCmd.Flags().StringVarP(&utfile, "utfile", "u", "ut.txt", "uptime tracker data file") RootCmd.Flags().StringVarP(&hwSurveyPath, "lpath", "p", "log_collecting", "path to the surveys") RootCmd.Flags().StringVarP(&sConfig, "svcconf", "f", "/opt/skywire/services-config.json", "path to the services-config.json") @@ -92,7 +127,12 @@ var RootCmd = &cobra.Command{ Short: "calculate rewards from uptime data & collected surveys", Long: ` Collect surveys: skywire-cli log -Fetch uptimes: skywire-cli ut > ut.txt`, +Fetch uptimes: skywire-cli ut > ut.txt + +Architectures: +` + fmt.Sprintf("%v", append(skywire.Architectures, "null", "all")) + ` + +`, Run: func(_ *cobra.Command, _ []string) { var err error if log == nil { @@ -130,12 +170,54 @@ Fetch uptimes: skywire-cli ut > ut.txt`, log.Fatal("uptime tracker data file not found\n", err, "\nfetch the uptime tracker data with:\n$ skywire-cli ut > ut.txt") } - archMap := make(map[string]struct{}) - for _, disallowedarch := range strings.Split(disallowArchitectures, ",") { - if disallowedarch != "" { - archMap[disallowedarch] = struct{}{} + // Create a map for disallowed architectures + disallowedMap := make(map[string]struct{}) + for _, disallowedArch := range disallowArchitectures { + disallowedMap[disallowedArch] = struct{}{} + } + + // Create maps for allowed architectures for pool 1 and pool 2 + allowArchMap1 := make(map[string]struct{}) + allowArchMap2 := make(map[string]struct{}) + + // Create a map for quick lookup of skywire architectures + supportedArchitecturesMap := make(map[string]struct{}) + for _, arch := range skywire.Architectures { + supportedArchitecturesMap[arch] = struct{}{} + } + + // Populate allowed architecture maps for pool 1 and pool 2, excluding disallowed ones + for _, arch := range allowArchitectures1 { + if _, isDisallowed := disallowedMap[arch]; !isDisallowed { + allowArchMap1[arch] = struct{}{} + } + } + for _, arch := range allowArchitectures2 { + if _, isDisallowed := disallowedMap[arch]; !isDisallowed { + allowArchMap2[arch] = struct{}{} } } + + // Check for common architectures between the two allowed slices + for arch := range allowArchMap1 { + if _, exists := allowArchMap2[arch]; exists { + log.Fatal("Error: Architecture cannot be specified in both pools: " + arch) + } + } + + // Validate each allowed architecture against the supported architectures + for arch := range allowArchMap1 { + if _, isValid := supportedArchitecturesMap[arch]; !isValid { + log.Fatal("Error: Architecture is not valid: ", arch) + } + } + + for arch := range allowArchMap2 { + if _, isValid := supportedArchitecturesMap[arch]; !isValid { + log.Fatal("Error: Architecture is not valid: ", arch) + } + } + var res []string if pubkey == "" { res, _ = script.File(utfile).Match(strings.TrimRight(wdate, "\n")).Column(1).Slice() //nolint @@ -148,7 +230,8 @@ Fetch uptimes: skywire-cli ut > ut.txt`, log.Fatal("Specified key " + pubkey + "\n did not achieve minimum uptime on " + wdate + " !") } } - var nodesInfos []nodeinfo + var nodesInfos1 []nodeinfo + var nodesInfos2 []nodeinfo var grrInfos []nodeinfo for _, pk := range res { nodeInfoDotJSON := fmt.Sprintf("%s/%s/node-info.json", hwSurveyPath, pk) @@ -190,10 +273,6 @@ Fetch uptimes: skywire-cli ut > ut.txt`, ip, _ = script.File(nodeInfoDotJSON).JQ(`.ip_address`).Replace(" ", "").Replace(`"`, "").String() //nolint ip = strings.TrimRight(ip, "\n") - if strings.Count(ip, ".") != 3 { - ip, _ = script.File(nodeInfoDotJSON).JQ(`."ip.skycoin.com".ip_address`).Replace(" ", "").Replace(`"`, "").String() //nolint - ip = strings.TrimRight(ip, "\n") - } sky, _ = script.File(nodeInfoDotJSON).JQ(".skycoin_address").Replace(" ", "").Replace(`"`, "").String() //nolint sky = strings.TrimRight(sky, "\n") arch, _ = script.File(nodeInfoDotJSON).JQ(`.go_arch`).Replace(" ", "").Replace(`"`, "").String() //nolint @@ -225,8 +304,17 @@ Fetch uptimes: skywire-cli ut > ut.txt`, SvcConf: svcconf, } //enforce all requirements for rewards - if _, disallowed := archMap[arch]; !disallowed && ip != "" && strings.Count(ip, ".") == 3 && sky != "" && uu != "" && ifc != "" && len(macs) > 0 && macs[0] != "" { - nodesInfos = append(nodesInfos, ni) + _, allowed1 := allowArchMap1[arch] + _, allowed2 := allowArchMap2[arch] + _, err := coincipher.DecodeBase58Address(sky) + + if (allowed1 || allowed2) && strings.Count(ip, ".") == 3 && uu != "" && ifc != "" && len(macs) > 0 && macs[0] != "" && err == nil { + if allowed1 { + nodesInfos1 = append(nodesInfos1, ni) + } + if allowed2 { + nodesInfos2 = append(nodesInfos2, ni) + } } else { if grr { grrInfos = append(grrInfos, ni) @@ -249,11 +337,14 @@ Fetch uptimes: skywire-cli ut > ut.txt`, fmt.Printf("days this month: %d\n", daysThisMonth) fmt.Printf("days in the year: %d\n", daysThisYear) fmt.Printf("this month's rewards: %.6f\n", monthReward) - fmt.Printf("reward total: %.6f\n", dayReward) + fmt.Printf("reward total per pool: %.6f\n", dayReward) } uniqueIP, _ := script.Echo(func() string { //nolint var inputStr strings.Builder - for _, ni := range nodesInfos { + for _, ni := range nodesInfos1 { + inputStr.WriteString(fmt.Sprintf("%s\n", ni.IPAddr)) + } + for _, ni := range nodesInfos2 { inputStr.WriteString(fmt.Sprintf("%s\n", ni.IPAddr)) } return inputStr.String() @@ -273,7 +364,10 @@ Fetch uptimes: skywire-cli ut > ut.txt`, } uniqueUUID, _ := script.Echo(func() string { //nolint var inputStr strings.Builder - for _, ni := range nodesInfos { + for _, ni := range nodesInfos1 { + inputStr.WriteString(fmt.Sprintf("%s\n", ni.UUID)) + } + for _, ni := range nodesInfos2 { inputStr.WriteString(fmt.Sprintf("%s\n", ni.UUID)) } return inputStr.String() @@ -282,7 +376,10 @@ Fetch uptimes: skywire-cli ut > ut.txt`, // look at the first non loopback interface macaddress uniqueMac, _ := script.Echo(func() string { //nolint var inputStr strings.Builder - for _, ni := range nodesInfos { + for _, ni := range nodesInfos1 { + inputStr.WriteString(fmt.Sprintf("%s\n", ni.MacAddr)) + } + for _, ni := range nodesInfos2 { inputStr.WriteString(fmt.Sprintf("%s\n", ni.MacAddr)) } return inputStr.String() @@ -303,8 +400,11 @@ Fetch uptimes: skywire-cli ut > ut.txt`, } } - totalValidShares := 0.0 - for _, ni := range nodesInfos { + totalValidShares1 := 0.0 + totalValidShares2 := 0.0 + + // Calculate shares and rewards for nodesInfos1 + for _, ni := range nodesInfos1 { share := 1.0 for _, ipCount := range ipCounts { if ni.IPAddr == ipCount.Name { @@ -318,42 +418,89 @@ Fetch uptimes: skywire-cli ut > ut.txt`, share = share / float64(macCount.Count) } } - totalValidShares += share + totalValidShares1 += share } + // Calculate shares and rewards for nodesInfos2 + for _, ni := range nodesInfos2 { + share := 1.0 + for _, ipCount := range ipCounts { + if ni.IPAddr == ipCount.Name { + if ipCount.Count >= 8 { + share = 8.0 / float64(ipCount.Count) + } + } + } + for _, macCount := range macCounts { + if macCount.Name == ni.MacAddr { + share = share / float64(macCount.Count) + } + } + totalValidShares2 += share + } + + // Output information for both pools if !h0 { - fmt.Printf("Visors meeting uptime & other requirements: %d\n", len(nodesInfos)) + fmt.Printf("Visors meeting uptime & other requirements (Pool 1): %d\n", len(nodesInfos1)) + fmt.Printf("Visors meeting uptime & other requirements (Pool 2): %d\n", len(nodesInfos2)) fmt.Printf("Unique mac addresses for first interface after lo: %d\n", len(uniqueMac)) - fmt.Printf("Unique Ip Addresses: %d\n", len(uniqueIP)) + fmt.Printf("Unique IP Addresses: %d\n", len(uniqueIP)) fmt.Printf("Unique UUIDs: %d\n", len(uniqueUUID)) - fmt.Printf("Total valid shares: %.6f\n", totalValidShares) - fmt.Printf("Skycoin Per Share: %.6f\n", dayReward/totalValidShares) + fmt.Printf("Total valid shares (Pool 1): %.6f\n", totalValidShares1) + fmt.Printf("Total valid shares (Pool 2): %.6f\n", totalValidShares2) + fmt.Printf("Skycoin Per Share (Pool 1): %.6f\n", dayReward/totalValidShares1) + fmt.Printf("Skycoin Per Share (Pool 2): %.6f\n", dayReward/totalValidShares2) } - for i, ni := range nodesInfos { - nodesInfos[i].Share = 1.0 + + // Calculate rewards for nodesInfos1 + for i, ni := range nodesInfos1 { + nodesInfos1[i].Share = 1.0 for _, ipCount := range ipCounts { if ni.IPAddr == ipCount.Name { if ipCount.Count >= 8 { - nodesInfos[i].Share = 8.0 / float64(ipCount.Count) + nodesInfos1[i].Share = 8.0 / float64(ipCount.Count) } } } for _, macCount := range macCounts { if macCount.Name == ni.MacAddr { - nodesInfos[i].Share = nodesInfos[i].Share / float64(macCount.Count) + nodesInfos1[i].Share = nodesInfos1[i].Share / float64(macCount.Count) } } - nodesInfos[i].Reward = nodesInfos[i].Share * dayReward / float64(totalValidShares) + nodesInfos1[i].Reward = nodesInfos1[i].Share * dayReward / float64(totalValidShares1) } + // Calculate rewards for nodesInfos2 + for i, ni := range nodesInfos2 { + nodesInfos2[i].Share = 1.0 + for _, ipCount := range ipCounts { + if ni.IPAddr == ipCount.Name { + if ipCount.Count >= 8 { + nodesInfos2[i].Share = 8.0 / float64(ipCount.Count) + } + } + } + for _, macCount := range macCounts { + if macCount.Name == ni.MacAddr { + nodesInfos2[i].Share = nodesInfos2[i].Share / float64(macCount.Count) + } + } + nodesInfos2[i].Reward = nodesInfos2[i].Share * dayReward / float64(totalValidShares2) + } + + // Combine nodesInfos1 and nodesInfos2 for output + combinedNodesInfos := append(nodesInfos1, nodesInfos2...) + if !h1 { fmt.Println("Skycoin Address, Skywire Public Key, Reward Shares, Reward SKY Amount, IP, Architecture, UUID, Interfaces") - for _, ni := range nodesInfos { + for _, ni := range combinedNodesInfos { fmt.Printf("%s, %s, %.6f, %.6f, %s, %s, %s, %s \n", ni.SkyAddr, ni.PK, ni.Share, ni.Reward, ni.IPAddr, ni.Arch, ni.UUID, ni.Interfaces) } } + + // Calculate reward sum by Skycoin Address rewardSumBySkyAddr := make(map[string]float64) - for _, ni := range nodesInfos { + for _, ni := range combinedNodesInfos { rewardSumBySkyAddr[ni.SkyAddr] += ni.Reward } var sortedSkyAddrs []rewardData @@ -376,7 +523,6 @@ Fetch uptimes: skywire-cli ut > ut.txt`, for _, skyAddrReward := range sortedSkyAddrs { fmt.Printf("%s, %.6f\n", skyAddrReward.SkyAddr, skyAddrReward.Reward) } - } }, } diff --git a/cmd/skywire-cli/commands/rewards/services.go b/cmd/skywire-cli/commands/rewards/services.go new file mode 100644 index 0000000000..bff0b9b881 --- /dev/null +++ b/cmd/skywire-cli/commands/rewards/services.go @@ -0,0 +1,181 @@ +// Package clirewards cmd/skywire-cli/commands/rewards/services.go +package clirewards + +import ( + "bytes" + "fmt" + "os" + "os/user" + "syscall" + "text/template" + + "github.com/bitfield/script" + "github.com/spf13/cobra" +) + +var ( + userName string + workingDir string + skyenvConf string + outputPath string + outPath string +) + +func init() { + RootCmd.AddCommand( + systemdServicesCmd, + ) + currentDir, err := os.Getwd() + if err != nil { + log.Fatal(err) + } + + fileInfo, err := os.Stat(currentDir) + if err != nil { + log.Fatal(err) + } + stat := fileInfo.Sys().(*syscall.Stat_t) + owner, err := user.LookupId(fmt.Sprint(stat.Uid)) + if err != nil { + log.Fatal(err) + } + + systemdServicesCmd.Flags().StringVarP(&userName, "user", "u", owner.Username, "user to set - should have write permission on path") + systemdServicesCmd.Flags().StringVarP(&workingDir, "path", "p", currentDir, "reward system data dir path") + systemdServicesCmd.Flags().StringVarP(&skyenvConf, "skyenv", "s", "fr.conf", "env config file path") + systemdServicesCmd.Flags().StringVarP(&outputPath, "out", "o", "/etc/systemd/system", "path to output systemd services") +} + +var systemdServicesCmd = &cobra.Command{ + Use: "systemd", + Short: "set up systemd services for reward system", + Long: `set up systemd services for reward system +must be run with sufficient permissions to write to output path`, + Run: func(_ *cobra.Command, _ []string) { + // Get the current user + + // Prepare the data for the template + serviceConfig := svcConfig{ + User: userName, + Dir: workingDir, + Conf: skyenvConf, + } + + // Create a new template and parse the service file template into it + tmpl, err := template.New("").Parse(skywireRewardSvcTpl) + if err != nil { + log.Fatal(err) + } + + var renderedServiceFile bytes.Buffer + var renderedServiceFile1 bytes.Buffer + err = tmpl.Execute(&renderedServiceFile, serviceConfig) + if err != nil { + log.Fatal(err) + } + + outPath = outputPath + + if outputPath != "/dev/stdout" { + outPath = outputPath + "/skywire-reward.service" + } + + _, err = script.Echo(renderedServiceFile.String()).Tee().WriteFile(outPath) + if err != nil { + log.Fatal(err) + } + fmt.Println("Wrote to: " + outPath) + + // Create a new template and parse the service file template into it + tmpl1, err := template.New("").Parse(fiberRewardSvcTpl) + if err != nil { + log.Fatal(err) + } + + // Execute the template with the data and output the result to stdout + err = tmpl1.Execute(&renderedServiceFile1, serviceConfig) + if err != nil { + log.Fatal(err) + } + + outPath = outputPath + + if outputPath != "/dev/stdout" { + outPath = outputPath + "/fiberreward.service" + } + + _, err = script.Echo(renderedServiceFile1.String()).Tee().WriteFile(outPath) + if err != nil { + log.Fatal(err) + } + fmt.Println("Wrote to: " + outPath) + + outPath = outputPath + + if outputPath != "/dev/stdout" { + outPath = outputPath + "/skywire-reward.timer" + } + _, err = script.Echo(skywireRewardTimerTpl).Tee().WriteFile(outPath) + if err != nil { + log.Fatal(err) + } + fmt.Println("Wrote to: " + outPath) + + }, +} + +// Timer for log collection & reward calculation + +const skywireRewardTimerTpl = `[Unit] +Description=skywire reward timer +After=network.target + +[Timer] +OnUnitActiveSec=1h +Unit=skywire-reward.service + +[Install] +WantedBy=multi-user.target +` + +// Log Collection & reward calculation +const skywireRewardSvcTpl = ` +[Unit] +Description=skywire reward service +After=network.target + +[Service] +Type=simple +User={{.User}} +WorkingDirectory={{.Dir}}/rewards +ExecStart=/bin/bash -c './getlogs.sh && ./reward.sh ; exit 0' + +[Install] +WantedBy=multi-user.target +` + +// UI / Frontend +const fiberRewardSvcTpl = ` +[Unit] +Description=skywire cli rewards ui +After=network.target + +[Service] +Type=simple +User={{.User}} +WorkingDirectory={{.Dir}} +Environment='SKYENV={{.Conf}}' +ExecStart=/usr/bin/bash -c 'skywire cli rewards ui' +Restart=always +RestartSec=20 +TimeoutSec=30 + +[Install] +WantedBy=multi-user.target +` + +type svcConfig struct { + User string + Dir string + Conf string +} diff --git a/cmd/skywire-cli/commands/rewards/ui.go b/cmd/skywire-cli/commands/rewards/ui.go index fde050be7a..150c86449b 100644 --- a/cmd/skywire-cli/commands/rewards/ui.go +++ b/cmd/skywire-cli/commands/rewards/ui.go @@ -5,24 +5,19 @@ import ( "bytes" "context" "encoding/base64" - "encoding/csv" - "encoding/json" "fmt" htmpl "html/template" "io" "net" "net/http" "os" - "path/filepath" "regexp" "runtime" - "sort" "strconv" "strings" "sync" "time" - "github.com/alecthomas/chroma/quick" "github.com/bitfield/script" "github.com/gin-gonic/gin" "github.com/robert-nix/ansihtml" @@ -37,13 +32,6 @@ import ( ) func init() { - var envServices skywire.EnvServices - var services skywire.Services - if err := json.Unmarshal([]byte(skywire.ServicesJSON), &envServices); err == nil { - if err := json.Unmarshal(envServices.Prod, &services); err == nil { - dmsgDiscURL = services.DmsgDiscovery - } - } RootCmd.CompletionOptions.DisableDefaultCmd = true RootCmd.AddCommand( uiCmd, @@ -56,7 +44,7 @@ func init() { msg += "\n\r" } uiCmd.Flags().StringVarP(&wl, "wl", "w", scriptExecArray("${REWARDPKS[@]}"), msg) - uiCmd.Flags().StringVarP(&dmsgDisc, "dmsg-disc", "D", "", "dmsg discovery url default:\n"+dmsgDiscURL) + uiCmd.Flags().StringVarP(&dmsgDisc, "dmsg-disc", "D", skywire.Prod.DmsgDiscovery, "dmsg discovery url") uiCmd.Flags().StringVarP(&ensureOnlineURL, "ensure-online", "O", scriptExecString("${ENSUREONLINE}"), "Exit when the specified URL cannot be fetched;\ni.e. https://fiber.skywire.dev") if os.Getenv("DMSGHTTP_SK") != "" { sk.Set(os.Getenv("DMSGHTTP_SK")) //nolint @@ -101,24 +89,10 @@ var ( wlkeys []cipher.PubKey webPort uint ensureOnlineURL string - dmsgDiscURL string ) var skyenvfile = os.Getenv("SKYENV") -func tploghtmlfunc() (l string) { - l = "
" - l += navlinks - l += "Blue = Verified Bandwidth
" - l += "Yellow = Transport bandwidth inconsistent
" - l += "Red = Error: sent or received is zero
" - tp, _ := script.Exec(`skywire cli log tp -d rewards/log_backups`).String() //nolint - l += fmt.Sprintf("%s\n", ansihtml.ConvertToHTML([]byte(tp))) - l += htmltoplink - l += htmlend - return l -} - func mainPage(c *gin.Context) { c.Writer.Header().Set("Server", "") tmpl0, err1 := tmpl.Clone() @@ -149,101 +123,6 @@ func mainPage(c *gin.Context) { var htmlRewardPageTemplate = ` {{.Page.Content}} ` - -/* -{{.Page.RewardCalc}}-{{.Page.DaysCalc}} -{{.Page.RCal}} --reward distribution transaction for {{.Page.MostRecentRewardForDate}} distributed on {{.Page.MostRecentTxIDDate}} -{{.Page.MostRecentTxID}} -{{.Page.CSVCheck}} -Previous distributions: -{{.Page.CSVCheck}} -top of page --reward distribution CSV for {{.Page.MostRecentRewardForDate}} distributed on {{.Page.MostRecentTxIDDate}} -{{.Page.LocalCSVCheckWithAnchor}} -Reward Shares Skycoin per share: {{.Page.SkyPerShare}} -Reward system status: -{{.Page.NextSkywireCliLogRun}} -{{.Page.MostRecentTxnInfo}} -
- - -type transaction struct { - Status struct { - Confirmed bool `json:"confirmed"` - Unconfirmed bool `json:"unconfirmed"` - Height int `json:"height"` - BlockSeq int `json:"block_seq"` - } `json:"status"` - Time int `json:"time"` - Txn struct { - Timestamp int `json:"timestamp"` - Length int `json:"length"` - Type int `json:"type"` - Txid string `json:"txid"` - InnerHash string `json:"inner_hash"` - Sigs []string `json:"sigs"` - Inputs []string `json:"inputs"` - Outputs []struct { - Uxid string `json:"uxid"` - Dst string `json:"dst"` - Coins string `json:"coins"` - Hours int `json:"hours"` - } `json:"outputs"` - } `json:"txn"` -} - -func csvcheck(txid string) string { - // Make HTTP GET request to locally running instance of Skycoin Explorer API - resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:8001/api/transaction?txid=%s", strings.TrimSuffix(txid, "\n"))) - if err != nil { - fmt.Fprintf(os.Stderr, "Error making HTTP request to local skycoin-explorer instance: %v\nTrying with explorer.skycoin.com", err) - // Make HTTP GET request to Skycoin Explorer API - resp, err = http.Get(fmt.Sprintf("https://explorer.skycoin.com/api/transaction?txid=%s", txid)) - if err != nil { - msg := fmt.Sprint("Error making HTTP request to explorer.skycoin.com: %v\nCannot check transaction ; %v", err) - fmt.Fprintf(os.Stderr, msg) - return msg - } - } - defer resp.Body.Close() //nolint - - // Decode JSON response into transaction struct - var tx transaction - err = json.NewDecoder(resp.Body).Decode(&tx) - if err != nil { - msg := fmt.Sprint("Error decoding JSON response: %v\n", err) - fmt.Fprintf(os.Stderr, msg) - return msg - } - - var csvOutputBuilder strings.Builder - w := csv.NewWriter(&csvOutputBuilder) - for i, output := range tx.Txn.Outputs { - if i == len(tx.Txn.Outputs)-1 { - // skip last line of output containing the from address - continue - } - record := []string{output.Dst, output.Coins} - if err := w.Write(record); err != nil { - msg := fmt.Sprint("Error writing CSV record: %v\n", err) - fmt.Fprintf(os.Stderr, msg) - return msg - } - } - w.Flush() - if err := w.Error(); err != nil { - msg := fmt.Sprint("Error flushing CSV writer: %v\n", err) - fmt.Fprintf(os.Stderr, msg) - return msg - } - return csvOutputBuilder.String() -} -*/ - var htmlPageTemplateData htmlTemplateData var tmpl *htmpl.Template @@ -441,67 +320,6 @@ func server() { c.Writer.Flush() }) - r1.GET("/tpsn", func(c *gin.Context) { - c.Writer.Header().Set("Server", "") - c.Writer.Header().Set("Content-Type", "text/html;charset=utf-8") - c.Writer.Header().Set("Transfer-Encoding", "chunked") - c.Writer.WriteHeader(http.StatusOK) - c.Writer.Flush() - c.Writer.Write([]byte("Skywire Transport Setup-Node ")) //nolint - c.Writer.Flush() - c.Writer.Write([]byte("\n\n")) //nolint - c.Writer.Flush() - c.Writer.Write([]byte(navlinks)) //nolint - c.Writer.Flush() - tmpFile, err := os.CreateTemp(os.TempDir(), "*.sh") - if err != nil { - return - } - if err := tmpFile.Close(); err != nil { - return - } - _, _ = script.Exec(`chmod +x ` + tmpFile.Name()).String() //nolint - _, _ = script.Echo(nextlogrun).WriteFile(tmpFile.Name()) //nolint - res, _ := script.Exec(`bash -c 'source ` + tmpFile.Name() + ` ; _nextskywireclilogrun'`).String() //nolint - os.Remove(tmpFile.Name()) //nolint - c.Writer.Write([]byte(fmt.Sprintf("%s\n", res))) //nolint - c.Writer.Flush() - - // Initial line count - initialLineCount, _ := script.File("rewards/transport-setup-node.txt").CountLines() //nolint - // Read and print the initial lines - initialContent, _ := script.File("rewards/transport-setup-node.txt").First(initialLineCount).Bytes() //nolint - c.Writer.Write(ansihtml.ConvertToHTML(initialContent)) //nolint - c.Writer.Flush() - for { - select { - case <-c.Writer.CloseNotify(): - return - default: - } - // Sleep for a short duration - time.Sleep(100 * time.Millisecond) - // Get the current line count - currentLineCount, _ := script.File("rewards/transport-setup-node.txt").CountLines() //nolint - // Check if there are new lines - if currentLineCount > initialLineCount { - newContent, _ := script.File("rewards/transport-setup-node.txt").Last(currentLineCount - initialLineCount).Bytes() //nolint - initialLineCount = currentLineCount - c.Writer.Write(ansihtml.ConvertToHTML(newContent)) //nolint - c.Writer.Flush() - } - finished, _ := script.Exec("bash -c 'systemctl is-active --quiet skywire-reward.service || echo true'").String() //nolint - if finished != "" { - break - } - } - - c.Writer.Write([]byte(htmltoplink)) //nolint - c.Writer.Flush() - c.Writer.Write([]byte(htmlend)) //nolint - c.Writer.Flush() - }) - r1.GET("/log-collection/tree", func(c *gin.Context) { c.Writer.Header().Set("Server", "") c.Writer.Header().Set("Transfer-Encoding", "chunked") @@ -513,73 +331,18 @@ func server() { surveycount, _ := script.FindFiles("rewards/log_backups/").Match("node-info.json").CountLines() //nolint c.Writer.Write([]byte(fmt.Sprintf("Total surveys: %v\n", surveycount))) //nolint c.Writer.Flush() - st, _ := script.Exec(`skywire cli log st -d rewards/log_backups -e rewards/tp_setup -x rewards/proxy_test/proxies.csv -r`).Bytes() //nolint - c.Writer.Write(ansihtml.ConvertToHTML(st)) //nolint + st, err := script.Exec(`skywire cli log st -d rewards/log_backups -r`).Bytes() //nolint + if err != nil { + log.WithError(err).Error() + c.Writer.Write([]byte(err.Error())) //nolint + } + c.Writer.Write(ansihtml.ConvertToHTML(st)) //nolint c.Writer.Flush() c.Writer.Write([]byte(htmltoplink)) //nolint c.Writer.Flush() c.Writer.Write([]byte(htmlend)) //nolint c.Writer.Flush() }) - r1.GET("/proxy-test", func(c *gin.Context) { - c.Writer.Header().Set("Server", "") - c.Writer.Header().Set("Transfer-Encoding", "chunked") - c.Writer.WriteHeader(http.StatusOK) - c.Writer.Write([]byte("Proxy Tests \n\n")) //nolint - c.Writer.Flush() - c.Writer.Write([]byte(navlinks)) //nolint - c.Writer.Flush() - c.Writer.Write([]byte("testproxies.sh\n")) //nolint - c.Writer.Flush() - proxytestcsvdata, _ := script.File("rewards/proxy_test/proxies.csv").Replace("00pk", "server_pk").String() //nolint - proxytestdata := "" - reader := csv.NewReader(strings.NewReader(proxytestcsvdata)) - records, err := reader.ReadAll() - if err != nil { - c.Writer.Write([]byte(htmltoplink + htmlend)) //nolint - c.Writer.Flush() - } - maxWidths := make([]int, len(records[0])) - for _, record := range records { - for i, field := range record { - if len(field) > maxWidths[i] { - maxWidths[i] = len(field) - } - } - } - sort.Slice(records[1:], func(i, j int) bool { - if records[i+1][0] == "N/A" { - return true - } - if records[j+1][0] == "N/A" { - return false - } - if records[i+1][0] == "self_transport" { - return true - } - if records[j+1][0] == "self_transport" { - return false - } - val1, _ := strconv.ParseFloat(strings.TrimSuffix(records[i+1][2], "s"), 64) //nolint - val2, _ := strconv.ParseFloat(strings.TrimSuffix(records[j+1][2], "s"), 64) //nolint - return val1 < val2 - }) - - for _, record := range records { - for i, field := range record { - proxytestdata += fmt.Sprintf("%*s ", maxWidths[i], field) - } - proxytestdata += "\n" - } - c.Writer.Write([]byte(proxytestdata)) //nolint - c.Writer.Flush() - c.Writer.Write([]byte(htmltoplink + htmlend)) //nolint - c.Writer.Flush() - }) - - r1.GET("/rewards/testproxies.sh", func(c *gin.Context) { - serveSyntaxHighlighted(c) - }) r1.GET("/log-collection/tree/:pk", func(c *gin.Context) { c.Writer.Header().Set("Server", "") @@ -623,7 +386,18 @@ func server() { r1.GET("/log-collection/tplogs", func(c *gin.Context) { c.Writer.Header().Set("Server", "") c.Writer.WriteHeader(http.StatusOK) - c.Writer.Write([]byte([]byte(tploghtmlfunc()))) //nolint + c.Writer.Write([]byte(func() (l string) { //nolint + l = "Skywire Transport Bandwidth Logs By Day \n\n" + l += navlinks + l += "Blue = Verified Bandwidth
" + l += "Yellow = Transport bandwidth inconsistent
" + l += "Red = Error: sent or received is zero
" + tp, _ := script.Exec(`skywire cli log tp -d rewards/log_backups`).String() //nolint + l += fmt.Sprintf("%s\n", ansihtml.ConvertToHTML([]byte(tp))) + l += htmltoplink + l += htmlend + return l + }())) //nolint }) r1.GET("/skycoin-rewards", func(c *gin.Context) { @@ -633,11 +407,9 @@ func server() { c.Writer.Flush() l := fmt.Sprintf("%s", func() string { yearlyTotal := 408000.0 - result := fmt.Sprintf("%g annual reward distribution\nReward total per month:\n", yearlyTotal) - + result := fmt.Sprintf("Annual reward distribution per pool:\n%g Skycoin\nMonthly rewards per pool:\n", yearlyTotal) currentMonth := time.Now().Month() currentYear := time.Now().Year() - for month := time.January; month <= time.December; month++ { daysInMonth := time.Date(currentYear, month+1, 0, 0, 0, 0, 0, time.UTC).Day() monthlyRewards := (yearlyTotal / 365) * float64(daysInMonth) @@ -647,45 +419,200 @@ func server() { } result += fmt.Sprintf(format, monthlyRewards, currentYear, month) } - firstDayOfNextYear := time.Date(currentYear+1, time.January, 1, 0, 0, 0, 0, time.UTC) lastDayOfYear := firstDayOfNextYear.Add(-time.Second) totalDaysInYear := int(lastDayOfYear.YearDay()) - skycoinPerDay := yearlyTotal / float64(totalDaysInYear) - result += fmt.Sprintf("%g Skycoin per day\n
", skycoinPerDay) - utstats, _ := script.Exec(`skywire cli ut -t`).String() //nolint - result += fmt.Sprintf("Uptime tracker version statistics:\n%s", utstats) - return result + result += fmt.Sprintf("%g Skycoin per day\n
", skycoinPerDay) + utstats, err := script.Exec(`skywire cli ut -t`).String() + if err == nil { + result += fmt.Sprintf("Uptime tracker version statistics:\n%s\n
", utstats) + } + nis, err := script.FindFiles("rewards/log_backups").Match("node-info.json").Slice() //nolint + if err == nil { + var surveyarches string + for _, ni := range nis { + surveyarch, err := script.File(ni).JQ(".go_arch").Replace(`"`, "").String() + if err == nil { + surveyarches += surveyarch + } + } + archstats, err := script.Echo(surveyarches).Freq().String() //nolint + if err == nil { + result += fmt.Sprintf("Survey architecture statistics:\n%s\n
", archstats) + } + var surveyOSNames string + for _, ni := range nis { + surveyOSName, err := script.File(ni).JQ(".zcalusic_sysinfo.os.name").Replace(`"`, "").String() + if err == nil { + surveyOSNames += surveyOSName + } + } + namestats, err := script.Echo(surveyOSNames).Freq().String() //nolint + if err == nil { + result += fmt.Sprintf("Survey OS name statistics:\n%s\n
", namestats) + } + var surveycpus string + for _, ni := range nis { + surveycpu, err := script.File(ni).JQ(".zcalusic_sysinfo.cpu.model").Replace(`"`, "").String() + if err == nil { + surveycpus += surveycpu + } + } + cpustats, err := script.Echo(surveycpus).Freq().String() //nolint + if err == nil { + result += fmt.Sprintf("Survey CPU statistics:\n%s\n
", cpustats) + } + + var totalBytes int64 + for _, ni := range nis { + surveytbs, err := script.File(ni).JQ(".ghw_blockinfo.total_size_bytes").Reject("null").Replace(`"`, "").String() + if err == nil { + if surveytbs != "\n" && surveytbs != "" { + byteValue, err := strconv.ParseInt(strings.TrimRight(surveytbs, "\n"), 10, 64) + if err != nil { + result += fmt.Sprintf("Non nil error from strconv.ParseInt: %v\n", err) + } + totalBytes += byteValue + } + } + } + + // Get stats for terabytes and gigabytes + bsstatsTB, _ := script.Exec(`bash -c 'jq '.ghw_blockinfo.total_size_bytes' rewards/log_backups/*/node-info.json | grep -v null | sort -n | numfmt --to=iec | sort -h | uniq -c'`).Reject("G").Slice() //nolint + bsstatsGB, _ := script.Exec(`bash -c 'jq '.ghw_blockinfo.total_size_bytes' rewards/log_backups/*/node-info.json | grep -v null | sort -n | numfmt --to=iec | sort -h | uniq -c'`).Reject("T").Slice() //nolint + formattedTotal, err := script.Echo(fmt.Sprintf("%d", totalBytes)).ExecForEach("numfmt --to=iec {{.}}").String() + if err != nil { + result += fmt.Sprintf("%v\n", err) + } + result += fmt.Sprintf("Survey total byte size (cumulative): %s\n", formattedTotal) + result += "Survey total byte size statistics:\n" + result += `
GB | TB | ||
---|---|---|---|
%s | `+"\n", bsstatsGB[i]) + } else { + result += `` + "\n" // Empty centered cell + } + if i < len(bsstatsTB) { + result += fmt.Sprintf(` | %s | `+"\n", bsstatsTB[i]) + } else { + result += `` + "\n" // Empty centered cell + } + result += " |
GB | MB | ||
---|---|---|---|
%s | `+"\n", statsGB[i]) + } else { + result += `` + "\n" // Empty centered cell + } + if i < len(statsMB) { + result += fmt.Sprintf(` | %s | `+"\n", statsMB[i]) + } else { + result += `` + "\n" // Empty centered cell + } + result += " |
RewardDate | Pool 1 SKY/VISOR | Pool 2 SKY/VISOR | Distributed [❌/✔] | \n"
+ l += "|
---|---|---|---|---|
" + rewardtxncsvs[i] + " | \n" + l += "" + skycoinpershare1 + " | \n" + if skycoinpershare2 != "" { + l += "" + skycoinpershare2 + " | \n" } else { - l += "" + rewardtxncsvs[i] + " " + strings.ReplaceAll(skycoinpershare, "\n", "") + " ❌\n" + l += "\n" } + l += " | " + distributedIcon + " | \n" + l += "
fiber skycoin rewards log collection survey index transport logging transport stats transport map uptime tracker address resolver transport discovery dmsgd entries all dmsg servers available dmsg servers
++ + +
+{{template "this" .}} diff --git a/go.mod b/go.mod index 794b8715a9..590e694323 100644 --- a/go.mod +++ b/go.mod @@ -2,53 +2,56 @@ module github.com/skycoin/skywire go 1.23 +toolchain go1.23.1 + require ( github.com/AudriusButkevicius/pfilter v0.0.11 - github.com/VictoriaMetrics/metrics v1.24.0 - github.com/alecthomas/chroma v0.10.0 + github.com/MichaelMure/go-term-markdown v0.1.4 + github.com/VictoriaMetrics/metrics v1.35.1 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 - github.com/bitfield/script v0.22.1 + github.com/bitfield/script v0.23.0 github.com/blang/semver/v4 v4.0.0 github.com/ccding/go-stun/stun v0.0.0-20200514191101-4dc67bcdb029 - github.com/elazarl/goproxy v0.0.0-20231117061959-7cc037d33fb5 - github.com/fatih/color v1.15.0 + github.com/elazarl/goproxy v0.0.0-20240909085733-6741dbfc16a1 + github.com/fatih/color v1.17.0 github.com/gen2brain/dlgs v0.0.0-20220603100644-40c77870fa8d - github.com/gin-gonic/gin v1.9.1 - github.com/go-chi/chi/v5 v5.0.11 - github.com/gocarina/gocsv v0.0.0-20230616125104-99d496ca653d - github.com/google/uuid v1.3.1 - github.com/gorilla/securecookie v1.1.1 - github.com/hashicorp/go-version v1.6.0 - github.com/hashicorp/yamux v0.1.1 + github.com/gin-gonic/gin v1.10.0 + github.com/go-chi/chi/v5 v5.1.0 + github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 + github.com/google/uuid v1.6.0 + github.com/gorilla/securecookie v1.1.2 + github.com/hashicorp/go-version v1.7.0 + github.com/hashicorp/yamux v0.1.2 github.com/ivanpirog/coloredcobra v1.0.1 github.com/james-barrow/golang-ipc v1.2.4 - github.com/jaypipes/ghw v0.12.0 + github.com/jaypipes/ghw v0.13.0 github.com/lib/pq v1.10.9 github.com/orandin/lumberjackrus v1.0.1 - github.com/pterm/pterm v0.12.66 + github.com/pterm/pterm v0.12.79 github.com/robert-nix/ansihtml v1.0.1 github.com/sirupsen/logrus v1.9.3 - github.com/skycoin/dmsg v1.3.29-0.20241004205035-c45754b16934 + github.com/skycoin/dmsg v1.3.29-0.20241019182716-022283c93835 github.com/skycoin/skycoin v0.28.0 - github.com/skycoin/skycoin-service-discovery v1.3.28 - github.com/skycoin/skywire-services v1.3.28 + github.com/skycoin/skycoin-service-discovery v1.3.29-0.20241019190420-233c2d68bce0 + github.com/skycoin/skywire-services v1.3.29-0.20241019183828-0d760251ca55 github.com/skycoin/skywire-utilities v1.3.25 github.com/skycoin/systray v1.10.0 github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 - github.com/spf13/cobra v1.7.0 + github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 github.com/tidwall/pretty v1.2.1 github.com/toqueteos/webbrowser v1.2.0 github.com/xtaci/kcp-go v5.4.20+incompatible - github.com/zcalusic/sysinfo v1.0.1 - go.etcd.io/bbolt v1.3.7 - golang.org/x/net v0.21.0 - golang.org/x/sync v0.3.0 - golang.org/x/sys v0.20.0 - golang.zx2c4.com/wireguard v0.0.0-20230223181233-21636207a675 - gopkg.in/telebot.v3 v3.2.1 + github.com/zcalusic/sysinfo v1.1.2 + go.etcd.io/bbolt v1.3.11 + golang.org/x/net v0.30.0 + golang.org/x/sync v0.8.0 + golang.org/x/sys v0.26.0 + golang.org/x/term v0.25.0 + golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 + gopkg.in/telebot.v3 v3.3.8 ) require ( @@ -57,112 +60,115 @@ require ( atomicgo.dev/schedule v0.1.0 // indirect github.com/ActiveState/termtest/conpty v0.5.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/MichaelMure/go-term-text v0.3.1 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/StackExchange/wmi v1.2.1 // indirect - github.com/bytedance/sonic v1.10.0 // indirect + github.com/alecthomas/chroma v0.10.0 // indirect + github.com/bytedance/sonic v1.12.3 // indirect + github.com/bytedance/sonic/loader v0.2.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect - github.com/chenzhuoyu/iasm v0.9.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/chen3feng/safecast v0.0.0-20220908170618-81b2ecd47937 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/coder/websocket v1.8.12 // indirect github.com/confiant-inc/go-socks5 v0.0.0-20210816151940-c1124825b1d6 // indirect - github.com/containerd/console v1.0.3 // indirect - github.com/creack/pty v1.1.18 // indirect + github.com/containerd/console v1.0.4 // indirect + github.com/creack/pty v1.1.23 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgraph-io/badger/v3 v3.2103.2 // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/dlclark/regexp2 v1.4.0 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect - github.com/ghodss/yaml v1.0.0 // indirect + github.com/disintegration/imaging v1.6.2 // indirect + github.com/dlclark/regexp2 v1.11.4 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/eliukblau/pixterm/pkg/ansimage v0.0.0-20191210081756-9fb6cf8c2f75 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.6 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-chi/chi v4.1.2+incompatible // indirect github.com/go-chi/cors v1.2.1 // indirect - github.com/go-chi/httprate v0.9.0 // indirect - github.com/go-echarts/go-echarts/v2 v2.3.3 // indirect + github.com/go-chi/httprate v0.14.1 // indirect + github.com/go-echarts/go-echarts/v2 v2.4.3 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.15.1 // indirect + github.com/go-playground/validator/v10 v10.22.1 // indirect github.com/go-redis/redis/v8 v8.11.5 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect - github.com/goccy/go-json v0.10.2 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/goccy/go-json v0.10.3 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/golang/snappy v0.0.3 // indirect - github.com/google/flatbuffers v1.12.1 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/gomarkdown/markdown v0.0.0-20240930133441-72d49d9543d8 // indirect + github.com/google/flatbuffers v24.3.25+incompatible // indirect + github.com/google/pprof v0.0.0-20241017200806-017d972448fc // indirect github.com/gookit/color v1.5.4 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/itchyny/gojq v0.12.13 // indirect - github.com/itchyny/timefmt-go v0.1.5 // indirect - github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.14.3 // indirect - github.com/jackc/pgio v1.0.0 // indirect + github.com/itchyny/gojq v0.12.16 // indirect + github.com/itchyny/timefmt-go v0.1.6 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.3.3 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgtype v1.14.0 // indirect - github.com/jackc/pgx/v4 v4.18.2 // indirect - github.com/jaypipes/pcidb v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jaypipes/pcidb v1.0.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect - github.com/klauspost/reedsolomon v1.11.8 // indirect - github.com/leodido/go-urn v1.2.4 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/klauspost/reedsolomon v1.12.4 // indirect + github.com/kyokomi/emoji/v2 v2.2.13 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/lithammer/fuzzysearch v1.1.8 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect - github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/onsi/ginkgo/v2 v2.12.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.9 // indirect - github.com/pires/go-proxyproto v0.6.2 // indirect + github.com/onsi/ginkgo/v2 v2.20.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/pires/go-proxyproto v0.8.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/quic-go/quic-go v0.42.0 // indirect - github.com/rivo/uniseg v0.4.4 // indirect - github.com/rs/cors v1.8.2 // indirect + github.com/quic-go/quic-go v0.48.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rs/cors v1.11.1 // indirect github.com/skycoin/noise v0.0.0-20180327030543-2492fe189ae6 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161 // indirect github.com/templexxx/xor v0.0.0-20191217153810-f85b25db303b // indirect github.com/tjfoc/gmsm v1.4.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.11 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect github.com/valyala/fastrand v1.1.0 // indirect github.com/valyala/histogram v1.2.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/xtaci/lossyconn v0.0.0-20200209145036-adba10fffc37 // indirect - go.opencensus.io v0.23.0 // indirect - go.uber.org/mock v0.4.0 // indirect - golang.org/x/arch v0.4.0 // indirect - golang.org/x/crypto v0.20.0 // indirect - golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect - golang.org/x/mod v0.12.0 // indirect - golang.org/x/term v0.17.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect + go.opencensus.io v0.24.0 // indirect + go.uber.org/mock v0.5.0 // indirect + golang.org/x/arch v0.11.0 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect + golang.org/x/image v0.21.0 // indirect + golang.org/x/mod v0.21.0 // indirect + golang.org/x/text v0.19.0 // indirect + golang.org/x/tools v0.26.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/protobuf v1.35.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gorm.io/driver/postgres v1.3.8 // indirect - gorm.io/gorm v1.23.8 // indirect - howett.net/plist v1.0.0 // indirect - mvdan.cc/sh/v3 v3.7.0 // indirect - nhooyr.io/websocket v1.8.7 // indirect + gorm.io/driver/postgres v1.5.9 // indirect + gorm.io/gorm v1.25.12 // indirect + howett.net/plist v1.0.1 // indirect + mvdan.cc/sh/v3 v3.9.0 // indirect ) replace github.com/xxxserxxx/gotop/v4 => github.com/ersonp/gotop/v4 v4.2.1 diff --git a/go.sum b/go.sum index afb3683a6c..6bffa48e4c 100644 --- a/go.sum +++ b/go.sum @@ -81,18 +81,26 @@ github.com/MarvinJWendt/testza v0.3.0/go.mod h1:eFcL4I0idjtIx8P9C6KkAuLgATNKpX4/ github.com/MarvinJWendt/testza v0.4.2/go.mod h1:mSdhXiKH8sg/gQehJ63bINcCKp7RtYewEjXsvsVUPbE= github.com/MarvinJWendt/testza v0.5.2 h1:53KDo64C1z/h/d/stCYCPY69bt/OSwjq5KpFNwi+zB4= github.com/MarvinJWendt/testza v0.5.2/go.mod h1:xu53QFE5sCdjtMCKk8YMQ2MnymimEctc4n3EjyIYvEY= -github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= -github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/MichaelMure/go-term-markdown v0.1.4 h1:Ir3kBXDUtOX7dEv0EaQV8CNPpH+T7AfTh0eniMOtNcs= +github.com/MichaelMure/go-term-markdown v0.1.4/go.mod h1:EhcA3+pKYnlUsxYKBJ5Sn1cTQmmBMjeNlpV8nRb+JxA= +github.com/MichaelMure/go-term-text v0.3.1 h1:Kw9kZanyZWiCHOYu9v/8pWEgDQ6UVN9/ix2Vd2zzWf0= +github.com/MichaelMure/go-term-text v0.3.1/go.mod h1:QgVjAEDUnRMlzpS6ky5CGblux7ebeiLnuy9dAaFZu8o= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= -github.com/VictoriaMetrics/metrics v1.24.0 h1:ILavebReOjYctAGY5QU2F9X0MYvkcrG3aEn2RKa1Zkw= -github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys= +github.com/VictoriaMetrics/metrics v1.35.1 h1:o84wtBKQbzLdDy14XeskkCZih6anG+veZ1SwJHFGwrU= +github.com/VictoriaMetrics/metrics v1.35.1/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8= +github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= +github.com/alecthomas/chroma v0.7.1/go.mod h1:gHw09mkX1Qp80JlYbmN9L3+4R5o6DJJ3GRShh+AICNc= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= +github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= +github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI= +github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -113,16 +121,17 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bitfield/script v0.22.1 h1:DphxoC5ssYciwd0ZS+N0Xae46geAD/0mVWh6a2NUxM4= -github.com/bitfield/script v0.22.1/go.mod h1:fv+6x4OzVsRs6qAlc7wiGq8fq1b5orhtQdtW0dwjUHI= +github.com/bitfield/script v0.23.0 h1:N0R5yLEl6wJIS9PR/A6xXwjMsplMubyxdi05N5l0X28= +github.com/bitfield/script v0.23.0/go.mod h1:fv+6x4OzVsRs6qAlc7wiGq8fq1b5orhtQdtW0dwjUHI= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= -github.com/bytedance/sonic v1.10.0 h1:qtNZduETEIWJVIyDl01BeNxur2rW9OwTQ/yBqFRkKEk= -github.com/bytedance/sonic v1.10.0/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= +github.com/bytedance/sonic v1.12.3 h1:W2MGa7RCU1QTeYRTPE3+88mVC0yXmsRQRChiyVocVjU= +github.com/bytedance/sonic v1.12.3/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= +github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/ccding/go-stun/stun v0.0.0-20200514191101-4dc67bcdb029 h1:POmUHfxXdeyM8Aomg4tKDcwATCFuW+cYLkj6pwsw9pc= github.com/ccding/go-stun/stun v0.0.0-20200514191101-4dc67bcdb029/go.mod h1:Rpr5n9cGHYdM3S3IK8ROSUUUYjQOu+MSUCZDcJbYWi8= github.com/cenkalti/backoff v1.1.0/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -130,20 +139,20 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= -github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= -github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= -github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo= -github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chen3feng/safecast v0.0.0-20220908170618-81b2ecd47937 h1:gJMTUTnqa+f2GzdU1p3UVa3E39hogZdiRHEgBBnqtVc= +github.com/chen3feng/safecast v0.0.0-20220908170618-81b2ecd47937/go.mod h1:HPBMB1GC+eBfIUWhh9IJKdL/mVhIBZbJzjvijHxG3F0= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -153,26 +162,25 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= -github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= +github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= github.com/confiant-inc/go-socks5 v0.0.0-20210816151940-c1124825b1d6 h1:sRQemCQ+r6Ht7uIT0D9Xcyjed4lKpDhNKarBEPFZp3c= github.com/confiant-inc/go-socks5 v0.0.0-20210816151940-c1124825b1d6/go.mod h1:S4w2wY39ZYaWQLXNMZ6uVfIyYrKmLP2N/S2/5YIFU6o= -github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= +github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/creack/pty v1.1.23 h1:4M6+isWdcStXEf15G/RbrMPOQj1dZ7HPZCGwE4kOeP0= +github.com/creack/pty v1.1.23/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -184,14 +192,21 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= +github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= +github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= +github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20231117061959-7cc037d33fb5 h1:m62nsMU279qRD9PQSWD1l66kmkXzuYcnVJqL4XLeV2M= -github.com/elazarl/goproxy v0.0.0-20231117061959-7cc037d33fb5/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/elazarl/goproxy v0.0.0-20240909085733-6741dbfc16a1 h1:g7YUigN4dW2+zpdusdTTghZ+5Py3BaUMAStvL8Nk+FY= +github.com/elazarl/goproxy v0.0.0-20240909085733-6741dbfc16a1/go.mod h1:thX175TtLTzLj3p7N/Q9IiKZ7NF+p72cvL91emV0hzo= github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/eliukblau/pixterm/pkg/ansimage v0.0.0-20191210081756-9fb6cf8c2f75 h1:vbix8DDQ/rfatfFr/8cf/sJfIL69i4BcZfjrVOxsMqk= +github.com/eliukblau/pixterm/pkg/ansimage v0.0.0-20191210081756-9fb6cf8c2f75/go.mod h1:0gZuvTO1ikSA5LtTI6E13LEOdWQNjIo5MTQOvrV0eFg= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -206,35 +221,32 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= -github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= -github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= +github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= github.com/gen2brain/dlgs v0.0.0-20220603100644-40c77870fa8d h1:dHYKX8CBAs1zSGXm3q3M15CLAEwPEkwrK1ed8FCo+Xo= github.com/gen2brain/dlgs v0.0.0-20220603100644-40c77870fa8d/go.mod h1:/eFcjDXaU2THSOOqLxOPETIbHETnamk8FA/hMjhg/gU= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= -github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= -github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA= -github.com/go-chi/chi/v5 v5.0.11/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= +github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4= github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= -github.com/go-chi/httprate v0.9.0 h1:21A+4WDMDA5FyWcg7mNrhj63aNT8CGh+Z1alOE/piU8= -github.com/go-chi/httprate v0.9.0/go.mod h1:6GOYBSwnpra4CQfAKXu8sQZg+nZ0M1g9QnyFvxrAB8A= -github.com/go-echarts/go-echarts/v2 v2.3.3 h1:uImZAk6qLkC6F9ju6mZ5SPBqTyK8xjZKwSmwnCg4bxg= -github.com/go-echarts/go-echarts/v2 v2.3.3/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI= +github.com/go-chi/httprate v0.14.1 h1:EKZHYEZ58Cg6hWcYzoZILsv7ppb46Wt4uQ738IRtpZs= +github.com/go-chi/httprate v0.14.1/go.mod h1:TUepLXaz/pCjmCtf/obgOQJ2Sz6rC8fSf5cAt5cnTt0= +github.com/go-echarts/go-echarts/v2 v2.4.3 h1:JVxD8idXAZHIGGvrmDT2LQKGKe2HgZLUfMxqmyJRdqY= +github.com/go-echarts/go-echarts/v2 v2.4.3/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -244,8 +256,8 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -258,36 +270,26 @@ github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/Nu github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= -github.com/go-playground/validator/v10 v10.15.1 h1:BSe8uhN+xQ4r5guV/ywQI4gO59C2raYcGffYWZEjZzM= -github.com/go-playground/validator/v10 v10.15.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= +github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= -github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= -github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= -github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= -github.com/gobwas/ws v1.2.1 h1:F2aeBZrm2NDsc7vbovKrWSogd4wvfAxg0FQ89/iqOTk= -github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= -github.com/gocarina/gocsv v0.0.0-20230616125104-99d496ca653d h1:KbPOUXFUDJxwZ04vbmDOc3yuruGvVO+LOa7cVER3yWw= -github.com/gocarina/gocsv v0.0.0-20230616125104-99d496ca653d/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= -github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= -github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= -github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -324,14 +326,21 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomarkdown/markdown v0.0.0-20191123064959-2c17d62f5098/go.mod h1:aii0r/K0ZnHv7G0KF7xy1v0A7s2Ljrb5byB7MO5p6TU= +github.com/gomarkdown/markdown v0.0.0-20240930133441-72d49d9543d8 h1:4txT5G2kqVAKMjzidIabL/8KqjIK71yj30YOeuxLn10= +github.com/gomarkdown/markdown v0.0.0-20240930133441-72d49d9543d8/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= +github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -349,6 +358,8 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -368,12 +379,12 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= -github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20241017200806-017d972448fc h1:NGyrhhFhwvRAZg02jnYVg3GBQy0qGBKmFQJwaPmpmxs= +github.com/google/pprof v0.0.0-20241017200806-017d972448fc/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -388,10 +399,8 @@ github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= -github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= -github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA= +github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0= @@ -413,8 +422,8 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= -github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -424,81 +433,36 @@ github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/ github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= -github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= -github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= +github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8= +github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/itchyny/gojq v0.12.13 h1:IxyYlHYIlspQHHTE0f3cJF0NKDMfajxViuhBLnHd/QU= -github.com/itchyny/gojq v0.12.13/go.mod h1:JzwzAqenfhrPUuwbmEz3nu3JQmFLlQTQMUcOdnu/Sf4= -github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE= -github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= +github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g= +github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM= +github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q= +github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg= github.com/ivanpirog/coloredcobra v1.0.1 h1:aURSdEmlR90/tSiWS0dMjdwOvCVUeYLfltLfbgNxrN4= github.com/ivanpirog/coloredcobra v1.0.1/go.mod h1:iho4nEKcnwZFiniGSdcgdvRgZNjxm+h20acv8vqmN6Q= -github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= -github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= -github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= -github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= -github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= -github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= -github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= -github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.12.1/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono= -github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= -github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= -github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= -github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= -github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= -github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= -github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= -github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= -github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= -github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= -github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= -github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.11.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= -github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= -github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= -github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= -github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= -github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= -github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ= -github.com/jackc/pgx/v4 v4.18.2 h1:xVpYkNR5pk5bMCZGfClbO962UIqVABcAGt7ha1s/FeU= -github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= -github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= +github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/james-barrow/golang-ipc v1.2.4 h1:d4NXRQxq6OWviWU8uAaob8R0YZGy/PhAkXGLpBNpkA4= github.com/james-barrow/golang-ipc v1.2.4/go.mod h1:+egiWSbOWmiPucFGSl4GNB1YSzrVGehyl7/7pW4N8F0= -github.com/jaypipes/ghw v0.12.0 h1:xU2/MDJfWmBhJnujHY9qwXQLs3DBsf0/Xa9vECY0Tho= -github.com/jaypipes/ghw v0.12.0/go.mod h1:jeJGbkRB2lL3/gxYzNYzEDETV1ZJ56OKr+CSeSEym+g= -github.com/jaypipes/pcidb v1.0.0 h1:vtZIfkiCUE42oYbJS0TAq9XSfSmcsgo9IdxSm9qzYU8= -github.com/jaypipes/pcidb v1.0.0/go.mod h1:TnYUvqhPBzCKnH34KrIX22kAeEbDCSRJ9cqLRCuNDfk= +github.com/jaypipes/ghw v0.13.0 h1:log8MXuB8hzTNnSktqpXMHc0c/2k/WgjOMSUtnI1RV4= +github.com/jaypipes/ghw v0.13.0/go.mod h1:In8SsaDqlb1oTyrbmTC14uy+fbBMvp+xdqX51MidlD8= +github.com/jaypipes/pcidb v1.0.1 h1:WB2zh27T3nwg8AE8ei81sNRb9yWBii3JGNJtT7K9Oic= +github.com/jaypipes/pcidb v1.0.1/go.mod h1:6xYUz/yYEyOkIkUt2t2J2folIuZ4Yg6uByCGFXMCeE4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -514,21 +478,19 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= -github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/reedsolomon v1.12.4 h1:5aDr3ZGoJbgu/8+j45KtUJxzYm8k08JGtB9Wx1VQ4OA= +github.com/klauspost/reedsolomon v1.12.4/go.mod h1:d3CzOMOt0JXGIFZm1StgkyF14EYr3xneR2rNWo7NcMU= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -538,25 +500,25 @@ github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NB github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kyokomi/emoji/v2 v2.2.8/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE= +github.com/kyokomi/emoji/v2 v2.2.13 h1:GhTfQa67venUUvmleTNFnb+bi7S3aocF7ZCXU9fSO7U= +github.com/kyokomi/emoji/v2 v2.2.13/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4= github.com/lithammer/fuzzysearch v1.1.8/go.mod h1:IdqeyBClc3FFqSzYq/MXESsS4S0FsZ5ajtkr5xPLts4= +github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -566,19 +528,18 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= @@ -607,10 +568,10 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= -github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4= +github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/orandin/lumberjackrus v1.0.1 h1:7ysDQ0MHD79zIFN9/EiDHjUcgopNi5ehtxFDy8rUkWo= github.com/orandin/lumberjackrus v1.0.1/go.mod h1:xYLt6H8W93pKnQgUQaxsApS0Eb4BwHLOkxk5DVzf5H0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -618,10 +579,10 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= -github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= -github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= -github.com/pires/go-proxyproto v0.6.2 h1:KAZ7UteSOt6urjme6ZldyFm4wDe/z0ZUP0Yv0Dos0d8= -github.com/pires/go-proxyproto v0.6.2/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pires/go-proxyproto v0.8.0 h1:5unRmEAPbHXHuLjDg01CxJWf91cw3lKHc/0xzKpXEe0= +github.com/pires/go-proxyproto v0.8.0/go.mod h1:iknsfgnH8EkjrMeMyvfKByp9TiBZCKZM0jx2xmKqnVY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -656,58 +617,51 @@ github.com/pterm/pterm v0.12.31/go.mod h1:32ZAWZVXD7ZfG0s8qqHXePte42kdz8ECtRyEej github.com/pterm/pterm v0.12.33/go.mod h1:x+h2uL+n7CP/rel9+bImHD5lF3nM9vJj80k9ybiiTTE= github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5bUw8T8= github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s= -github.com/pterm/pterm v0.12.66 h1:bjsoMyUstaarzJ1NG7+1HGT7afR0JVMYsR3ooPeh4bo= -github.com/pterm/pterm v0.12.66/go.mod h1:nFuT9ZVkkCi8o4L1dtWuYPwDQxggLh4C263qG5nTLpQ= -github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= -github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= +github.com/pterm/pterm v0.12.79 h1:lH3yrYMhdpeqX9y5Ep1u7DejyHy7NSQg9qrBjF9dFT4= +github.com/pterm/pterm v0.12.79/go.mod h1:1v/gzOF1N0FsjbgTHZ1wVycRkKiatFvJSJC4IGaQAAo= +github.com/quic-go/quic-go v0.48.0 h1:2TCyvBrMu1Z25rvIAlnp2dPT4lgh/uTqLqiXVpp5AeU= +github.com/quic-go/quic-go v0.48.0/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robert-nix/ansihtml v1.0.1 h1:VTiyQ6/+AxSJoSSLsMecnkh8i0ZqOEdiRl/odOc64fc= github.com/robert-nix/ansihtml v1.0.1/go.mod h1:CJwclxYaTPc2RfcxtanEACsYuTksh4yDXcNeHHKZINE= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= -github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= -github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= +github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/skycoin/dmsg v1.3.29-0.20241004205035-c45754b16934 h1:WHQRftvtSYDj0AvLLbAdOiI20GGTTG/xtxuCowJsBx4= -github.com/skycoin/dmsg v1.3.29-0.20241004205035-c45754b16934/go.mod h1:AxsiUhS7FpllJv/gHO/OlI3mGyeznIp9zitXwKWQXeQ= +github.com/skycoin/dmsg v1.3.29-0.20241019182716-022283c93835 h1:29jU+cxqQGJCWoxGsrgk3nIF+JDipvJsQU89x6FKSZw= +github.com/skycoin/dmsg v1.3.29-0.20241019182716-022283c93835/go.mod h1:hFJtHnX+pDC6v5Z81QPbpVktMMfMNzL/kgcZmCKu3Bg= github.com/skycoin/encodertest v0.0.0-20190217072920-14c2e31898b9/go.mod h1:OQz8NXVJUWEw7PWYASZ/1BIw5GXgVMTGvrCGDlZa9+k= github.com/skycoin/noise v0.0.0-20180327030543-2492fe189ae6 h1:1Nc5EBY6pjfw1kwW0duwyG+7WliWz5u9kgk1h5MnLuA= github.com/skycoin/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:UXghlricA7J3aRD/k7p/zBObQfmBawwCxIVPVjz2Q3o= github.com/skycoin/skycoin v0.28.0 h1:0TN2tsU8CbCnGANKKxMr4jcG8RuBa8hWREeYtjDDtlk= github.com/skycoin/skycoin v0.28.0/go.mod h1:E9RvlMn8iQGX/upaIQvkHHcw/180JBqAQbRIzkLrkSQ= -github.com/skycoin/skycoin-service-discovery v1.3.28 h1:S63+IUGYCazwaw4siJ2puqQsML5PQx03m+ueN6Sgsyo= -github.com/skycoin/skycoin-service-discovery v1.3.28/go.mod h1:Qn8Iv3HWjSdUoRFpzyJl87d7WMj1E9V8qhuxGS2CawY= -github.com/skycoin/skywire-services v1.3.28 h1:Q6Cxcmf3Mn30ACYlODyaHJ6PjxRjPFxuPChbZSbWriU= -github.com/skycoin/skywire-services v1.3.28/go.mod h1:UsOT9WeQXTe2dXPpZUneqp7JWsMTjrWodB3OQH6nOoQ= +github.com/skycoin/skycoin-service-discovery v1.3.29-0.20241019190420-233c2d68bce0 h1:2F9JSOHefkyTLsREj8bMMXk1z13fJhnVa0R/XCxbUwM= +github.com/skycoin/skycoin-service-discovery v1.3.29-0.20241019190420-233c2d68bce0/go.mod h1:3/WomvXyOLcjEMV4GiLv8b+bueohWa9ceoNwtQSB9KI= +github.com/skycoin/skywire-services v1.3.29-0.20241019183828-0d760251ca55 h1:BqMp8eU21Wg7CCs7IxL8/gGqfGtmta93W/NFfEsYBiY= +github.com/skycoin/skywire-services v1.3.29-0.20241019183828-0d760251ca55/go.mod h1:Q3s3Ea+tcT3M4zkMT89ywk5NmLCdXZWHRsPhNBEo7Zw= github.com/skycoin/skywire-utilities v1.3.25 h1:mk8dUonFdhVopFF3d9wbOyXXoiuAO+mN1y+ve6SzgX4= github.com/skycoin/skywire-utilities v1.3.25/go.mod h1:yFKWpL1bDRPKU3uK+cTF4PnYUMe+eyIj5N2bk4sF5Cw= github.com/skycoin/systray v1.10.0 h1:fQZJHMylpVvfmOOTLvUssfyHVDoC8Idx6Ba2BlLEuGg= @@ -725,8 +679,8 @@ github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -738,7 +692,6 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/spf13/viper v1.13.0/go.mod h1:Icm2xNL3/8uyh/wFuB1jI7TiTNKp8632Nwegu+zgdYw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= @@ -753,8 +706,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= @@ -774,11 +725,9 @@ github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= @@ -798,11 +747,10 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zcalusic/sysinfo v1.0.1 h1:cVh8q3codjh43AGRTa54dJ2Zq+qPejv8n2VWpxKViwc= -github.com/zcalusic/sysinfo v1.0.1/go.mod h1:LxwKwtQdbTIQc65drhjQzYzt0o7jfB80LrrZm7SWn8o= -github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= -go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +github.com/zcalusic/sysinfo v1.1.2 h1:38KUgZQmCxlN9vUTt4miis4rU5ISJXGXOJ2rY7bMC8g= +github.com/zcalusic/sysinfo v1.1.2/go.mod h1:NX+qYnWGtJVPV0yWldff9uppNKU4h40hJIRPf/pGLv4= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU= @@ -813,49 +761,34 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= -go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.4.0 h1:A8WCeEWhLwPBKNbFi5Wv5UTCBx5zzubnXDlMOFAzFMc= -golang.org/x/arch v0.4.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/dl v0.0.0-20190829154251-82a15e2f2ead/go.mod h1:IUMfjQLJQd4UTqG1Z90tenwKoCX93Gn3MAQJMOSBsDQ= +golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= +golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg= -golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -866,10 +799,14 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20191206065243-da761ea9ff43/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.21.0 h1:c5qV36ajHpdj4Qi0GnE0jUc/yuo33OLFaa0d+crTD5s= +golang.org/x/image v0.21.0/go.mod h1:vUbsLavqK/W303ZroQQVKQ+Af3Yl6Uz1Ppu5J/cLz78= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -896,8 +833,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -951,8 +888,8 @@ golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -988,19 +925,19 @@ golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181023152157-44b849a8bc13/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1008,7 +945,6 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1080,21 +1016,21 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1106,8 +1042,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1120,19 +1056,15 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1140,7 +1072,6 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1176,10 +1107,8 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= -golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E= -golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= -golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1188,8 +1117,8 @@ golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= -golang.zx2c4.com/wireguard v0.0.0-20230223181233-21636207a675 h1:/J/RVnr7ng4fWPRH3xa4WtBJ1Jp+Auu4YNLmGiPv5QU= -golang.zx2c4.com/wireguard v0.0.0-20230223181233-21636207a675/go.mod h1:whfbyDBt09xhCYQWtO2+3UVjlaq6/9hDZrjg2ZE6SyA= +golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uIfPMv78iAJGcPKDeqAFnaLBropIC4= +golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1359,8 +1288,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1368,12 +1297,11 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -gopkg.in/telebot.v3 v3.2.1 h1:3I4LohaAyJBiivGmkfB+CiVu7QFOWkuZ4+KHgO/G3rs= -gopkg.in/telebot.v3 v3.2.1/go.mod h1:GJKwwWqp9nSkIVN51eRKU78aB5f5OnQuWdwiIZfPbko= +gopkg.in/telebot.v3 v3.3.8 h1:uVDGjak9l824FN9YARWUHMsiNZnlohAVwUycw21k6t8= +gopkg.in/telebot.v3 v3.3.8/go.mod h1:1mlbqcLTVSfK9dx7fdp+Nb5HZsy4LLPtpZTKmwhwtzM= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= @@ -1384,17 +1312,17 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/postgres v1.3.8 h1:8bEphSAB69t3odsCR4NDzt581iZEWQuRM27Cg6KgfPY= -gorm.io/driver/postgres v1.3.8/go.mod h1:qB98Aj6AhRO/oyu/jmZsi/YM9g6UzVCjMxO/6frFvcA= -gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE= -gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= +gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= +gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= +gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259 h1:TbRPT0HtzFP3Cno1zZo7yPzEEnfu8EjLfl6IU9VfqkQ= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259/go.mod h1:AVgIgHMwK63XvmAzWG9vLQ41YnVHN0du0tEC46fI7yY= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1402,15 +1330,12 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= -howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= -mvdan.cc/sh/v3 v3.7.0 h1:lSTjdP/1xsddtaKfGg7Myu7DnlHItd3/M2tomOcNNBg= -mvdan.cc/sh/v3 v3.7.0/go.mod h1:K2gwkaesF/D7av7Kxl0HbF5kGOd2ArupNTX3X44+8l8= -nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= -nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= +howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM= +howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= +mvdan.cc/sh/v3 v3.9.0 h1:it14fyjCdQUk4jf/aYxLO3FG8jFarR9GzMCtnlvvD7c= +mvdan.cc/sh/v3 v3.9.0/go.mod h1:cdBk8bgoiBI7lSZqK5JhUuq7OB64VQ7fgm85xelw3Nk= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/mainnet_rules.md b/mainnet_rules.md index 1e93f17d44..22545022c6 100644 --- a/mainnet_rules.md +++ b/mainnet_rules.md @@ -87,17 +87,21 @@ To receive Skycoin rewards for running skywire, the following requirements must All the production deployment services may be accessed by the visor over the dmsg network when the visor runs with a dmsghttp config. -This type of config is generated automatically based on region (via `skywire cli config gen -b --bestproto`), to circumvent ISP blocking of http requests. +This type of config is generated automatically based on region via: +``` +skywire cli config gen -b --bestproto +``` +to circumvent ISP blocking of http requests. -In order to bootstrap the visor's to connection to the dmsg network (via TCP connection to an individual dmsg server) the `dmsghttp-config.json` is provided with the skywire binary release. +In order to bootstrap the visor's to connection to the dmsg network (via TCP connection to an individual dmsg server) the [dmsghttp-config.json](/dmsghttp-config.json) is provided with the skywire binary release. -In the instance that the skywire production deployment changes - specifically the dmsg servers - it will be necessary to update to the next version or package release which fixes the dmsg servers - or else to manually update the `dmsghttp-config.json` which is provided by your skywire installation. +In the instance that the skywire production deployment changes - specifically the dmsg servers - it will be necessary to update to the next version or package release which fixes the dmsg servers - or else to manually update the [dmsghttp-config.json](/dmsghttp-config.json) which is provided by your skywire installation. Currently, **there is no mechanism for updating the dmsghttp-config.json which does not require an http request** ; a request which may be blocked depending on region. -In this instance, the visor will not connect to any service because it is not connected to the dmsg network, so it will not be possible for the visor to accumulate uptime or for the reward system to collect the survey, which are prerequisite for reward eligibility. +In this instance, the visor will not connect to any service because it is not connected to the dmsg network, so it will not be possible for the visor to accumulate uptime or for the reward system to collect the survey, which are prerequisites for reward eligibility. -As a consequence of this; any visors running a dmsghttp-config, and hence any visors running in regions such as China, the minimum version requirement for obtaining rewards is not only the latest available version, but __the latest release of the package__ unless the dmsghttp-config.json is updated manually. +As a consequence of this; any visors running a dmsghttp-config, and hence any visors running in regions such as China, the minimum version requirement for obtaining rewards is not only the latest available version, but __the latest release of the package__ unless the dmsghttp-config.json is updated manually within your installation. ## Verifying Requirements & Eligibility @@ -126,7 +130,7 @@ Rewards Cutoff date for updating 10-1-2024 ### Deployment -The deployment your visor is running on can be verified by comparing the services configured in the visor's `.json` config against [conf.skywire.skycoin.com](https://conf.skywire.skycoin.com) +The deployment your visor is running on can be verified by comparing the services configured in the visor's .json config against [conf.skywire.skycoin.com](https://conf.skywire.skycoin.com) ``` cat /opt/skywire/skywire.json @@ -136,13 +140,25 @@ The service configuration will be automatically updated any time a config is gen For those visors in china or those running a dmsghttp-config, compare the dmsghttp-config of your current installation with the dmsghttp-config on the develop branch of [github.com/skycoin/skywire](https://github.com/skycoin/skywire) +The same data in a different format should be displayed in the [dmsg-discovery all_servers](https://dmsgd.skywire.skycoin.com/dmsg-discovery/all_servers) page. Ensure that the dmsghttp-config.json in your installation has the same ip addresses and ports for the dmsg server keys. The data from the dmsg discovery should be considered authoritative or current. ### Uptime Daily uptime statistics for all visors may be accessed via the + - [uptime tracker](https://ut.skywire.skycoin.com/uptimes?v=v2) + or using skywire cli -- `skywire cli ut -n0 -k` + +``` +skywire cli ut -n0 -k +``` + +For example with a locally running visor: +``` +skywire cli ut -n0 -k $(skywire cli visor pk) +``` + ### Skycoin Address @@ -155,23 +171,76 @@ skywire cli reward ![image](https://user-images.githubusercontent.com/36607567/213941582-f57213b8-2acd-4c9a-a2c0-9089a8c2604e.png) +``` +$ skywire cli reward --help + + reward address setting + + Sets the skycoin reward address for the visor. + The config is written to the root of the default local directory + + this config is served via dmsghttp along with transport logs + and the system hardware survey for automating reward distribution + +Flags: + --all show all flags + +``` + +``` +$ skywire cli reward 2jBbGxZRGoQG1mqhPBnXnLTxK6oxsTf8os6 +Reward address: + 2jBbGxZRGoQG1mqhPBnXnLTxK6oxsTf8os6 +``` + +``` +$ skywire cli reward --help + + skycoin reward address set to: + 2jBbGxZRGoQG1mqhPBnXnLTxK6oxsTf8os6 + +Flags: + --all show all flags +``` + or via the hypervisor UI. ![image](https://user-images.githubusercontent.com/36607567/213941478-34c81493-9c3c-40c2-ac22-e33e3683a16f.png) the example above shows the genesis address for the skycoin blockchain. **Please do not use the genesis address.** -It is __highly recommended__ to set the reward address in the file `/etc/skywire.conf` by adding this line to the file: +It is __highly recommended__ to set the reward address in the file '/etc/skywire.conf' by adding this line to the file: ``` -REWARDSKYADDR=('') +echo "REWARDSKYADDR=('2jBbGxZRGoQG1mqhPBnXnLTxK6oxsTf8os6')" | sudo tee -a /etc/skywire.conf ``` -Add your skycoin address there and run `skywire-autoconfig` on linux (assumes you have installed the package) +**PLEASE DO NOT USE THE GENESIS ADDRESS!** -If this file does not exist for you, it can be created with `skywire cli config gen -q | tee /etc/skywire.conf` -**If you do this, please uncomment `PKGENV=true` before saving the file** +Add your skycoin address there and run: +``` +skywire-autoconfig +``` + +on linux (assumes you have installed the package) + +If this file does not exist for you, it can be created with +``` +skywire cli config gen -q | sudo tee /etc/skywire.conf +``` + +**If you do this, YOU MUST UNCOMMENT THE FOLLOWING LINES OF THE FILE:** +``` +PKGENV=true +BESTPROTO=TRUE +``` + +you can open the file in an editor to make that change, for instance nano + +``` +sudo nano /etc/skywire.conf +``` ### Connection to DMSG network @@ -181,6 +250,12 @@ This can be verified by examining the visor's logging: ![image](https://github.com/skycoin/skywire/assets/36607567/eb66bca1-fc9e-4c80-a38a-e00a73f675d0) +``` +[DMSGHTTP] 2024/10/09 - 22:31:45 | 200 | 47.2µs | | 03714c8bdaee0fb48f47babbc47c33e1880752b6620317c9d56b30f3b0ff58a9c3:51405 | GET /health +[DMSGHTTP] 2024/10/09 - 22:31:46 | 200 | 193.325µs | | 03714c8bdaee0fb48f47babbc47c33e1880752b6620317c9d56b30f3b0ff58a9c3:51457 | GET /node-info +[DMSGHTTP] 2024/10/09 - 22:31:47 | 200 | 98.93µs | | 03714c8bdaee0fb48f47babbc47c33e1880752b6620317c9d56b30f3b0ff58a9c3:51503 | GET /2024-10-10.csv +``` + The collected surveys and transport bandwidth log files should be visible in the survey index here: [fiber.skywire.dev/log-collection/tree](https://fiber.skywire.dev/log-collection/tree) @@ -227,12 +302,21 @@ In the future, it is anticipated that the transport bandwidth logs and ping metr ### Survey -On setting the skycoin reward address, the visor will generate and serve a sysytem survey over dmsg. Only keys which are whitelisted in the survey_whitelist array of the visor's config will have access to collect the survey. +On setting the skycoin reward address, the visor will generate and serve a system survey over dmsg. + +Only keys which are whitelisted in the survey_whitelist array of the visor's config will have access to collect the survey. -To print the survey (approximately) as it would be served, one can use `skywire cli survey` +To print the survey (approximately) as it would be served, one can use: +``` +skywire cli survey -p +``` **The purpose of the survey is strictly for checking [eligibility requirements](#rules--requirements) numbers 3 through 7.** +**Setting a skycoin address is considered as consent for collecting the survey ; the survey is not generated unless you set a skycoin address** + +We respect your privacy. + ### Verifying other requirements If the visor is not able to meet the [eligibility requirements](#rules--requirements) numbers 8 through 13, that is usually not the fault of the user - nor is it something the user is expected to troubleshoot on their own at this time. Please ask for assistance on telegram [@skywire](https://t.me/skywire) @@ -245,9 +329,13 @@ The skycoin reward address is in a text file contained in the "local" folder (lo The skycoin reward address is also included with the [system hardware survey](https://github.com/skycoin/skywire/tree/develop/cmd/skywire/README.md#survey) and served, along with transport logs, via dmsghttp. -The system survey (`local/node-info.json`) is fetched hourly by the reward system via `skywire cli log`; along with transport bandwidth logs. +The system survey ('local/node-info.json') is fetched hourly by the reward system via +``` +skywire cli log +``` +along with transport bandwidth logs. -A survey of transports which have been set by transport setup-nodes are also collected hourly, from all visors which have had surveys collected +~~A survey of transports which have been set by transport setup-nodes are also collected hourly, from all visors which have had surveys collected~~ The index of the collected files may be viewed at [fiber.skywire.dev/log-collection/tree](https://fiber.skywire.dev/log-collection/tree) @@ -261,7 +349,7 @@ The public keys which require to be whitelisted in order to collect the surveys, ## Reward System Funding & Distributions -The reward system is funded on a monthly basis. Sometimes there are unexpected or unavoidable delays in the funding. In these instances, rewards will be distributed based on the data generated wen the system is funded +The reward system is funded on a monthly basis. **Sometimes there are unexpected or unavoidable delays in the funding.** In these instances, rewards will be distributed based on the data generated when the system is funded. In some instances, it's necessary to discard previous reward data and do multiple distributions to handle backlog of reward system funds. We do our best to ensure fair reward distribution, but the system itself is not infinitely flexible. If there is no good way to rectify historical or undistributed rewards backlog, it will be distributed going forward as multiple distributions on the same day. ## Deployment Outages @@ -271,166 +359,4 @@ The policy for handling rewards in the instance of a deployment outage is to rep ## Hardware -**Virtual Machines, servers, and personal computers are currently not eligible to collect rewards** - -_NOTE: this list of hardware was used with the initial implementation of the reward system - which required an application with pictures to be submitted. This no longer applies. The new reward system does not differentiate hardware except by architecture - only x86_64 / amd64 hardware is excluded from rewards currently. Every visor will receive the same reward amount, regardless of the hardware specs. It should be noted that the main difference between using cheap or low-end hardware and high-end hardware is the ease of maintenance and speed of updates for the node operator._ - -The following hardware is eligible for rewards: - -#### Orange Pi - - Prime - - 2G-IOT - - 4G-IOT - - i96 - - Lite - - Lite2 - - One - - One-Plus - - PC - - PC-Plus - - PC2 - - Plus - - Plus2 - - Plus2E - - RK3399 - - Win - - Win-Plus - - Zero - - Zero LTS - - Zero-Plus - - Zero-Plus2 - - 3 - -#### Raspberry Pi - - 1-Model-A+ - - 1-Model-B+ - - 2-Model-B - - 3-Model-B - - 3-Model-B+ - - 4-Model-B - - Compute Module 3 - - Compute Module 4 - - Zero-W - - Zero - -#### Asus - - Tinkerboard - -#### Banana Pi - - BPI-D1 - - BPI-G1 - - BPI-M1 - - BPI-M1+ - - BPI-M2 - - BPI-M2+ - - BPI-M2-Berry - - BPI-M2M - - BPI-M2U - - BPI-M64 - - BPI-R2 - - BPI-R3 - - BPI-Zero - -#### Beelink - - X2 - -#### Cubieboard - - Cubietruck - - Cubietruck-Plus - - 1 - - 2 - - 4 - -#### Geniatech - - Developer Board IV - -#### Helios - - 4 - -#### Libre Computer - - Le-Potato-AML-S905X-CC - - Renegade-ROC-RK3328-CC - - Tritium-ALL-H3-CC - -#### MQMaker - - MiQi - -#### NanoPi - - NanoPi - - 2 - - 2-Fire - - A64 - - K2 - - M1 - - M1-plus - - M2 - - M2A - - M3 - - M4 - - NEO - - NEO-Air - - NEO-Core - - NEO-Core2 - - NEO2 - - NEO2-Black - - S2 - - Smart4418 - -#### Odroid - - C2 - - C4 - - HC1 - - HC2 - - MC1 - - XU4 - -#### Olimex - - Lime1 - - Lime2 - - Lime2-eMMC - - LimeA33 - - Micro - -#### Pine - - Pine-A64 - - Pinebook-A64 - - Sopine-A64 - - Rock64 - - ROCKPro64 - -#### ROCKPI - - Rockpi 4 - - Rockpi S - - Rockpi E - - Rockpi N10 - -#### SolidRun - - CuBox-i - - CuBox-Pulse - - Humming-Board - - Humming-Board-Pulse - - ClearCloud-8K - - ClearFog-A38 - - ClearFog-GT-8K - -#### Udoo - - Blu - - Bricks - - Dual - - Neo - - Quad - - X86 - -#### X96 Android TV Box - - X96 mini - -#### Dolamee - - A95X F1 Smart TV Box - -#### Radxa - - ROCK Pi S - -#### ZTE - - ZXV10 B860H - -~~If you would like to use other boards please contact the team first for approval ; only the boards on the list are guaranteed to be eligible for rewards.~~ +As of November 2024, skywire rewards are open to all computer hardware and architectures. diff --git a/skywire.go b/skywire.go index ee99b8062b..47a18175b9 100644 --- a/skywire.go +++ b/skywire.go @@ -1,4 +1,6 @@ -// Package skywire skywire.go +// Package skywire github.com/skycoin/skywire/skywire.go +// +//go:generate go run cmd/gen/gen.go -jo arches.json package skywire import ( @@ -25,6 +27,22 @@ var ServicesJSON []byte //go:embed dmsghttp-config.json var DmsghttpJSON []byte +// ArchesJSON is the embedded arches.json file +// go run cmd/gen/gen.go -jo arches.json +// ["amd64","arm64","386","arm","ppc64","riscv64","wasm","loong64","mips","mips64","mips64le","mipsle","ppc64le","s390x"] +// +//go:embed arches.json +var ArchesJSON []byte + +// Architectures is an array of GOARCH architectures +var Architectures []string + +// MainnetRules is the mainnet_rules.md ; it shouldn't have to be embedded here but goland doesn't allow 'embed ../../../../mainnet_rules.md' +// print the mainnet rules with `skywire cli reward rules` +// +//go:embed mainnet_rules.md +var MainnetRules string + // EnvServices is the wrapper struct for the outer JSON - i.e. 'prod' or 'test' deployment config type EnvServices struct { Test json.RawMessage `json:"test"` @@ -64,8 +82,12 @@ var Test Services var TestConf Conf func init() { + err := json.Unmarshal(ArchesJSON, &Architectures) + if err != nil { + log.Panic("arches.json ", err) + } var js interface{} - err := json.Unmarshal([]byte(ServicesJSON), &js) + err = json.Unmarshal([]byte(ServicesJSON), &js) if err != nil { log.Panic("services-config.json ", err) } diff --git a/vendor/github.com/MichaelMure/go-term-markdown/.gitignore b/vendor/github.com/MichaelMure/go-term-markdown/.gitignore new file mode 100644 index 0000000000..9f11b755a1 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-markdown/.gitignore @@ -0,0 +1 @@ +.idea/ diff --git a/vendor/github.com/MichaelMure/go-term-markdown/.travis.yml b/vendor/github.com/MichaelMure/go-term-markdown/.travis.yml new file mode 100644 index 0000000000..bd3e355621 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-markdown/.travis.yml @@ -0,0 +1,15 @@ +language: go + +go: + - 1.12.x + - 1.13.x + +env: + - GO111MODULE=on + +script: + - go build + - go test -v -bench=. -race -coverprofile=coverage.txt -covermode=atomic ./... + +after_success: + - bash <(curl -s https://codecov.io/bash) diff --git a/vendor/github.com/MichaelMure/go-term-markdown/LICENSE b/vendor/github.com/MichaelMure/go-term-markdown/LICENSE new file mode 100644 index 0000000000..5ba12bf4c5 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-markdown/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Michael Muré + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/MichaelMure/go-term-markdown/Readme.md b/vendor/github.com/MichaelMure/go-term-markdown/Readme.md new file mode 100644 index 0000000000..942f06553c --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-markdown/Readme.md @@ -0,0 +1,67 @@ +# go-term-markdown + +[![Build Status](https://travis-ci.com/MichaelMure/go-term-markdown.svg?branch=master)](https://travis-ci.com/MichaelMure/go-term-markdown) +[![GoDoc](https://godoc.org/github.com/MichaelMure/go-term-markdown?status.svg)](https://godoc.org/github.com/MichaelMure/go-term-markdown) +[![Go Report Card](https://goreportcard.com/badge/github.com/MichaelMure/go-term-markdown)](https://goreportcard.com/report/github.com/MichaelMure/go-term-markdown) +[![codecov](https://codecov.io/gh/MichaelMure/go-term-markdown/branch/master/graph/badge.svg)](https://codecov.io/gh/MichaelMure/go-term-markdown) +[![GitHub license](https://img.shields.io/github/license/MichaelMure/go-term-markdown.svg)](https://github.com/MichaelMure/go-term-markdown/blob/master/LICENSE) +[![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/the-git-bug/Lobby) + +`go-term-markdown` is a go package implementing a Markdown renderer for the terminal. + +Note: Markdown being originally designed to render as HTML, rendering in a terminal is occasionally challenging and some adaptation had to be made. + +Features: +- formatting +- lists +- tables +- images +- code blocks with syntax highlighting +- basic HTML support + +Note: this renderer is packaged as a standalone terminal viewer at https://github.com/MichaelMure/mdr/ + +## Usage + +```go +import ( + "fmt" + "io/ioutil" + + markdown "github.com/MichaelMure/go-term-markdown" +) + +func main() { + path := "Readme.md" + source, err := ioutil.ReadFile(path) + if err != nil { + panic(err) + } + + result := markdown.Render(string(source), 80, 6) + + fmt.Println(result) +} +``` + +## Example + +Here is the [Readme](https://github.com/MichaelMure/go-term-text/blob/v0.2.4/Readme.md) of `go-term-text` rendered with `go-term-markdown`: + +![rendering example](misc/result.png) + +Here is an example of table rendering: + +![table rendering](misc/table.png) + +## Origin + +This package has been extracted from the [git-bug](https://github.com/MichaelMure/git-bug) project. As such, its aim is to support this project and not to provide an all-in-one solution. Contributions or full-on takeover as welcome though. + +## Contribute + +PRs accepted. + +## License + +MIT diff --git a/vendor/github.com/MichaelMure/go-term-markdown/colors.go b/vendor/github.com/MichaelMure/go-term-markdown/colors.go new file mode 100644 index 0000000000..de1339cd09 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-markdown/colors.go @@ -0,0 +1,24 @@ +package markdown + +import "github.com/fatih/color" + +var ( + // we need a bunch of escape code for manual formatting + boldOn = "\x1b[1m" + // boldOff = "\x1b[21m" --> use resetAll + snapshot with bold off instead + italicOn = "\x1b[3m" + italicOff = "\x1b[23m" + crossedOutOn = "\x1b[9m" + crossedOutOff = "\x1b[29m" + greenOn = "\x1b[32m" + + resetAll = "\x1b[0m" + colorOff = "\x1b[39m" + + Green = color.New(color.FgGreen).SprintFunc() + HiGreen = color.New(color.FgHiGreen).SprintFunc() + GreenBold = color.New(color.FgGreen, color.Bold).SprintFunc() + Blue = color.New(color.FgBlue).SprintFunc() + BlueBgItalic = color.New(color.BgBlue, color.Italic).SprintFunc() + Red = color.New(color.FgRed).SprintFunc() +) diff --git a/vendor/github.com/MichaelMure/go-term-markdown/html/html.go b/vendor/github.com/MichaelMure/go-term-markdown/html/html.go new file mode 100644 index 0000000000..9245192e66 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-markdown/html/html.go @@ -0,0 +1,78 @@ +package html + +import . "golang.org/x/net/html" + +// WalkStatus allows NodeVisitor to have some control over the tree traversal. +// It is returned from NodeVisitor and different values allow Node.Walk to +// decide which node to go to next. +type WalkStatus int + +const ( + // GoToNext is the default traversal of every node. + GoToNext WalkStatus = iota + // SkipChildren tells walker to skip all children of current node. + SkipChildren + // Terminate tells walker to terminate the traversal. + Terminate +) + +// NodeVisitor is a callback to be called when traversing the syntax tree. +// Called twice for every node: once with entering=true when the branch is +// first visited, then with entering=false after all the children are done. +type NodeVisitor interface { + Visit(node *Node, entering bool) WalkStatus +} + +func Walk(n *Node, visitor NodeVisitor) WalkStatus { + isContainer := n.FirstChild != nil + + // some special case that are container but can be self closing + if n.Type == ElementNode { + switch n.Data { + case "td": + isContainer = true + } + } + + status := visitor.Visit(n, true) + + if status == Terminate { + // even if terminating, close container node + if isContainer { + visitor.Visit(n, false) + } + } + + if isContainer && status != SkipChildren { + child := n.FirstChild + for child != nil { + status = Walk(child, visitor) + if status == Terminate { + return status + } + child = child.NextSibling + } + } + + if isContainer { + status = visitor.Visit(n, false) + if status == Terminate { + return status + } + } + + return GoToNext +} + +// NodeVisitorFunc casts a function to match NodeVisitor interface +type NodeVisitorFunc func(node *Node, entering bool) WalkStatus + +// Visit calls visitor function +func (f NodeVisitorFunc) Visit(node *Node, entering bool) WalkStatus { + return f(node, entering) +} + +// WalkFunc is like Walk but accepts just a callback function +func WalkFunc(n *Node, f NodeVisitorFunc) { + Walk(n, f) +} diff --git a/vendor/github.com/MichaelMure/go-term-markdown/markdown.go b/vendor/github.com/MichaelMure/go-term-markdown/markdown.go new file mode 100644 index 0000000000..0a3f771d72 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-markdown/markdown.go @@ -0,0 +1,33 @@ +package markdown + +import ( + md "github.com/gomarkdown/markdown" + "github.com/gomarkdown/markdown/parser" +) + +// Extensions returns the bitmask of extensions supported by this renderer. +// The output of this function can be used to instantiate a new markdown +// parser using the `NewWithExtensions` function. +func Extensions() parser.Extensions { + extensions := parser.NoIntraEmphasis // Ignore emphasis markers inside words + extensions |= parser.Tables // Parse tables + extensions |= parser.FencedCode // Parse fenced code blocks + extensions |= parser.Autolink // Detect embedded URLs that are not explicitly marked + extensions |= parser.Strikethrough // Strikethrough text using ~~test~~ + extensions |= parser.SpaceHeadings // Be strict about prefix heading rules + extensions |= parser.HeadingIDs // specify heading IDs with {#id} + extensions |= parser.BackslashLineBreak // Translate trailing backslashes into line breaks + extensions |= parser.DefinitionLists // Parse definition lists + extensions |= parser.LaxHTMLBlocks // more in HTMLBlock, less in HTMLSpan + extensions |= parser.NoEmptyLineBeforeBlock // no need for new line before a list + + return extensions +} + +func Render(source string, lineWidth int, leftPad int, opts ...Options) []byte { + p := parser.NewWithExtensions(Extensions()) + nodes := md.Parse([]byte(source), p) + renderer := NewRenderer(lineWidth, leftPad, opts...) + + return md.Render(nodes, renderer) +} diff --git a/vendor/github.com/MichaelMure/go-term-markdown/numbering.go b/vendor/github.com/MichaelMure/go-term-markdown/numbering.go new file mode 100644 index 0000000000..94e4ba8764 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-markdown/numbering.go @@ -0,0 +1,47 @@ +package markdown + +import "strconv" + +type headingNumbering struct { + levels [6]int +} + +// Observe register the event of a new level with the given depth and +// adjust the numbering accordingly +func (hn *headingNumbering) Observe(level int) { + if level <= 0 { + panic("level start at 1, ask blackfriday why") + } + if level > 6 { + panic("Markdown is limited to 6 levels of heading") + } + + hn.levels[level-1]++ + for i := level; i < 6; i++ { + hn.levels[i] = 0 + } +} + +// Render render the current headings numbering. +func (hn *headingNumbering) Render() string { + slice := hn.levels[:] + + // pop the last zero levels + for i := 5; i >= 0; i-- { + if hn.levels[i] != 0 { + break + } + slice = slice[:len(slice)-1] + } + + var result string + + for i := range slice { + if i > 0 { + result += "." + } + result += strconv.Itoa(slice[i]) + } + + return result +} diff --git a/vendor/github.com/MichaelMure/go-term-markdown/options.go b/vendor/github.com/MichaelMure/go-term-markdown/options.go new file mode 100644 index 0000000000..9edd1580f9 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-markdown/options.go @@ -0,0 +1,37 @@ +package markdown + +import "github.com/eliukblau/pixterm/pkg/ansimage" + +type Options func(r *renderer) + +// DitheringMode type is used for image scale dithering mode constants. +type DitheringMode uint8 + +const ( + NoDithering = DitheringMode(iota) + DitheringWithBlocks + DitheringWithChars +) + +// Dithering mode for ansimage +// Default is fine directly through a terminal +// DitheringWithBlocks is recommended if a terminal UI library is used +func WithImageDithering(mode DitheringMode) Options { + return func(r *renderer) { + r.imageDithering = ansimage.DitheringMode(mode) + } +} + +// Use a custom collection of ANSI colors for the headings +func WithHeadingShades(shades []shadeFmt) Options { + return func(r *renderer) { + r.headingShade = shade(shades) + } +} + +// Use a custom collection of ANSI colors for the blockquotes +func WithBlockquoteShades(shades []shadeFmt) Options { + return func(r *renderer) { + r.blockQuoteShade = shade(shades) + } +} diff --git a/vendor/github.com/MichaelMure/go-term-markdown/renderer.go b/vendor/github.com/MichaelMure/go-term-markdown/renderer.go new file mode 100644 index 0000000000..719063a622 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-markdown/renderer.go @@ -0,0 +1,975 @@ +package markdown + +import ( + "bytes" + "fmt" + stdcolor "image/color" + "io" + "math" + "net/http" + "os" + "strings" + "time" + "unicode" + + "github.com/MichaelMure/go-term-text" + "github.com/alecthomas/chroma" + "github.com/alecthomas/chroma/formatters" + "github.com/alecthomas/chroma/lexers" + "github.com/alecthomas/chroma/styles" + "github.com/eliukblau/pixterm/pkg/ansimage" + "github.com/fatih/color" + md "github.com/gomarkdown/markdown" + "github.com/gomarkdown/markdown/ast" + "github.com/kyokomi/emoji/v2" + "golang.org/x/net/html" + + htmlWalker "github.com/MichaelMure/go-term-markdown/html" +) + +/* + +Here are the possible cases for the AST. You can render it using PlantUML. + +@startuml + +(*) --> Document +BlockQuote --> BlockQuote +BlockQuote --> CodeBlock +BlockQuote --> List +BlockQuote --> Paragraph +Del --> Emph +Del --> Strong +Del --> Text +Document --> BlockQuote +Document --> CodeBlock +Document --> Heading +Document --> HorizontalRule +Document --> HTMLBlock +Document --> List +Document --> Paragraph +Document --> Table +Emph --> Text +Heading --> Code +Heading --> Del +Heading --> Emph +Heading --> HTMLSpan +Heading --> Image +Heading --> Link +Heading --> Strong +Heading --> Text +Image --> Text +Link --> Image +Link --> Text +ListItem --> List +ListItem --> Paragraph +List --> ListItem +Paragraph --> Code +Paragraph --> Del +Paragraph --> Emph +Paragraph --> Hardbreak +Paragraph --> HTMLSpan +Paragraph --> Image +Paragraph --> Link +Paragraph --> Strong +Paragraph --> Text +Strong --> Emph +Strong --> Text +TableBody --> TableRow +TableCell --> Code +TableCell --> Del +TableCell --> Emph +TableCell --> HTMLSpan +TableCell --> Image +TableCell --> Link +TableCell --> Strong +TableCell --> Text +TableHeader --> TableRow +TableRow --> TableCell +Table --> TableBody +Table --> TableHeader + +@enduml + +*/ + +var _ md.Renderer = &renderer{} + +type renderer struct { + // maximum line width allowed + lineWidth int + // constant left padding to apply + leftPad int + + // Dithering mode for ansimage + // Default is fine directly through a terminal + // DitheringWithBlocks is recommended if a terminal UI library is used + imageDithering ansimage.DitheringMode + + // all the custom left paddings, without the fixed space from leftPad + padAccumulator []string + + // one-shot indent for the first line of the inline content + indent string + + // for Heading, Paragraph, HTMLBlock and TableCell, accumulate the content of + // the child nodes (Link, Text, Image, formatting ...). The result + // is then rendered appropriately when exiting the node. + inlineAccumulator strings.Builder + + // record and render the heading numbering + headingNumbering headingNumbering + headingShade levelShadeFmt + + blockQuoteLevel int + blockQuoteShade levelShadeFmt + + table *tableRenderer +} + +/// NewRenderer creates a new instance of the console renderer +func NewRenderer(lineWidth int, leftPad int, opts ...Options) *renderer { + r := &renderer{ + lineWidth: lineWidth, + leftPad: leftPad, + padAccumulator: make([]string, 0, 10), + headingShade: shade(defaultHeadingShades), + blockQuoteShade: shade(defaultQuoteShades), + } + for _, opt := range opts { + opt(r) + } + return r +} + +func (r *renderer) pad() string { + return strings.Repeat(" ", r.leftPad) + strings.Join(r.padAccumulator, "") +} + +func (r *renderer) addPad(pad string) { + r.padAccumulator = append(r.padAccumulator, pad) +} + +func (r *renderer) popPad() { + r.padAccumulator = r.padAccumulator[:len(r.padAccumulator)-1] +} + +func (r *renderer) RenderNode(w io.Writer, node ast.Node, entering bool) ast.WalkStatus { + // TODO: remove + // if node.AsLeaf() != nil { + // fmt.Printf("%T, %v (%s)\n", node, entering, string(node.AsLeaf().Literal)) + // } else { + // fmt.Printf("%T, %v\n", node, entering) + // } + + switch node := node.(type) { + case *ast.Document: + // Nothing to do + + case *ast.BlockQuote: + // set and remove a colored bar on the left + if entering { + r.blockQuoteLevel++ + r.addPad(r.blockQuoteShade(r.blockQuoteLevel)("┃ ")) + } else { + r.blockQuoteLevel-- + r.popPad() + } + + case *ast.List: + // extra new line at the end of a list *if* next is not a list + if next := ast.GetNextNode(node); !entering && next != nil { + _, parentIsListItem := node.GetParent().(*ast.ListItem) + _, nextIsList := next.(*ast.List) + if !nextIsList && !parentIsListItem { + _, _ = fmt.Fprintln(w) + } + } + + case *ast.ListItem: + // write the prefix, add a padding if needed, and let Paragraph handle the rest + if entering { + switch { + // numbered list + case node.ListFlags&ast.ListTypeOrdered != 0: + itemNumber := 1 + siblings := node.GetParent().GetChildren() + for _, sibling := range siblings { + if sibling == node { + break + } + itemNumber++ + } + prefix := fmt.Sprintf("%d. ", itemNumber) + r.indent = r.pad() + Green(prefix) + r.addPad(strings.Repeat(" ", text.Len(prefix))) + + // header of a definition + case node.ListFlags&ast.ListTypeTerm != 0: + r.inlineAccumulator.WriteString(greenOn) + + // content of a definition + case node.ListFlags&ast.ListTypeDefinition != 0: + r.addPad(" ") + + // no flags means it's the normal bullet point list + default: + r.indent = r.pad() + Green("• ") + r.addPad(" ") + } + } else { + switch { + // numbered list + case node.ListFlags&ast.ListTypeOrdered != 0: + r.popPad() + + // header of a definition + case node.ListFlags&ast.ListTypeTerm != 0: + r.inlineAccumulator.WriteString(colorOff) + + // content of a definition + case node.ListFlags&ast.ListTypeDefinition != 0: + r.popPad() + _, _ = fmt.Fprintln(w) + + // no flags means it's the normal bullet point list + default: + r.popPad() + } + } + + case *ast.Paragraph: + // on exiting, collect and format the accumulated content + if !entering { + content := r.inlineAccumulator.String() + r.inlineAccumulator.Reset() + + var out string + if r.indent != "" { + out, _ = text.WrapWithPadIndent(content, r.lineWidth, r.indent, r.pad()) + r.indent = "" + } else { + out, _ = text.WrapWithPad(content, r.lineWidth, r.pad()) + } + _, _ = fmt.Fprint(w, out, "\n") + + // extra line break in some cases + if next := ast.GetNextNode(node); next != nil { + switch next.(type) { + case *ast.Paragraph, *ast.Heading, *ast.HorizontalRule, + *ast.CodeBlock, *ast.HTMLBlock: + _, _ = fmt.Fprintln(w) + } + } + } + + case *ast.Heading: + if !entering { + r.renderHeading(w, node.Level) + } + + case *ast.HorizontalRule: + r.renderHorizontalRule(w) + + case *ast.Emph: + if entering { + r.inlineAccumulator.WriteString(italicOn) + } else { + r.inlineAccumulator.WriteString(italicOff) + } + + case *ast.Strong: + if entering { + r.inlineAccumulator.WriteString(boldOn) + } else { + // This is super silly but some terminals, instead of having + // the ANSI code SGR 21 do "bold off" like the logic would guide, + // do "double underline" instead. This is madness. + + // To resolve that problem, we take a snapshot of the escape state, + // remove the bold, then output "reset all" + snapshot + es := text.EscapeState{} + es.Witness(r.inlineAccumulator.String()) + es.Bold = false + r.inlineAccumulator.WriteString(resetAll) + r.inlineAccumulator.WriteString(es.FormatString()) + } + + case *ast.Del: + if entering { + r.inlineAccumulator.WriteString(crossedOutOn) + } else { + r.inlineAccumulator.WriteString(crossedOutOff) + } + + case *ast.Link: + if entering { + r.inlineAccumulator.WriteString("[") + r.inlineAccumulator.WriteString(string(ast.GetFirstChild(node).AsLeaf().Literal)) + r.inlineAccumulator.WriteString("](") + r.inlineAccumulator.WriteString(Blue(string(node.Destination))) + if len(node.Title) > 0 { + r.inlineAccumulator.WriteString(" ") + r.inlineAccumulator.WriteString(string(node.Title)) + } + r.inlineAccumulator.WriteString(")") + return ast.SkipChildren + } + + case *ast.Image: + if entering { + // the alt text/title is weirdly parsed and is actually + // a child text of this node + var title string + if len(node.Children) == 1 { + if t, ok := node.Children[0].(*ast.Text); ok { + title = string(t.Literal) + } + } + + str, rendered := r.renderImage( + string(node.Destination), title, + r.lineWidth-r.leftPad, + ) + + if rendered { + r.inlineAccumulator.WriteString("\n") + r.inlineAccumulator.WriteString(str) + r.inlineAccumulator.WriteString("\n\n") + } else { + r.inlineAccumulator.WriteString(str) + r.inlineAccumulator.WriteString("\n") + } + + return ast.SkipChildren + } + + case *ast.Text: + if string(node.Literal) == "\n" { + break + } + content := string(node.Literal) + if shouldCleanText(node) { + content = removeLineBreak(content) + } + // emoji support ! + emojed := emoji.Sprint(content) + r.inlineAccumulator.WriteString(emojed) + + case *ast.HTMLBlock: + r.renderHTMLBlock(w, node) + + case *ast.CodeBlock: + r.renderCodeBlock(w, node) + + case *ast.Softbreak: + // not actually implemented in gomarkdown + r.inlineAccumulator.WriteString("\n") + + case *ast.Hardbreak: + r.inlineAccumulator.WriteString("\n") + + case *ast.Code: + r.inlineAccumulator.WriteString(BlueBgItalic(string(node.Literal))) + + case *ast.HTMLSpan: + r.inlineAccumulator.WriteString(Red(string(node.Literal))) + + case *ast.Table: + if entering { + r.table = newTableRenderer() + } else { + r.table.Render(w, r.leftPad, r.lineWidth) + r.table = nil + } + + case *ast.TableCell: + if !entering { + content := r.inlineAccumulator.String() + r.inlineAccumulator.Reset() + + align := CellAlignLeft + switch node.Align { + case ast.TableAlignmentRight: + align = CellAlignRight + case ast.TableAlignmentCenter: + align = CellAlignCenter + } + + if node.IsHeader { + r.table.AddHeaderCell(content, align) + } else { + r.table.AddBodyCell(content, CellAlignCopyHeader) + } + } + + case *ast.TableHeader, *ast.TableBody, *ast.TableFooter: + // nothing to do + + case *ast.TableRow: + if _, ok := node.Parent.(*ast.TableBody); ok && entering { + r.table.NextBodyRow() + } + if _, ok := node.Parent.(*ast.TableFooter); ok && entering { + r.table.NextBodyRow() + } + + default: + panic(fmt.Sprintf("Unknown node type %T", node)) + } + + return ast.GoToNext +} + +func (*renderer) RenderHeader(w io.Writer, node ast.Node) {} + +func (*renderer) RenderFooter(w io.Writer, node ast.Node) {} + +func (r *renderer) renderHorizontalRule(w io.Writer) { + _, _ = fmt.Fprintf(w, "%s%s\n\n", r.pad(), strings.Repeat("─", r.lineWidth-r.leftPad)) +} + +func (r *renderer) renderHeading(w io.Writer, level int) { + content := r.inlineAccumulator.String() + r.inlineAccumulator.Reset() + + // render the full line with the headingNumbering + r.headingNumbering.Observe(level) + content = fmt.Sprintf("%s %s", r.headingNumbering.Render(), content) + content = r.headingShade(level)(content) + + // wrap if needed + wrapped, _ := text.WrapWithPad(content, r.lineWidth, r.pad()) + _, _ = fmt.Fprintln(w, wrapped) + + // render the underline, if any + if level == 1 { + _, _ = fmt.Fprintf(w, "%s%s\n", r.pad(), strings.Repeat("─", r.lineWidth-r.leftPad)) + } + + _, _ = fmt.Fprintln(w) +} + +func (r *renderer) renderCodeBlock(w io.Writer, node *ast.CodeBlock) { + code := string(node.Literal) + var lexer chroma.Lexer + // try to get the lexer from the language tag if any + if len(node.Info) > 0 { + lexer = lexers.Get(string(node.Info)) + } + // fallback on detection + if lexer == nil { + lexer = lexers.Analyse(code) + } + // all failed :-( + if lexer == nil { + lexer = lexers.Fallback + } + // simplify the lexer output + lexer = chroma.Coalesce(lexer) + + var formatter chroma.Formatter + if color.NoColor { + formatter = formatters.Fallback + } else { + formatter = formatters.TTY8 + } + + iterator, err := lexer.Tokenise(nil, code) + if err != nil { + // Something failed, falling back to no highlight render + r.renderFormattedCodeBlock(w, code) + return + } + + buf := &bytes.Buffer{} + + err = formatter.Format(buf, styles.Pygments, iterator) + if err != nil { + // Something failed, falling back to no highlight render + r.renderFormattedCodeBlock(w, code) + return + } + + r.renderFormattedCodeBlock(w, buf.String()) +} + +func (r *renderer) renderFormattedCodeBlock(w io.Writer, code string) { + // remove the trailing line break + code = strings.TrimRight(code, "\n") + + r.addPad(GreenBold("┃ ")) + output, _ := text.WrapWithPad(code, r.lineWidth, r.pad()) + r.popPad() + + _, _ = fmt.Fprint(w, output) + + _, _ = fmt.Fprintf(w, "\n\n") +} + +func (r *renderer) renderHTMLBlock(w io.Writer, node *ast.HTMLBlock) { + var buf bytes.Buffer + + flushInline := func() { + if r.inlineAccumulator.Len() <= 0 { + return + } + content := r.inlineAccumulator.String() + r.inlineAccumulator.Reset() + out, _ := text.WrapWithPad(content, r.lineWidth, r.pad()) + _, _ = fmt.Fprint(&buf, out, "\n\n") + } + + doc, err := html.Parse(bytes.NewReader(node.Literal)) + if err != nil { + // if there is a parsing error, fallback to a simple render + r.inlineAccumulator.Reset() + content := Red(string(node.Literal)) + out, _ := text.WrapWithPad(content, r.lineWidth, r.pad()) + _, _ = fmt.Fprint(w, out, "\n\n") + return + } + + htmlWalker.WalkFunc(doc, func(node *html.Node, entering bool) htmlWalker.WalkStatus { + // if node.Type != html.TextNode { + // fmt.Println(node.Type, "(", node.Data, ")", entering) + // } + + switch node.Type { + case html.CommentNode, html.DoctypeNode: + // Not rendered + + case html.DocumentNode: + + case html.ElementNode: + switch node.Data { + case "html", "body": + return htmlWalker.GoToNext + + case "head": + return htmlWalker.SkipChildren + + case "div", "p": + if entering { + flushInline() + } else { + content := r.inlineAccumulator.String() + r.inlineAccumulator.Reset() + if len(content) == 0 { + return htmlWalker.GoToNext + } + // remove all line breaks, those are fully managed in HTML + content = strings.Replace(content, "\n", "", -1) + align := getDivHTMLAttr(node.Attr) + content, _ = text.WrapWithPadAlign(content, r.lineWidth, r.pad(), align) + _, _ = fmt.Fprint(&buf, content, "\n\n") + } + + case "h1": + if !entering { + r.renderHeading(&buf, 1) + } + case "h2": + if !entering { + r.renderHeading(&buf, 2) + } + case "h3": + if !entering { + r.renderHeading(&buf, 3) + } + case "h4": + if !entering { + r.renderHeading(&buf, 4) + } + case "h5": + if !entering { + r.renderHeading(&buf, 5) + } + case "h6": + if !entering { + r.renderHeading(&buf, 6) + } + + case "img": + flushInline() + src, title := getImgHTMLAttr(node.Attr) + str, _ := r.renderImage(src, title, r.lineWidth-len(r.pad())) + r.inlineAccumulator.WriteString(str) + + case "hr": + flushInline() + r.renderHorizontalRule(&buf) + + case "ul", "ol": + if !entering { + if node.NextSibling == nil { + _, _ = fmt.Fprint(&buf, "\n") + return htmlWalker.GoToNext + } + switch node.NextSibling.Data { + case "ul", "ol": + default: + _, _ = fmt.Fprint(&buf, "\n") + } + } + + case "li": + if entering { + switch node.Parent.Data { + case "ul": + r.indent = r.pad() + Green("• ") + r.addPad(" ") + + case "ol": + itemNumber := 1 + previous := node.PrevSibling + for previous != nil { + itemNumber++ + previous = previous.PrevSibling + } + prefix := fmt.Sprintf("%d. ", itemNumber) + r.indent = r.pad() + Green(prefix) + r.addPad(strings.Repeat(" ", text.Len(prefix))) + + default: + r.inlineAccumulator.WriteString(Red(renderRawHtml(node))) + return htmlWalker.SkipChildren + } + } else { + switch node.Parent.Data { + case "ul", "ol": + content := r.inlineAccumulator.String() + r.inlineAccumulator.Reset() + out, _ := text.WrapWithPadIndent(content, r.lineWidth, r.indent, r.pad()) + r.indent = "" + _, _ = fmt.Fprint(&buf, out, "\n") + r.popPad() + } + } + + case "a": + if entering { + r.inlineAccumulator.WriteString("[") + } else { + href, alt := getAHTMLAttr(node.Attr) + r.inlineAccumulator.WriteString("](") + r.inlineAccumulator.WriteString(Blue(href)) + if len(alt) > 0 { + r.inlineAccumulator.WriteString(" ") + r.inlineAccumulator.WriteString(alt) + } + r.inlineAccumulator.WriteString(")") + } + + case "br": + if entering { + r.inlineAccumulator.WriteString("\n") + } + + case "table": + if entering { + flushInline() + r.table = newTableRenderer() + } else { + r.table.Render(&buf, r.leftPad, r.lineWidth) + r.table = nil + } + + case "thead", "tbody": + // nothing to do + + case "tr": + if entering && node.Parent.Data != "thead" { + r.table.NextBodyRow() + } + + case "th": + if !entering { + content := r.inlineAccumulator.String() + r.inlineAccumulator.Reset() + + align := getTdHTMLAttr(node.Attr) + r.table.AddHeaderCell(content, align) + } + + case "td": + if !entering { + content := r.inlineAccumulator.String() + r.inlineAccumulator.Reset() + + align := getTdHTMLAttr(node.Attr) + r.table.AddBodyCell(content, align) + } + + case "strong", "b": + if entering { + r.inlineAccumulator.WriteString(boldOn) + } else { + // This is super silly but some terminals, instead of having + // the ANSI code SGR 21 do "bold off" like the logic would guide, + // do "double underline" instead. This is madness. + + // To resolve that problem, we take a snapshot of the escape state, + // remove the bold, then output "reset all" + snapshot + es := text.EscapeState{} + es.Witness(r.inlineAccumulator.String()) + es.Bold = false + r.inlineAccumulator.WriteString(resetAll) + r.inlineAccumulator.WriteString(es.FormatString()) + } + + case "i", "em": + if entering { + r.inlineAccumulator.WriteString(italicOn) + } else { + r.inlineAccumulator.WriteString(italicOff) + } + + case "s": + if entering { + r.inlineAccumulator.WriteString(crossedOutOn) + } else { + r.inlineAccumulator.WriteString(crossedOutOff) + } + + default: + r.inlineAccumulator.WriteString(Red(renderRawHtml(node))) + } + + case html.TextNode: + t := strings.TrimSpace(node.Data) + t = strings.ReplaceAll(t, "\n", "") + r.inlineAccumulator.WriteString(t) + + default: + panic("unhandled case") + } + + return htmlWalker.GoToNext + }) + + flushInline() + _, _ = fmt.Fprint(w, buf.String()) + r.inlineAccumulator.Reset() + + // // dl + (dt+dd) + // + // // details + // // summary + // +} + +func getDivHTMLAttr(attrs []html.Attribute) text.Alignment { + for _, attr := range attrs { + switch attr.Key { + case "align": + switch attr.Val { + case "left": + return text.AlignLeft + case "center": + return text.AlignCenter + case "right": + return text.AlignRight + } + } + } + return text.AlignLeft +} + +func getImgHTMLAttr(attrs []html.Attribute) (src, title string) { + for _, attr := range attrs { + switch attr.Key { + case "src": + src = attr.Val + case "alt": + title = attr.Val + } + } + return +} + +func getAHTMLAttr(attrs []html.Attribute) (href, alt string) { + for _, attr := range attrs { + switch attr.Key { + case "href": + href = attr.Val + case "alt": + alt = attr.Val + } + } + return +} + +func getTdHTMLAttr(attrs []html.Attribute) CellAlign { + for _, attr := range attrs { + switch attr.Key { + case "align": + switch attr.Val { + case "right": + return CellAlignRight + case "left": + return CellAlignLeft + case "center": + return CellAlignCenter + } + + case "style": + for _, pair := range strings.Split(attr.Val, " ") { + split := strings.Split(pair, ":") + if split[0] != "text-align" || len(split) != 2 { + continue + } + switch split[1] { + case "right": + return CellAlignRight + case "left": + return CellAlignLeft + case "center": + return CellAlignCenter + } + } + } + } + return CellAlignLeft +} + +func renderRawHtml(node *html.Node) string { + var result strings.Builder + openContent := make([]string, 0, 8) + + openContent = append(openContent, node.Data) + for _, attr := range node.Attr { + openContent = append(openContent, fmt.Sprintf("%s=\"%s\"", attr.Key, attr.Val)) + } + + result.WriteString("<") + result.WriteString(strings.Join(openContent, " ")) + + if node.FirstChild == nil { + result.WriteString("/>") + return result.String() + } + + result.WriteString(">") + + child := node.FirstChild + for child != nil { + if child.Type == html.TextNode { + t := strings.TrimSpace(child.Data) + result.WriteString(t) + child = child.NextSibling + continue + } + + switch node.Data { + case "ul", "p": + result.WriteString("\n ") + } + + result.WriteString(renderRawHtml(child)) + child = child.NextSibling + } + + switch node.Data { + case "ul", "p": + result.WriteString("\n") + } + + result.WriteString("") + result.WriteString(node.Data) + result.WriteString(">") + + return result.String() +} + +func (r *renderer) renderImage(dest string, title string, lineWidth int) (result string, rendered bool) { + title = strings.ReplaceAll(title, "\n", "") + title = strings.TrimSpace(title) + dest = strings.ReplaceAll(dest, "\n", "") + dest = strings.TrimSpace(dest) + + fallback := func() (string, bool) { + return fmt.Sprintf("![%s](%s)", title, Blue(dest)), false + } + + reader, err := imageFromDestination(dest) + if err != nil { + return fallback() + } + + x := lineWidth + + if r.imageDithering == ansimage.DitheringWithChars || r.imageDithering == ansimage.DitheringWithBlocks { + // not sure why this is needed by ansimage + // x *= 4 + } + + img, err := ansimage.NewScaledFromReader(reader, math.MaxInt32, x, + stdcolor.Black, ansimage.ScaleModeFit, r.imageDithering) + + if err != nil { + return fallback() + } + + if title != "" { + return fmt.Sprintf("%s%s: %s", img.Render(), title, Blue(dest)), true + } + return fmt.Sprintf("%s%s", img.Render(), Blue(dest)), true +} + +func imageFromDestination(dest string) (io.ReadCloser, error) { + client := http.Client{ + Timeout: 5 * time.Second, + } + + if strings.HasPrefix(dest, "http://") || strings.HasPrefix(dest, "https://") { + res, err := client.Get(dest) + if err != nil { + return nil, err + } + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("http: %v", http.StatusText(res.StatusCode)) + } + + return res.Body, nil + } + + return os.Open(dest) +} + +func removeLineBreak(text string) string { + lines := strings.Split(text, "\n") + + if len(lines) <= 1 { + return text + } + + for i, l := range lines { + switch i { + case 0: + lines[i] = strings.TrimRightFunc(l, unicode.IsSpace) + case len(lines) - 1: + lines[i] = strings.TrimLeftFunc(l, unicode.IsSpace) + default: + lines[i] = strings.TrimFunc(l, unicode.IsSpace) + } + } + return strings.Join(lines, " ") +} + +func shouldCleanText(node ast.Node) bool { + for node != nil { + switch node.(type) { + case *ast.BlockQuote: + return false + + case *ast.Heading, *ast.Image, *ast.Link, + *ast.TableCell, *ast.Document, *ast.ListItem: + return true + } + + node = node.GetParent() + } + + panic("bad markdown document or missing case") +} diff --git a/vendor/github.com/MichaelMure/go-term-markdown/shades.go b/vendor/github.com/MichaelMure/go-term-markdown/shades.go new file mode 100644 index 0000000000..b91f00c69e --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-markdown/shades.go @@ -0,0 +1,33 @@ +package markdown + +var defaultHeadingShades = []shadeFmt{ + GreenBold, + GreenBold, + HiGreen, + Green, +} + +var defaultQuoteShades = []shadeFmt{ + GreenBold, + GreenBold, + HiGreen, + Green, +} + +type shadeFmt func(a ...interface{}) string + +type levelShadeFmt func(level int) shadeFmt + +// Return a function giving the color function corresponding to the level. +// Beware, level start counting from 1. +func shade(shades []shadeFmt) levelShadeFmt { + return func(level int) shadeFmt { + if level < 1 { + level = 1 + } + if level > len(shades) { + level = len(shades) + } + return shades[level-1] + } +} diff --git a/vendor/github.com/MichaelMure/go-term-markdown/tables.go b/vendor/github.com/MichaelMure/go-term-markdown/tables.go new file mode 100644 index 0000000000..f9c1ad6897 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-markdown/tables.go @@ -0,0 +1,350 @@ +package markdown + +import ( + "io" + "strings" + + "github.com/MichaelMure/go-term-text" +) + +const minColumnCompactedWidth = 5 + +type CellAlign int + +const ( + CellAlignLeft CellAlign = iota + CellAlignRight + CellAlignCenter + CellAlignCopyHeader +) + +type tableCell struct { + content string + alignment CellAlign +} + +type tableRenderer struct { + header []tableCell + body [][]tableCell +} + +func newTableRenderer() *tableRenderer { + return &tableRenderer{} +} + +func (tr *tableRenderer) AddHeaderCell(content string, alignment CellAlign) { + tr.header = append(tr.header, tableCell{ + content: content, + alignment: alignment, + }) +} + +func (tr *tableRenderer) NextBodyRow() { + tr.body = append(tr.body, nil) +} + +func (tr *tableRenderer) AddBodyCell(content string, alignement CellAlign) { + row := tr.body[len(tr.body)-1] + row = append(row, tableCell{ + content: content, + alignment: alignement, + }) + tr.body[len(tr.body)-1] = row +} + +// normalize ensure that the table has the same number of cells +// in each rows, header or not. +func (tr *tableRenderer) normalize() { + width := len(tr.header) + for _, row := range tr.body { + width = max(width, len(row)) + } + + // grow the header if needed + for len(tr.header) < width { + tr.header = append(tr.header, tableCell{}) + } + + // grow lines if needed + for i := range tr.body { + for len(tr.body[i]) < width { + tr.body[i] = append(tr.body[i], tableCell{}) + } + } +} + +func (tr *tableRenderer) copyAlign() { + for i, row := range tr.body { + for j, cell := range row { + if cell.alignment == CellAlignCopyHeader { + tr.body[i][j].alignment = tr.header[j].alignment + } + } + } +} + +func (tr *tableRenderer) Render(w io.Writer, leftPad int, lineWidth int) { + tr.normalize() + tr.copyAlign() + + columnWidths, truncated := tr.columnWidths(lineWidth - leftPad) + pad := strings.Repeat(" ", leftPad) + + drawTopLine(w, pad, columnWidths, truncated) + + drawRow(w, pad, tr.header, columnWidths, truncated) + + drawHeaderUnderline(w, pad, columnWidths, truncated) + + for i, row := range tr.body { + drawRow(w, pad, row, columnWidths, truncated) + if i != len(tr.body)-1 { + drawRowLine(w, pad, columnWidths, truncated) + } + } + + drawBottomLine(w, pad, columnWidths, truncated) +} + +func (tr *tableRenderer) columnWidths(lineWidth int) (widths []int, truncated bool) { + l := len(tr.header) + if len(tr.body) > 0 { + l = max(l, len(tr.body[0])) + } + + maxWidth := make([]int, l) + + for i, cell := range tr.header { + maxWidth[i] = max(maxWidth[i], text.MaxLineLen(cell.content)) + } + + for _, row := range tr.body { + for i, cell := range row { + maxWidth[i] = max(maxWidth[i], text.MaxLineLen(cell.content)) + } + } + + sumWidth := 1 + minWidth := 1 + for _, width := range maxWidth { + sumWidth += width + 1 + minWidth += min(width, minColumnCompactedWidth) + 1 + } + + // Strategy 1: the easy case, content is not large enough to overflow + if sumWidth <= lineWidth { + return maxWidth, false + } + + // Strategy 2: overflow, but still enough room + if minWidth < lineWidth { + return tr.overflowColumnWidths(lineWidth, maxWidth), false + } + + // Strategy 3: too much columns, we need to truncate + return tr.truncateColumnWidths(lineWidth, maxWidth), true +} + +func (tr *tableRenderer) overflowColumnWidths(lineWidth int, maxWidth []int) []int { + // We have an overflow. First, we take as is the columns that are thinner + // than the space equally divided. + // Integer division, rounded lower. + available := lineWidth - len(tr.header) - 1 + fairSpace := available / len(tr.header) + + result := make([]int, len(tr.header)) + remainingColumn := len(tr.header) + + for i, width := range maxWidth { + if width <= fairSpace { + result[i] = width + available -= width + remainingColumn-- + } else { + // Mark the column as non-allocated yet + result[i] = -1 + } + } + + // Now we allocate evenly the remaining space to the remaining columns + for i, width := range result { + if width == -1 { + width = available / remainingColumn + result[i] = width + available -= width + remainingColumn-- + } + } + + return result +} + +func (tr *tableRenderer) truncateColumnWidths(lineWidth int, maxWidth []int) []int { + var result []int + used := 1 + + // Pack as much column as possible without compacting them too much + for _, width := range maxWidth { + w := min(width, minColumnCompactedWidth) + + if used+w+1 > lineWidth { + return result + } + + result = append(result, w) + used += w + 1 + } + + return result +} + +func drawTopLine(w io.Writer, pad string, columnWidths []int, truncated bool) { + _, _ = w.Write([]byte(pad)) + _, _ = w.Write([]byte("┌")) + for i, width := range columnWidths { + _, _ = w.Write([]byte(strings.Repeat("─", width))) + if i != len(columnWidths)-1 { + _, _ = w.Write([]byte("┬")) + } + } + _, _ = w.Write([]byte("┐")) + if truncated { + _, _ = w.Write([]byte("…")) + } + _, _ = w.Write([]byte("\n")) +} + +func drawHeaderUnderline(w io.Writer, pad string, columnWidths []int, truncated bool) { + _, _ = w.Write([]byte(pad)) + _, _ = w.Write([]byte("╞")) + for i, width := range columnWidths { + _, _ = w.Write([]byte(strings.Repeat("═", width))) + if i != len(columnWidths)-1 { + _, _ = w.Write([]byte("╪")) + } + } + _, _ = w.Write([]byte("╡")) + if truncated { + _, _ = w.Write([]byte("…")) + } + _, _ = w.Write([]byte("\n")) +} + +func drawBottomLine(w io.Writer, pad string, columnWidths []int, truncated bool) { + _, _ = w.Write([]byte(pad)) + _, _ = w.Write([]byte("└")) + for i, width := range columnWidths { + _, _ = w.Write([]byte(strings.Repeat("─", width))) + if i != len(columnWidths)-1 { + _, _ = w.Write([]byte("┴")) + } + } + _, _ = w.Write([]byte("┘")) + if truncated { + _, _ = w.Write([]byte("…")) + } + _, _ = w.Write([]byte("\n")) +} + +func drawRowLine(w io.Writer, pad string, columnWidths []int, truncated bool) { + _, _ = w.Write([]byte(pad)) + _, _ = w.Write([]byte("├")) + for i, width := range columnWidths { + _, _ = w.Write([]byte(strings.Repeat("─", width))) + if i != len(columnWidths)-1 { + _, _ = w.Write([]byte("┼")) + } + } + _, _ = w.Write([]byte("┤")) + if truncated { + _, _ = w.Write([]byte("…")) + } + _, _ = w.Write([]byte("\n")) +} + +func drawRow(w io.Writer, pad string, cells []tableCell, columnWidths []int, truncated bool) { + contents := make([][]string, len(cells)) + + // As we draw the row line by line, we need a way to reset and recover + // the formatting when we alternate between cells. To do that, we witness + // the ongoing series of ANSI escape sequence for each cell into a EscapeState. + // This component will be able to merge them and to give us a snapshot sequence + // that we can use when we start the cell again + formatting := make([]text.EscapeState, len(columnWidths)) + + maxHeight := 0 + + // Wrap each cell content into multiple lines, depending on + // how wide each cell is. + for i, cell := range cells[:len(columnWidths)] { + if columnWidths[i] == 0 { + continue + } + wrapped, lines := text.Wrap(cell.content, columnWidths[i]) + contents[i] = strings.Split(wrapped, "\n") + maxHeight = max(maxHeight, lines) + } + + // Draw the row line by line + for i := 0; i < maxHeight; i++ { + _, _ = w.Write([]byte(pad)) + _, _ = w.Write([]byte("│")) + for j, width := range columnWidths { + content := "" + if len(contents[j]) > i { + content = contents[j][i] + trimmed := text.TrimSpace(content) + + switch cells[j].alignment { + case CellAlignLeft: + _, _ = w.Write([]byte(formatting[j].FormatString())) + // accumulate the formatting + formatting[j].Witness(trimmed) + _, _ = w.Write([]byte(trimmed)) + _, _ = w.Write([]byte(formatting[j].ResetString())) + _, _ = w.Write([]byte(strings.Repeat(" ", width-text.Len(trimmed)))) + + case CellAlignCenter: + spaces := width - text.Len(trimmed) + _, _ = w.Write([]byte(strings.Repeat(" ", spaces/2))) + _, _ = w.Write([]byte(formatting[j].FormatString())) + // accumulate the formatting + formatting[j].Witness(trimmed) + _, _ = w.Write([]byte(trimmed)) + _, _ = w.Write([]byte(formatting[j].ResetString())) + _, _ = w.Write([]byte(strings.Repeat(" ", spaces-(spaces/2)))) + + case CellAlignRight: + _, _ = w.Write([]byte(strings.Repeat(" ", width-text.Len(trimmed)))) + _, _ = w.Write([]byte(formatting[j].FormatString())) + // accumulate the formatting + formatting[j].Witness(trimmed) + _, _ = w.Write([]byte(trimmed)) + _, _ = w.Write([]byte(formatting[j].ResetString())) + } + } else { + padding := strings.Repeat(" ", width-text.Len(content)) + _, _ = w.Write([]byte(padding)) + } + _, _ = w.Write([]byte("│")) + } + if truncated { + _, _ = w.Write([]byte("…")) + } + _, _ = w.Write([]byte("\n")) + } +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/vendor/github.com/MichaelMure/go-term-text/.gitignore b/vendor/github.com/MichaelMure/go-term-text/.gitignore new file mode 100644 index 0000000000..9f11b755a1 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-text/.gitignore @@ -0,0 +1 @@ +.idea/ diff --git a/vendor/github.com/MichaelMure/go-term-text/LICENSE b/vendor/github.com/MichaelMure/go-term-text/LICENSE new file mode 100644 index 0000000000..5ba12bf4c5 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-text/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Michael Muré + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/MichaelMure/go-term-text/Readme.md b/vendor/github.com/MichaelMure/go-term-text/Readme.md new file mode 100644 index 0000000000..03acfba94a --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-text/Readme.md @@ -0,0 +1,74 @@ +# go-term-text + +![Build Status](https://github.com/MichaelMure/go-term-text/workflows/Go%20build%20and%20test/badge.svg) +[![GoDoc](https://godoc.org/github.com/MichaelMure/go-term-text?status.svg)](https://godoc.org/github.com/MichaelMure/go-term-text) +[![Go Report Card](https://goreportcard.com/badge/github.com/MichaelMure/go-term-text)](https://goreportcard.com/report/github.com/MichaelMure/go-term-text) +[![codecov](https://codecov.io/gh/MichaelMure/go-term-text/branch/master/graph/badge.svg)](https://codecov.io/gh/MichaelMure/go-term-text) +[![GitHub license](https://img.shields.io/github/license/MichaelMure/go-term-text.svg)](https://github.com/MichaelMure/go-term-text/blob/master/LICENSE) +[![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/the-git-bug/Lobby) + +`go-term-text` is a go package implementing a collection of algorithms to help format and manipulate text for the terminal. + +In particular, `go-term-text`: +- support wide characters (chinese, japanese ...) and emoji +- handle properly ANSI escape sequences + +Included algorithms cover: +- wrapping with padding and indentation +- padding +- text length +- trimming +- alignment +- escape sequence extraction and reapplication +- escape sequence snapshot and simplification +- truncation + +## Example + +```go +package main + +import ( + "fmt" + "strings" + + "github.com/MichaelMure/go-term-text" +) + +func main() { + input := "The \x1b[1mLorem ipsum\x1b[0m text is typically composed of " + + "pseudo-Latin words. It is commonly used as \x1b[3mplaceholder\x1b[0m" + + " text to examine or demonstrate the \x1b[9mvisual effects\x1b[0m of " + + "various graphic design. 一只 A Quick \x1b[31m敏捷的狐 Fox " + + "狸跳过了\x1b[0mDog一只懒狗。" + + output, n := text.Wrap(input, 60, + text.WrapIndent("\x1b[34m<-indent-> \x1b[0m"), + text.WrapPad("\x1b[33m<-pad-> \x1b[0m"), + ) + + fmt.Printf("output has %d lines\n\n", n) + + fmt.Println("|" + strings.Repeat("-", 58) + "|") + fmt.Println(output) + fmt.Println("|" + strings.Repeat("-", 58) + "|") +} +``` + +This will print: + +![example output](/img/example.png) + +For more details, have a look at the [GoDoc](https://godoc.org/github.com/MichaelMure/go-term-text). + +## Origin + +This package has been extracted from the [git-bug](https://github.com/MichaelMure/git-bug) project. As such, its aim is to support this project and not to provide an all-in-one solution. Contributions as welcome though. + +## Contribute + +PRs accepted. + +## License + +MIT diff --git a/vendor/github.com/MichaelMure/go-term-text/align.go b/vendor/github.com/MichaelMure/go-term-text/align.go new file mode 100644 index 0000000000..8262a4de5d --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-text/align.go @@ -0,0 +1,67 @@ +package text + +import ( + "strings" +) + +type Alignment int + +const ( + NoAlign Alignment = iota + AlignLeft + AlignCenter + AlignRight +) + +// LineAlign align the given line as asked and apply the needed padding to match the given +// lineWidth, while ignoring the terminal escape sequences. +// If the given lineWidth is too small to fit the given line, it's returned without +// padding, overflowing lineWidth. +func LineAlign(line string, lineWidth int, align Alignment) string { + switch align { + case NoAlign: + return line + case AlignLeft: + return LineAlignLeft(line, lineWidth) + case AlignCenter: + return LineAlignCenter(line, lineWidth) + case AlignRight: + return LineAlignRight(line, lineWidth) + } + panic("unknown alignment") +} + +// LineAlignLeft align the given line on the left while ignoring the terminal escape sequences. +// If the given lineWidth is too small to fit the given line, it's returned without +// padding, overflowing lineWidth. +func LineAlignLeft(line string, lineWidth int) string { + return TrimSpace(line) +} + +// LineAlignCenter align the given line on the center and apply the needed left +// padding, while ignoring the terminal escape sequences. +// If the given lineWidth is too small to fit the given line, it's returned without +// padding, overflowing lineWidth. +func LineAlignCenter(line string, lineWidth int) string { + trimmed := TrimSpace(line) + totalPadLen := lineWidth - Len(trimmed) + if totalPadLen < 0 { + totalPadLen = 0 + } + pad := strings.Repeat(" ", totalPadLen/2) + return pad + trimmed +} + +// LineAlignRight align the given line on the right and apply the needed left +// padding to match the given lineWidth, while ignoring the terminal escape sequences. +// If the given lineWidth is too small to fit the given line, it's returned without +// padding, overflowing lineWidth. +func LineAlignRight(line string, lineWidth int) string { + trimmed := TrimSpace(line) + padLen := lineWidth - Len(trimmed) + if padLen < 0 { + padLen = 0 + } + pad := strings.Repeat(" ", padLen) + return pad + trimmed +} diff --git a/vendor/github.com/MichaelMure/go-term-text/escape_state.go b/vendor/github.com/MichaelMure/go-term-text/escape_state.go new file mode 100644 index 0000000000..c4bd60e849 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-text/escape_state.go @@ -0,0 +1,288 @@ +package text + +import ( + "fmt" + "strconv" + "strings" +) + +const Escape = '\x1b' + +type EscapeState struct { + Bold bool + Dim bool + Italic bool + Underlined bool + Blink bool + Reverse bool + Hidden bool + CrossedOut bool + + FgColor Color + BgColor Color +} + +type Color interface { + Codes() []string +} + +func (es *EscapeState) Witness(s string) { + inEscape := false + var start int + + runes := []rune(s) + + for i, r := range runes { + if r == Escape { + inEscape = true + start = i + continue + } + if inEscape { + if r == 'm' { + inEscape = false + es.witnessCode(string(runes[start+1 : i])) + } + continue + } + } +} + +func (es *EscapeState) witnessCode(s string) { + if s == "" { + return + } + if s == "[" { + es.reset() + return + } + if len(s) < 2 { + return + } + if s[0] != '[' { + return + } + + s = s[1:] + split := strings.Split(s, ";") + + dequeue := func() { + split = split[1:] + } + + color := func(ground int) Color { + if len(split) < 1 { + // the whole sequence is broken, ignoring the rest + return nil + } + + subCode := split[0] + dequeue() + + switch subCode { + case "2": + if len(split) < 3 { + return nil + } + r, err := strconv.Atoi(split[0]) + dequeue() + if err != nil { + return nil + } + g, err := strconv.Atoi(split[0]) + dequeue() + if err != nil { + return nil + } + b, err := strconv.Atoi(split[0]) + dequeue() + if err != nil { + return nil + } + return &ColorRGB{ground: ground, R: r, G: g, B: b} + + case "5": + if len(split) < 1 { + return nil + } + index, err := strconv.Atoi(split[0]) + dequeue() + if err != nil { + return nil + } + return &Color256{ground: ground, Index: index} + + } + return nil + } + + for len(split) > 0 { + code, err := strconv.Atoi(split[0]) + if err != nil { + return + } + dequeue() + + switch { + case code == 0: + es.reset() + + case code == 1: + es.Bold = true + case code == 2: + es.Dim = true + case code == 3: + es.Italic = true + case code == 4: + es.Underlined = true + case code == 5: + es.Blink = true + // case code == 6: + case code == 7: + es.Reverse = true + case code == 8: + es.Hidden = true + case code == 9: + es.CrossedOut = true + + case code == 21: + es.Bold = false + case code == 22: + es.Dim = false + case code == 23: + es.Italic = false + case code == 24: + es.Underlined = false + case code == 25: + es.Blink = false + // case code == 26: + case code == 27: + es.Reverse = false + case code == 28: + es.Hidden = false + case code == 29: + es.CrossedOut = false + + case (code >= 30 && code <= 37) || code == 39 || (code >= 90 && code <= 97): + es.FgColor = ColorIndex(code) + + case (code >= 40 && code <= 47) || code == 49 || (code >= 100 && code <= 107): + es.BgColor = ColorIndex(code) + + case code == 38: + es.FgColor = color(code) + if es.FgColor == nil { + return + } + + case code == 48: + es.BgColor = color(code) + if es.BgColor == nil { + return + } + } + } +} + +func (es *EscapeState) reset() { + *es = EscapeState{} +} + +// FormatString return the escape codes to enable that formatting. +func (es *EscapeState) FormatString() string { + var codes []string + + if es.Bold { + codes = append(codes, strconv.Itoa(1)) + } + if es.Dim { + codes = append(codes, strconv.Itoa(2)) + } + if es.Italic { + codes = append(codes, strconv.Itoa(3)) + } + if es.Underlined { + codes = append(codes, strconv.Itoa(4)) + } + if es.Blink { + codes = append(codes, strconv.Itoa(5)) + } + if es.Reverse { + codes = append(codes, strconv.Itoa(7)) + } + if es.Hidden { + codes = append(codes, strconv.Itoa(8)) + } + if es.CrossedOut { + codes = append(codes, strconv.Itoa(9)) + } + + if es.FgColor != nil { + codes = append(codes, es.FgColor.Codes()...) + } + if es.BgColor != nil { + codes = append(codes, es.BgColor.Codes()...) + } + + if len(codes) == 0 { + return "" + } + + return fmt.Sprintf("\x1b[%sm", strings.Join(codes, ";")) +} + +// ResetString return either the global reset code or nothing, depending on if +// this state has something to reset or not. +func (es *EscapeState) ResetString() string { + if es.IsZero() { + return "" + } + return "\x1b[0m" +} + +func (es *EscapeState) IsZero() bool { + return !es.Bold && + !es.Dim && + !es.Italic && + !es.Underlined && + !es.Blink && + !es.Reverse && + !es.Hidden && + !es.CrossedOut && + es.FgColor == nil && + es.BgColor == nil +} + +type ColorIndex int + +func (cInd ColorIndex) Codes() []string { + return []string{strconv.Itoa(int(cInd))} +} + +type Color256 struct { + ground int + Index int +} + +func (c256 Color256) Codes() []string { + return []string{ + strconv.Itoa(c256.ground), + "5", + strconv.Itoa(c256.Index), + } +} + +type ColorRGB struct { + ground int + R, G, B int +} + +func (cRGB ColorRGB) Codes() []string { + return []string{ + strconv.Itoa(cRGB.ground), + "2", + strconv.Itoa(cRGB.R), + strconv.Itoa(cRGB.G), + strconv.Itoa(cRGB.B), + } +} diff --git a/vendor/github.com/MichaelMure/go-term-text/escapes.go b/vendor/github.com/MichaelMure/go-term-text/escapes.go new file mode 100644 index 0000000000..19f78c9269 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-text/escapes.go @@ -0,0 +1,95 @@ +package text + +import ( + "strings" + "unicode/utf8" +) + +// EscapeItem hold the description of terminal escapes in a line. +// 'item' is the actual escape command +// 'pos' is the index in the rune array where the 'item' shall be inserted back. +// For example, the escape item in "F\x1b33mox" is {"\x1b33m", 1}. +type EscapeItem struct { + Item string + Pos int +} + +// ExtractTermEscapes extract terminal escapes out of a line and returns a new +// line without terminal escapes and a slice of escape items. The terminal escapes +// can be inserted back into the new line at rune index 'item.pos' to recover the +// original line. +// +// Required: The line shall not contain "\n" +func ExtractTermEscapes(line string) (string, []EscapeItem) { + var termEscapes []EscapeItem + var line1 strings.Builder + + pos := 0 + item := "" + occupiedRuneCount := 0 + inEscape := false + for i, r := range []rune(line) { + if r == '\x1b' { + pos = i + item = string(r) + inEscape = true + continue + } + if inEscape { + item += string(r) + if r == 'm' { + termEscapes = append(termEscapes, EscapeItem{item, pos - occupiedRuneCount}) + occupiedRuneCount += utf8.RuneCountInString(item) + inEscape = false + } + continue + } + line1.WriteRune(r) + } + + return line1.String(), termEscapes +} + +// ApplyTermEscapes apply the extracted terminal escapes to the edited line. +// Escape sequences need to be ordered by their position. +// If the position is < 0, the escape is applied at the beginning of the line. +// If the position is > len(line), the escape is applied at the end of the line. +func ApplyTermEscapes(line string, escapes []EscapeItem) string { + if len(escapes) == 0 { + return line + } + + var out strings.Builder + + currPos := 0 + currItem := 0 + for _, r := range line { + for currItem < len(escapes) && currPos >= escapes[currItem].Pos { + out.WriteString(escapes[currItem].Item) + currItem++ + } + out.WriteRune(r) + currPos++ + } + + // Don't forget the trailing escapes, if any. + for currItem < len(escapes) { + out.WriteString(escapes[currItem].Item) + currItem++ + } + + return out.String() +} + +// OffsetEscapes is a utility function to offset the position of a +// collection of EscapeItem. +func OffsetEscapes(escapes []EscapeItem, offset int) []EscapeItem { + result := make([]EscapeItem, len(escapes)) + for i, e := range escapes { + result[i] = EscapeItem{ + Item: e.Item, + Pos: e.Pos + offset, + } + } + return result +} diff --git a/vendor/github.com/MichaelMure/go-term-text/left_pad.go b/vendor/github.com/MichaelMure/go-term-text/left_pad.go new file mode 100644 index 0000000000..a63fedb9bf --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-text/left_pad.go @@ -0,0 +1,50 @@ +package text + +import ( + "bytes" + "strings" + + "github.com/mattn/go-runewidth" +) + +// LeftPadMaxLine pads a line on the left by a specified amount and pads the +// string on the right to fill the maxLength. +// If the given string is too long, it is truncated with an ellipsis. +// Handle properly terminal color escape code +func LeftPadMaxLine(line string, length, leftPad int) string { + cleaned, escapes := ExtractTermEscapes(line) + + scrWidth := runewidth.StringWidth(cleaned) + // truncate and ellipse if needed + if scrWidth+leftPad > length { + cleaned = runewidth.Truncate(cleaned, length-leftPad, "…") + } else if scrWidth+leftPad < length { + cleaned = runewidth.FillRight(cleaned, length-leftPad) + } + + rightPart := ApplyTermEscapes(cleaned, escapes) + pad := strings.Repeat(" ", leftPad) + + return pad + rightPart +} + +// LeftPad left pad each line of the given text +func LeftPadLines(text string, leftPad int) string { + var result bytes.Buffer + + pad := strings.Repeat(" ", leftPad) + + lines := strings.Split(text, "\n") + + for i, line := range lines { + result.WriteString(pad) + result.WriteString(line) + + // no additional line break at the end + if i < len(lines)-1 { + result.WriteString("\n") + } + } + + return result.String() +} diff --git a/vendor/github.com/MichaelMure/go-term-text/len.go b/vendor/github.com/MichaelMure/go-term-text/len.go new file mode 100644 index 0000000000..c6bcaeaca8 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-text/len.go @@ -0,0 +1,45 @@ +package text + +import ( + "strings" + + "github.com/mattn/go-runewidth" +) + +// Len return the length of a string in a terminal, while ignoring the terminal +// escape sequences. +func Len(text string) int { + length := 0 + escape := false + + for _, char := range text { + if char == '\x1b' { + escape = true + } + if !escape { + length += runewidth.RuneWidth(char) + } + if char == 'm' { + escape = false + } + } + + return length +} + +// MaxLineLen return the length in a terminal of the longest line, while +// ignoring the terminal escape sequences. +func MaxLineLen(text string) int { + lines := strings.Split(text, "\n") + + max := 0 + + for _, line := range lines { + length := Len(line) + if length > max { + max = length + } + } + + return max +} diff --git a/vendor/github.com/MichaelMure/go-term-text/trim.go b/vendor/github.com/MichaelMure/go-term-text/trim.go new file mode 100644 index 0000000000..eaf2ca0c07 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-text/trim.go @@ -0,0 +1,28 @@ +package text + +import ( + "strings" + "unicode" +) + +// TrimSpace remove the leading and trailing whitespace while ignoring the +// terminal escape sequences. +// Returns the number of trimmed space on both side. +func TrimSpace(line string) string { + cleaned, escapes := ExtractTermEscapes(line) + + // trim left while counting + left := 0 + trimmed := strings.TrimLeftFunc(cleaned, func(r rune) bool { + if unicode.IsSpace(r) { + left++ + return true + } + return false + }) + + trimmed = strings.TrimRightFunc(trimmed, unicode.IsSpace) + + escapes = OffsetEscapes(escapes, -left) + return ApplyTermEscapes(trimmed, escapes) +} diff --git a/vendor/github.com/MichaelMure/go-term-text/truncate.go b/vendor/github.com/MichaelMure/go-term-text/truncate.go new file mode 100644 index 0000000000..b51bb39e37 --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-text/truncate.go @@ -0,0 +1,24 @@ +package text + +import "github.com/mattn/go-runewidth" + +// TruncateMax truncate a line if its length is greater +// than the given length. Otherwise, the line is returned +// as is. If truncating occur, an ellipsis is inserted at +// the end. +// Handle properly terminal color escape code +func TruncateMax(line string, length int) string { + if length <= 0 { + return "…" + } + + l := Len(line) + if l <= length || l == 0 { + return line + } + + cleaned, escapes := ExtractTermEscapes(line) + truncated := runewidth.Truncate(cleaned, length-1, "") + + return ApplyTermEscapes(truncated, escapes) + "…" +} diff --git a/vendor/github.com/MichaelMure/go-term-text/wrap.go b/vendor/github.com/MichaelMure/go-term-text/wrap.go new file mode 100644 index 0000000000..f63ee84fcd --- /dev/null +++ b/vendor/github.com/MichaelMure/go-term-text/wrap.go @@ -0,0 +1,419 @@ +package text + +import ( + "strings" + + "github.com/mattn/go-runewidth" +) + +type wrapOpts struct { + indent string + pad string + align Alignment +} + +// WrapOption is a functional option for the Wrap() function +type WrapOption func(opts *wrapOpts) + +// WrapPad configure the padding with a string for Wrap() +func WrapPad(pad string) WrapOption { + return func(opts *wrapOpts) { + opts.pad = pad + } +} + +// WrapPadded configure the padding with a number of space characters for Wrap() +func WrapPadded(padLen int) WrapOption { + return func(opts *wrapOpts) { + opts.pad = strings.Repeat(" ", padLen) + } +} + +// WrapPad configure the indentation on the first line for Wrap() +func WrapIndent(indent string) WrapOption { + return func(opts *wrapOpts) { + opts.indent = indent + } +} + +// WrapAlign configure the text alignment for Wrap() +func WrapAlign(align Alignment) WrapOption { + return func(opts *wrapOpts) { + opts.align = align + } +} + +// allWrapOpts compile the set of WrapOption into a final wrapOpts +// from the default values. +func allWrapOpts(opts []WrapOption) *wrapOpts { + wrapOpts := &wrapOpts{ + indent: "", + pad: "", + align: NoAlign, + } + for _, opt := range opts { + opt(wrapOpts) + } + if wrapOpts.indent == "" { + wrapOpts.indent = wrapOpts.pad + } + return wrapOpts +} + +// Wrap a text for a given line size. +// Handle properly terminal color escape code +// Options are accepted to configure things like indent, padding or alignment. +// Return the wrapped text and the number of lines +func Wrap(text string, lineWidth int, opts ...WrapOption) (string, int) { + wrapOpts := allWrapOpts(opts) + + if lineWidth <= 0 { + return "", 1 + } + + var result strings.Builder + var state EscapeState + nbLine := 0 + + // output function to: + // - set the endlines (same as strings.Join()) + // - reset and set again the escape state around the padding/indent + output := func(padding string, content string) { + zeroState := state.IsZero() + if !zeroState && len(padding) > 0 { + result.WriteString("\x1b[0m") + } + if nbLine > 0 { + result.WriteString("\n") + } + result.WriteString(padding) + if !zeroState && len(padding) > 0 { + result.WriteString(state.FormatString()) + } + result.WriteString(content) + nbLine++ + state.Witness(content) + } + + if Len(wrapOpts.indent) >= lineWidth { + // indent is too wide, fallback rendering + output(strings.Repeat("⭬", lineWidth), "") + wrapOpts.indent = wrapOpts.pad + } + if Len(wrapOpts.pad) >= lineWidth { + // padding is too wide, fallback rendering + line := strings.Repeat("⭬", lineWidth) + return strings.Repeat(line+"\n", 5), 6 + } + + // Start with the indent + padStr := wrapOpts.indent + padLen := Len(wrapOpts.indent) + + // tabs are formatted as 4 spaces + text = strings.Replace(text, "\t", " ", -1) + + // NOTE: text is first segmented into lines so that softwrapLine can handle individually + for i, line := range strings.Split(text, "\n") { + // on the second line, switch to use the padding instead + if i == 1 { + padStr = wrapOpts.pad + padLen = Len(wrapOpts.pad) + } + + if line == "" || strings.TrimSpace(line) == "" { + // nothing in the line, we just add the non-empty part of the padding + output(strings.TrimRight(padStr, " "), "") + continue + } + + wrapped := softwrapLine(line, lineWidth-padLen) + split := strings.Split(wrapped, "\n") + + if i == 0 && len(split) > 1 { + // the very first line got wrapped. + // that means we need to use the indent, use the first wrapped line, discard the rest + // switch to the normal padding, do the softwrap again with the remainder, + // and fallback to the normal wrapping flow + + content := LineAlign(strings.TrimRight(split[0], " "), lineWidth-padLen, wrapOpts.align) + output(padStr, content) + + line = strings.TrimPrefix(line, split[0]) + line = strings.TrimLeft(line, " ") + + padStr = wrapOpts.pad + padLen = Len(wrapOpts.pad) + + wrapped = softwrapLine(line, lineWidth-padLen) + split = strings.Split(wrapped, "\n") + } + + for j, seg := range split { + if j == 0 { + // keep the left padding of the wrapped line + content := LineAlign(strings.TrimRight(seg, " "), lineWidth-padLen, wrapOpts.align) + output(padStr, content) + } else { + content := LineAlign(strings.TrimSpace(seg), lineWidth-padLen, wrapOpts.align) + output(padStr, content) + } + } + } + + return result.String(), nbLine +} + +// WrapLeftPadded wrap a text for a given line size with a left padding. +// Handle properly terminal color escape code +func WrapLeftPadded(text string, lineWidth int, leftPad int) (string, int) { + return Wrap(text, lineWidth, WrapPadded(leftPad)) +} + +// WrapWithPad wrap a text for a given line size with a custom left padding +// Handle properly terminal color escape code +func WrapWithPad(text string, lineWidth int, pad string) (string, int) { + return Wrap(text, lineWidth, WrapPad(pad)) +} + +// WrapWithPad wrap a text for a given line size with a custom left padding +// This function also align the result depending on the requested alignment. +// Handle properly terminal color escape code +func WrapWithPadAlign(text string, lineWidth int, pad string, align Alignment) (string, int) { + return Wrap(text, lineWidth, WrapPad(pad), WrapAlign(align)) +} + +// WrapWithPadIndent wrap a text for a given line size with a custom left padding +// and a first line indent. The padding is not effective on the first line, indent +// is used instead, which allow to implement indents and outdents. +// Handle properly terminal color escape code +func WrapWithPadIndent(text string, lineWidth int, indent string, pad string) (string, int) { + return Wrap(text, lineWidth, WrapIndent(indent), WrapPad(pad)) +} + +// WrapWithPadIndentAlign wrap a text for a given line size with a custom left padding +// and a first line indent. The padding is not effective on the first line, indent +// is used instead, which allow to implement indents and outdents. +// This function also align the result depending on the requested alignment. +// Handle properly terminal color escape code +func WrapWithPadIndentAlign(text string, lineWidth int, indent string, pad string, align Alignment) (string, int) { + return Wrap(text, lineWidth, WrapIndent(indent), WrapPad(pad), WrapAlign(align)) +} + +// Break a line into several lines so that each line consumes at most +// 'lineWidth' cells. Lines break at groups of white spaces and multicell +// chars. Nothing is removed from the original text so that it behaves like a +// softwrap. +// +// Required: The line shall not contain '\n' +// +// WRAPPING ALGORITHM: The line is broken into non-breakable chunks, then line +// breaks ("\n") are inserted between these groups so that the total length +// between breaks does not exceed the required width. Words that are longer than +// the textWidth are broken into pieces no longer than textWidth. +func softwrapLine(line string, lineWidth int) string { + escaped, escapes := ExtractTermEscapes(line) + + chunks := segmentLine(escaped) + // Reverse the chunk array so we can use it as a stack. + for i, j := 0, len(chunks)-1; i < j; i, j = i+1, j-1 { + chunks[i], chunks[j] = chunks[j], chunks[i] + } + + // for readability, minimal implementation of a stack: + + pop := func() string { + result := chunks[len(chunks)-1] + chunks = chunks[:len(chunks)-1] + return result + } + + push := func(chunk string) { + chunks = append(chunks, chunk) + } + + peek := func() string { + return chunks[len(chunks)-1] + } + + empty := func() bool { + return len(chunks) == 0 + } + + var out strings.Builder + + // helper to write in the output while interleaving the escape + // sequence at the correct places. + // note: the final algorithm will add additional line break in the original + // text. Those line break are *not* fed to this helper so the positions don't + // need to be offset, which make the whole thing much easier. + currPos := 0 + currItem := 0 + outputString := func(s string) { + for _, r := range s { + for currItem < len(escapes) && currPos == escapes[currItem].Pos { + out.WriteString(escapes[currItem].Item) + currItem++ + } + out.WriteRune(r) + currPos++ + } + } + + width := 0 + + for !empty() { + wl := Len(peek()) + + if width+wl <= lineWidth { + // the chunk fit in the available space + outputString(pop()) + width += wl + if width == lineWidth && !empty() { + // only add line break when there is more chunk to come + out.WriteRune('\n') + width = 0 + } + } else if wl > lineWidth { + // words too long for a full line are split to fill the remaining space. + // But if the long words is the first non-space word in the middle of the + // line, preceding spaces shall not be counted in word splitting. + splitWidth := lineWidth - width + if strings.HasSuffix(out.String(), "\n"+strings.Repeat(" ", width)) { + splitWidth += width + } + left, right := splitWord(pop(), splitWidth) + // remainder is pushed back to the stack for next round + push(right) + outputString(left) + out.WriteRune('\n') + width = 0 + } else { + // normal line overflow, we add a line break and try again + out.WriteRune('\n') + width = 0 + } + } + + // Don't forget the trailing escapes, if any. + for currItem < len(escapes) && currPos >= escapes[currItem].Pos { + out.WriteString(escapes[currItem].Item) + currItem++ + } + + return out.String() +} + +// Segment a line into chunks, where each chunk consists of chars with the same +// type and is not breakable. +func segmentLine(s string) []string { + var chunks []string + + var word string + wordType := none + flushWord := func() { + chunks = append(chunks, word) + word = "" + wordType = none + } + + for _, r := range s { + // A WIDE_CHAR itself constitutes a chunk. + thisType := runeType(r) + if thisType == wideChar { + if wordType != none { + flushWord() + } + chunks = append(chunks, string(r)) + continue + } + // Other type of chunks starts with a char of that type, and ends with a + // char with different type or end of string. + if thisType != wordType { + if wordType != none { + flushWord() + } + word = string(r) + wordType = thisType + } else { + word += string(r) + } + } + if word != "" { + flushWord() + } + + return chunks +} + +type RuneType int + +// Rune categories +// +// These categories are so defined that each category forms a non-breakable +// chunk. It IS NOT the same as unicode code point categories. +const ( + none RuneType = iota + wideChar + invisible + shortUnicode + space + visibleAscii +) + +// Determine the category of a rune. +func runeType(r rune) RuneType { + rw := runewidth.RuneWidth(r) + if rw > 1 { + return wideChar + } else if rw == 0 { + return invisible + } else if r > 127 { + return shortUnicode + } else if r == ' ' { + return space + } else { + return visibleAscii + } +} + +// splitWord split a word at the given length, while ignoring the terminal escape sequences +func splitWord(word string, length int) (string, string) { + runes := []rune(word) + var result []rune + added := 0 + escape := false + + if length == 0 { + return "", word + } + + for _, r := range runes { + if r == '\x1b' { + escape = true + } + + width := runewidth.RuneWidth(r) + if width+added > length { + // wide character made the length overflow + break + } + + result = append(result, r) + + if !escape { + added += width + if added >= length { + break + } + } + + if r == 'm' { + escape = false + } + } + + leftover := runes[len(result):] + + return string(result), string(leftover) +} diff --git a/vendor/github.com/Microsoft/go-winio/.golangci.yml b/vendor/github.com/Microsoft/go-winio/.golangci.yml index 7b503d26a3..faedfe937a 100644 --- a/vendor/github.com/Microsoft/go-winio/.golangci.yml +++ b/vendor/github.com/Microsoft/go-winio/.golangci.yml @@ -1,7 +1,3 @@ -run: - skip-dirs: - - pkg/etw/sample - linters: enable: # style @@ -20,9 +16,13 @@ linters: - gofmt # files are gofmt'ed - gosec # security - nilerr # returns nil even with non-nil error + - thelper # test helpers without t.Helper() - unparam # unused function params issues: + exclude-dirs: + - pkg/etw/sample + exclude-rules: # err is very often shadowed in nested scopes - linters: @@ -69,9 +69,7 @@ linters-settings: # struct order is often for Win32 compat # also, ignore pointer bytes/GC issues for now until performance becomes an issue - fieldalignment - check-shadowing: true nolintlint: - allow-leading-space: false require-explanation: true require-specific: true revive: diff --git a/vendor/github.com/Microsoft/go-winio/backup.go b/vendor/github.com/Microsoft/go-winio/backup.go index 09621c8846..b54341daac 100644 --- a/vendor/github.com/Microsoft/go-winio/backup.go +++ b/vendor/github.com/Microsoft/go-winio/backup.go @@ -10,14 +10,14 @@ import ( "io" "os" "runtime" - "syscall" "unicode/utf16" + "github.com/Microsoft/go-winio/internal/fs" "golang.org/x/sys/windows" ) -//sys backupRead(h syscall.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupRead -//sys backupWrite(h syscall.Handle, b []byte, bytesWritten *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupWrite +//sys backupRead(h windows.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupRead +//sys backupWrite(h windows.Handle, b []byte, bytesWritten *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupWrite const ( BackupData = uint32(iota + 1) @@ -104,7 +104,7 @@ func (r *BackupStreamReader) Next() (*BackupHeader, error) { if err := binary.Read(r.r, binary.LittleEndian, name); err != nil { return nil, err } - hdr.Name = syscall.UTF16ToString(name) + hdr.Name = windows.UTF16ToString(name) } if wsi.StreamID == BackupSparseBlock { if err := binary.Read(r.r, binary.LittleEndian, &hdr.Offset); err != nil { @@ -205,7 +205,7 @@ func NewBackupFileReader(f *os.File, includeSecurity bool) *BackupFileReader { // Read reads a backup stream from the file by calling the Win32 API BackupRead(). func (r *BackupFileReader) Read(b []byte) (int, error) { var bytesRead uint32 - err := backupRead(syscall.Handle(r.f.Fd()), b, &bytesRead, false, r.includeSecurity, &r.ctx) + err := backupRead(windows.Handle(r.f.Fd()), b, &bytesRead, false, r.includeSecurity, &r.ctx) if err != nil { return 0, &os.PathError{Op: "BackupRead", Path: r.f.Name(), Err: err} } @@ -220,7 +220,7 @@ func (r *BackupFileReader) Read(b []byte) (int, error) { // the underlying file. func (r *BackupFileReader) Close() error { if r.ctx != 0 { - _ = backupRead(syscall.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx) + _ = backupRead(windows.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx) runtime.KeepAlive(r.f) r.ctx = 0 } @@ -244,7 +244,7 @@ func NewBackupFileWriter(f *os.File, includeSecurity bool) *BackupFileWriter { // Write restores a portion of the file using the provided backup stream. func (w *BackupFileWriter) Write(b []byte) (int, error) { var bytesWritten uint32 - err := backupWrite(syscall.Handle(w.f.Fd()), b, &bytesWritten, false, w.includeSecurity, &w.ctx) + err := backupWrite(windows.Handle(w.f.Fd()), b, &bytesWritten, false, w.includeSecurity, &w.ctx) if err != nil { return 0, &os.PathError{Op: "BackupWrite", Path: w.f.Name(), Err: err} } @@ -259,7 +259,7 @@ func (w *BackupFileWriter) Write(b []byte) (int, error) { // close the underlying file. func (w *BackupFileWriter) Close() error { if w.ctx != 0 { - _ = backupWrite(syscall.Handle(w.f.Fd()), nil, nil, true, false, &w.ctx) + _ = backupWrite(windows.Handle(w.f.Fd()), nil, nil, true, false, &w.ctx) runtime.KeepAlive(w.f) w.ctx = 0 } @@ -271,17 +271,14 @@ func (w *BackupFileWriter) Close() error { // // If the file opened was a directory, it cannot be used with Readdir(). func OpenForBackup(path string, access uint32, share uint32, createmode uint32) (*os.File, error) { - winPath, err := syscall.UTF16FromString(path) - if err != nil { - return nil, err - } - h, err := syscall.CreateFile(&winPath[0], - access, - share, + h, err := fs.CreateFile(path, + fs.AccessMask(access), + fs.FileShareMode(share), nil, - createmode, - syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OPEN_REPARSE_POINT, - 0) + fs.FileCreationDisposition(createmode), + fs.FILE_FLAG_BACKUP_SEMANTICS|fs.FILE_FLAG_OPEN_REPARSE_POINT, + 0, + ) if err != nil { err = &os.PathError{Op: "open", Path: path, Err: err} return nil, err diff --git a/vendor/github.com/Microsoft/go-winio/file.go b/vendor/github.com/Microsoft/go-winio/file.go index 175a99d3f4..fe82a180db 100644 --- a/vendor/github.com/Microsoft/go-winio/file.go +++ b/vendor/github.com/Microsoft/go-winio/file.go @@ -15,26 +15,11 @@ import ( "golang.org/x/sys/windows" ) -//sys cancelIoEx(file syscall.Handle, o *syscall.Overlapped) (err error) = CancelIoEx -//sys createIoCompletionPort(file syscall.Handle, port syscall.Handle, key uintptr, threadCount uint32) (newport syscall.Handle, err error) = CreateIoCompletionPort -//sys getQueuedCompletionStatus(port syscall.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) = GetQueuedCompletionStatus -//sys setFileCompletionNotificationModes(h syscall.Handle, flags uint8) (err error) = SetFileCompletionNotificationModes -//sys wsaGetOverlappedResult(h syscall.Handle, o *syscall.Overlapped, bytes *uint32, wait bool, flags *uint32) (err error) = ws2_32.WSAGetOverlappedResult - -type atomicBool int32 - -func (b *atomicBool) isSet() bool { return atomic.LoadInt32((*int32)(b)) != 0 } -func (b *atomicBool) setFalse() { atomic.StoreInt32((*int32)(b), 0) } -func (b *atomicBool) setTrue() { atomic.StoreInt32((*int32)(b), 1) } - -//revive:disable-next-line:predeclared Keep "new" to maintain consistency with "atomic" pkg -func (b *atomicBool) swap(new bool) bool { - var newInt int32 - if new { - newInt = 1 - } - return atomic.SwapInt32((*int32)(b), newInt) == 1 -} +//sys cancelIoEx(file windows.Handle, o *windows.Overlapped) (err error) = CancelIoEx +//sys createIoCompletionPort(file windows.Handle, port windows.Handle, key uintptr, threadCount uint32) (newport windows.Handle, err error) = CreateIoCompletionPort +//sys getQueuedCompletionStatus(port windows.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) = GetQueuedCompletionStatus +//sys setFileCompletionNotificationModes(h windows.Handle, flags uint8) (err error) = SetFileCompletionNotificationModes +//sys wsaGetOverlappedResult(h windows.Handle, o *windows.Overlapped, bytes *uint32, wait bool, flags *uint32) (err error) = ws2_32.WSAGetOverlappedResult var ( ErrFileClosed = errors.New("file has already been closed") @@ -50,7 +35,7 @@ func (*timeoutError) Temporary() bool { return true } type timeoutChan chan struct{} var ioInitOnce sync.Once -var ioCompletionPort syscall.Handle +var ioCompletionPort windows.Handle // ioResult contains the result of an asynchronous IO operation. type ioResult struct { @@ -60,12 +45,12 @@ type ioResult struct { // ioOperation represents an outstanding asynchronous Win32 IO. type ioOperation struct { - o syscall.Overlapped + o windows.Overlapped ch chan ioResult } func initIO() { - h, err := createIoCompletionPort(syscall.InvalidHandle, 0, 0, 0xffffffff) + h, err := createIoCompletionPort(windows.InvalidHandle, 0, 0, 0xffffffff) if err != nil { panic(err) } @@ -76,10 +61,10 @@ func initIO() { // win32File implements Reader, Writer, and Closer on a Win32 handle without blocking in a syscall. // It takes ownership of this handle and will close it if it is garbage collected. type win32File struct { - handle syscall.Handle + handle windows.Handle wg sync.WaitGroup wgLock sync.RWMutex - closing atomicBool + closing atomic.Bool socket bool readDeadline deadlineHandler writeDeadline deadlineHandler @@ -90,11 +75,11 @@ type deadlineHandler struct { channel timeoutChan channelLock sync.RWMutex timer *time.Timer - timedout atomicBool + timedout atomic.Bool } // makeWin32File makes a new win32File from an existing file handle. -func makeWin32File(h syscall.Handle) (*win32File, error) { +func makeWin32File(h windows.Handle) (*win32File, error) { f := &win32File{handle: h} ioInitOnce.Do(initIO) _, err := createIoCompletionPort(h, ioCompletionPort, 0, 0xffffffff) @@ -110,7 +95,12 @@ func makeWin32File(h syscall.Handle) (*win32File, error) { return f, nil } +// Deprecated: use NewOpenFile instead. func MakeOpenFile(h syscall.Handle) (io.ReadWriteCloser, error) { + return NewOpenFile(windows.Handle(h)) +} + +func NewOpenFile(h windows.Handle) (io.ReadWriteCloser, error) { // If we return the result of makeWin32File directly, it can result in an // interface-wrapped nil, rather than a nil interface value. f, err := makeWin32File(h) @@ -124,13 +114,13 @@ func MakeOpenFile(h syscall.Handle) (io.ReadWriteCloser, error) { func (f *win32File) closeHandle() { f.wgLock.Lock() // Atomically set that we are closing, releasing the resources only once. - if !f.closing.swap(true) { + if !f.closing.Swap(true) { f.wgLock.Unlock() // cancel all IO and wait for it to complete _ = cancelIoEx(f.handle, nil) f.wg.Wait() // at this point, no new IO can start - syscall.Close(f.handle) + windows.Close(f.handle) f.handle = 0 } else { f.wgLock.Unlock() @@ -145,14 +135,14 @@ func (f *win32File) Close() error { // IsClosed checks if the file has been closed. func (f *win32File) IsClosed() bool { - return f.closing.isSet() + return f.closing.Load() } // prepareIO prepares for a new IO operation. // The caller must call f.wg.Done() when the IO is finished, prior to Close() returning. func (f *win32File) prepareIO() (*ioOperation, error) { f.wgLock.RLock() - if f.closing.isSet() { + if f.closing.Load() { f.wgLock.RUnlock() return nil, ErrFileClosed } @@ -164,12 +154,12 @@ func (f *win32File) prepareIO() (*ioOperation, error) { } // ioCompletionProcessor processes completed async IOs forever. -func ioCompletionProcessor(h syscall.Handle) { +func ioCompletionProcessor(h windows.Handle) { for { var bytes uint32 var key uintptr var op *ioOperation - err := getQueuedCompletionStatus(h, &bytes, &key, &op, syscall.INFINITE) + err := getQueuedCompletionStatus(h, &bytes, &key, &op, windows.INFINITE) if op == nil { panic(err) } @@ -182,11 +172,11 @@ func ioCompletionProcessor(h syscall.Handle) { // asyncIO processes the return value from ReadFile or WriteFile, blocking until // the operation has actually completed. func (f *win32File) asyncIO(c *ioOperation, d *deadlineHandler, bytes uint32, err error) (int, error) { - if err != syscall.ERROR_IO_PENDING { //nolint:errorlint // err is Errno + if err != windows.ERROR_IO_PENDING { //nolint:errorlint // err is Errno return int(bytes), err } - if f.closing.isSet() { + if f.closing.Load() { _ = cancelIoEx(f.handle, &c.o) } @@ -201,8 +191,8 @@ func (f *win32File) asyncIO(c *ioOperation, d *deadlineHandler, bytes uint32, er select { case r = <-c.ch: err = r.err - if err == syscall.ERROR_OPERATION_ABORTED { //nolint:errorlint // err is Errno - if f.closing.isSet() { + if err == windows.ERROR_OPERATION_ABORTED { //nolint:errorlint // err is Errno + if f.closing.Load() { err = ErrFileClosed } } else if err != nil && f.socket { @@ -214,7 +204,7 @@ func (f *win32File) asyncIO(c *ioOperation, d *deadlineHandler, bytes uint32, er _ = cancelIoEx(f.handle, &c.o) r = <-c.ch err = r.err - if err == syscall.ERROR_OPERATION_ABORTED { //nolint:errorlint // err is Errno + if err == windows.ERROR_OPERATION_ABORTED { //nolint:errorlint // err is Errno err = ErrTimeout } } @@ -235,23 +225,22 @@ func (f *win32File) Read(b []byte) (int, error) { } defer f.wg.Done() - if f.readDeadline.timedout.isSet() { + if f.readDeadline.timedout.Load() { return 0, ErrTimeout } var bytes uint32 - err = syscall.ReadFile(f.handle, b, &bytes, &c.o) + err = windows.ReadFile(f.handle, b, &bytes, &c.o) n, err := f.asyncIO(c, &f.readDeadline, bytes, err) runtime.KeepAlive(b) // Handle EOF conditions. if err == nil && n == 0 && len(b) != 0 { return 0, io.EOF - } else if err == syscall.ERROR_BROKEN_PIPE { //nolint:errorlint // err is Errno + } else if err == windows.ERROR_BROKEN_PIPE { //nolint:errorlint // err is Errno return 0, io.EOF - } else { - return n, err } + return n, err } // Write writes to a file handle. @@ -262,12 +251,12 @@ func (f *win32File) Write(b []byte) (int, error) { } defer f.wg.Done() - if f.writeDeadline.timedout.isSet() { + if f.writeDeadline.timedout.Load() { return 0, ErrTimeout } var bytes uint32 - err = syscall.WriteFile(f.handle, b, &bytes, &c.o) + err = windows.WriteFile(f.handle, b, &bytes, &c.o) n, err := f.asyncIO(c, &f.writeDeadline, bytes, err) runtime.KeepAlive(b) return n, err @@ -282,7 +271,7 @@ func (f *win32File) SetWriteDeadline(deadline time.Time) error { } func (f *win32File) Flush() error { - return syscall.FlushFileBuffers(f.handle) + return windows.FlushFileBuffers(f.handle) } func (f *win32File) Fd() uintptr { @@ -299,7 +288,7 @@ func (d *deadlineHandler) set(deadline time.Time) error { } d.timer = nil } - d.timedout.setFalse() + d.timedout.Store(false) select { case <-d.channel: @@ -314,7 +303,7 @@ func (d *deadlineHandler) set(deadline time.Time) error { } timeoutIO := func() { - d.timedout.setTrue() + d.timedout.Store(true) close(d.channel) } diff --git a/vendor/github.com/Microsoft/go-winio/fileinfo.go b/vendor/github.com/Microsoft/go-winio/fileinfo.go index 702950e72a..c860eb9917 100644 --- a/vendor/github.com/Microsoft/go-winio/fileinfo.go +++ b/vendor/github.com/Microsoft/go-winio/fileinfo.go @@ -18,9 +18,18 @@ type FileBasicInfo struct { _ uint32 // padding } +// alignedFileBasicInfo is a FileBasicInfo, but aligned to uint64 by containing +// uint64 rather than windows.Filetime. Filetime contains two uint32s. uint64 +// alignment is necessary to pass this as FILE_BASIC_INFO. +type alignedFileBasicInfo struct { + CreationTime, LastAccessTime, LastWriteTime, ChangeTime uint64 + FileAttributes uint32 + _ uint32 // padding +} + // GetFileBasicInfo retrieves times and attributes for a file. func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) { - bi := &FileBasicInfo{} + bi := &alignedFileBasicInfo{} if err := windows.GetFileInformationByHandleEx( windows.Handle(f.Fd()), windows.FileBasicInfo, @@ -30,16 +39,21 @@ func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) { return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err} } runtime.KeepAlive(f) - return bi, nil + // Reinterpret the alignedFileBasicInfo as a FileBasicInfo so it matches the + // public API of this module. The data may be unnecessarily aligned. + return (*FileBasicInfo)(unsafe.Pointer(bi)), nil } // SetFileBasicInfo sets times and attributes for a file. func SetFileBasicInfo(f *os.File, bi *FileBasicInfo) error { + // Create an alignedFileBasicInfo based on a FileBasicInfo. The copy is + // suitable to pass to GetFileInformationByHandleEx. + biAligned := *(*alignedFileBasicInfo)(unsafe.Pointer(bi)) if err := windows.SetFileInformationByHandle( windows.Handle(f.Fd()), windows.FileBasicInfo, - (*byte)(unsafe.Pointer(bi)), - uint32(unsafe.Sizeof(*bi)), + (*byte)(unsafe.Pointer(&biAligned)), + uint32(unsafe.Sizeof(biAligned)), ); err != nil { return &os.PathError{Op: "SetFileInformationByHandle", Path: f.Name(), Err: err} } diff --git a/vendor/github.com/Microsoft/go-winio/hvsock.go b/vendor/github.com/Microsoft/go-winio/hvsock.go index c881916583..c4fdd9d4ae 100644 --- a/vendor/github.com/Microsoft/go-winio/hvsock.go +++ b/vendor/github.com/Microsoft/go-winio/hvsock.go @@ -10,7 +10,6 @@ import ( "io" "net" "os" - "syscall" "time" "unsafe" @@ -181,13 +180,13 @@ type HvsockConn struct { var _ net.Conn = &HvsockConn{} func newHVSocket() (*win32File, error) { - fd, err := syscall.Socket(afHVSock, syscall.SOCK_STREAM, 1) + fd, err := windows.Socket(afHVSock, windows.SOCK_STREAM, 1) if err != nil { return nil, os.NewSyscallError("socket", err) } f, err := makeWin32File(fd) if err != nil { - syscall.Close(fd) + windows.Close(fd) return nil, err } f.socket = true @@ -197,16 +196,24 @@ func newHVSocket() (*win32File, error) { // ListenHvsock listens for connections on the specified hvsock address. func ListenHvsock(addr *HvsockAddr) (_ *HvsockListener, err error) { l := &HvsockListener{addr: *addr} - sock, err := newHVSocket() + + var sock *win32File + sock, err = newHVSocket() if err != nil { return nil, l.opErr("listen", err) } + defer func() { + if err != nil { + _ = sock.Close() + } + }() + sa := addr.raw() - err = socket.Bind(windows.Handle(sock.handle), &sa) + err = socket.Bind(sock.handle, &sa) if err != nil { return nil, l.opErr("listen", os.NewSyscallError("socket", err)) } - err = syscall.Listen(sock.handle, 16) + err = windows.Listen(sock.handle, 16) if err != nil { return nil, l.opErr("listen", os.NewSyscallError("listen", err)) } @@ -246,7 +253,7 @@ func (l *HvsockListener) Accept() (_ net.Conn, err error) { var addrbuf [addrlen * 2]byte var bytes uint32 - err = syscall.AcceptEx(l.sock.handle, sock.handle, &addrbuf[0], 0 /* rxdatalen */, addrlen, addrlen, &bytes, &c.o) + err = windows.AcceptEx(l.sock.handle, sock.handle, &addrbuf[0], 0 /* rxdatalen */, addrlen, addrlen, &bytes, &c.o) if _, err = l.sock.asyncIO(c, nil, bytes, err); err != nil { return nil, l.opErr("accept", os.NewSyscallError("acceptex", err)) } @@ -263,7 +270,7 @@ func (l *HvsockListener) Accept() (_ net.Conn, err error) { conn.remote.fromRaw((*rawHvsockAddr)(unsafe.Pointer(&addrbuf[addrlen]))) // initialize the accepted socket and update its properties with those of the listening socket - if err = windows.Setsockopt(windows.Handle(sock.handle), + if err = windows.Setsockopt(sock.handle, windows.SOL_SOCKET, windows.SO_UPDATE_ACCEPT_CONTEXT, (*byte)(unsafe.Pointer(&l.sock.handle)), int32(unsafe.Sizeof(l.sock.handle))); err != nil { return nil, conn.opErr("accept", os.NewSyscallError("setsockopt", err)) @@ -334,7 +341,7 @@ func (d *HvsockDialer) Dial(ctx context.Context, addr *HvsockAddr) (conn *Hvsock }() sa := addr.raw() - err = socket.Bind(windows.Handle(sock.handle), &sa) + err = socket.Bind(sock.handle, &sa) if err != nil { return nil, conn.opErr(op, os.NewSyscallError("bind", err)) } @@ -347,7 +354,7 @@ func (d *HvsockDialer) Dial(ctx context.Context, addr *HvsockAddr) (conn *Hvsock var bytes uint32 for i := uint(0); i <= d.Retries; i++ { err = socket.ConnectEx( - windows.Handle(sock.handle), + sock.handle, &sa, nil, // sendBuf 0, // sendDataLen @@ -367,7 +374,7 @@ func (d *HvsockDialer) Dial(ctx context.Context, addr *HvsockAddr) (conn *Hvsock // update the connection properties, so shutdown can be used if err = windows.Setsockopt( - windows.Handle(sock.handle), + sock.handle, windows.SOL_SOCKET, windows.SO_UPDATE_CONNECT_CONTEXT, nil, // optvalue @@ -378,7 +385,7 @@ func (d *HvsockDialer) Dial(ctx context.Context, addr *HvsockAddr) (conn *Hvsock // get the local name var sal rawHvsockAddr - err = socket.GetSockName(windows.Handle(sock.handle), &sal) + err = socket.GetSockName(sock.handle, &sal) if err != nil { return nil, conn.opErr(op, os.NewSyscallError("getsockname", err)) } @@ -421,7 +428,7 @@ func (d *HvsockDialer) redialWait(ctx context.Context) (err error) { return ctx.Err() } -// assumes error is a plain, unwrapped syscall.Errno provided by direct syscall. +// assumes error is a plain, unwrapped windows.Errno provided by direct syscall. func canRedial(err error) bool { //nolint:errorlint // guaranteed to be an Errno switch err { @@ -447,9 +454,9 @@ func (conn *HvsockConn) Read(b []byte) (int, error) { return 0, conn.opErr("read", err) } defer conn.sock.wg.Done() - buf := syscall.WSABuf{Buf: &b[0], Len: uint32(len(b))} + buf := windows.WSABuf{Buf: &b[0], Len: uint32(len(b))} var flags, bytes uint32 - err = syscall.WSARecv(conn.sock.handle, &buf, 1, &bytes, &flags, &c.o, nil) + err = windows.WSARecv(conn.sock.handle, &buf, 1, &bytes, &flags, &c.o, nil) n, err := conn.sock.asyncIO(c, &conn.sock.readDeadline, bytes, err) if err != nil { var eno windows.Errno @@ -482,9 +489,9 @@ func (conn *HvsockConn) write(b []byte) (int, error) { return 0, conn.opErr("write", err) } defer conn.sock.wg.Done() - buf := syscall.WSABuf{Buf: &b[0], Len: uint32(len(b))} + buf := windows.WSABuf{Buf: &b[0], Len: uint32(len(b))} var bytes uint32 - err = syscall.WSASend(conn.sock.handle, &buf, 1, &bytes, 0, &c.o, nil) + err = windows.WSASend(conn.sock.handle, &buf, 1, &bytes, 0, &c.o, nil) n, err := conn.sock.asyncIO(c, &conn.sock.writeDeadline, bytes, err) if err != nil { var eno windows.Errno @@ -511,7 +518,7 @@ func (conn *HvsockConn) shutdown(how int) error { return socket.ErrSocketClosed } - err := syscall.Shutdown(conn.sock.handle, how) + err := windows.Shutdown(conn.sock.handle, how) if err != nil { // If the connection was closed, shutdowns fail with "not connected" if errors.Is(err, windows.WSAENOTCONN) || @@ -525,7 +532,7 @@ func (conn *HvsockConn) shutdown(how int) error { // CloseRead shuts down the read end of the socket, preventing future read operations. func (conn *HvsockConn) CloseRead() error { - err := conn.shutdown(syscall.SHUT_RD) + err := conn.shutdown(windows.SHUT_RD) if err != nil { return conn.opErr("closeread", err) } @@ -535,7 +542,7 @@ func (conn *HvsockConn) CloseRead() error { // CloseWrite shuts down the write end of the socket, preventing future write operations and // notifying the other endpoint that no more data will be written. func (conn *HvsockConn) CloseWrite() error { - err := conn.shutdown(syscall.SHUT_WR) + err := conn.shutdown(windows.SHUT_WR) if err != nil { return conn.opErr("closewrite", err) } diff --git a/vendor/github.com/Microsoft/go-winio/internal/fs/fs.go b/vendor/github.com/Microsoft/go-winio/internal/fs/fs.go index 509b3ec641..0cd9621df7 100644 --- a/vendor/github.com/Microsoft/go-winio/internal/fs/fs.go +++ b/vendor/github.com/Microsoft/go-winio/internal/fs/fs.go @@ -11,12 +11,14 @@ import ( //go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go fs.go // https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew -//sys CreateFile(name string, access AccessMask, mode FileShareMode, sa *syscall.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) [failretval==windows.InvalidHandle] = CreateFileW +//sys CreateFile(name string, access AccessMask, mode FileShareMode, sa *windows.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) [failretval==windows.InvalidHandle] = CreateFileW const NullHandle windows.Handle = 0 // AccessMask defines standard, specific, and generic rights. // +// Used with CreateFile and NtCreateFile (and co.). +// // Bitmask: // 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 // 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 @@ -47,6 +49,12 @@ const ( // https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew#parameters FILE_ANY_ACCESS AccessMask = 0 + GENERIC_READ AccessMask = 0x8000_0000 + GENERIC_WRITE AccessMask = 0x4000_0000 + GENERIC_EXECUTE AccessMask = 0x2000_0000 + GENERIC_ALL AccessMask = 0x1000_0000 + ACCESS_SYSTEM_SECURITY AccessMask = 0x0100_0000 + // Specific Object Access // from ntioapi.h @@ -124,14 +132,32 @@ const ( TRUNCATE_EXISTING FileCreationDisposition = 0x05 ) +// Create disposition values for NtCreate* +type NTFileCreationDisposition uint32 + +//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. +const ( + // From ntioapi.h + + FILE_SUPERSEDE NTFileCreationDisposition = 0x00 + FILE_OPEN NTFileCreationDisposition = 0x01 + FILE_CREATE NTFileCreationDisposition = 0x02 + FILE_OPEN_IF NTFileCreationDisposition = 0x03 + FILE_OVERWRITE NTFileCreationDisposition = 0x04 + FILE_OVERWRITE_IF NTFileCreationDisposition = 0x05 + FILE_MAXIMUM_DISPOSITION NTFileCreationDisposition = 0x05 +) + // CreateFile and co. take flags or attributes together as one parameter. // Define alias until we can use generics to allow both - +// // https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants type FileFlagOrAttribute uint32 //nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. -const ( // from winnt.h +const ( + // from winnt.h + FILE_FLAG_WRITE_THROUGH FileFlagOrAttribute = 0x8000_0000 FILE_FLAG_OVERLAPPED FileFlagOrAttribute = 0x4000_0000 FILE_FLAG_NO_BUFFERING FileFlagOrAttribute = 0x2000_0000 @@ -145,17 +171,51 @@ const ( // from winnt.h FILE_FLAG_FIRST_PIPE_INSTANCE FileFlagOrAttribute = 0x0008_0000 ) +// NtCreate* functions take a dedicated CreateOptions parameter. +// +// https://learn.microsoft.com/en-us/windows/win32/api/Winternl/nf-winternl-ntcreatefile +// +// https://learn.microsoft.com/en-us/windows/win32/devnotes/nt-create-named-pipe-file +type NTCreateOptions uint32 + +//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. +const ( + // From ntioapi.h + + FILE_DIRECTORY_FILE NTCreateOptions = 0x0000_0001 + FILE_WRITE_THROUGH NTCreateOptions = 0x0000_0002 + FILE_SEQUENTIAL_ONLY NTCreateOptions = 0x0000_0004 + FILE_NO_INTERMEDIATE_BUFFERING NTCreateOptions = 0x0000_0008 + + FILE_SYNCHRONOUS_IO_ALERT NTCreateOptions = 0x0000_0010 + FILE_SYNCHRONOUS_IO_NONALERT NTCreateOptions = 0x0000_0020 + FILE_NON_DIRECTORY_FILE NTCreateOptions = 0x0000_0040 + FILE_CREATE_TREE_CONNECTION NTCreateOptions = 0x0000_0080 + + FILE_COMPLETE_IF_OPLOCKED NTCreateOptions = 0x0000_0100 + FILE_NO_EA_KNOWLEDGE NTCreateOptions = 0x0000_0200 + FILE_DISABLE_TUNNELING NTCreateOptions = 0x0000_0400 + FILE_RANDOM_ACCESS NTCreateOptions = 0x0000_0800 + + FILE_DELETE_ON_CLOSE NTCreateOptions = 0x0000_1000 + FILE_OPEN_BY_FILE_ID NTCreateOptions = 0x0000_2000 + FILE_OPEN_FOR_BACKUP_INTENT NTCreateOptions = 0x0000_4000 + FILE_NO_COMPRESSION NTCreateOptions = 0x0000_8000 +) + type FileSQSFlag = FileFlagOrAttribute //nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. -const ( // from winbase.h +const ( + // from winbase.h + SECURITY_ANONYMOUS FileSQSFlag = FileSQSFlag(SecurityAnonymous << 16) SECURITY_IDENTIFICATION FileSQSFlag = FileSQSFlag(SecurityIdentification << 16) SECURITY_IMPERSONATION FileSQSFlag = FileSQSFlag(SecurityImpersonation << 16) SECURITY_DELEGATION FileSQSFlag = FileSQSFlag(SecurityDelegation << 16) - SECURITY_SQOS_PRESENT FileSQSFlag = 0x00100000 - SECURITY_VALID_SQOS_FLAGS FileSQSFlag = 0x001F0000 + SECURITY_SQOS_PRESENT FileSQSFlag = 0x0010_0000 + SECURITY_VALID_SQOS_FLAGS FileSQSFlag = 0x001F_0000 ) // GetFinalPathNameByHandle flags diff --git a/vendor/github.com/Microsoft/go-winio/internal/fs/zsyscall_windows.go b/vendor/github.com/Microsoft/go-winio/internal/fs/zsyscall_windows.go index e2f7bb24e5..a94e234c70 100644 --- a/vendor/github.com/Microsoft/go-winio/internal/fs/zsyscall_windows.go +++ b/vendor/github.com/Microsoft/go-winio/internal/fs/zsyscall_windows.go @@ -33,9 +33,6 @@ func errnoErr(e syscall.Errno) error { case errnoERROR_IO_PENDING: return errERROR_IO_PENDING } - // TODO: add more here, after collecting data on the common - // error values see on Windows. (perhaps when running - // all.bat?) return e } @@ -45,7 +42,7 @@ var ( procCreateFileW = modkernel32.NewProc("CreateFileW") ) -func CreateFile(name string, access AccessMask, mode FileShareMode, sa *syscall.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) { +func CreateFile(name string, access AccessMask, mode FileShareMode, sa *windows.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) { var _p0 *uint16 _p0, err = syscall.UTF16PtrFromString(name) if err != nil { @@ -54,8 +51,8 @@ func CreateFile(name string, access AccessMask, mode FileShareMode, sa *syscall. return _CreateFile(_p0, access, mode, sa, createmode, attrs, templatefile) } -func _CreateFile(name *uint16, access AccessMask, mode FileShareMode, sa *syscall.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) { - r0, _, e1 := syscall.Syscall9(procCreateFileW.Addr(), 7, uintptr(unsafe.Pointer(name)), uintptr(access), uintptr(mode), uintptr(unsafe.Pointer(sa)), uintptr(createmode), uintptr(attrs), uintptr(templatefile), 0, 0) +func _CreateFile(name *uint16, access AccessMask, mode FileShareMode, sa *windows.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) { + r0, _, e1 := syscall.SyscallN(procCreateFileW.Addr(), uintptr(unsafe.Pointer(name)), uintptr(access), uintptr(mode), uintptr(unsafe.Pointer(sa)), uintptr(createmode), uintptr(attrs), uintptr(templatefile)) handle = windows.Handle(r0) if handle == windows.InvalidHandle { err = errnoErr(e1) diff --git a/vendor/github.com/Microsoft/go-winio/internal/socket/socket.go b/vendor/github.com/Microsoft/go-winio/internal/socket/socket.go index aeb7b7250f..88580d974e 100644 --- a/vendor/github.com/Microsoft/go-winio/internal/socket/socket.go +++ b/vendor/github.com/Microsoft/go-winio/internal/socket/socket.go @@ -156,9 +156,7 @@ func connectEx( bytesSent *uint32, overlapped *windows.Overlapped, ) (err error) { - // todo: after upgrading to 1.18, switch from syscall.Syscall9 to syscall.SyscallN - r1, _, e1 := syscall.Syscall9(connectExFunc.addr, - 7, + r1, _, e1 := syscall.SyscallN(connectExFunc.addr, uintptr(s), uintptr(name), uintptr(namelen), @@ -166,8 +164,8 @@ func connectEx( uintptr(sendDataLen), uintptr(unsafe.Pointer(bytesSent)), uintptr(unsafe.Pointer(overlapped)), - 0, - 0) + ) + if r1 == 0 { if e1 != 0 { err = error(e1) diff --git a/vendor/github.com/Microsoft/go-winio/internal/socket/zsyscall_windows.go b/vendor/github.com/Microsoft/go-winio/internal/socket/zsyscall_windows.go index 6d2e1a9e44..e1504126aa 100644 --- a/vendor/github.com/Microsoft/go-winio/internal/socket/zsyscall_windows.go +++ b/vendor/github.com/Microsoft/go-winio/internal/socket/zsyscall_windows.go @@ -33,9 +33,6 @@ func errnoErr(e syscall.Errno) error { case errnoERROR_IO_PENDING: return errERROR_IO_PENDING } - // TODO: add more here, after collecting data on the common - // error values see on Windows. (perhaps when running - // all.bat?) return e } @@ -48,7 +45,7 @@ var ( ) func bind(s windows.Handle, name unsafe.Pointer, namelen int32) (err error) { - r1, _, e1 := syscall.Syscall(procbind.Addr(), 3, uintptr(s), uintptr(name), uintptr(namelen)) + r1, _, e1 := syscall.SyscallN(procbind.Addr(), uintptr(s), uintptr(name), uintptr(namelen)) if r1 == socketError { err = errnoErr(e1) } @@ -56,7 +53,7 @@ func bind(s windows.Handle, name unsafe.Pointer, namelen int32) (err error) { } func getpeername(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) { - r1, _, e1 := syscall.Syscall(procgetpeername.Addr(), 3, uintptr(s), uintptr(name), uintptr(unsafe.Pointer(namelen))) + r1, _, e1 := syscall.SyscallN(procgetpeername.Addr(), uintptr(s), uintptr(name), uintptr(unsafe.Pointer(namelen))) if r1 == socketError { err = errnoErr(e1) } @@ -64,7 +61,7 @@ func getpeername(s windows.Handle, name unsafe.Pointer, namelen *int32) (err err } func getsockname(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) { - r1, _, e1 := syscall.Syscall(procgetsockname.Addr(), 3, uintptr(s), uintptr(name), uintptr(unsafe.Pointer(namelen))) + r1, _, e1 := syscall.SyscallN(procgetsockname.Addr(), uintptr(s), uintptr(name), uintptr(unsafe.Pointer(namelen))) if r1 == socketError { err = errnoErr(e1) } diff --git a/vendor/github.com/Microsoft/go-winio/internal/stringbuffer/wstring.go b/vendor/github.com/Microsoft/go-winio/internal/stringbuffer/wstring.go index 7ad5057024..42ebc019fc 100644 --- a/vendor/github.com/Microsoft/go-winio/internal/stringbuffer/wstring.go +++ b/vendor/github.com/Microsoft/go-winio/internal/stringbuffer/wstring.go @@ -62,7 +62,7 @@ func (b *WString) Free() { // ResizeTo grows the buffer to at least c and returns the new capacity, freeing the // previous buffer back into pool. func (b *WString) ResizeTo(c uint32) uint32 { - // allready sufficient (or n is 0) + // already sufficient (or n is 0) if c <= b.Cap() { return b.Cap() } diff --git a/vendor/github.com/Microsoft/go-winio/pipe.go b/vendor/github.com/Microsoft/go-winio/pipe.go index 25cc811031..a2da6639d0 100644 --- a/vendor/github.com/Microsoft/go-winio/pipe.go +++ b/vendor/github.com/Microsoft/go-winio/pipe.go @@ -11,7 +11,6 @@ import ( "net" "os" "runtime" - "syscall" "time" "unsafe" @@ -20,20 +19,44 @@ import ( "github.com/Microsoft/go-winio/internal/fs" ) -//sys connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) = ConnectNamedPipe -//sys createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) [failretval==syscall.InvalidHandle] = CreateNamedPipeW -//sys getNamedPipeInfo(pipe syscall.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) = GetNamedPipeInfo -//sys getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) = GetNamedPipeHandleStateW -//sys localAlloc(uFlags uint32, length uint32) (ptr uintptr) = LocalAlloc -//sys ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntStatus) = ntdll.NtCreateNamedPipeFile +//sys connectNamedPipe(pipe windows.Handle, o *windows.Overlapped) (err error) = ConnectNamedPipe +//sys createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *windows.SecurityAttributes) (handle windows.Handle, err error) [failretval==windows.InvalidHandle] = CreateNamedPipeW +//sys disconnectNamedPipe(pipe windows.Handle) (err error) = DisconnectNamedPipe +//sys getNamedPipeInfo(pipe windows.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) = GetNamedPipeInfo +//sys getNamedPipeHandleState(pipe windows.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) = GetNamedPipeHandleStateW +//sys ntCreateNamedPipeFile(pipe *windows.Handle, access ntAccessMask, oa *objectAttributes, iosb *ioStatusBlock, share ntFileShareMode, disposition ntFileCreationDisposition, options ntFileOptions, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntStatus) = ntdll.NtCreateNamedPipeFile //sys rtlNtStatusToDosError(status ntStatus) (winerr error) = ntdll.RtlNtStatusToDosErrorNoTeb //sys rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntStatus) = ntdll.RtlDosPathNameToNtPathName_U //sys rtlDefaultNpAcl(dacl *uintptr) (status ntStatus) = ntdll.RtlDefaultNpAcl +type PipeConn interface { + net.Conn + Disconnect() error + Flush() error +} + +// type aliases for mkwinsyscall code +type ( + ntAccessMask = fs.AccessMask + ntFileShareMode = fs.FileShareMode + ntFileCreationDisposition = fs.NTFileCreationDisposition + ntFileOptions = fs.NTCreateOptions +) + type ioStatusBlock struct { Status, Information uintptr } +// typedef struct _OBJECT_ATTRIBUTES { +// ULONG Length; +// HANDLE RootDirectory; +// PUNICODE_STRING ObjectName; +// ULONG Attributes; +// PVOID SecurityDescriptor; +// PVOID SecurityQualityOfService; +// } OBJECT_ATTRIBUTES; +// +// https://learn.microsoft.com/en-us/windows/win32/api/ntdef/ns-ntdef-_object_attributes type objectAttributes struct { Length uintptr RootDirectory uintptr @@ -49,6 +72,17 @@ type unicodeString struct { Buffer uintptr } +// typedef struct _SECURITY_DESCRIPTOR { +// BYTE Revision; +// BYTE Sbz1; +// SECURITY_DESCRIPTOR_CONTROL Control; +// PSID Owner; +// PSID Group; +// PACL Sacl; +// PACL Dacl; +// } SECURITY_DESCRIPTOR, *PISECURITY_DESCRIPTOR; +// +// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-security_descriptor type securityDescriptor struct { Revision byte Sbz1 byte @@ -80,6 +114,8 @@ type win32Pipe struct { path string } +var _ PipeConn = (*win32Pipe)(nil) + type win32MessageBytePipe struct { win32Pipe writeClosed bool @@ -103,6 +139,10 @@ func (f *win32Pipe) SetDeadline(t time.Time) error { return f.SetWriteDeadline(t) } +func (f *win32Pipe) Disconnect() error { + return disconnectNamedPipe(f.win32File.handle) +} + // CloseWrite closes the write side of a message pipe in byte mode. func (f *win32MessageBytePipe) CloseWrite() error { if f.writeClosed { @@ -146,7 +186,7 @@ func (f *win32MessageBytePipe) Read(b []byte) (int, error) { // zero-byte message, ensure that all future Read() calls // also return EOF. f.readEOF = true - } else if err == syscall.ERROR_MORE_DATA { //nolint:errorlint // err is Errno + } else if err == windows.ERROR_MORE_DATA { //nolint:errorlint // err is Errno // ERROR_MORE_DATA indicates that the pipe's read mode is message mode // and the message still has more bytes. Treat this as a success, since // this package presents all named pipes as byte streams. @@ -164,21 +204,20 @@ func (s pipeAddress) String() string { } // tryDialPipe attempts to dial the pipe at `path` until `ctx` cancellation or timeout. -func tryDialPipe(ctx context.Context, path *string, access fs.AccessMask) (syscall.Handle, error) { +func tryDialPipe(ctx context.Context, path *string, access fs.AccessMask, impLevel PipeImpLevel) (windows.Handle, error) { for { select { case <-ctx.Done(): - return syscall.Handle(0), ctx.Err() + return windows.Handle(0), ctx.Err() default: - wh, err := fs.CreateFile(*path, + h, err := fs.CreateFile(*path, access, 0, // mode nil, // security attributes fs.OPEN_EXISTING, - fs.FILE_FLAG_OVERLAPPED|fs.SECURITY_SQOS_PRESENT|fs.SECURITY_ANONYMOUS, + fs.FILE_FLAG_OVERLAPPED|fs.SECURITY_SQOS_PRESENT|fs.FileSQSFlag(impLevel), 0, // template file handle ) - h := syscall.Handle(wh) if err == nil { return h, nil } @@ -214,15 +253,33 @@ func DialPipe(path string, timeout *time.Duration) (net.Conn, error) { // DialPipeContext attempts to connect to a named pipe by `path` until `ctx` // cancellation or timeout. func DialPipeContext(ctx context.Context, path string) (net.Conn, error) { - return DialPipeAccess(ctx, path, syscall.GENERIC_READ|syscall.GENERIC_WRITE) + return DialPipeAccess(ctx, path, uint32(fs.GENERIC_READ|fs.GENERIC_WRITE)) } +// PipeImpLevel is an enumeration of impersonation levels that may be set +// when calling DialPipeAccessImpersonation. +type PipeImpLevel uint32 + +const ( + PipeImpLevelAnonymous = PipeImpLevel(fs.SECURITY_ANONYMOUS) + PipeImpLevelIdentification = PipeImpLevel(fs.SECURITY_IDENTIFICATION) + PipeImpLevelImpersonation = PipeImpLevel(fs.SECURITY_IMPERSONATION) + PipeImpLevelDelegation = PipeImpLevel(fs.SECURITY_DELEGATION) +) + // DialPipeAccess attempts to connect to a named pipe by `path` with `access` until `ctx` // cancellation or timeout. func DialPipeAccess(ctx context.Context, path string, access uint32) (net.Conn, error) { + return DialPipeAccessImpLevel(ctx, path, access, PipeImpLevelAnonymous) +} + +// DialPipeAccessImpLevel attempts to connect to a named pipe by `path` with +// `access` at `impLevel` until `ctx` cancellation or timeout. The other +// DialPipe* implementations use PipeImpLevelAnonymous. +func DialPipeAccessImpLevel(ctx context.Context, path string, access uint32, impLevel PipeImpLevel) (net.Conn, error) { var err error - var h syscall.Handle - h, err = tryDialPipe(ctx, &path, fs.AccessMask(access)) + var h windows.Handle + h, err = tryDialPipe(ctx, &path, fs.AccessMask(access), impLevel) if err != nil { return nil, err } @@ -235,7 +292,7 @@ func DialPipeAccess(ctx context.Context, path string, access uint32) (net.Conn, f, err := makeWin32File(h) if err != nil { - syscall.Close(h) + windows.Close(h) return nil, err } @@ -255,7 +312,7 @@ type acceptResponse struct { } type win32PipeListener struct { - firstHandle syscall.Handle + firstHandle windows.Handle path string config PipeConfig acceptCh chan (chan acceptResponse) @@ -263,8 +320,8 @@ type win32PipeListener struct { doneCh chan int } -func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (syscall.Handle, error) { - path16, err := syscall.UTF16FromString(path) +func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (windows.Handle, error) { + path16, err := windows.UTF16FromString(path) if err != nil { return 0, &os.PathError{Op: "open", Path: path, Err: err} } @@ -280,16 +337,20 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy ).Err(); err != nil { return 0, &os.PathError{Op: "open", Path: path, Err: err} } - defer localFree(ntPath.Buffer) + defer windows.LocalFree(windows.Handle(ntPath.Buffer)) //nolint:errcheck oa.ObjectName = &ntPath oa.Attributes = windows.OBJ_CASE_INSENSITIVE // The security descriptor is only needed for the first pipe. if first { if sd != nil { + //todo: does `sdb` need to be allocated on the heap, or can go allocate it? l := uint32(len(sd)) - sdb := localAlloc(0, l) - defer localFree(sdb) + sdb, err := windows.LocalAlloc(0, l) + if err != nil { + return 0, fmt.Errorf("LocalAlloc for security descriptor with of length %d: %w", l, err) + } + defer windows.LocalFree(windows.Handle(sdb)) //nolint:errcheck copy((*[0xffff]byte)(unsafe.Pointer(sdb))[:], sd) oa.SecurityDescriptor = (*securityDescriptor)(unsafe.Pointer(sdb)) } else { @@ -298,7 +359,7 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy if err := rtlDefaultNpAcl(&dacl).Err(); err != nil { return 0, fmt.Errorf("getting default named pipe ACL: %w", err) } - defer localFree(dacl) + defer windows.LocalFree(windows.Handle(dacl)) //nolint:errcheck sdb := &securityDescriptor{ Revision: 1, @@ -314,27 +375,27 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy typ |= windows.FILE_PIPE_MESSAGE_TYPE } - disposition := uint32(windows.FILE_OPEN) - access := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE | syscall.SYNCHRONIZE) + disposition := fs.FILE_OPEN + access := fs.GENERIC_READ | fs.GENERIC_WRITE | fs.SYNCHRONIZE if first { - disposition = windows.FILE_CREATE + disposition = fs.FILE_CREATE // By not asking for read or write access, the named pipe file system // will put this pipe into an initially disconnected state, blocking // client connections until the next call with first == false. - access = syscall.SYNCHRONIZE + access = fs.SYNCHRONIZE } timeout := int64(-50 * 10000) // 50ms var ( - h syscall.Handle + h windows.Handle iosb ioStatusBlock ) err = ntCreateNamedPipeFile(&h, access, &oa, &iosb, - syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE, + fs.FILE_SHARE_READ|fs.FILE_SHARE_WRITE, disposition, 0, typ, @@ -359,7 +420,7 @@ func (l *win32PipeListener) makeServerPipe() (*win32File, error) { } f, err := makeWin32File(h) if err != nil { - syscall.Close(h) + windows.Close(h) return nil, err } return f, nil @@ -418,7 +479,7 @@ func (l *win32PipeListener) listenerRoutine() { closed = err == ErrPipeListenerClosed //nolint:errorlint // err is Errno } } - syscall.Close(l.firstHandle) + windows.Close(l.firstHandle) l.firstHandle = 0 // Notify Close() and Accept() callers that the handle has been closed. close(l.doneCh) diff --git a/vendor/github.com/Microsoft/go-winio/privilege.go b/vendor/github.com/Microsoft/go-winio/privilege.go index 0ff9dac906..d9b90b6e86 100644 --- a/vendor/github.com/Microsoft/go-winio/privilege.go +++ b/vendor/github.com/Microsoft/go-winio/privilege.go @@ -9,7 +9,6 @@ import ( "fmt" "runtime" "sync" - "syscall" "unicode/utf16" "golang.org/x/sys/windows" @@ -18,8 +17,8 @@ import ( //sys adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (success bool, err error) [true] = advapi32.AdjustTokenPrivileges //sys impersonateSelf(level uint32) (err error) = advapi32.ImpersonateSelf //sys revertToSelf() (err error) = advapi32.RevertToSelf -//sys openThreadToken(thread syscall.Handle, accessMask uint32, openAsSelf bool, token *windows.Token) (err error) = advapi32.OpenThreadToken -//sys getCurrentThread() (h syscall.Handle) = GetCurrentThread +//sys openThreadToken(thread windows.Handle, accessMask uint32, openAsSelf bool, token *windows.Token) (err error) = advapi32.OpenThreadToken +//sys getCurrentThread() (h windows.Handle) = GetCurrentThread //sys lookupPrivilegeValue(systemName string, name string, luid *uint64) (err error) = advapi32.LookupPrivilegeValueW //sys lookupPrivilegeName(systemName string, luid *uint64, buffer *uint16, size *uint32) (err error) = advapi32.LookupPrivilegeNameW //sys lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) = advapi32.LookupPrivilegeDisplayNameW @@ -29,7 +28,7 @@ const ( SE_PRIVILEGE_ENABLED = windows.SE_PRIVILEGE_ENABLED //revive:disable-next-line:var-naming ALL_CAPS - ERROR_NOT_ALL_ASSIGNED syscall.Errno = windows.ERROR_NOT_ALL_ASSIGNED + ERROR_NOT_ALL_ASSIGNED windows.Errno = windows.ERROR_NOT_ALL_ASSIGNED SeBackupPrivilege = "SeBackupPrivilege" SeRestorePrivilege = "SeRestorePrivilege" @@ -177,7 +176,7 @@ func newThreadToken() (windows.Token, error) { } var token windows.Token - err = openThreadToken(getCurrentThread(), syscall.TOKEN_ADJUST_PRIVILEGES|syscall.TOKEN_QUERY, false, &token) + err = openThreadToken(getCurrentThread(), windows.TOKEN_ADJUST_PRIVILEGES|windows.TOKEN_QUERY, false, &token) if err != nil { rerr := revertToSelf() if rerr != nil { diff --git a/vendor/github.com/Microsoft/go-winio/sd.go b/vendor/github.com/Microsoft/go-winio/sd.go index 5550ef6b61..c3685e98e1 100644 --- a/vendor/github.com/Microsoft/go-winio/sd.go +++ b/vendor/github.com/Microsoft/go-winio/sd.go @@ -5,7 +5,7 @@ package winio import ( "errors" - "syscall" + "fmt" "unsafe" "golang.org/x/sys/windows" @@ -15,10 +15,6 @@ import ( //sys lookupAccountSid(systemName *uint16, sid *byte, name *uint16, nameSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) = advapi32.LookupAccountSidW //sys convertSidToStringSid(sid *byte, str **uint16) (err error) = advapi32.ConvertSidToStringSidW //sys convertStringSidToSid(str *uint16, sid **byte) (err error) = advapi32.ConvertStringSidToSidW -//sys convertStringSecurityDescriptorToSecurityDescriptor(str string, revision uint32, sd *uintptr, size *uint32) (err error) = advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW -//sys convertSecurityDescriptorToStringSecurityDescriptor(sd *byte, revision uint32, secInfo uint32, sddl **uint16, sddlSize *uint32) (err error) = advapi32.ConvertSecurityDescriptorToStringSecurityDescriptorW -//sys localFree(mem uintptr) = LocalFree -//sys getSecurityDescriptorLength(sd uintptr) (len uint32) = advapi32.GetSecurityDescriptorLength type AccountLookupError struct { Name string @@ -64,7 +60,7 @@ func LookupSidByName(name string) (sid string, err error) { var sidSize, sidNameUse, refDomainSize uint32 err = lookupAccountName(nil, name, nil, &sidSize, nil, &refDomainSize, &sidNameUse) - if err != nil && err != syscall.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // err is Errno + if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // err is Errno return "", &AccountLookupError{name, err} } sidBuffer := make([]byte, sidSize) @@ -78,8 +74,8 @@ func LookupSidByName(name string) (sid string, err error) { if err != nil { return "", &AccountLookupError{name, err} } - sid = syscall.UTF16ToString((*[0xffff]uint16)(unsafe.Pointer(strBuffer))[:]) - localFree(uintptr(unsafe.Pointer(strBuffer))) + sid = windows.UTF16ToString((*[0xffff]uint16)(unsafe.Pointer(strBuffer))[:]) + _, _ = windows.LocalFree(windows.Handle(unsafe.Pointer(strBuffer))) return sid, nil } @@ -100,7 +96,7 @@ func LookupNameBySid(sid string) (name string, err error) { if err = convertStringSidToSid(sidBuffer, &sidPtr); err != nil { return "", &AccountLookupError{sid, err} } - defer localFree(uintptr(unsafe.Pointer(sidPtr))) + defer windows.LocalFree(windows.Handle(unsafe.Pointer(sidPtr))) //nolint:errcheck var nameSize, refDomainSize, sidNameUse uint32 err = lookupAccountSid(nil, sidPtr, nil, &nameSize, nil, &refDomainSize, &sidNameUse) @@ -120,25 +116,18 @@ func LookupNameBySid(sid string) (name string, err error) { } func SddlToSecurityDescriptor(sddl string) ([]byte, error) { - var sdBuffer uintptr - err := convertStringSecurityDescriptorToSecurityDescriptor(sddl, 1, &sdBuffer, nil) + sd, err := windows.SecurityDescriptorFromString(sddl) if err != nil { - return nil, &SddlConversionError{sddl, err} + return nil, &SddlConversionError{Sddl: sddl, Err: err} } - defer localFree(sdBuffer) - sd := make([]byte, getSecurityDescriptorLength(sdBuffer)) - copy(sd, (*[0xffff]byte)(unsafe.Pointer(sdBuffer))[:len(sd)]) - return sd, nil + b := unsafe.Slice((*byte)(unsafe.Pointer(sd)), sd.Length()) + return b, nil } func SecurityDescriptorToSddl(sd []byte) (string, error) { - var sddl *uint16 - // The returned string length seems to include an arbitrary number of terminating NULs. - // Don't use it. - err := convertSecurityDescriptorToStringSecurityDescriptor(&sd[0], 1, 0xff, &sddl, nil) - if err != nil { - return "", err + if l := int(unsafe.Sizeof(windows.SECURITY_DESCRIPTOR{})); len(sd) < l { + return "", fmt.Errorf("SecurityDescriptor (%d) smaller than expected (%d): %w", len(sd), l, windows.ERROR_INCORRECT_SIZE) } - defer localFree(uintptr(unsafe.Pointer(sddl))) - return syscall.UTF16ToString((*[0xffff]uint16)(unsafe.Pointer(sddl))[:]), nil + s := (*windows.SECURITY_DESCRIPTOR)(unsafe.Pointer(&sd[0])) + return s.String(), nil } diff --git a/vendor/github.com/Microsoft/go-winio/tools.go b/vendor/github.com/Microsoft/go-winio/tools.go deleted file mode 100644 index 2aa045843e..0000000000 --- a/vendor/github.com/Microsoft/go-winio/tools.go +++ /dev/null @@ -1,5 +0,0 @@ -//go:build tools - -package winio - -import _ "golang.org/x/tools/cmd/stringer" diff --git a/vendor/github.com/Microsoft/go-winio/zsyscall_windows.go b/vendor/github.com/Microsoft/go-winio/zsyscall_windows.go index 469b16f639..89b66eda8c 100644 --- a/vendor/github.com/Microsoft/go-winio/zsyscall_windows.go +++ b/vendor/github.com/Microsoft/go-winio/zsyscall_windows.go @@ -33,9 +33,6 @@ func errnoErr(e syscall.Errno) error { case errnoERROR_IO_PENDING: return errERROR_IO_PENDING } - // TODO: add more here, after collecting data on the common - // error values see on Windows. (perhaps when running - // all.bat?) return e } @@ -45,38 +42,34 @@ var ( modntdll = windows.NewLazySystemDLL("ntdll.dll") modws2_32 = windows.NewLazySystemDLL("ws2_32.dll") - procAdjustTokenPrivileges = modadvapi32.NewProc("AdjustTokenPrivileges") - procConvertSecurityDescriptorToStringSecurityDescriptorW = modadvapi32.NewProc("ConvertSecurityDescriptorToStringSecurityDescriptorW") - procConvertSidToStringSidW = modadvapi32.NewProc("ConvertSidToStringSidW") - procConvertStringSecurityDescriptorToSecurityDescriptorW = modadvapi32.NewProc("ConvertStringSecurityDescriptorToSecurityDescriptorW") - procConvertStringSidToSidW = modadvapi32.NewProc("ConvertStringSidToSidW") - procGetSecurityDescriptorLength = modadvapi32.NewProc("GetSecurityDescriptorLength") - procImpersonateSelf = modadvapi32.NewProc("ImpersonateSelf") - procLookupAccountNameW = modadvapi32.NewProc("LookupAccountNameW") - procLookupAccountSidW = modadvapi32.NewProc("LookupAccountSidW") - procLookupPrivilegeDisplayNameW = modadvapi32.NewProc("LookupPrivilegeDisplayNameW") - procLookupPrivilegeNameW = modadvapi32.NewProc("LookupPrivilegeNameW") - procLookupPrivilegeValueW = modadvapi32.NewProc("LookupPrivilegeValueW") - procOpenThreadToken = modadvapi32.NewProc("OpenThreadToken") - procRevertToSelf = modadvapi32.NewProc("RevertToSelf") - procBackupRead = modkernel32.NewProc("BackupRead") - procBackupWrite = modkernel32.NewProc("BackupWrite") - procCancelIoEx = modkernel32.NewProc("CancelIoEx") - procConnectNamedPipe = modkernel32.NewProc("ConnectNamedPipe") - procCreateIoCompletionPort = modkernel32.NewProc("CreateIoCompletionPort") - procCreateNamedPipeW = modkernel32.NewProc("CreateNamedPipeW") - procGetCurrentThread = modkernel32.NewProc("GetCurrentThread") - procGetNamedPipeHandleStateW = modkernel32.NewProc("GetNamedPipeHandleStateW") - procGetNamedPipeInfo = modkernel32.NewProc("GetNamedPipeInfo") - procGetQueuedCompletionStatus = modkernel32.NewProc("GetQueuedCompletionStatus") - procLocalAlloc = modkernel32.NewProc("LocalAlloc") - procLocalFree = modkernel32.NewProc("LocalFree") - procSetFileCompletionNotificationModes = modkernel32.NewProc("SetFileCompletionNotificationModes") - procNtCreateNamedPipeFile = modntdll.NewProc("NtCreateNamedPipeFile") - procRtlDefaultNpAcl = modntdll.NewProc("RtlDefaultNpAcl") - procRtlDosPathNameToNtPathName_U = modntdll.NewProc("RtlDosPathNameToNtPathName_U") - procRtlNtStatusToDosErrorNoTeb = modntdll.NewProc("RtlNtStatusToDosErrorNoTeb") - procWSAGetOverlappedResult = modws2_32.NewProc("WSAGetOverlappedResult") + procAdjustTokenPrivileges = modadvapi32.NewProc("AdjustTokenPrivileges") + procConvertSidToStringSidW = modadvapi32.NewProc("ConvertSidToStringSidW") + procConvertStringSidToSidW = modadvapi32.NewProc("ConvertStringSidToSidW") + procImpersonateSelf = modadvapi32.NewProc("ImpersonateSelf") + procLookupAccountNameW = modadvapi32.NewProc("LookupAccountNameW") + procLookupAccountSidW = modadvapi32.NewProc("LookupAccountSidW") + procLookupPrivilegeDisplayNameW = modadvapi32.NewProc("LookupPrivilegeDisplayNameW") + procLookupPrivilegeNameW = modadvapi32.NewProc("LookupPrivilegeNameW") + procLookupPrivilegeValueW = modadvapi32.NewProc("LookupPrivilegeValueW") + procOpenThreadToken = modadvapi32.NewProc("OpenThreadToken") + procRevertToSelf = modadvapi32.NewProc("RevertToSelf") + procBackupRead = modkernel32.NewProc("BackupRead") + procBackupWrite = modkernel32.NewProc("BackupWrite") + procCancelIoEx = modkernel32.NewProc("CancelIoEx") + procConnectNamedPipe = modkernel32.NewProc("ConnectNamedPipe") + procCreateIoCompletionPort = modkernel32.NewProc("CreateIoCompletionPort") + procCreateNamedPipeW = modkernel32.NewProc("CreateNamedPipeW") + procDisconnectNamedPipe = modkernel32.NewProc("DisconnectNamedPipe") + procGetCurrentThread = modkernel32.NewProc("GetCurrentThread") + procGetNamedPipeHandleStateW = modkernel32.NewProc("GetNamedPipeHandleStateW") + procGetNamedPipeInfo = modkernel32.NewProc("GetNamedPipeInfo") + procGetQueuedCompletionStatus = modkernel32.NewProc("GetQueuedCompletionStatus") + procSetFileCompletionNotificationModes = modkernel32.NewProc("SetFileCompletionNotificationModes") + procNtCreateNamedPipeFile = modntdll.NewProc("NtCreateNamedPipeFile") + procRtlDefaultNpAcl = modntdll.NewProc("RtlDefaultNpAcl") + procRtlDosPathNameToNtPathName_U = modntdll.NewProc("RtlDosPathNameToNtPathName_U") + procRtlNtStatusToDosErrorNoTeb = modntdll.NewProc("RtlNtStatusToDosErrorNoTeb") + procWSAGetOverlappedResult = modws2_32.NewProc("WSAGetOverlappedResult") ) func adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (success bool, err error) { @@ -84,7 +77,7 @@ func adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, ou if releaseAll { _p0 = 1 } - r0, _, e1 := syscall.Syscall6(procAdjustTokenPrivileges.Addr(), 6, uintptr(token), uintptr(_p0), uintptr(unsafe.Pointer(input)), uintptr(outputSize), uintptr(unsafe.Pointer(output)), uintptr(unsafe.Pointer(requiredSize))) + r0, _, e1 := syscall.SyscallN(procAdjustTokenPrivileges.Addr(), uintptr(token), uintptr(_p0), uintptr(unsafe.Pointer(input)), uintptr(outputSize), uintptr(unsafe.Pointer(output)), uintptr(unsafe.Pointer(requiredSize))) success = r0 != 0 if true { err = errnoErr(e1) @@ -92,33 +85,8 @@ func adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, ou return } -func convertSecurityDescriptorToStringSecurityDescriptor(sd *byte, revision uint32, secInfo uint32, sddl **uint16, sddlSize *uint32) (err error) { - r1, _, e1 := syscall.Syscall6(procConvertSecurityDescriptorToStringSecurityDescriptorW.Addr(), 5, uintptr(unsafe.Pointer(sd)), uintptr(revision), uintptr(secInfo), uintptr(unsafe.Pointer(sddl)), uintptr(unsafe.Pointer(sddlSize)), 0) - if r1 == 0 { - err = errnoErr(e1) - } - return -} - func convertSidToStringSid(sid *byte, str **uint16) (err error) { - r1, _, e1 := syscall.Syscall(procConvertSidToStringSidW.Addr(), 2, uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(str)), 0) - if r1 == 0 { - err = errnoErr(e1) - } - return -} - -func convertStringSecurityDescriptorToSecurityDescriptor(str string, revision uint32, sd *uintptr, size *uint32) (err error) { - var _p0 *uint16 - _p0, err = syscall.UTF16PtrFromString(str) - if err != nil { - return - } - return _convertStringSecurityDescriptorToSecurityDescriptor(_p0, revision, sd, size) -} - -func _convertStringSecurityDescriptorToSecurityDescriptor(str *uint16, revision uint32, sd *uintptr, size *uint32) (err error) { - r1, _, e1 := syscall.Syscall6(procConvertStringSecurityDescriptorToSecurityDescriptorW.Addr(), 4, uintptr(unsafe.Pointer(str)), uintptr(revision), uintptr(unsafe.Pointer(sd)), uintptr(unsafe.Pointer(size)), 0, 0) + r1, _, e1 := syscall.SyscallN(procConvertSidToStringSidW.Addr(), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(str))) if r1 == 0 { err = errnoErr(e1) } @@ -126,21 +94,15 @@ func _convertStringSecurityDescriptorToSecurityDescriptor(str *uint16, revision } func convertStringSidToSid(str *uint16, sid **byte) (err error) { - r1, _, e1 := syscall.Syscall(procConvertStringSidToSidW.Addr(), 2, uintptr(unsafe.Pointer(str)), uintptr(unsafe.Pointer(sid)), 0) + r1, _, e1 := syscall.SyscallN(procConvertStringSidToSidW.Addr(), uintptr(unsafe.Pointer(str)), uintptr(unsafe.Pointer(sid))) if r1 == 0 { err = errnoErr(e1) } return } -func getSecurityDescriptorLength(sd uintptr) (len uint32) { - r0, _, _ := syscall.Syscall(procGetSecurityDescriptorLength.Addr(), 1, uintptr(sd), 0, 0) - len = uint32(r0) - return -} - func impersonateSelf(level uint32) (err error) { - r1, _, e1 := syscall.Syscall(procImpersonateSelf.Addr(), 1, uintptr(level), 0, 0) + r1, _, e1 := syscall.SyscallN(procImpersonateSelf.Addr(), uintptr(level)) if r1 == 0 { err = errnoErr(e1) } @@ -157,7 +119,7 @@ func lookupAccountName(systemName *uint16, accountName string, sid *byte, sidSiz } func _lookupAccountName(systemName *uint16, accountName *uint16, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) { - r1, _, e1 := syscall.Syscall9(procLookupAccountNameW.Addr(), 7, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(accountName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(sidSize)), uintptr(unsafe.Pointer(refDomain)), uintptr(unsafe.Pointer(refDomainSize)), uintptr(unsafe.Pointer(sidNameUse)), 0, 0) + r1, _, e1 := syscall.SyscallN(procLookupAccountNameW.Addr(), uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(accountName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(sidSize)), uintptr(unsafe.Pointer(refDomain)), uintptr(unsafe.Pointer(refDomainSize)), uintptr(unsafe.Pointer(sidNameUse))) if r1 == 0 { err = errnoErr(e1) } @@ -165,7 +127,7 @@ func _lookupAccountName(systemName *uint16, accountName *uint16, sid *byte, sidS } func lookupAccountSid(systemName *uint16, sid *byte, name *uint16, nameSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) { - r1, _, e1 := syscall.Syscall9(procLookupAccountSidW.Addr(), 7, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(nameSize)), uintptr(unsafe.Pointer(refDomain)), uintptr(unsafe.Pointer(refDomainSize)), uintptr(unsafe.Pointer(sidNameUse)), 0, 0) + r1, _, e1 := syscall.SyscallN(procLookupAccountSidW.Addr(), uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(nameSize)), uintptr(unsafe.Pointer(refDomain)), uintptr(unsafe.Pointer(refDomainSize)), uintptr(unsafe.Pointer(sidNameUse))) if r1 == 0 { err = errnoErr(e1) } @@ -182,7 +144,7 @@ func lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, } func _lookupPrivilegeDisplayName(systemName *uint16, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) { - r1, _, e1 := syscall.Syscall6(procLookupPrivilegeDisplayNameW.Addr(), 5, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size)), uintptr(unsafe.Pointer(languageId)), 0) + r1, _, e1 := syscall.SyscallN(procLookupPrivilegeDisplayNameW.Addr(), uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size)), uintptr(unsafe.Pointer(languageId))) if r1 == 0 { err = errnoErr(e1) } @@ -199,7 +161,7 @@ func lookupPrivilegeName(systemName string, luid *uint64, buffer *uint16, size * } func _lookupPrivilegeName(systemName *uint16, luid *uint64, buffer *uint16, size *uint32) (err error) { - r1, _, e1 := syscall.Syscall6(procLookupPrivilegeNameW.Addr(), 4, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(luid)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size)), 0, 0) + r1, _, e1 := syscall.SyscallN(procLookupPrivilegeNameW.Addr(), uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(luid)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size))) if r1 == 0 { err = errnoErr(e1) } @@ -221,19 +183,19 @@ func lookupPrivilegeValue(systemName string, name string, luid *uint64) (err err } func _lookupPrivilegeValue(systemName *uint16, name *uint16, luid *uint64) (err error) { - r1, _, e1 := syscall.Syscall(procLookupPrivilegeValueW.Addr(), 3, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(luid))) + r1, _, e1 := syscall.SyscallN(procLookupPrivilegeValueW.Addr(), uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(luid))) if r1 == 0 { err = errnoErr(e1) } return } -func openThreadToken(thread syscall.Handle, accessMask uint32, openAsSelf bool, token *windows.Token) (err error) { +func openThreadToken(thread windows.Handle, accessMask uint32, openAsSelf bool, token *windows.Token) (err error) { var _p0 uint32 if openAsSelf { _p0 = 1 } - r1, _, e1 := syscall.Syscall6(procOpenThreadToken.Addr(), 4, uintptr(thread), uintptr(accessMask), uintptr(_p0), uintptr(unsafe.Pointer(token)), 0, 0) + r1, _, e1 := syscall.SyscallN(procOpenThreadToken.Addr(), uintptr(thread), uintptr(accessMask), uintptr(_p0), uintptr(unsafe.Pointer(token))) if r1 == 0 { err = errnoErr(e1) } @@ -241,14 +203,14 @@ func openThreadToken(thread syscall.Handle, accessMask uint32, openAsSelf bool, } func revertToSelf() (err error) { - r1, _, e1 := syscall.Syscall(procRevertToSelf.Addr(), 0, 0, 0, 0) + r1, _, e1 := syscall.SyscallN(procRevertToSelf.Addr()) if r1 == 0 { err = errnoErr(e1) } return } -func backupRead(h syscall.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) { +func backupRead(h windows.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) { var _p0 *byte if len(b) > 0 { _p0 = &b[0] @@ -261,14 +223,14 @@ func backupRead(h syscall.Handle, b []byte, bytesRead *uint32, abort bool, proce if processSecurity { _p2 = 1 } - r1, _, e1 := syscall.Syscall9(procBackupRead.Addr(), 7, uintptr(h), uintptr(unsafe.Pointer(_p0)), uintptr(len(b)), uintptr(unsafe.Pointer(bytesRead)), uintptr(_p1), uintptr(_p2), uintptr(unsafe.Pointer(context)), 0, 0) + r1, _, e1 := syscall.SyscallN(procBackupRead.Addr(), uintptr(h), uintptr(unsafe.Pointer(_p0)), uintptr(len(b)), uintptr(unsafe.Pointer(bytesRead)), uintptr(_p1), uintptr(_p2), uintptr(unsafe.Pointer(context))) if r1 == 0 { err = errnoErr(e1) } return } -func backupWrite(h syscall.Handle, b []byte, bytesWritten *uint32, abort bool, processSecurity bool, context *uintptr) (err error) { +func backupWrite(h windows.Handle, b []byte, bytesWritten *uint32, abort bool, processSecurity bool, context *uintptr) (err error) { var _p0 *byte if len(b) > 0 { _p0 = &b[0] @@ -281,39 +243,39 @@ func backupWrite(h syscall.Handle, b []byte, bytesWritten *uint32, abort bool, p if processSecurity { _p2 = 1 } - r1, _, e1 := syscall.Syscall9(procBackupWrite.Addr(), 7, uintptr(h), uintptr(unsafe.Pointer(_p0)), uintptr(len(b)), uintptr(unsafe.Pointer(bytesWritten)), uintptr(_p1), uintptr(_p2), uintptr(unsafe.Pointer(context)), 0, 0) + r1, _, e1 := syscall.SyscallN(procBackupWrite.Addr(), uintptr(h), uintptr(unsafe.Pointer(_p0)), uintptr(len(b)), uintptr(unsafe.Pointer(bytesWritten)), uintptr(_p1), uintptr(_p2), uintptr(unsafe.Pointer(context))) if r1 == 0 { err = errnoErr(e1) } return } -func cancelIoEx(file syscall.Handle, o *syscall.Overlapped) (err error) { - r1, _, e1 := syscall.Syscall(procCancelIoEx.Addr(), 2, uintptr(file), uintptr(unsafe.Pointer(o)), 0) +func cancelIoEx(file windows.Handle, o *windows.Overlapped) (err error) { + r1, _, e1 := syscall.SyscallN(procCancelIoEx.Addr(), uintptr(file), uintptr(unsafe.Pointer(o))) if r1 == 0 { err = errnoErr(e1) } return } -func connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) { - r1, _, e1 := syscall.Syscall(procConnectNamedPipe.Addr(), 2, uintptr(pipe), uintptr(unsafe.Pointer(o)), 0) +func connectNamedPipe(pipe windows.Handle, o *windows.Overlapped) (err error) { + r1, _, e1 := syscall.SyscallN(procConnectNamedPipe.Addr(), uintptr(pipe), uintptr(unsafe.Pointer(o))) if r1 == 0 { err = errnoErr(e1) } return } -func createIoCompletionPort(file syscall.Handle, port syscall.Handle, key uintptr, threadCount uint32) (newport syscall.Handle, err error) { - r0, _, e1 := syscall.Syscall6(procCreateIoCompletionPort.Addr(), 4, uintptr(file), uintptr(port), uintptr(key), uintptr(threadCount), 0, 0) - newport = syscall.Handle(r0) +func createIoCompletionPort(file windows.Handle, port windows.Handle, key uintptr, threadCount uint32) (newport windows.Handle, err error) { + r0, _, e1 := syscall.SyscallN(procCreateIoCompletionPort.Addr(), uintptr(file), uintptr(port), uintptr(key), uintptr(threadCount)) + newport = windows.Handle(r0) if newport == 0 { err = errnoErr(e1) } return } -func createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) { +func createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *windows.SecurityAttributes) (handle windows.Handle, err error) { var _p0 *uint16 _p0, err = syscall.UTF16PtrFromString(name) if err != nil { @@ -322,96 +284,93 @@ func createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances ui return _createNamedPipe(_p0, flags, pipeMode, maxInstances, outSize, inSize, defaultTimeout, sa) } -func _createNamedPipe(name *uint16, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) { - r0, _, e1 := syscall.Syscall9(procCreateNamedPipeW.Addr(), 8, uintptr(unsafe.Pointer(name)), uintptr(flags), uintptr(pipeMode), uintptr(maxInstances), uintptr(outSize), uintptr(inSize), uintptr(defaultTimeout), uintptr(unsafe.Pointer(sa)), 0) - handle = syscall.Handle(r0) - if handle == syscall.InvalidHandle { +func _createNamedPipe(name *uint16, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *windows.SecurityAttributes) (handle windows.Handle, err error) { + r0, _, e1 := syscall.SyscallN(procCreateNamedPipeW.Addr(), uintptr(unsafe.Pointer(name)), uintptr(flags), uintptr(pipeMode), uintptr(maxInstances), uintptr(outSize), uintptr(inSize), uintptr(defaultTimeout), uintptr(unsafe.Pointer(sa))) + handle = windows.Handle(r0) + if handle == windows.InvalidHandle { err = errnoErr(e1) } return } -func getCurrentThread() (h syscall.Handle) { - r0, _, _ := syscall.Syscall(procGetCurrentThread.Addr(), 0, 0, 0, 0) - h = syscall.Handle(r0) - return -} - -func getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) { - r1, _, e1 := syscall.Syscall9(procGetNamedPipeHandleStateW.Addr(), 7, uintptr(pipe), uintptr(unsafe.Pointer(state)), uintptr(unsafe.Pointer(curInstances)), uintptr(unsafe.Pointer(maxCollectionCount)), uintptr(unsafe.Pointer(collectDataTimeout)), uintptr(unsafe.Pointer(userName)), uintptr(maxUserNameSize), 0, 0) +func disconnectNamedPipe(pipe windows.Handle) (err error) { + r1, _, e1 := syscall.SyscallN(procDisconnectNamedPipe.Addr(), uintptr(pipe)) if r1 == 0 { err = errnoErr(e1) } return } -func getNamedPipeInfo(pipe syscall.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) { - r1, _, e1 := syscall.Syscall6(procGetNamedPipeInfo.Addr(), 5, uintptr(pipe), uintptr(unsafe.Pointer(flags)), uintptr(unsafe.Pointer(outSize)), uintptr(unsafe.Pointer(inSize)), uintptr(unsafe.Pointer(maxInstances)), 0) - if r1 == 0 { - err = errnoErr(e1) - } +func getCurrentThread() (h windows.Handle) { + r0, _, _ := syscall.SyscallN(procGetCurrentThread.Addr()) + h = windows.Handle(r0) return } -func getQueuedCompletionStatus(port syscall.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) { - r1, _, e1 := syscall.Syscall6(procGetQueuedCompletionStatus.Addr(), 5, uintptr(port), uintptr(unsafe.Pointer(bytes)), uintptr(unsafe.Pointer(key)), uintptr(unsafe.Pointer(o)), uintptr(timeout), 0) +func getNamedPipeHandleState(pipe windows.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) { + r1, _, e1 := syscall.SyscallN(procGetNamedPipeHandleStateW.Addr(), uintptr(pipe), uintptr(unsafe.Pointer(state)), uintptr(unsafe.Pointer(curInstances)), uintptr(unsafe.Pointer(maxCollectionCount)), uintptr(unsafe.Pointer(collectDataTimeout)), uintptr(unsafe.Pointer(userName)), uintptr(maxUserNameSize)) if r1 == 0 { err = errnoErr(e1) } return } -func localAlloc(uFlags uint32, length uint32) (ptr uintptr) { - r0, _, _ := syscall.Syscall(procLocalAlloc.Addr(), 2, uintptr(uFlags), uintptr(length), 0) - ptr = uintptr(r0) +func getNamedPipeInfo(pipe windows.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) { + r1, _, e1 := syscall.SyscallN(procGetNamedPipeInfo.Addr(), uintptr(pipe), uintptr(unsafe.Pointer(flags)), uintptr(unsafe.Pointer(outSize)), uintptr(unsafe.Pointer(inSize)), uintptr(unsafe.Pointer(maxInstances))) + if r1 == 0 { + err = errnoErr(e1) + } return } -func localFree(mem uintptr) { - syscall.Syscall(procLocalFree.Addr(), 1, uintptr(mem), 0, 0) +func getQueuedCompletionStatus(port windows.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) { + r1, _, e1 := syscall.SyscallN(procGetQueuedCompletionStatus.Addr(), uintptr(port), uintptr(unsafe.Pointer(bytes)), uintptr(unsafe.Pointer(key)), uintptr(unsafe.Pointer(o)), uintptr(timeout)) + if r1 == 0 { + err = errnoErr(e1) + } return } -func setFileCompletionNotificationModes(h syscall.Handle, flags uint8) (err error) { - r1, _, e1 := syscall.Syscall(procSetFileCompletionNotificationModes.Addr(), 2, uintptr(h), uintptr(flags), 0) +func setFileCompletionNotificationModes(h windows.Handle, flags uint8) (err error) { + r1, _, e1 := syscall.SyscallN(procSetFileCompletionNotificationModes.Addr(), uintptr(h), uintptr(flags)) if r1 == 0 { err = errnoErr(e1) } return } -func ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntStatus) { - r0, _, _ := syscall.Syscall15(procNtCreateNamedPipeFile.Addr(), 14, uintptr(unsafe.Pointer(pipe)), uintptr(access), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(share), uintptr(disposition), uintptr(options), uintptr(typ), uintptr(readMode), uintptr(completionMode), uintptr(maxInstances), uintptr(inboundQuota), uintptr(outputQuota), uintptr(unsafe.Pointer(timeout)), 0) +func ntCreateNamedPipeFile(pipe *windows.Handle, access ntAccessMask, oa *objectAttributes, iosb *ioStatusBlock, share ntFileShareMode, disposition ntFileCreationDisposition, options ntFileOptions, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntStatus) { + r0, _, _ := syscall.SyscallN(procNtCreateNamedPipeFile.Addr(), uintptr(unsafe.Pointer(pipe)), uintptr(access), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(share), uintptr(disposition), uintptr(options), uintptr(typ), uintptr(readMode), uintptr(completionMode), uintptr(maxInstances), uintptr(inboundQuota), uintptr(outputQuota), uintptr(unsafe.Pointer(timeout))) status = ntStatus(r0) return } func rtlDefaultNpAcl(dacl *uintptr) (status ntStatus) { - r0, _, _ := syscall.Syscall(procRtlDefaultNpAcl.Addr(), 1, uintptr(unsafe.Pointer(dacl)), 0, 0) + r0, _, _ := syscall.SyscallN(procRtlDefaultNpAcl.Addr(), uintptr(unsafe.Pointer(dacl))) status = ntStatus(r0) return } func rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntStatus) { - r0, _, _ := syscall.Syscall6(procRtlDosPathNameToNtPathName_U.Addr(), 4, uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(ntName)), uintptr(filePart), uintptr(reserved), 0, 0) + r0, _, _ := syscall.SyscallN(procRtlDosPathNameToNtPathName_U.Addr(), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(ntName)), uintptr(filePart), uintptr(reserved)) status = ntStatus(r0) return } func rtlNtStatusToDosError(status ntStatus) (winerr error) { - r0, _, _ := syscall.Syscall(procRtlNtStatusToDosErrorNoTeb.Addr(), 1, uintptr(status), 0, 0) + r0, _, _ := syscall.SyscallN(procRtlNtStatusToDosErrorNoTeb.Addr(), uintptr(status)) if r0 != 0 { winerr = syscall.Errno(r0) } return } -func wsaGetOverlappedResult(h syscall.Handle, o *syscall.Overlapped, bytes *uint32, wait bool, flags *uint32) (err error) { +func wsaGetOverlappedResult(h windows.Handle, o *windows.Overlapped, bytes *uint32, wait bool, flags *uint32) (err error) { var _p0 uint32 if wait { _p0 = 1 } - r1, _, e1 := syscall.Syscall6(procWSAGetOverlappedResult.Addr(), 5, uintptr(h), uintptr(unsafe.Pointer(o)), uintptr(unsafe.Pointer(bytes)), uintptr(_p0), uintptr(unsafe.Pointer(flags)), 0) + r1, _, e1 := syscall.SyscallN(procWSAGetOverlappedResult.Addr(), uintptr(h), uintptr(unsafe.Pointer(o)), uintptr(unsafe.Pointer(bytes)), uintptr(_p0), uintptr(unsafe.Pointer(flags))) if r1 == 0 { err = errnoErr(e1) } diff --git a/vendor/github.com/VictoriaMetrics/metrics/README.md b/vendor/github.com/VictoriaMetrics/metrics/README.md index e1a2537cb3..b01d81e850 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/README.md +++ b/vendor/github.com/VictoriaMetrics/metrics/README.md @@ -73,8 +73,11 @@ http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) { metrics.InitPush("http://victoria-metrics:8428/api/v1/import/prometheus", 10*time.Second, `instance="foobar"`, true) ``` -See [docs](http://godoc.org/github.com/VictoriaMetrics/metrics) for more info. +By default, exposed metrics [do not have](https://github.com/VictoriaMetrics/metrics/issues/48#issuecomment-1620765811) +`TYPE` or `HELP` meta information. Call [`ExposeMetadata(true)`](https://pkg.go.dev/github.com/VictoriaMetrics/metrics#ExposeMetadata) +in order to generate `TYPE` and `HELP` meta information per each metric. +See [docs](https://pkg.go.dev/github.com/VictoriaMetrics/metrics) for more info. ### Users diff --git a/vendor/github.com/VictoriaMetrics/metrics/counter.go b/vendor/github.com/VictoriaMetrics/metrics/counter.go index dfe947794a..1076e80c24 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/counter.go +++ b/vendor/github.com/VictoriaMetrics/metrics/counter.go @@ -42,6 +42,11 @@ func (c *Counter) Add(n int) { atomic.AddUint64(&c.n, uint64(n)) } +// AddInt64 adds n to c. +func (c *Counter) AddInt64(n int64) { + atomic.AddUint64(&c.n, uint64(n)) +} + // Get returns the current value for c. func (c *Counter) Get() uint64 { return atomic.LoadUint64(&c.n) @@ -58,6 +63,10 @@ func (c *Counter) marshalTo(prefix string, w io.Writer) { fmt.Fprintf(w, "%s %d\n", prefix, v) } +func (c *Counter) metricType() string { + return "counter" +} + // GetOrCreateCounter returns registered counter with the given name // or creates new counter if the registry doesn't contain counter with // the given name. diff --git a/vendor/github.com/VictoriaMetrics/metrics/floatcounter.go b/vendor/github.com/VictoriaMetrics/metrics/floatcounter.go index f898790995..8bd9fa67af 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/floatcounter.go +++ b/vendor/github.com/VictoriaMetrics/metrics/floatcounter.go @@ -63,6 +63,10 @@ func (fc *FloatCounter) marshalTo(prefix string, w io.Writer) { fmt.Fprintf(w, "%s %g\n", prefix, v) } +func (fc *FloatCounter) metricType() string { + return "counter" +} + // GetOrCreateFloatCounter returns registered FloatCounter with the given name // or creates new FloatCounter if the registry doesn't contain FloatCounter with // the given name. diff --git a/vendor/github.com/VictoriaMetrics/metrics/gauge.go b/vendor/github.com/VictoriaMetrics/metrics/gauge.go index 9084fc4d7d..3573e14458 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/gauge.go +++ b/vendor/github.com/VictoriaMetrics/metrics/gauge.go @@ -3,10 +3,11 @@ package metrics import ( "fmt" "io" + "math" + "sync/atomic" ) -// NewGauge registers and returns gauge with the given name, which calls f -// to obtain gauge value. +// NewGauge registers and returns gauge with the given name, which calls f to obtain gauge value. // // name must be valid Prometheus-compatible metric with possible labels. // For instance, @@ -16,6 +17,7 @@ import ( // - foo{bar="baz",aaa="b"} // // f must be safe for concurrent calls. +// if f is nil, then it is expected that the gauge value is changed via Set(), Inc(), Dec() and Add() calls. // // The returned gauge is safe to use from concurrent goroutines. // @@ -25,19 +27,68 @@ func NewGauge(name string, f func() float64) *Gauge { } // Gauge is a float64 gauge. -// -// See also Counter, which could be used as a gauge with Set and Dec calls. type Gauge struct { + // valueBits contains uint64 representation of float64 passed to Gauge.Set. + valueBits uint64 + + // f is a callback, which is called for returning the gauge value. f func() float64 } // Get returns the current value for g. func (g *Gauge) Get() float64 { - return g.f() + if f := g.f; f != nil { + return f() + } + n := atomic.LoadUint64(&g.valueBits) + return math.Float64frombits(n) +} + +// Set sets g value to v. +// +// The g must be created with nil callback in order to be able to call this function. +func (g *Gauge) Set(v float64) { + if g.f != nil { + panic(fmt.Errorf("cannot call Set on gauge created with non-nil callback")) + } + n := math.Float64bits(v) + atomic.StoreUint64(&g.valueBits, n) +} + +// Inc increments g by 1. +// +// The g must be created with nil callback in order to be able to call this function. +func (g *Gauge) Inc() { + g.Add(1) +} + +// Dec decrements g by 1. +// +// The g must be created with nil callback in order to be able to call this function. +func (g *Gauge) Dec() { + g.Add(-1) +} + +// Add adds fAdd to g. fAdd may be positive and negative. +// +// The g must be created with nil callback in order to be able to call this function. +func (g *Gauge) Add(fAdd float64) { + if g.f != nil { + panic(fmt.Errorf("cannot call Set on gauge created with non-nil callback")) + } + for { + n := atomic.LoadUint64(&g.valueBits) + f := math.Float64frombits(n) + fNew := f + fAdd + nNew := math.Float64bits(fNew) + if atomic.CompareAndSwapUint64(&g.valueBits, n, nNew) { + break + } + } } func (g *Gauge) marshalTo(prefix string, w io.Writer) { - v := g.f() + v := g.Get() if float64(int64(v)) == v { // Marshal integer values without scientific notation fmt.Fprintf(w, "%s %d\n", prefix, int64(v)) @@ -46,6 +97,10 @@ func (g *Gauge) marshalTo(prefix string, w io.Writer) { } } +func (g *Gauge) metricType() string { + return "gauge" +} + // GetOrCreateGauge returns registered gauge with the given name // or creates new gauge if the registry doesn't contain gauge with // the given name. diff --git a/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go b/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go index f8b606731e..d8b949de7d 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go +++ b/vendor/github.com/VictoriaMetrics/metrics/go_metrics.go @@ -3,41 +3,78 @@ package metrics import ( "fmt" "io" + "log" + "math" "runtime" + runtimemetrics "runtime/metrics" + "strings" "github.com/valyala/histogram" ) +// See https://pkg.go.dev/runtime/metrics#hdr-Supported_metrics +var runtimeMetrics = [][2]string{ + {"/sched/latencies:seconds", "go_sched_latencies_seconds"}, + {"/sync/mutex/wait/total:seconds", "go_mutex_wait_seconds_total"}, + {"/cpu/classes/gc/mark/assist:cpu-seconds", "go_gc_mark_assist_cpu_seconds_total"}, + {"/cpu/classes/gc/total:cpu-seconds", "go_gc_cpu_seconds_total"}, + {"/gc/pauses:seconds", "go_gc_pauses_seconds"}, + {"/cpu/classes/scavenge/total:cpu-seconds", "go_scavenge_cpu_seconds_total"}, + {"/gc/gomemlimit:bytes", "go_memlimit_bytes"}, +} + +var supportedRuntimeMetrics = initSupportedRuntimeMetrics(runtimeMetrics) + +func initSupportedRuntimeMetrics(rms [][2]string) [][2]string { + exposedMetrics := make(map[string]struct{}) + for _, d := range runtimemetrics.All() { + exposedMetrics[d.Name] = struct{}{} + } + var supportedMetrics [][2]string + for _, rm := range rms { + metricName := rm[0] + if _, ok := exposedMetrics[metricName]; ok { + supportedMetrics = append(supportedMetrics, rm) + } else { + log.Printf("github.com/VictoriaMetrics/metrics: do not expose %s metric, since the corresponding metric %s isn't supported in the current Go runtime", rm[1], metricName) + } + } + return supportedMetrics +} + func writeGoMetrics(w io.Writer) { + writeRuntimeMetrics(w) + var ms runtime.MemStats runtime.ReadMemStats(&ms) - fmt.Fprintf(w, "go_memstats_alloc_bytes %d\n", ms.Alloc) - fmt.Fprintf(w, "go_memstats_alloc_bytes_total %d\n", ms.TotalAlloc) - fmt.Fprintf(w, "go_memstats_buck_hash_sys_bytes %d\n", ms.BuckHashSys) - fmt.Fprintf(w, "go_memstats_frees_total %d\n", ms.Frees) - fmt.Fprintf(w, "go_memstats_gc_cpu_fraction %g\n", ms.GCCPUFraction) - fmt.Fprintf(w, "go_memstats_gc_sys_bytes %d\n", ms.GCSys) - fmt.Fprintf(w, "go_memstats_heap_alloc_bytes %d\n", ms.HeapAlloc) - fmt.Fprintf(w, "go_memstats_heap_idle_bytes %d\n", ms.HeapIdle) - fmt.Fprintf(w, "go_memstats_heap_inuse_bytes %d\n", ms.HeapInuse) - fmt.Fprintf(w, "go_memstats_heap_objects %d\n", ms.HeapObjects) - fmt.Fprintf(w, "go_memstats_heap_released_bytes %d\n", ms.HeapReleased) - fmt.Fprintf(w, "go_memstats_heap_sys_bytes %d\n", ms.HeapSys) - fmt.Fprintf(w, "go_memstats_last_gc_time_seconds %g\n", float64(ms.LastGC)/1e9) - fmt.Fprintf(w, "go_memstats_lookups_total %d\n", ms.Lookups) - fmt.Fprintf(w, "go_memstats_mallocs_total %d\n", ms.Mallocs) - fmt.Fprintf(w, "go_memstats_mcache_inuse_bytes %d\n", ms.MCacheInuse) - fmt.Fprintf(w, "go_memstats_mcache_sys_bytes %d\n", ms.MCacheSys) - fmt.Fprintf(w, "go_memstats_mspan_inuse_bytes %d\n", ms.MSpanInuse) - fmt.Fprintf(w, "go_memstats_mspan_sys_bytes %d\n", ms.MSpanSys) - fmt.Fprintf(w, "go_memstats_next_gc_bytes %d\n", ms.NextGC) - fmt.Fprintf(w, "go_memstats_other_sys_bytes %d\n", ms.OtherSys) - fmt.Fprintf(w, "go_memstats_stack_inuse_bytes %d\n", ms.StackInuse) - fmt.Fprintf(w, "go_memstats_stack_sys_bytes %d\n", ms.StackSys) - fmt.Fprintf(w, "go_memstats_sys_bytes %d\n", ms.Sys) - - fmt.Fprintf(w, "go_cgo_calls_count %d\n", runtime.NumCgoCall()) - fmt.Fprintf(w, "go_cpu_count %d\n", runtime.NumCPU()) + WriteGaugeUint64(w, "go_memstats_alloc_bytes", ms.Alloc) + WriteCounterUint64(w, "go_memstats_alloc_bytes_total", ms.TotalAlloc) + WriteGaugeUint64(w, "go_memstats_buck_hash_sys_bytes", ms.BuckHashSys) + WriteCounterUint64(w, "go_memstats_frees_total", ms.Frees) + WriteGaugeFloat64(w, "go_memstats_gc_cpu_fraction", ms.GCCPUFraction) + WriteGaugeUint64(w, "go_memstats_gc_sys_bytes", ms.GCSys) + + WriteGaugeUint64(w, "go_memstats_heap_alloc_bytes", ms.HeapAlloc) + WriteGaugeUint64(w, "go_memstats_heap_idle_bytes", ms.HeapIdle) + WriteGaugeUint64(w, "go_memstats_heap_inuse_bytes", ms.HeapInuse) + WriteGaugeUint64(w, "go_memstats_heap_objects", ms.HeapObjects) + WriteGaugeUint64(w, "go_memstats_heap_released_bytes", ms.HeapReleased) + WriteGaugeUint64(w, "go_memstats_heap_sys_bytes", ms.HeapSys) + WriteGaugeFloat64(w, "go_memstats_last_gc_time_seconds", float64(ms.LastGC)/1e9) + WriteCounterUint64(w, "go_memstats_lookups_total", ms.Lookups) + WriteCounterUint64(w, "go_memstats_mallocs_total", ms.Mallocs) + WriteGaugeUint64(w, "go_memstats_mcache_inuse_bytes", ms.MCacheInuse) + WriteGaugeUint64(w, "go_memstats_mcache_sys_bytes", ms.MCacheSys) + WriteGaugeUint64(w, "go_memstats_mspan_inuse_bytes", ms.MSpanInuse) + WriteGaugeUint64(w, "go_memstats_mspan_sys_bytes", ms.MSpanSys) + WriteGaugeUint64(w, "go_memstats_next_gc_bytes", ms.NextGC) + WriteGaugeUint64(w, "go_memstats_other_sys_bytes", ms.OtherSys) + WriteGaugeUint64(w, "go_memstats_stack_inuse_bytes", ms.StackInuse) + WriteGaugeUint64(w, "go_memstats_stack_sys_bytes", ms.StackSys) + WriteGaugeUint64(w, "go_memstats_sys_bytes", ms.Sys) + + WriteCounterUint64(w, "go_cgo_calls_count", uint64(runtime.NumCgoCall())) + WriteGaugeUint64(w, "go_cpu_count", uint64(runtime.NumCPU())) gcPauses := histogram.NewFast() for _, pauseNs := range ms.PauseNs[:] { @@ -45,20 +82,103 @@ func writeGoMetrics(w io.Writer) { } phis := []float64{0, 0.25, 0.5, 0.75, 1} quantiles := make([]float64, 0, len(phis)) + WriteMetadataIfNeeded(w, "go_gc_duration_seconds", "summary") for i, q := range gcPauses.Quantiles(quantiles[:0], phis) { fmt.Fprintf(w, `go_gc_duration_seconds{quantile="%g"} %g`+"\n", phis[i], q) } - fmt.Fprintf(w, `go_gc_duration_seconds_sum %g`+"\n", float64(ms.PauseTotalNs)/1e9) - fmt.Fprintf(w, `go_gc_duration_seconds_count %d`+"\n", ms.NumGC) - fmt.Fprintf(w, `go_gc_forced_count %d`+"\n", ms.NumForcedGC) + fmt.Fprintf(w, "go_gc_duration_seconds_sum %g\n", float64(ms.PauseTotalNs)/1e9) + fmt.Fprintf(w, "go_gc_duration_seconds_count %d\n", ms.NumGC) + + WriteCounterUint64(w, "go_gc_forced_count", uint64(ms.NumForcedGC)) - fmt.Fprintf(w, `go_gomaxprocs %d`+"\n", runtime.GOMAXPROCS(0)) - fmt.Fprintf(w, `go_goroutines %d`+"\n", runtime.NumGoroutine()) + WriteGaugeUint64(w, "go_gomaxprocs", uint64(runtime.GOMAXPROCS(0))) + WriteGaugeUint64(w, "go_goroutines", uint64(runtime.NumGoroutine())) numThread, _ := runtime.ThreadCreateProfile(nil) - fmt.Fprintf(w, `go_threads %d`+"\n", numThread) + WriteGaugeUint64(w, "go_threads", uint64(numThread)) // Export build details. + WriteMetadataIfNeeded(w, "go_info", "gauge") fmt.Fprintf(w, "go_info{version=%q} 1\n", runtime.Version()) + + WriteMetadataIfNeeded(w, "go_info_ext", "gauge") fmt.Fprintf(w, "go_info_ext{compiler=%q, GOARCH=%q, GOOS=%q, GOROOT=%q} 1\n", runtime.Compiler, runtime.GOARCH, runtime.GOOS, runtime.GOROOT()) } + +func writeRuntimeMetrics(w io.Writer) { + samples := make([]runtimemetrics.Sample, len(supportedRuntimeMetrics)) + for i, rm := range supportedRuntimeMetrics { + samples[i].Name = rm[0] + } + runtimemetrics.Read(samples) + for i, rm := range supportedRuntimeMetrics { + writeRuntimeMetric(w, rm[1], &samples[i]) + } +} + +func writeRuntimeMetric(w io.Writer, name string, sample *runtimemetrics.Sample) { + kind := sample.Value.Kind() + switch kind { + case runtimemetrics.KindBad: + panic(fmt.Errorf("BUG: unexpected runtimemetrics.KindBad for sample.Name=%q", sample.Name)) + case runtimemetrics.KindUint64: + v := sample.Value.Uint64() + if strings.HasSuffix(name, "_total") { + WriteCounterUint64(w, name, v) + } else { + WriteGaugeUint64(w, name, v) + } + case runtimemetrics.KindFloat64: + v := sample.Value.Float64() + if isCounterName(name) { + WriteCounterFloat64(w, name, v) + } else { + WriteGaugeFloat64(w, name, v) + } + case runtimemetrics.KindFloat64Histogram: + h := sample.Value.Float64Histogram() + writeRuntimeHistogramMetric(w, name, h) + default: + panic(fmt.Errorf("unexpected metric kind=%d", kind)) + } +} + +func writeRuntimeHistogramMetric(w io.Writer, name string, h *runtimemetrics.Float64Histogram) { + buckets := h.Buckets + counts := h.Counts + if len(buckets) != len(counts)+1 { + panic(fmt.Errorf("the number of buckets must be bigger than the number of counts by 1 in histogram %s; got buckets=%d, counts=%d", name, len(buckets), len(counts))) + } + tailCount := uint64(0) + if strings.HasSuffix(name, "_seconds") { + // Limit the maximum bucket to 1 second, since Go runtime exposes buckets with 10K seconds, + // which have little sense. At the same time such buckets may lead to high cardinality issues + // at the scraper side. + for len(buckets) > 0 && buckets[len(buckets)-1] > 1 { + buckets = buckets[:len(buckets)-1] + tailCount += counts[len(counts)-1] + counts = counts[:len(counts)-1] + } + } + + iStep := float64(len(buckets)) / maxRuntimeHistogramBuckets + + totalCount := uint64(0) + iNext := 0.0 + WriteMetadataIfNeeded(w, name, "histogram") + for i, count := range counts { + totalCount += count + if float64(i) >= iNext { + iNext += iStep + le := buckets[i+1] + if !math.IsInf(le, 1) { + fmt.Fprintf(w, `%s_bucket{le="%g"} %d`+"\n", name, le, totalCount) + } + } + } + totalCount += tailCount + fmt.Fprintf(w, `%s_bucket{le="+Inf"} %d`+"\n", name, totalCount) +} + +// Limit the number of buckets for Go runtime histograms in order to prevent from high cardinality issues at scraper side. +const maxRuntimeHistogramBuckets = 30 diff --git a/vendor/github.com/VictoriaMetrics/metrics/histogram.go b/vendor/github.com/VictoriaMetrics/metrics/histogram.go index a576681778..d703ae82fb 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/histogram.go +++ b/vendor/github.com/VictoriaMetrics/metrics/histogram.go @@ -36,7 +36,7 @@ var bucketMultiplier = math.Pow(10, 1.0/bucketsPerDecimal) // // Histogram buckets can be converted to Prometheus-like buckets with `le` labels // with `prometheus_buckets( _bucket)` function from PromQL extensions in VictoriaMetrics. -// (see https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL ): +// (see https://docs.victoriametrics.com/metricsql/ ): // // prometheus_buckets(request_duration_bucket) // @@ -47,13 +47,21 @@ var bucketMultiplier = math.Pow(10, 1.0/bucketsPerDecimal) // Zero histogram is usable. type Histogram struct { // Mu gurantees synchronous update for all the counters and sum. + // + // Do not use sync.RWMutex, since it has zero sense from performance PoV. + // It only complicates the code. mu sync.Mutex + // decimalBuckets contains counters for histogram buckets decimalBuckets [decimalBucketsCount]*[bucketsPerDecimal]uint64 + // lower is the number of values, which hit the lower bucket lower uint64 + + // upper is the number of values, which hit the upper bucket upper uint64 + // sum is the sum of all the values put into Histogram sum float64 } @@ -109,6 +117,34 @@ func (h *Histogram) Update(v float64) { h.mu.Unlock() } +// Merge merges src to h +func (h *Histogram) Merge(src *Histogram) { + h.mu.Lock() + defer h.mu.Unlock() + + src.mu.Lock() + defer src.mu.Unlock() + + h.lower += src.lower + h.upper += src.upper + h.sum += src.sum + + for i, dbSrc := range src.decimalBuckets { + if dbSrc == nil { + continue + } + dbDst := h.decimalBuckets[i] + if dbDst == nil { + var b [bucketsPerDecimal]uint64 + dbDst = &b + h.decimalBuckets[i] = dbDst + } + for j := range dbSrc { + dbDst[j] += dbSrc[j] + } + } +} + // VisitNonZeroBuckets calls f for all buckets with non-zero counters. // // vmrange contains " ... " string with bucket bounds. The lower bound @@ -228,3 +264,7 @@ func (h *Histogram) getSum() float64 { h.mu.Unlock() return sum } + +func (h *Histogram) metricType() string { + return "histogram" +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/metrics.go b/vendor/github.com/VictoriaMetrics/metrics/metrics.go index 7dfa97219a..74e97352cd 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/metrics.go +++ b/vendor/github.com/VictoriaMetrics/metrics/metrics.go @@ -13,9 +13,12 @@ package metrics import ( + "fmt" "io" "sort" + "strings" "sync" + "sync/atomic" "unsafe" ) @@ -27,6 +30,7 @@ type namedMetric struct { type metric interface { marshalTo(prefix string, w io.Writer) + metricType() string } var defaultSet = NewSet() @@ -51,16 +55,33 @@ func RegisterSet(s *Set) { // UnregisterSet stops exporting metrics for the given s via global WritePrometheus() call. // -// Call s.UnregisterAllMetrics() after unregistering s if it is no longer used. -func UnregisterSet(s *Set) { +// If destroySet is set to true, then s.UnregisterAllMetrics() is called on s after unregistering it, +// so s becomes destroyed. Otherwise the s can be registered again in the set by passing it to RegisterSet(). +func UnregisterSet(s *Set, destroySet bool) { registeredSetsLock.Lock() delete(registeredSets, s) registeredSetsLock.Unlock() + + if destroySet { + s.UnregisterAllMetrics() + } } -// WritePrometheus writes all the metrics from default set and all the registered sets in Prometheus format to w. +// RegisterMetricsWriter registers writeMetrics callback for including metrics in the output generated by WritePrometheus. +// +// The writeMetrics callback must write metrics to w in Prometheus text exposition format without timestamps and trailing comments. +// The last line generated by writeMetrics must end with \n. +// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format +// +// It is OK to register multiple writeMetrics callbacks - all of them will be called sequentially for gererating the output at WritePrometheus. +func RegisterMetricsWriter(writeMetrics func(w io.Writer)) { + defaultSet.RegisterMetricsWriter(writeMetrics) +} + +// WritePrometheus writes all the metrics in Prometheus format from the default set, all the added sets and metrics writers to w. // // Additional sets can be registered via RegisterSet() call. +// Additional metric writers can be registered via RegisterMetricsWriter() call. // // If exposeProcessMetrics is true, then various `go_*` and `process_*` metrics // are exposed for the current process. @@ -134,10 +155,26 @@ func WritePrometheus(w io.Writer, exposeProcessMetrics bool) { // // - process_io_storage_written_bytes_total - the number of bytes actually written to disk // +// - go_sched_latencies_seconds - time spent by goroutines in ready state before they start execution +// +// - go_mutex_wait_seconds_total - summary time spent by all the goroutines while waiting for locked mutex +// +// - go_gc_mark_assist_cpu_seconds_total - summary CPU time spent by goroutines in GC mark assist state +// +// - go_gc_cpu_seconds_total - summary time spent in GC +// +// - go_gc_pauses_seconds - duration of GC pauses +// +// - go_scavenge_cpu_seconds_total - CPU time spent on returning the memory to OS +// +// - go_memlimit_bytes - the GOMEMLIMIT env var value +// // - go_memstats_alloc_bytes - memory usage for Go objects in the heap // // - go_memstats_alloc_bytes_total - the cumulative counter for total size of allocated Go objects // +// - go_memstats_buck_hash_sys_bytes - bytes of memory in profiling bucket hash tables +// // - go_memstats_frees_total - the cumulative counter for number of freed Go objects // // - go_memstats_gc_cpu_fraction - the fraction of CPU spent in Go garbage collector @@ -148,20 +185,42 @@ func WritePrometheus(w io.Writer, exposeProcessMetrics bool) { // // - go_memstats_heap_idle_bytes - idle memory ready for new Go object allocations // +// - go_memstats_heap_inuse_bytes - bytes in in-use spans +// // - go_memstats_heap_objects - the number of Go objects in the heap // +// - go_memstats_heap_released_bytes - bytes of physical memory returned to the OS +// // - go_memstats_heap_sys_bytes - memory requested for Go objects from the OS // +// - go_memstats_last_gc_time_seconds - unix timestamp the last garbage collection finished +// +// - go_memstats_lookups_total - the number of pointer lookups performed by the runtime +// // - go_memstats_mallocs_total - the number of allocations for Go objects // +// - go_memstats_mcache_inuse_bytes - bytes of allocated mcache structures +// +// - go_memstats_mcache_sys_bytes - bytes of memory obtained from the OS for mcache structures +// +// - go_memstats_mspan_inuse_bytes - bytes of allocated mspan structures +// +// - go_memstats_mspan_sys_bytes - bytes of memory obtained from the OS for mspan structures +// // - go_memstats_next_gc_bytes - the target heap size when the next garbage collection should start // +// - go_memstats_other_sys_bytes - bytes of memory in miscellaneous off-heap runtime allocations +// // - go_memstats_stack_inuse_bytes - memory used for goroutine stacks // // - go_memstats_stack_sys_bytes - memory requested fromthe OS for goroutine stacks // // - go_memstats_sys_bytes - memory requested by Go runtime from the OS // +// - go_cgo_calls_count - the total number of CGO calls +// +// - go_cpu_count - the number of CPU cores on the host where the app runs +// // The WriteProcessMetrics func is usually called in combination with writing Set metrics // inside "/metrics" handler: // @@ -170,7 +229,7 @@ func WritePrometheus(w io.Writer, exposeProcessMetrics bool) { // metrics.WriteProcessMetrics(w) // }) // -// See also WrteFDMetrics. +// See also WriteFDMetrics. func WriteProcessMetrics(w io.Writer) { writeGoMetrics(w) writeProcessMetrics(w) @@ -190,6 +249,8 @@ func UnregisterMetric(name string) bool { } // UnregisterAllMetrics unregisters all the metrics from default set. +// +// It also unregisters writeMetrics callbacks passed to RegisterMetricsWriter. func UnregisterAllMetrics() { defaultSet.UnregisterAllMetrics() } @@ -203,3 +264,76 @@ func ListMetricNames() []string { func GetDefaultSet() *Set { return defaultSet } + +// ExposeMetadata allows enabling adding TYPE and HELP metadata to the exposed metrics globally. +// +// It is safe to call this method multiple times. It is allowed to change it in runtime. +// ExposeMetadata is set to false by default. +func ExposeMetadata(v bool) { + n := 0 + if v { + n = 1 + } + atomic.StoreUint32(&exposeMetadata, uint32(n)) +} + +func isMetadataEnabled() bool { + n := atomic.LoadUint32(&exposeMetadata) + return n != 0 +} + +var exposeMetadata uint32 + +func isCounterName(name string) bool { + return strings.HasSuffix(name, "_total") +} + +// WriteGaugeUint64 writes gauge metric with the given name and value to w in Prometheus text exposition format. +func WriteGaugeUint64(w io.Writer, name string, value uint64) { + writeMetricUint64(w, name, "gauge", value) +} + +// WriteGaugeFloat64 writes gauge metric with the given name and value to w in Prometheus text exposition format. +func WriteGaugeFloat64(w io.Writer, name string, value float64) { + writeMetricFloat64(w, name, "gauge", value) +} + +// WriteCounterUint64 writes counter metric with the given name and value to w in Prometheus text exposition format. +func WriteCounterUint64(w io.Writer, name string, value uint64) { + writeMetricUint64(w, name, "counter", value) +} + +// WriteCounterFloat64 writes counter metric with the given name and value to w in Prometheus text exposition format. +func WriteCounterFloat64(w io.Writer, name string, value float64) { + writeMetricFloat64(w, name, "counter", value) +} + +func writeMetricUint64(w io.Writer, metricName, metricType string, value uint64) { + WriteMetadataIfNeeded(w, metricName, metricType) + fmt.Fprintf(w, "%s %d\n", metricName, value) +} + +func writeMetricFloat64(w io.Writer, metricName, metricType string, value float64) { + WriteMetadataIfNeeded(w, metricName, metricType) + fmt.Fprintf(w, "%s %g\n", metricName, value) +} + +// WriteMetadataIfNeeded writes HELP and TYPE metadata for the given metricName and metricType if this is globally enabled via ExposeMetadata(). +// +// If the metadata exposition isn't enabled, then this function is no-op. +func WriteMetadataIfNeeded(w io.Writer, metricName, metricType string) { + if !isMetadataEnabled() { + return + } + metricFamily := getMetricFamily(metricName) + fmt.Fprintf(w, "# HELP %s\n", metricFamily) + fmt.Fprintf(w, "# TYPE %s %s\n", metricFamily, metricType) +} + +func getMetricFamily(metricName string) string { + n := strings.IndexByte(metricName, '{') + if n < 0 { + return metricName + } + return metricName[:n] +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go index 48def1cba7..e4587b717a 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go +++ b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go @@ -16,6 +16,11 @@ import ( // See https://github.com/prometheus/procfs/blob/a4ac0826abceb44c40fc71daed2b301db498b93e/proc_stat.go#L40 . const userHZ = 100 +// Different environments may have different page size. +// +// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6457 +var pageSizeBytes = uint64(os.Getpagesize()) + // See http://man7.org/linux/man-pages/man5/proc.5.html type procStat struct { State byte @@ -74,15 +79,15 @@ func writeProcessMetrics(w io.Writer) { utime := float64(p.Utime) / userHZ stime := float64(p.Stime) / userHZ - fmt.Fprintf(w, "process_cpu_seconds_system_total %g\n", stime) - fmt.Fprintf(w, "process_cpu_seconds_total %g\n", utime+stime) - fmt.Fprintf(w, "process_cpu_seconds_user_total %g\n", utime) - fmt.Fprintf(w, "process_major_pagefaults_total %d\n", p.Majflt) - fmt.Fprintf(w, "process_minor_pagefaults_total %d\n", p.Minflt) - fmt.Fprintf(w, "process_num_threads %d\n", p.NumThreads) - fmt.Fprintf(w, "process_resident_memory_bytes %d\n", p.Rss*4096) - fmt.Fprintf(w, "process_start_time_seconds %d\n", startTimeSeconds) - fmt.Fprintf(w, "process_virtual_memory_bytes %d\n", p.Vsize) + WriteCounterFloat64(w, "process_cpu_seconds_system_total", stime) + WriteCounterFloat64(w, "process_cpu_seconds_total", utime+stime) + WriteCounterFloat64(w, "process_cpu_seconds_user_total", utime) + WriteCounterUint64(w, "process_major_pagefaults_total", uint64(p.Majflt)) + WriteCounterUint64(w, "process_minor_pagefaults_total", uint64(p.Minflt)) + WriteGaugeUint64(w, "process_num_threads", uint64(p.NumThreads)) + WriteGaugeUint64(w, "process_resident_memory_bytes", uint64(p.Rss)*pageSizeBytes) + WriteGaugeUint64(w, "process_start_time_seconds", uint64(startTimeSeconds)) + WriteGaugeUint64(w, "process_virtual_memory_bytes", uint64(p.Vsize)) writeProcessMemMetrics(w) writeIOMetrics(w) } @@ -133,12 +138,12 @@ func writeIOMetrics(w io.Writer) { writeBytes = getInt(s) } } - fmt.Fprintf(w, "process_io_read_bytes_total %d\n", rchar) - fmt.Fprintf(w, "process_io_written_bytes_total %d\n", wchar) - fmt.Fprintf(w, "process_io_read_syscalls_total %d\n", syscr) - fmt.Fprintf(w, "process_io_write_syscalls_total %d\n", syscw) - fmt.Fprintf(w, "process_io_storage_read_bytes_total %d\n", readBytes) - fmt.Fprintf(w, "process_io_storage_written_bytes_total %d\n", writeBytes) + WriteGaugeUint64(w, "process_io_read_bytes_total", uint64(rchar)) + WriteGaugeUint64(w, "process_io_written_bytes_total", uint64(wchar)) + WriteGaugeUint64(w, "process_io_read_syscalls_total", uint64(syscr)) + WriteGaugeUint64(w, "process_io_write_syscalls_total", uint64(syscw)) + WriteGaugeUint64(w, "process_io_storage_read_bytes_total", uint64(readBytes)) + WriteGaugeUint64(w, "process_io_storage_written_bytes_total", uint64(writeBytes)) } var startTimeSeconds = time.Now().Unix() @@ -155,8 +160,8 @@ func writeFDMetrics(w io.Writer) { log.Printf("ERROR: metrics: cannot determine the limit on open file descritors: %s", err) return } - fmt.Fprintf(w, "process_max_fds %d\n", maxOpenFDs) - fmt.Fprintf(w, "process_open_fds %d\n", totalOpenFDs) + WriteGaugeUint64(w, "process_max_fds", maxOpenFDs) + WriteGaugeUint64(w, "process_open_fds", totalOpenFDs) } func getOpenFDsCount(path string) (uint64, error) { @@ -224,11 +229,11 @@ func writeProcessMemMetrics(w io.Writer) { log.Printf("ERROR: metrics: cannot determine memory status: %s", err) return } - fmt.Fprintf(w, "process_virtual_memory_peak_bytes %d\n", ms.vmPeak) - fmt.Fprintf(w, "process_resident_memory_peak_bytes %d\n", ms.rssPeak) - fmt.Fprintf(w, "process_resident_memory_anon_bytes %d\n", ms.rssAnon) - fmt.Fprintf(w, "process_resident_memory_file_bytes %d\n", ms.rssFile) - fmt.Fprintf(w, "process_resident_memory_shared_bytes %d\n", ms.rssShmem) + WriteGaugeUint64(w, "process_virtual_memory_peak_bytes", ms.vmPeak) + WriteGaugeUint64(w, "process_resident_memory_peak_bytes", ms.rssPeak) + WriteGaugeUint64(w, "process_resident_memory_anon_bytes", ms.rssAnon) + WriteGaugeUint64(w, "process_resident_memory_file_bytes", ms.rssFile) + WriteGaugeUint64(w, "process_resident_memory_shared_bytes", ms.rssShmem) } diff --git a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_windows.go b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_windows.go index e824ada945..bda7c82f73 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_windows.go +++ b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_windows.go @@ -4,7 +4,6 @@ package metrics import ( - "fmt" "io" "log" "syscall" @@ -55,16 +54,16 @@ func writeProcessMetrics(w io.Writer) { log.Printf("ERROR: metrics: cannot read process memory information: %s", err) return } - stimeSeconds := (uint64(stime.HighDateTime)<<32 + uint64(stime.LowDateTime)) / 1e7 - utimeSeconds := (uint64(utime.HighDateTime)<<32 + uint64(utime.LowDateTime)) / 1e7 - fmt.Fprintf(w, "process_cpu_seconds_system_total %d\n", stimeSeconds) - fmt.Fprintf(w, "process_cpu_seconds_total %d\n", stimeSeconds+utimeSeconds) - fmt.Fprintf(w, "process_cpu_seconds_user_total %d\n", stimeSeconds) - fmt.Fprintf(w, "process_pagefaults_total %d\n", mc.PageFaultCount) - fmt.Fprintf(w, "process_start_time_seconds %d\n", startTime.Nanoseconds()/1e9) - fmt.Fprintf(w, "process_virtual_memory_bytes %d\n", mc.PrivateUsage) - fmt.Fprintf(w, "process_resident_memory_peak_bytes %d\n", mc.PeakWorkingSetSize) - fmt.Fprintf(w, "process_resident_memory_bytes %d\n", mc.WorkingSetSize) + stimeSeconds := float64(uint64(stime.HighDateTime)<<32+uint64(stime.LowDateTime)) / 1e7 + utimeSeconds := float64(uint64(utime.HighDateTime)<<32+uint64(utime.LowDateTime)) / 1e7 + WriteCounterFloat64(w, "process_cpu_seconds_system_total", stimeSeconds) + WriteCounterFloat64(w, "process_cpu_seconds_total", stimeSeconds+utimeSeconds) + WriteCounterFloat64(w, "process_cpu_seconds_user_total", stimeSeconds) + WriteCounterUint64(w, "process_pagefaults_total", uint64(mc.PageFaultCount)) + WriteGaugeUint64(w, "process_start_time_seconds", uint64(startTime.Nanoseconds())/1e9) + WriteGaugeUint64(w, "process_virtual_memory_bytes", uint64(mc.PrivateUsage)) + WriteGaugeUint64(w, "process_resident_memory_peak_bytes", uint64(mc.PeakWorkingSetSize)) + WriteGaugeUint64(w, "process_resident_memory_bytes", uint64(mc.WorkingSetSize)) } func writeFDMetrics(w io.Writer) { @@ -80,6 +79,6 @@ func writeFDMetrics(w io.Writer) { } // it seems to be hard-coded limit for 64-bit systems // https://learn.microsoft.com/en-us/archive/blogs/markrussinovich/pushing-the-limits-of-windows-handles#maximum-number-of-handles - fmt.Fprintf(w, "process_max_fds %d\n", 16777216) - fmt.Fprintf(w, "process_open_fds %d\n", count) + WriteGaugeUint64(w, "process_max_fds", 16777216) + WriteGaugeUint64(w, "process_open_fds", uint64(count)) } diff --git a/vendor/github.com/VictoriaMetrics/metrics/push.go b/vendor/github.com/VictoriaMetrics/metrics/push.go index 4215f48ab6..f33886f9b6 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/push.go +++ b/vendor/github.com/VictoriaMetrics/metrics/push.go @@ -2,17 +2,69 @@ package metrics import ( "bytes" + "context" + "errors" "fmt" "io" "io/ioutil" "log" "net/http" "net/url" + "strings" + "sync" "time" "compress/gzip" ) +// PushOptions is the list of options, which may be applied to InitPushWithOptions(). +type PushOptions struct { + // ExtraLabels is an optional comma-separated list of `label="value"` labels, which must be added to all the metrics before pushing them to pushURL. + ExtraLabels string + + // Headers is an optional list of HTTP headers to add to every push request to pushURL. + // + // Every item in the list must have the form `Header: value`. For example, `Authorization: Custom my-top-secret`. + Headers []string + + // Whether to disable HTTP request body compression before sending the metrics to pushURL. + // + // By default the compression is enabled. + DisableCompression bool + + // Method is HTTP request method to use when pushing metrics to pushURL. + // + // By default the Method is GET. + Method string + + // Optional WaitGroup for waiting until all the push workers created with this WaitGroup are stopped. + WaitGroup *sync.WaitGroup +} + +// InitPushWithOptions sets up periodic push for globally registered metrics to the given pushURL with the given interval. +// +// The periodic push is stopped when ctx is canceled. +// It is possible to wait until the background metrics push worker is stopped on a WaitGroup passed via opts.WaitGroup. +// +// If pushProcessMetrics is set to true, then 'process_*' and `go_*` metrics are also pushed to pushURL. +// +// opts may contain additional configuration options if non-nil. +// +// The metrics are pushed to pushURL in Prometheus text exposition format. +// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format +// +// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to +// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format +// +// It is OK calling InitPushWithOptions multiple times with different pushURL - +// in this case metrics are pushed to all the provided pushURL urls. +func InitPushWithOptions(ctx context.Context, pushURL string, interval time.Duration, pushProcessMetrics bool, opts *PushOptions) error { + writeMetrics := func(w io.Writer) { + WritePrometheus(w, pushProcessMetrics) + } + return InitPushExtWithOptions(ctx, pushURL, interval, writeMetrics, opts) +} + // InitPushProcessMetrics sets up periodic push for 'process_*' metrics to the given pushURL with the given interval. // // extraLabels may contain comma-separated list of `label="value"` labels, which will be added @@ -27,10 +79,7 @@ import ( // It is OK calling InitPushProcessMetrics multiple times with different pushURL - // in this case metrics are pushed to all the provided pushURL urls. func InitPushProcessMetrics(pushURL string, interval time.Duration, extraLabels string) error { - writeMetrics := func(w io.Writer) { - WriteProcessMetrics(w) - } - return InitPushExt(pushURL, interval, extraLabels, writeMetrics) + return InitPushExt(pushURL, interval, extraLabels, WriteProcessMetrics) } // InitPush sets up periodic push for globally registered metrics to the given pushURL with the given interval. @@ -38,7 +87,7 @@ func InitPushProcessMetrics(pushURL string, interval time.Duration, extraLabels // extraLabels may contain comma-separated list of `label="value"` labels, which will be added // to all the metrics before pushing them to pushURL. // -// If pushProcessMetrics is set to true, then 'process_*' metrics are also pushed to pushURL. +// If pushProcessMetrics is set to true, then 'process_*' and `go_*` metrics are also pushed to pushURL. // // The metrics are pushed to pushURL in Prometheus text exposition format. // See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format @@ -55,12 +104,46 @@ func InitPush(pushURL string, interval time.Duration, extraLabels string, pushPr return InitPushExt(pushURL, interval, extraLabels, writeMetrics) } +// PushMetrics pushes globally registered metrics to pushURL. +// +// If pushProcessMetrics is set to true, then 'process_*' and `go_*` metrics are also pushed to pushURL. +// +// opts may contain additional configuration options if non-nil. +// +// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to +// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format +func PushMetrics(ctx context.Context, pushURL string, pushProcessMetrics bool, opts *PushOptions) error { + writeMetrics := func(w io.Writer) { + WritePrometheus(w, pushProcessMetrics) + } + return PushMetricsExt(ctx, pushURL, writeMetrics, opts) +} + +// InitPushWithOptions sets up periodic push for metrics from s to the given pushURL with the given interval. +// +// The periodic push is stopped when the ctx is canceled. +// It is possible to wait until the background metrics push worker is stopped on a WaitGroup passed via opts.WaitGroup. +// +// opts may contain additional configuration options if non-nil. +// +// The metrics are pushed to pushURL in Prometheus text exposition format. +// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format +// +// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to +// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format +// +// It is OK calling InitPushWithOptions multiple times with different pushURL - +// in this case metrics are pushed to all the provided pushURL urls. +func (s *Set) InitPushWithOptions(ctx context.Context, pushURL string, interval time.Duration, opts *PushOptions) error { + return InitPushExtWithOptions(ctx, pushURL, interval, s.WritePrometheus, opts) +} + // InitPush sets up periodic push for metrics from s to the given pushURL with the given interval. // // extraLabels may contain comma-separated list of `label="value"` labels, which will be added // to all the metrics before pushing them to pushURL. // -// / The metrics are pushed to pushURL in Prometheus text exposition format. +// The metrics are pushed to pushURL in Prometheus text exposition format. // See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format // // It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to @@ -69,10 +152,17 @@ func InitPush(pushURL string, interval time.Duration, extraLabels string, pushPr // It is OK calling InitPush multiple times with different pushURL - // in this case metrics are pushed to all the provided pushURL urls. func (s *Set) InitPush(pushURL string, interval time.Duration, extraLabels string) error { - writeMetrics := func(w io.Writer) { - s.WritePrometheus(w) - } - return InitPushExt(pushURL, interval, extraLabels, writeMetrics) + return InitPushExt(pushURL, interval, extraLabels, s.WritePrometheus) +} + +// PushMetrics pushes s metrics to pushURL. +// +// opts may contain additional configuration options if non-nil. +// +// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to +// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format +func (s *Set) PushMetrics(ctx context.Context, pushURL string, opts *PushOptions) error { + return PushMetricsExt(ctx, pushURL, s.WritePrometheus, opts) } // InitPushExt sets up periodic push for metrics obtained by calling writeMetrics with the given interval. @@ -90,94 +180,246 @@ func (s *Set) InitPush(pushURL string, interval time.Duration, extraLabels strin // in this case metrics are pushed to all the provided pushURL urls. // // It is OK calling InitPushExt multiple times with different writeMetrics - -// in this case all the metrics generated by writeMetrics callbacks are writte to pushURL. +// in this case all the metrics generated by writeMetrics callbacks are written to pushURL. func InitPushExt(pushURL string, interval time.Duration, extraLabels string, writeMetrics func(w io.Writer)) error { + opts := &PushOptions{ + ExtraLabels: extraLabels, + } + return InitPushExtWithOptions(context.Background(), pushURL, interval, writeMetrics, opts) +} + +// InitPushExtWithOptions sets up periodic push for metrics obtained by calling writeMetrics with the given interval. +// +// The writeMetrics callback must write metrics to w in Prometheus text exposition format without timestamps and trailing comments. +// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format +// +// The periodic push is stopped when the ctx is canceled. +// It is possible to wait until the background metrics push worker is stopped on a WaitGroup passed via opts.WaitGroup. +// +// opts may contain additional configuration options if non-nil. +// +// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to +// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format +// +// It is OK calling InitPushExtWithOptions multiple times with different pushURL - +// in this case metrics are pushed to all the provided pushURL urls. +// +// It is OK calling InitPushExtWithOptions multiple times with different writeMetrics - +// in this case all the metrics generated by writeMetrics callbacks are written to pushURL. +func InitPushExtWithOptions(ctx context.Context, pushURL string, interval time.Duration, writeMetrics func(w io.Writer), opts *PushOptions) error { + pc, err := newPushContext(pushURL, opts) + if err != nil { + return err + } + + // validate interval if interval <= 0 { return fmt.Errorf("interval must be positive; got %s", interval) } - if err := validateTags(extraLabels); err != nil { - return fmt.Errorf("invalid extraLabels=%q: %w", extraLabels, err) + pushMetricsSet.GetOrCreateFloatCounter(fmt.Sprintf(`metrics_push_interval_seconds{url=%q}`, pc.pushURLRedacted)).Set(interval.Seconds()) + + var wg *sync.WaitGroup + if opts != nil { + wg = opts.WaitGroup + if wg != nil { + wg.Add(1) + } + } + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + stopCh := ctx.Done() + for { + select { + case <-ticker.C: + ctxLocal, cancel := context.WithTimeout(ctx, interval+time.Second) + err := pc.pushMetrics(ctxLocal, writeMetrics) + cancel() + if err != nil { + log.Printf("ERROR: metrics.push: %s", err) + } + case <-stopCh: + if wg != nil { + wg.Done() + } + return + } + } + }() + + return nil +} + +// PushMetricsExt pushes metrics generated by wirteMetrics to pushURL. +// +// The writeMetrics callback must write metrics to w in Prometheus text exposition format without timestamps and trailing comments. +// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format +// +// opts may contain additional configuration options if non-nil. +// +// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to +// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format +func PushMetricsExt(ctx context.Context, pushURL string, writeMetrics func(w io.Writer), opts *PushOptions) error { + pc, err := newPushContext(pushURL, opts) + if err != nil { + return err } + return pc.pushMetrics(ctx, writeMetrics) +} + +type pushContext struct { + pushURL *url.URL + method string + pushURLRedacted string + extraLabels string + headers http.Header + disableCompression bool + + client *http.Client + + pushesTotal *Counter + bytesPushedTotal *Counter + pushBlockSize *Histogram + pushDuration *Histogram + pushErrors *Counter +} + +func newPushContext(pushURL string, opts *PushOptions) (*pushContext, error) { + if opts == nil { + opts = &PushOptions{} + } + + // validate pushURL pu, err := url.Parse(pushURL) if err != nil { - return fmt.Errorf("cannot parse pushURL=%q: %w", pushURL, err) + return nil, fmt.Errorf("cannot parse pushURL=%q: %w", pushURL, err) } if pu.Scheme != "http" && pu.Scheme != "https" { - return fmt.Errorf("unsupported scheme in pushURL=%q; expecting 'http' or 'https'", pushURL) + return nil, fmt.Errorf("unsupported scheme in pushURL=%q; expecting 'http' or 'https'", pushURL) } if pu.Host == "" { - return fmt.Errorf("missing host in pushURL=%q", pushURL) + return nil, fmt.Errorf("missing host in pushURL=%q", pushURL) + } + + method := opts.Method + if method == "" { + method = http.MethodGet + } + + // validate ExtraLabels + extraLabels := opts.ExtraLabels + if err := validateTags(extraLabels); err != nil { + return nil, fmt.Errorf("invalid extraLabels=%q: %w", extraLabels, err) + } + + // validate Headers + headers := make(http.Header) + for _, h := range opts.Headers { + n := strings.IndexByte(h, ':') + if n < 0 { + return nil, fmt.Errorf("missing `:` delimiter in the header %q", h) + } + name := strings.TrimSpace(h[:n]) + value := strings.TrimSpace(h[n+1:]) + headers.Add(name, value) } + pushURLRedacted := pu.Redacted() - c := &http.Client{ - Timeout: interval, - } - pushesTotal := pushMetrics.GetOrCreateCounter(fmt.Sprintf(`metrics_push_total{url=%q}`, pushURLRedacted)) - pushErrorsTotal := pushMetrics.GetOrCreateCounter(fmt.Sprintf(`metrics_push_errors_total{url=%q}`, pushURLRedacted)) - bytesPushedTotal := pushMetrics.GetOrCreateCounter(fmt.Sprintf(`metrics_push_bytes_pushed_total{url=%q}`, pushURLRedacted)) - pushDuration := pushMetrics.GetOrCreateHistogram(fmt.Sprintf(`metrics_push_duration_seconds{url=%q}`, pushURLRedacted)) - pushBlockSize := pushMetrics.GetOrCreateHistogram(fmt.Sprintf(`metrics_push_block_size_bytes{url=%q}`, pushURLRedacted)) - pushMetrics.GetOrCreateFloatCounter(fmt.Sprintf(`metrics_push_interval_seconds{url=%q}`, pushURLRedacted)).Set(interval.Seconds()) - go func() { - ticker := time.NewTicker(interval) - var bb bytes.Buffer - var tmpBuf []byte - zw := gzip.NewWriter(&bb) - for range ticker.C { - bb.Reset() - writeMetrics(&bb) - if len(extraLabels) > 0 { - tmpBuf = addExtraLabels(tmpBuf[:0], bb.Bytes(), extraLabels) - bb.Reset() - if _, err := bb.Write(tmpBuf); err != nil { - panic(fmt.Errorf("BUG: cannot write %d bytes to bytes.Buffer: %s", len(tmpBuf), err)) - } - } - tmpBuf = append(tmpBuf[:0], bb.Bytes()...) - bb.Reset() - zw.Reset(&bb) - if _, err := zw.Write(tmpBuf); err != nil { - panic(fmt.Errorf("BUG: cannot write %d bytes to gzip writer: %s", len(tmpBuf), err)) - } - if err := zw.Close(); err != nil { - panic(fmt.Errorf("BUG: cannot flush metrics to gzip writer: %s", err)) - } - pushesTotal.Inc() - blockLen := bb.Len() - bytesPushedTotal.Add(blockLen) - pushBlockSize.Update(float64(blockLen)) - req, err := http.NewRequest("GET", pushURL, &bb) - if err != nil { - panic(fmt.Errorf("BUG: metrics.push: cannot initialize request for metrics push to %q: %w", pushURLRedacted, err)) - } - req.Header.Set("Content-Type", "text/plain") - req.Header.Set("Content-Encoding", "gzip") - startTime := time.Now() - resp, err := c.Do(req) - pushDuration.UpdateDuration(startTime) - if err != nil { - log.Printf("ERROR: metrics.push: cannot push metrics to %q: %s", pushURLRedacted, err) - pushErrorsTotal.Inc() - continue - } - if resp.StatusCode/100 != 2 { - body, _ := ioutil.ReadAll(resp.Body) - _ = resp.Body.Close() - log.Printf("ERROR: metrics.push: unexpected status code in response from %q: %d; expecting 2xx; response body: %q", - pushURLRedacted, resp.StatusCode, body) - pushErrorsTotal.Inc() - continue - } - _ = resp.Body.Close() + client := &http.Client{} + return &pushContext{ + pushURL: pu, + method: method, + pushURLRedacted: pushURLRedacted, + extraLabels: extraLabels, + headers: headers, + disableCompression: opts.DisableCompression, + + client: client, + + pushesTotal: pushMetricsSet.GetOrCreateCounter(fmt.Sprintf(`metrics_push_total{url=%q}`, pushURLRedacted)), + bytesPushedTotal: pushMetricsSet.GetOrCreateCounter(fmt.Sprintf(`metrics_push_bytes_pushed_total{url=%q}`, pushURLRedacted)), + pushBlockSize: pushMetricsSet.GetOrCreateHistogram(fmt.Sprintf(`metrics_push_block_size_bytes{url=%q}`, pushURLRedacted)), + pushDuration: pushMetricsSet.GetOrCreateHistogram(fmt.Sprintf(`metrics_push_duration_seconds{url=%q}`, pushURLRedacted)), + pushErrors: pushMetricsSet.GetOrCreateCounter(fmt.Sprintf(`metrics_push_errors_total{url=%q}`, pushURLRedacted)), + }, nil +} + +func (pc *pushContext) pushMetrics(ctx context.Context, writeMetrics func(w io.Writer)) error { + bb := getBytesBuffer() + defer putBytesBuffer(bb) + + writeMetrics(bb) + + if len(pc.extraLabels) > 0 { + bbTmp := getBytesBuffer() + bbTmp.B = append(bbTmp.B[:0], bb.B...) + bb.B = addExtraLabels(bb.B[:0], bbTmp.B, pc.extraLabels) + putBytesBuffer(bbTmp) + } + if !pc.disableCompression { + bbTmp := getBytesBuffer() + bbTmp.B = append(bbTmp.B[:0], bb.B...) + bb.B = bb.B[:0] + zw := getGzipWriter(bb) + if _, err := zw.Write(bbTmp.B); err != nil { + panic(fmt.Errorf("BUG: cannot write %d bytes to gzip writer: %s", len(bbTmp.B), err)) } - }() + if err := zw.Close(); err != nil { + panic(fmt.Errorf("BUG: cannot flush metrics to gzip writer: %s", err)) + } + putGzipWriter(zw) + putBytesBuffer(bbTmp) + } + + // Update metrics + pc.pushesTotal.Inc() + blockLen := len(bb.B) + pc.bytesPushedTotal.Add(blockLen) + pc.pushBlockSize.Update(float64(blockLen)) + + // Prepare the request to sent to pc.pushURL + reqBody := bytes.NewReader(bb.B) + req, err := http.NewRequestWithContext(ctx, pc.method, pc.pushURL.String(), reqBody) + if err != nil { + panic(fmt.Errorf("BUG: metrics.push: cannot initialize request for metrics push to %q: %w", pc.pushURLRedacted, err)) + } + + req.Header.Set("Content-Type", "text/plain") + // Set the needed headers, and `Content-Type` allowed be overwrited. + for name, values := range pc.headers { + for _, value := range values { + req.Header.Add(name, value) + } + } + if !pc.disableCompression { + req.Header.Set("Content-Encoding", "gzip") + } + + // Perform the request + startTime := time.Now() + resp, err := pc.client.Do(req) + pc.pushDuration.UpdateDuration(startTime) + if err != nil { + if errors.Is(err, context.Canceled) { + return nil + } + pc.pushErrors.Inc() + return fmt.Errorf("cannot push metrics to %q: %s", pc.pushURLRedacted, err) + } + if resp.StatusCode/100 != 2 { + body, _ := ioutil.ReadAll(resp.Body) + _ = resp.Body.Close() + pc.pushErrors.Inc() + return fmt.Errorf("unexpected status code in response from %q: %d; expecting 2xx; response body: %q", pc.pushURLRedacted, resp.StatusCode, body) + } + _ = resp.Body.Close() return nil } -var pushMetrics = NewSet() +var pushMetricsSet = NewSet() func writePushMetrics(w io.Writer) { - pushMetrics.WritePrometheus(w) + pushMetricsSet.WritePrometheus(w) } func addExtraLabels(dst, src []byte, extraLabels string) []byte { @@ -225,3 +467,44 @@ func addExtraLabels(dst, src []byte, extraLabels string) []byte { } var bashBytes = []byte("#") + +func getBytesBuffer() *bytesBuffer { + v := bytesBufferPool.Get() + if v == nil { + return &bytesBuffer{} + } + return v.(*bytesBuffer) +} + +func putBytesBuffer(bb *bytesBuffer) { + bb.B = bb.B[:0] + bytesBufferPool.Put(bb) +} + +var bytesBufferPool sync.Pool + +type bytesBuffer struct { + B []byte +} + +func (bb *bytesBuffer) Write(p []byte) (int, error) { + bb.B = append(bb.B, p...) + return len(p), nil +} + +func getGzipWriter(w io.Writer) *gzip.Writer { + v := gzipWriterPool.Get() + if v == nil { + return gzip.NewWriter(w) + } + zw := v.(*gzip.Writer) + zw.Reset(w) + return zw +} + +func putGzipWriter(zw *gzip.Writer) { + zw.Reset(io.Discard) + gzipWriterPool.Put(zw) +} + +var gzipWriterPool sync.Pool diff --git a/vendor/github.com/VictoriaMetrics/metrics/set.go b/vendor/github.com/VictoriaMetrics/metrics/set.go index 79355ea386..868a01c943 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/set.go +++ b/vendor/github.com/VictoriaMetrics/metrics/set.go @@ -19,6 +19,8 @@ type Set struct { a []*namedMetric m map[string]*namedMetric summaries []*Summary + + metricsWriters []func(w io.Writer) } // NewSet creates new set of metrics. @@ -45,14 +47,27 @@ func (s *Set) WritePrometheus(w io.Writer) { sort.Slice(s.a, lessFunc) } sa := append([]*namedMetric(nil), s.a...) + metricsWriters := s.metricsWriters s.mu.Unlock() - // Call marshalTo without the global lock, since certain metric types such as Gauge - // can call a callback, which, in turn, can try calling s.mu.Lock again. + prevMetricFamily := "" for _, nm := range sa { + metricFamily := getMetricFamily(nm.name) + if metricFamily != prevMetricFamily { + // write meta info only once per metric family + metricType := nm.metric.metricType() + WriteMetadataIfNeeded(&bb, nm.name, metricType) + prevMetricFamily = metricFamily + } + // Call marshalTo without the global lock, since certain metric types such as Gauge + // can call a callback, which, in turn, can try calling s.mu.Lock again. nm.metric.marshalTo(nm.name, &bb) } w.Write(bb.Bytes()) + + for _, writeMetrics := range metricsWriters { + writeMetrics(w) + } } // NewHistogram creates and returns new histogram in s with the given name. @@ -243,9 +258,6 @@ func (s *Set) GetOrCreateFloatCounter(name string) *FloatCounter { // // The returned gauge is safe to use from concurrent goroutines. func (s *Set) NewGauge(name string, f func() float64) *Gauge { - if f == nil { - panic(fmt.Errorf("BUG: f cannot be nil")) - } g := &Gauge{ f: f, } @@ -272,9 +284,6 @@ func (s *Set) GetOrCreateGauge(name string, f func() float64) *Gauge { s.mu.Unlock() if nm == nil { // Slow path - create and register missing gauge. - if f == nil { - panic(fmt.Errorf("BUG: f cannot be nil")) - } if err := validateMetric(name); err != nil { panic(fmt.Errorf("BUG: invalid metric name %q: %s", name, err)) } @@ -521,14 +530,22 @@ func (s *Set) unregisterMetricLocked(nm *namedMetric) bool { } // UnregisterAllMetrics de-registers all metrics registered in s. +// +// It also de-registers writeMetrics callbacks passed to RegisterMetricsWriter. func (s *Set) UnregisterAllMetrics() { metricNames := s.ListMetricNames() for _, name := range metricNames { s.UnregisterMetric(name) } + + s.mu.Lock() + s.metricsWriters = nil + s.mu.Unlock() } // ListMetricNames returns sorted list of all the metrics in s. +// +// The returned list doesn't include metrics generated by metricsWriter passed to RegisterMetricsWriter. func (s *Set) ListMetricNames() []string { s.mu.Lock() defer s.mu.Unlock() @@ -542,3 +559,17 @@ func (s *Set) ListMetricNames() []string { sort.Strings(metricNames) return metricNames } + +// RegisterMetricsWriter registers writeMetrics callback for including metrics in the output generated by s.WritePrometheus. +// +// The writeMetrics callback must write metrics to w in Prometheus text exposition format without timestamps and trailing comments. +// The last line generated by writeMetrics must end with \n. +// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format +// +// It is OK to reguster multiple writeMetrics callbacks - all of them will be called sequentially for gererating the output at s.WritePrometheus. +func (s *Set) RegisterMetricsWriter(writeMetrics func(w io.Writer)) { + s.mu.Lock() + defer s.mu.Unlock() + + s.metricsWriters = append(s.metricsWriters, writeMetrics) +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/summary.go b/vendor/github.com/VictoriaMetrics/metrics/summary.go index 52183d22bb..057b67bc5a 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/summary.go +++ b/vendor/github.com/VictoriaMetrics/metrics/summary.go @@ -119,6 +119,10 @@ func (sm *Summary) marshalTo(prefix string, w io.Writer) { } } +func (sm *Summary) metricType() string { + return "summary" +} + func splitMetricName(name string) (string, string) { n := strings.IndexByte(name, '{') if n < 0 { @@ -196,6 +200,10 @@ func (qv *quantileValue) marshalTo(prefix string, w io.Writer) { } } +func (qv *quantileValue) metricType() string { + return "unsupported" +} + func addTag(name, tag string) string { if len(name) == 0 || name[len(name)-1] != '}' { return fmt.Sprintf("%s{%s}", name, tag) diff --git a/vendor/github.com/alecthomas/chroma/quick/quick.go b/vendor/github.com/alecthomas/chroma/quick/quick.go deleted file mode 100644 index 1ed3db7e8d..0000000000 --- a/vendor/github.com/alecthomas/chroma/quick/quick.go +++ /dev/null @@ -1,44 +0,0 @@ -// Package quick provides simple, no-configuration source code highlighting. -package quick - -import ( - "io" - - "github.com/alecthomas/chroma" - "github.com/alecthomas/chroma/formatters" - "github.com/alecthomas/chroma/lexers" - "github.com/alecthomas/chroma/styles" -) - -// Highlight some text. -// -// Lexer, formatter and style may be empty, in which case a best-effort is made. -func Highlight(w io.Writer, source, lexer, formatter, style string) error { - // Determine lexer. - l := lexers.Get(lexer) - if l == nil { - l = lexers.Analyse(source) - } - if l == nil { - l = lexers.Fallback - } - l = chroma.Coalesce(l) - - // Determine formatter. - f := formatters.Get(formatter) - if f == nil { - f = formatters.Fallback - } - - // Determine style. - s := styles.Get(style) - if s == nil { - s = styles.Fallback - } - - it, err := l.Tokenise(nil, source) - if err != nil { - return err - } - return f.Format(w, s, it) -} diff --git a/vendor/github.com/bitfield/script/README.md b/vendor/github.com/bitfield/script/README.md index a06a70c19c..98e87ec4d4 100644 --- a/vendor/github.com/bitfield/script/README.md +++ b/vendor/github.com/bitfield/script/README.md @@ -1,7 +1,8 @@ [![Go Reference](https://pkg.go.dev/badge/github.com/bitfield/script.svg)](https://pkg.go.dev/github.com/bitfield/script) [![Go Report Card](https://goreportcard.com/badge/github.com/bitfield/script)](https://goreportcard.com/report/github.com/bitfield/script) [![Mentioned in Awesome Go](https://awesome.re/mentioned-badge-flat.svg)](https://github.com/avelino/awesome-go) -![Tests](https://github.com/bitfield/script/actions/workflows/test.yml/badge.svg) +![CI](https://github.com/bitfield/script/actions/workflows/ci.yml/badge.svg) +![Audit](https://github.com/bitfield/script/actions/workflows/audit.yml/badge.svg) ```go import "github.com/bitfield/script" @@ -33,6 +34,7 @@ If you're already familiar with shell scripting and the Unix toolset, here is a | `>` | [`WriteFile`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WriteFile) | | `>>` | [`AppendFile`](https://pkg.go.dev/github.com/bitfield/script#Pipe.AppendFile) | | `$*` | [`Args`](https://pkg.go.dev/github.com/bitfield/script#Args) | +| `base64` | [`DecodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.DecodeBase64) / [`EncodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.EncodeBase64) | | `basename` | [`Basename`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Basename) | | `cat` | [`File`](https://pkg.go.dev/github.com/bitfield/script#File) / [`Concat`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Concat) | | `curl` | [`Do`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Do) / [`Get`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Get) / [`Post`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Post) | @@ -267,18 +269,31 @@ These are functions that create a pipe with a given contents: | Source | Contents | | -------- | ------------- | -| [`Args`](https://pkg.go.dev/github.com/bitfield/script#Args) | command-line arguments -| [`Do`](https://pkg.go.dev/github.com/bitfield/script#Do) | HTTP response -| [`Echo`](https://pkg.go.dev/github.com/bitfield/script#Echo) | a string -| [`Exec`](https://pkg.go.dev/github.com/bitfield/script#Exec) | command output -| [`File`](https://pkg.go.dev/github.com/bitfield/script#File) | file contents -| [`FindFiles`](https://pkg.go.dev/github.com/bitfield/script#FindFiles) | recursive file listing -| [`Get`](https://pkg.go.dev/github.com/bitfield/script#Get) | HTTP response -| [`IfExists`](https://pkg.go.dev/github.com/bitfield/script#IfExists) | do something only if some file exists -| [`ListFiles`](https://pkg.go.dev/github.com/bitfield/script#ListFiles) | file listing (including wildcards) -| [`Post`](https://pkg.go.dev/github.com/bitfield/script#Post) | HTTP response -| [`Slice`](https://pkg.go.dev/github.com/bitfield/script#Slice) | slice elements, one per line -| [`Stdin`](https://pkg.go.dev/github.com/bitfield/script#Stdin) | standard input +| [`Args`](https://pkg.go.dev/github.com/bitfield/script#Args) | command-line arguments | +| [`Do`](https://pkg.go.dev/github.com/bitfield/script#Do) | HTTP response | +| [`Echo`](https://pkg.go.dev/github.com/bitfield/script#Echo) | a string | +| [`Exec`](https://pkg.go.dev/github.com/bitfield/script#Exec) | command output | +| [`File`](https://pkg.go.dev/github.com/bitfield/script#File) | file contents | +| [`FindFiles`](https://pkg.go.dev/github.com/bitfield/script#FindFiles) | recursive file listing | +| [`Get`](https://pkg.go.dev/github.com/bitfield/script#Get) | HTTP response | +| [`IfExists`](https://pkg.go.dev/github.com/bitfield/script#IfExists) | do something only if some file exists | +| [`ListFiles`](https://pkg.go.dev/github.com/bitfield/script#ListFiles) | file listing (including wildcards) | +| [`Post`](https://pkg.go.dev/github.com/bitfield/script#Post) | HTTP response | +| [`Slice`](https://pkg.go.dev/github.com/bitfield/script#Slice) | slice elements, one per line | +| [`Stdin`](https://pkg.go.dev/github.com/bitfield/script#Stdin) | standard input | + +## Modifiers + +These are methods on a pipe that change its configuration: + +| Source | Modifies | +| -------- | ------------- | +| [`WithEnv`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WithEnv) | environment for commands | +| [`WithError`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WithError) | pipe error status | +| [`WithHTTPClient`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WithHTTPClient) | client for HTTP requests | +| [`WithReader`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WithReader) | pipe source | +| [`WithStderr`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WithStderr) | standard error output stream for command | +| [`WithStdout`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WithStdout) | standard output stream for pipe | ## Filters @@ -289,9 +304,11 @@ Filters are methods on an existing pipe that also return a pipe, allowing you to | [`Basename`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Basename) | removes leading path components from each line, leaving only the filename | | [`Column`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Column) | Nth column of input | | [`Concat`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Concat) | contents of multiple files | +| [`DecodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.DecodeBase64) | input decoded from base64 | | [`Dirname`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Dirname) | removes filename from each line, leaving only leading path components | | [`Do`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Do) | response to supplied HTTP request | | [`Echo`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Echo) | all input replaced by given string | +| [`EncodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.EncodeBase64) | input encoded to base64 | | [`Exec`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Exec) | filtered through external command | | [`ExecForEach`](https://pkg.go.dev/github.com/bitfield/script#Pipe.ExecForEach) | execute given command template for each line of input | | [`Filter`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Filter) | user-supplied function filtering a reader to a writer | @@ -329,13 +346,16 @@ Sinks are methods that return some data from a pipe, ending the pipeline and ext | [`Slice`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Slice) | | data as `[]string`, error | | [`Stdout`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Stdout) | standard output | bytes written, error | | [`String`](https://pkg.go.dev/github.com/bitfield/script#Pipe.String) | | data as `string`, error | -| [`Wait`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Wait) | | none | +| [`Wait`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Wait) | | error | | [`WriteFile`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WriteFile) | specified file, truncating if it exists | bytes written, error | # What's new | Version | New | | ----------- | ------- | +| 0.23.0 | [`WithEnv`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WithEnv) | +| | [`DecodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.DecodeBase64) / [`EncodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.EncodeBase64) | +| | [`Wait`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Wait) returns error | | v0.22.0 | [`Tee`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Tee), [`WithStderr`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WithStderr) | | v0.21.0 | HTTP support: [`Do`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Do), [`Get`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Get), [`Post`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Post) | | v0.20.0 | [`JQ`](https://pkg.go.dev/github.com/bitfield/script#Pipe.JQ) | @@ -346,7 +366,7 @@ See the [contributor's guide](CONTRIBUTING.md) for some helpful tips if you'd li # Links -- [Scripting with Go](https://bitfieldconsulting.com/golang/scripting) +- [Scripting with Go](https://bitfieldconsulting.com/posts/scripting) - [Code Club: Script](https://www.youtube.com/watch?v=6S5EqzVwpEg) - [Bitfield Consulting](https://bitfieldconsulting.com/) - [Go books by John Arundel](https://bitfieldconsulting.com/books) diff --git a/vendor/github.com/bitfield/script/script.go b/vendor/github.com/bitfield/script/script.go index c471f74b49..e25b1c4883 100644 --- a/vendor/github.com/bitfield/script/script.go +++ b/vendor/github.com/bitfield/script/script.go @@ -4,6 +4,7 @@ import ( "bufio" "container/ring" "crypto/sha256" + "encoding/base64" "encoding/hex" "encoding/json" "fmt" @@ -27,13 +28,14 @@ import ( // Pipe represents a pipe object with an associated [ReadAutoCloser]. type Pipe struct { // Reader is the underlying reader. - Reader ReadAutoCloser - stdout, stderr io.Writer - httpClient *http.Client + Reader ReadAutoCloser + stdout io.Writer + httpClient *http.Client - // because pipe stages are concurrent, protect 'err' - mu *sync.Mutex - err error + mu *sync.Mutex + err error + stderr io.Writer + env []string } // Args creates a pipe containing the program's command-line arguments from @@ -166,6 +168,7 @@ func NewPipe() *Pipe { mu: new(sync.Mutex), stdout: os.Stdout, httpClient: http.DefaultClient, + env: nil, } } @@ -275,6 +278,18 @@ func (p *Pipe) CountLines() (lines int, err error) { return lines, p.Error() } +// DecodeBase64 produces the string represented by the base64 encoded input. +func (p *Pipe) DecodeBase64() *Pipe { + return p.Filter(func(r io.Reader, w io.Writer) error { + decoder := base64.NewDecoder(base64.StdEncoding, r) + _, err := io.Copy(w, decoder) + if err != nil { + return err + } + return nil + }) +} + // Dirname reads paths from the pipe, one per line, and produces only the // parent directories of each path. For example, /usr/local/bin/foo would // become just /usr/local/bin. This is the complementary operation to @@ -347,7 +362,29 @@ func (p *Pipe) Echo(s string) *Pipe { return p.WithReader(strings.NewReader(s)) } +// EncodeBase64 produces the base64 encoding of the input. +func (p *Pipe) EncodeBase64() *Pipe { + return p.Filter(func(r io.Reader, w io.Writer) error { + encoder := base64.NewEncoder(base64.StdEncoding, w) + defer encoder.Close() + _, err := io.Copy(encoder, r) + if err != nil { + return err + } + return nil + }) +} + +func (p *Pipe) environment() []string { + p.mu.Lock() + defer p.mu.Unlock() + return p.env +} + // Error returns any error present on the pipe, or nil otherwise. +// Error is not a sink and does not wait until the pipe reaches +// completion. To wait for completion before returning the error, +// see [Pipe.Wait]. func (p *Pipe) Error() error { if p.mu == nil { // uninitialised pipe return nil @@ -362,6 +399,11 @@ func (p *Pipe) Error() error { // error output). The effect of this is to filter the contents of the pipe // through the external command. // +// # Environment +// +// The command inherits the current process's environment, optionally modified +// by [Pipe.WithEnv]. +// // # Error handling // // If the command had a non-zero exit status, the pipe's error status will also @@ -385,8 +427,13 @@ func (p *Pipe) Exec(cmdLine string) *Pipe { cmd.Stdin = r cmd.Stdout = w cmd.Stderr = w - if p.stderr != nil { - cmd.Stderr = p.stderr + pipeStderr := p.stdErr() + if pipeStderr != nil { + cmd.Stderr = pipeStderr + } + pipeEnv := p.environment() + if pipeEnv != nil { + cmd.Env = pipeEnv } err = cmd.Start() if err != nil { @@ -399,7 +446,8 @@ func (p *Pipe) Exec(cmdLine string) *Pipe { // ExecForEach renders cmdLine as a Go template for each line of input, running // the resulting command, and produces the combined output of all these -// commands in sequence. See [Pipe.Exec] for error handling details. +// commands in sequence. See [Pipe.Exec] for details on error handling and +// environment variables. // // This is mostly useful for substituting data into commands using Go template // syntax. For example: @@ -425,8 +473,12 @@ func (p *Pipe) ExecForEach(cmdLine string) *Pipe { cmd := exec.Command(args[0], args[1:]...) cmd.Stdout = w cmd.Stderr = w - if p.stderr != nil { - cmd.Stderr = p.stderr + pipeStderr := p.stdErr() + if pipeStderr != nil { + cmd.Stderr = pipeStderr + } + if p.env != nil { + cmd.Env = p.env } err = cmd.Start() if err != nil { @@ -810,6 +862,18 @@ func (p *Pipe) Slice() ([]string, error) { return result, p.Error() } +// stdErr returns the pipe's configured standard error writer for commands run +// via [Pipe.Exec] and [Pipe.ExecForEach]. The default is nil, which means that +// error output will go to the pipe. +func (p *Pipe) stdErr() io.Writer { + if p.mu == nil { // uninitialised pipe + return nil + } + p.mu.Lock() + defer p.mu.Unlock() + return p.stderr +} + // Stdout copies the pipe's contents to its configured standard output (using // [Pipe.WithStdout]), or to [os.Stdout] otherwise, and returns the number of // bytes successfully written, together with any error. @@ -848,14 +912,25 @@ func (p *Pipe) Tee(writers ...io.Writer) *Pipe { return p.WithReader(io.TeeReader(p.Reader, teeWriter)) } -// Wait reads the pipe to completion and discards the result. This is mostly -// useful for waiting until concurrent filters have completed (see -// [Pipe.Filter]). -func (p *Pipe) Wait() { +// Wait reads the pipe to completion and returns any error present on +// the pipe, or nil otherwise. This is mostly useful for waiting until +// concurrent filters have completed (see [Pipe.Filter]). +func (p *Pipe) Wait() error { _, err := io.Copy(io.Discard, p) if err != nil { p.SetError(err) } + return p.Error() +} + +// WithEnv sets the environment for subsequent [Pipe.Exec] and [Pipe.ExecForEach] +// commands to the string slice env, using the same format as [os/exec.Cmd.Env]. +// An empty slice unsets all existing environment variables. +func (p *Pipe) WithEnv(env []string) *Pipe { + p.mu.Lock() + defer p.mu.Unlock() + p.env = env + return p } // WithError sets the error err on the pipe. @@ -883,10 +958,11 @@ func (p *Pipe) WithReader(r io.Reader) *Pipe { return p } -// WithStderr redirects the standard error output for commands run via -// [Pipe.Exec] or [Pipe.ExecForEach] to the writer w, instead of going to the -// pipe as it normally would. +// WithStderr sets the standard error output for [Pipe.Exec] or +// [Pipe.ExecForEach] commands to w, instead of the pipe. func (p *Pipe) WithStderr(w io.Writer) *Pipe { + p.mu.Lock() + defer p.mu.Unlock() p.stderr = w return p } diff --git a/vendor/github.com/bytedance/sonic/.gitignore b/vendor/github.com/bytedance/sonic/.gitignore index 0d8844705b..fa60f43a29 100644 --- a/vendor/github.com/bytedance/sonic/.gitignore +++ b/vendor/github.com/bytedance/sonic/.gitignore @@ -49,4 +49,7 @@ ast/bench.sh !testdata/*.json.gz fuzz/testdata -*__debug_bin \ No newline at end of file +*__debug_bin* +*pprof +*coverage.txt +tools/venv/* \ No newline at end of file diff --git a/vendor/github.com/bytedance/sonic/.gitmodules b/vendor/github.com/bytedance/sonic/.gitmodules index b8d11c9215..ea84b991a9 100644 --- a/vendor/github.com/bytedance/sonic/.gitmodules +++ b/vendor/github.com/bytedance/sonic/.gitmodules @@ -1,3 +1,6 @@ -[submodule "tools/asm2asm"] +[submodule "cloudwego"] path = tools/asm2asm - url = https://github.com/chenzhuoyu/asm2asm + url = https://github.com/cloudwego/asm2asm.git +[submodule "tools/simde"] + path = tools/simde + url = https://github.com/simd-everywhere/simde.git diff --git a/vendor/github.com/bytedance/sonic/Makefile b/vendor/github.com/bytedance/sonic/Makefile deleted file mode 100644 index c672c313e7..0000000000 --- a/vendor/github.com/bytedance/sonic/Makefile +++ /dev/null @@ -1,111 +0,0 @@ -# -# Copyright 2021 ByteDance Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -ARCH := avx avx2 sse -TMP_DIR := output -OUT_DIR := internal/native -SRC_FILE := native/native.c - -CPU_avx := amd64 -CPU_avx2 := amd64 -CPU_sse := amd64 - -TMPL_avx := fastint_amd64_test fastfloat_amd64_test native_amd64_test recover_amd64_test -TMPL_avx2 := fastint_amd64_test fastfloat_amd64_test native_amd64_test recover_amd64_test -TMPL_sse := fastint_amd64_test fastfloat_amd64_test native_amd64_test recover_amd64_test - -CFLAGS_avx := -msse -mno-sse4 -mavx -mpclmul -mno-avx2 -mstack-alignment=0 -DUSE_AVX=1 -DUSE_AVX2=0 -CFLAGS_avx2 := -msse -mno-sse4 -mavx -mpclmul -mavx2 -mstack-alignment=0 -DUSE_AVX=1 -DUSE_AVX2=1 -CFLAGS_sse := -msse -mno-sse4 -mno-avx -mno-avx2 -mpclmul - -CC_amd64 := clang -ASM2ASM_amd64 := tools/asm2asm/asm2asm.py - -CFLAGS := -mno-red-zone -CFLAGS += -target x86_64-apple-macos11 -CFLAGS += -fno-asynchronous-unwind-tables -CFLAGS += -fno-builtin -CFLAGS += -fno-exceptions -CFLAGS += -fno-rtti -CFLAGS += -fno-stack-protector -CFLAGS += -nostdlib -CFLAGS += -O3 -CFLAGS += -Wall -Werror - -NATIVE_SRC := $(wildcard native/*.h) -NATIVE_SRC += $(wildcard native/*.c) - -.PHONY: all clean ${ARCH} - -define build_tmpl - $(eval @arch := $(1)) - $(eval @tmpl := $(2)) - $(eval @dest := $(3)) - -${@dest}: ${@tmpl} - mkdir -p $(dir ${@dest}) - echo '// Code generated by Makefile, DO NOT EDIT.' > ${@dest} - echo >> ${@dest} - sed -e 's/{{PACKAGE}}/${@arch}/g' ${@tmpl} >> ${@dest} -endef - -define build_arch - $(eval @cpu := $(value CPU_$(1))) - $(eval @deps := $(foreach tmpl,$(value TMPL_$(1)),${OUT_DIR}/$(1)/${tmpl}.go)) - $(eval @asmin := ${TMP_DIR}/$(1)/native.s) - $(eval @asmout := ${OUT_DIR}/$(1)/native_text_${@cpu}.go) - $(eval @stubin := ${OUT_DIR}/native_${@cpu}.tmpl) - $(eval @stubout := ${OUT_DIR}/$(1)/native_${@cpu}.go) - -$(1): ${@asmout} ${@deps} - -${@asmout}: ${@stubout} ${NATIVE_SRC} - mkdir -p ${TMP_DIR}/$(1) - $${CC_${@cpu}} $${CFLAGS} $${CFLAGS_$(1)} -S -o ${TMP_DIR}/$(1)/native.s ${SRC_FILE} - python3 $${ASM2ASM_${@cpu}} -r ${@stubout} ${TMP_DIR}/$(1)/native.s - -$(eval $(call \ - build_tmpl, \ - $(1), \ - ${@stubin}, \ - ${@stubout} \ -)) - -$(foreach \ - tmpl, \ - $(value TMPL_$(1)), \ - $(eval $(call \ - build_tmpl, \ - $(1), \ - ${OUT_DIR}/${tmpl}.tmpl, \ - ${OUT_DIR}/$(1)/${tmpl}.go \ - )) \ -) -endef - -all: ${ARCH} - -clean: - for arch in ${ARCH}; do \ - rm -vfr ${TMP_DIR}/$${arch}; \ - rm -vfr ${OUT_DIR}/$${arch}; \ - done - -$(foreach \ - arch, \ - ${ARCH}, \ - $(eval $(call build_arch,${arch})) \ -) diff --git a/vendor/github.com/bytedance/sonic/README.md b/vendor/github.com/bytedance/sonic/README.md index 42772018c1..5f609b1c34 100644 --- a/vendor/github.com/bytedance/sonic/README.md +++ b/vendor/github.com/bytedance/sonic/README.md @@ -5,18 +5,27 @@ English | [中文](README_ZH_CN.md) A blazingly fast JSON serializing & deserializing library, accelerated by JIT (just-in-time compiling) and SIMD (single-instruction-multiple-data). ## Requirement -- Go 1.16~1.21 -- Linux / MacOS / Windows(need go1.17 above) -- Amd64 ARCH + +- Go: 1.17~1.23 +- OS: Linux / MacOS / Windows +- CPU: AMD64 / ARM64(need go1.20 above) ## Features + - Runtime object binding without code generation - Complete APIs for JSON value manipulation - Fast, fast, fast! +## APIs + +see [go.dev](https://pkg.go.dev/github.com/bytedance/sonic) + ## Benchmarks + For **all sizes** of json and **all scenarios** of usage, **Sonic performs best**. + - [Medium](https://github.com/bytedance/sonic/blob/main/decoder/testdata_test.go#L19) (13KB, 300+ key, 6 layers) + ```powershell goversion: 1.17.1 goos: darwin @@ -81,14 +90,16 @@ BenchmarkLoadNode_Parallel/LoadAll()-16 5493 ns/op 2370.6 BenchmarkLoadNode/Interface()-16 17722 ns/op 734.85 MB/s 13323 B/op 88 allocs/op BenchmarkLoadNode_Parallel/Interface()-16 10330 ns/op 1260.70 MB/s 15178 B/op 88 allocs/op ``` + - [Small](https://github.com/bytedance/sonic/blob/main/testdata/small.go) (400B, 11 keys, 3 layers) ![small benchmarks](./docs/imgs/bench-small.png) - [Large](https://github.com/bytedance/sonic/blob/main/testdata/twitter.json) (635KB, 10000+ key, 6 layers) ![large benchmarks](./docs/imgs/bench-large.png) -See [bench.sh](https://github.com/bytedance/sonic/blob/main/bench.sh) for benchmark codes. +See [bench.sh](https://github.com/bytedance/sonic/blob/main/scripts/bench.sh) for benchmark codes. ## How it works + See [INTRODUCTION.md](./docs/INTRODUCTION.md). ## Usage @@ -96,6 +107,7 @@ See [INTRODUCTION.md](./docs/INTRODUCTION.md). ### Marshal/Unmarshal Default behaviors are mostly consistent with `encoding/json`, except HTML escaping form (see [Escape HTML](https://github.com/bytedance/sonic/blob/main/README.md#escape-html)) and `SortKeys` feature (optional support see [Sort Keys](https://github.com/bytedance/sonic/blob/main/README.md#sort-keys)) that is **NOT** in conformity to [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259). + ```go import "github.com/bytedance/sonic" @@ -107,8 +119,11 @@ err := sonic.Unmarshal(output, &data) ``` ### Streaming IO -Sonic supports decoding json from `io.Reader` or encoding objects into `io.`Writer`, aims at handling multiple values as well as reducing memory consumption. + +Sonic supports decoding json from `io.Reader` or encoding objects into `io.Writer`, aims at handling multiple values as well as reducing memory consumption. + - encoder + ```go var o1 = map[string]interface{}{ "a": "b", @@ -123,7 +138,9 @@ fmt.Println(w.String()) // {"a":"b"} // 1 ``` + - decoder + ```go var o = map[string]interface{}{} var r = strings.NewReader(`{"a":"b"}{"1":"2"}`) @@ -136,6 +153,7 @@ fmt.Printf("%+v", o) ``` ### Use Number/Use Int64 + ```go import "github.com/bytedance/sonic/decoder" @@ -164,7 +182,9 @@ fm := root.Interface().(float64) // jn == jm ``` ### Sort Keys + On account of the performance loss from sorting (roughly 10%), sonic doesn't enable this feature by default. If your component depends on it to work (like [zstd](https://github.com/facebook/zstd)), Use it like this: + ```go import "github.com/bytedance/sonic" import "github.com/bytedance/sonic/encoder" @@ -177,19 +197,26 @@ v, err := encoder.Encode(m, encoder.SortMapKeys) var root := sonic.Get(JSON) err := root.SortKeys() ``` + ### Escape HTML + On account of the performance loss (roughly 15%), sonic doesn't enable this feature by default. You can use `encoder.EscapeHTML` option to open this feature (align with `encoding/json.HTMLEscape`). + ```go import "github.com/bytedance/sonic" v := map[string]string{"&&":"<>"} ret, err := Encode(v, EscapeHTML) // ret == `{"\u0026\u0026":{"X":"\u003c\u003e"}}` ``` + ### Compact Format + Sonic encodes primitive objects (struct/map...) as compact-format JSON by default, except marshaling `json.RawMessage` or `json.Marshaler`: sonic ensures validating their output JSON but **DONOT** compacting them for performance concerns. We provide the option `encoder.CompactMarshaler` to add compacting process. ### Print Error + If there invalid syntax in input JSON, sonic will return `decoder.SyntaxError`, which supports pretty-printing of error position + ```go import "github.com/bytedance/sonic" import "github.com/bytedance/sonic/decoder" @@ -215,7 +242,9 @@ if err != nil { ``` #### Mismatched Types [Sonic v1.6.0] + If there a **mismatch-typed** value for a given key, sonic will report `decoder.MismatchTypeError` (if there are many, report the last one), but still skip wrong the value and keep decoding next JSON. + ```go import "github.com/bytedance/sonic" import "github.com/bytedance/sonic/decoder" @@ -228,10 +257,15 @@ err := UnmarshalString(`{"A":"1","B":1}`, &data) println(err.Error()) // Mismatch type int with value string "at index 5: mismatched type with value\n\n\t{\"A\":\"1\",\"B\":1}\n\t.....^.........\n" fmt.Printf("%+v", data) // {A:0 B:1} ``` + ### Ast.Node + Sonic/ast.Node is a completely self-contained AST for JSON. It implements serialization and deserialization both and provides robust APIs for obtaining and modification of generic data. + #### Get/Index + Search partial JSON by given paths, which must be non-negative integer or string, or nil + ```go import "github.com/bytedance/sonic" @@ -245,10 +279,29 @@ raw := root.Raw() // == string(input) root, err := sonic.Get(input, "key1", 1, "key2") sub := root.Get("key3").Index(2).Int64() // == 3 ``` + **Tip**: since `Index()` uses offset to locate data, which is much faster than scanning like `Get()`, we suggest you use it as much as possible. And sonic also provides another API `IndexOrGet()` to underlying use offset as well as ensure the key is matched. +#### SearchOption + +`Searcher` provides some options for user to meet different needs: + +```go +opts := ast.SearchOption{ CopyReturn: true ... } +val, err := sonic.GetWithOptions(JSON, opts, "key") +``` + +- CopyReturn +Indicate the searcher to copy the result JSON string instead of refer from the input. This can help to reduce memory usage if you cache the results +- ConcurentRead +Since `ast.Node` use `Lazy-Load` design, it doesn't support Concurrently-Read by default. If you want to read it concurrently, please specify it. +- ValidateJSON +Indicate the searcher to validate the entire JSON. This option is enabled by default, which slow down the search speed a little. + #### Set/Unset + Modify the json content by Set()/Unset() + ```go import "github.com/bytedance/sonic" @@ -265,7 +318,9 @@ println(root.Get("key4").Check()) // "value not exist" ``` #### Serialize + To encode `ast.Node` as json, use `MarshalJson()` or `json.Marshal()` (MUST pass the node's pointer) + ```go import ( "encoding/json" @@ -279,6 +334,7 @@ println(string(buf) == string(exp)) // true ``` #### APIs + - validation: `Check()`, `Error()`, `Valid()`, `Exist()` - searching: `Index()`, `Get()`, `IndexPair()`, `IndexOrGet()`, `GetByPath()` - go-type casting: `Int64()`, `Float64()`, `String()`, `Number()`, `Bool()`, `Map[UseNumber|UseNode]()`, `Array[UseNumber|UseNode]()`, `Interface[UseNumber|UseNode]()` @@ -287,7 +343,9 @@ println(string(buf) == string(exp)) // true - modification: `Set()`, `SetByIndex()`, `Add()` ### Ast.Visitor + Sonic provides an advanced API for fully parsing JSON into non-standard types (neither `struct` not `map[string]interface{}`) without using any intermediate representation (`ast.Node` or `interface{}`). For example, you might have the following types which are like `interface{}` but actually not `interface{}`: + ```go type UserNode interface {} @@ -302,7 +360,9 @@ type ( UserArray struct{ Value []UserNode } ) ``` + Sonic provides the following API to return **the preorder traversal of a JSON AST**. The `ast.Visitor` is a SAX style interface which is used in some C++ JSON library. You should implement `ast.Visitor` by yourself and pass it to `ast.Preorder()` method. In your visitor you can make your custom types to represent JSON values. There may be an O(n) space container (such as stack) in your visitor to record the object / array hierarchy. + ```go func Preorder(str string, visitor Visitor, opts *VisitorOptions) error @@ -323,20 +383,20 @@ type Visitor interface { See [ast/visitor.go](https://github.com/bytedance/sonic/blob/main/ast/visitor.go) for detailed usage. We also implement a demo visitor for `UserNode` in [ast/visitor_test.go](https://github.com/bytedance/sonic/blob/main/ast/visitor_test.go). ## Compatibility -Sonic **DOES NOT** ensure to support all environments, due to the difficulty of developing high-performance codes. For developers who use sonic to build their applications in different environments, we have the following suggestions: -- Developing on **Mac M1**: Make sure you have Rosetta 2 installed on your machine, and set `GOARCH=amd64` when building your application. Rosetta 2 can automatically translate x86 binaries to arm64 binaries and run x86 applications on Mac M1. -- Developing on **Linux arm64**: You can install qemu and use the `qemu-x86_64 -cpu max` command to convert x86 binaries to amr64 binaries for applications built with sonic. The qemu can achieve a similar transfer effect to Rosetta 2 on Mac M1. +For developers who want to use sonic to meet diffirent scenarios, we provide some integrated configs as `sonic.API` -For developers who want to use sonic on Linux arm64 without qemu, or those who want to handle JSON strictly consistent with `encoding/json`, we provide some compatible APIs as `sonic.API` -- `ConfigDefault`: the sonic's default config (`EscapeHTML=false`,`SortKeys=false`...) to run on sonic-supporting environment. It will fall back to `encoding/json` with the corresponding config, and some options like `SortKeys=false` will be invalid. -- `ConfigStd`: the std-compatible config (`EscapeHTML=true`,`SortKeys=true`...) to run on sonic-supporting environment. It will fall back to `encoding/json`. -- `ConfigFastest`: the fastest config (`NoQuoteTextMarshaler=true`) to run on sonic-supporting environment. It will fall back to `encoding/json` with the corresponding config, and some options will be invalid. +- `ConfigDefault`: the sonic's default config (`EscapeHTML=false`,`SortKeys=false`...) to run sonic fast meanwhile ensure security. +- `ConfigStd`: the std-compatible config (`EscapeHTML=true`,`SortKeys=true`...) +- `ConfigFastest`: the fastest config (`NoQuoteTextMarshaler=true`) to run on sonic as fast as possible. +Sonic **DOES NOT** ensure to support all environments, due to the difficulty of developing high-performance codes. On non-sonic-supporting environment, the implementation will fall back to `encoding/json`. Thus beflow configs will all equal to `ConfigStd`. ## Tips ### Pretouch + Since Sonic uses [golang-asm](https://github.com/twitchyliquid64/golang-asm) as a JIT assembler, which is NOT very suitable for runtime compiling, first-hit running of a huge schema may cause request-timeout or even process-OOM. For better stability, we advise **using `Pretouch()` for huge-schema or compact-memory applications** before `Marshal()/Unmarshal()`. + ```go import ( "reflect" @@ -362,17 +422,23 @@ func init() { ``` ### Copy string -When decoding **string values without any escaped characters**, sonic references them from the origin JSON buffer instead of mallocing a new buffer to copy. This helps a lot for CPU performance but may leave the whole JSON buffer in memory as long as the decoded objects are being used. In practice, we found the extra memory introduced by referring JSON buffer is usually 20% ~ 80% of decoded objects. Once an application holds these objects for a long time (for example, cache the decoded objects for reusing), its in-use memory on the server may go up. We provide the option `decoder.CopyString()` for users to choose not to reference the JSON buffer, which may cause a decline in CPU performance to some degree. + +When decoding **string values without any escaped characters**, sonic references them from the origin JSON buffer instead of mallocing a new buffer to copy. This helps a lot for CPU performance but may leave the whole JSON buffer in memory as long as the decoded objects are being used. In practice, we found the extra memory introduced by referring JSON buffer is usually 20% ~ 80% of decoded objects. Once an application holds these objects for a long time (for example, cache the decoded objects for reusing), its in-use memory on the server may go up. - `Config.CopyString`/`decoder.CopyString()`: We provide the option for `Decode()` / `Unmarshal()` users to choose not to reference the JSON buffer, which may cause a decline in CPU performance to some degree. + +- `GetFromStringNoCopy()`: For memory safety, `sonic.Get()` / `sonic.GetFromString()` now copies return JSON. If users want to get json more quickly and not care about memory usage, you can use `GetFromStringNoCopy()` to return a JSON directly referenced from source. ### Pass string or []byte? + For alignment to `encoding/json`, we provide API to pass `[]byte` as an argument, but the string-to-bytes copy is conducted at the same time considering safety, which may lose performance when the origin JSON is huge. Therefore, you can use `UnmarshalString()` and `GetFromString()` to pass a string, as long as your origin data is a string or **nocopy-cast** is safe for your []byte. We also provide API `MarshalString()` for convenient **nocopy-cast** of encoded JSON []byte, which is safe since sonic's output bytes is always duplicated and unique. ### Accelerate `encoding.TextMarshaler` -To ensure data security, sonic.Encoder quotes and escapes string values from `encoding.TextMarshaler` interfaces by default, which may degrade performance much if most of your data is in form of them. We provide `encoder.NoQuoteTextMarshaler` to skip these operations, which means you **MUST** ensure their output string escaped and quoted following [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259). +To ensure data security, sonic.Encoder quotes and escapes string values from `encoding.TextMarshaler` interfaces by default, which may degrade performance much if most of your data is in form of them. We provide `encoder.NoQuoteTextMarshaler` to skip these operations, which means you **MUST** ensure their output string escaped and quoted following [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259). ### Better performance for generic data + In **fully-parsed** scenario, `Unmarshal()` performs better than `Get()`+`Node.Interface()`. But if you only have a part of the schema for specific json, you can combine `Get()` and `Unmarshal()` together: + ```go import "github.com/bytedance/sonic" @@ -380,7 +446,9 @@ node, err := sonic.GetFromString(_TwitterJson, "statuses", 3, "user") var user User // your partial schema... err = sonic.UnmarshalString(node.Raw(), &user) ``` + Even if you don't have any schema, use `ast.Node` as the container of generic values instead of `map` or `interface`: + ```go import "github.com/bytedance/sonic" @@ -391,7 +459,9 @@ err = user.Check() // err = user.LoadAll() // only call this when you want to use 'user' concurrently... go someFunc(user) ``` + Why? Because `ast.Node` stores its children using `array`: + - `Array`'s performance is **much better** than `Map` when Inserting (Deserialize) and Scanning (Serialize) data; - **Hashing** (`map[x]`) is not as efficient as **Indexing** (`array[x]`), which `ast.Node` can conduct on **both array and object**; - Using `Interface()`/`Map()` means Sonic must parse all the underlying values, while `ast.Node` can parse them **on demand**. @@ -399,6 +469,7 @@ Why? Because `ast.Node` stores its children using `array`: **CAUTION:** `ast.Node` **DOESN'T** ensure concurrent security directly, due to its **lazy-load** design. However, you can call `Node.Load()`/`Node.LoadAll()` to achieve that, which may bring performance reduction while it still works faster than converting to `map` or `interface{}` ### Ast.Node or Ast.Visitor? + For generic data, `ast.Node` should be enough for your needs in most cases. However, `ast.Node` is designed for partially processing JSON string. It has some special designs such as lazy-load which might not be suitable for directly parsing the whole JSON string like `Unmarshal()`. Although `ast.Node` is better then `map` or `interface{}`, it's also a kind of intermediate representation after all if your final types are customized and you have to convert the above types to your custom types after parsing. @@ -407,5 +478,18 @@ For better performance, in previous case the `ast.Visitor` will be the better ch But `ast.Visitor` is not a very handy API. You might need to write a lot of code to implement your visitor and carefully maintain the tree hierarchy during decoding. Please read the comments in [ast/visitor.go](https://github.com/bytedance/sonic/blob/main/ast/visitor.go) carefully if you decide to use this API. +### Buffer Size + +Sonic use memory pool in many places like `encoder.Encode`, `ast.Node.MarshalJSON` to improve performace, which may produce more memory usage (in-use) when server's load is high. See [issue 614](https://github.com/bytedance/sonic/issues/614). Therefore, we introduce some options to let user control the behavior of memory pool. See [option](https://pkg.go.dev/github.com/bytedance/sonic@v1.11.9/option#pkg-variables) package. + +### Faster JSON skip + +For security, sonic use [FSM](native/skip_one.c) algorithm to validate JSON when decoding raw JSON or encoding `json.Marshaler`, which is much slower (1~10x) than [SIMD-searching-pair](native/skip_one_fast.c) algorithm. If user has many redundant JSON value and DO NOT NEED to strictly validate JSON correctness, you can enable below options: + +- `Config.NoValidateSkipJSON`: for faster skipping JSON when decoding, such as unknown fields, json.Unmarshaler(json.RawMessage), mismatched values, and redundant array elements +- `Config.NoValidateJSONMarshaler`: avoid validating JSON when encoding `json.Marshaler` +- `SearchOption.ValidateJSON`: indicates if validate located JSON value when `Get` + ## Community + Sonic is a subproject of [CloudWeGo](https://www.cloudwego.io/). We are committed to building a cloud native ecosystem. diff --git a/vendor/github.com/bytedance/sonic/README_ZH_CN.md b/vendor/github.com/bytedance/sonic/README_ZH_CN.md index dc5cfb5df2..4f8980c53f 100644 --- a/vendor/github.com/bytedance/sonic/README_ZH_CN.md +++ b/vendor/github.com/bytedance/sonic/README_ZH_CN.md @@ -6,9 +6,13 @@ ## 依赖 -- Go 1.16~1.20 -- Linux / MacOS / Windows(需要 Go1.17 以上) -- Amd64 架构 +- Go: 1.17~1.23 +- OS: Linux / MacOS / Windows +- CPU: AMD64 / ARM64(需要 Go1.20 以上) + +## 接口 + +详见 [go.dev](https://pkg.go.dev/github.com/bytedance/sonic) ## 特色 @@ -19,7 +23,9 @@ ## 基准测试 对于**所有大小**的 json 和**所有使用场景**, **Sonic 表现均为最佳**。 + - [中型](https://github.com/bytedance/sonic/blob/main/decoder/testdata_test.go#L19) (13kB, 300+ 键, 6 层) + ```powershell goversion: 1.17.1 goos: darwin @@ -84,12 +90,13 @@ BenchmarkLoadNode_Parallel/LoadAll()-16 5493 ns/op 2370.6 BenchmarkLoadNode/Interface()-16 17722 ns/op 734.85 MB/s 13323 B/op 88 allocs/op BenchmarkLoadNode_Parallel/Interface()-16 10330 ns/op 1260.70 MB/s 15178 B/op 88 allocs/op ``` + - [小型](https://github.com/bytedance/sonic/blob/main/testdata/small.go) (400B, 11 个键, 3 层) ![small benchmarks](./docs/imgs/bench-small.png) - [大型](https://github.com/bytedance/sonic/blob/main/testdata/twitter.json) (635kB, 10000+ 个键, 6 层) ![large benchmarks](./docs/imgs/bench-large.png) -要查看基准测试代码,请参阅 [bench.sh](https://github.com/bytedance/sonic/blob/main/bench.sh) 。 +要查看基准测试代码,请参阅 [bench.sh](https://github.com/bytedance/sonic/blob/main/scripts/bench.sh) 。 ## 工作原理 @@ -100,6 +107,7 @@ BenchmarkLoadNode_Parallel/Interface()-16 10330 ns/op 1260.7 ### 序列化/反序列化 默认的行为基本上与 `encoding/json` 相一致,除了 HTML 转义形式(参见 [Escape HTML](https://github.com/bytedance/sonic/blob/main/README.md#escape-html)) 和 `SortKeys` 功能(参见 [Sort Keys](https://github.com/bytedance/sonic/blob/main/README.md#sort-keys))**没有**遵循 [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259) 。 + ```go import "github.com/bytedance/sonic" @@ -113,7 +121,9 @@ err := sonic.Unmarshal(output, &data) ### 流式输入输出 Sonic 支持解码 `io.Reader` 中输入的 json,或将对象编码为 json 后输出至 `io.Writer`,以处理多个值并减少内存消耗。 + - 编码器 + ```go var o1 = map[string]interface{}{ "a": "b", @@ -128,7 +138,9 @@ fmt.Println(w.String()) // {"a":"b"} // 1 ``` + - 解码器 + ```go var o = map[string]interface{}{} var r = strings.NewReader(`{"a":"b"}{"1":"2"}`) @@ -172,6 +184,7 @@ fm := root.Interface().(float64) // jn == jm ### 对键排序 考虑到排序带来的性能损失(约 10% ), sonic 默认不会启用这个功能。如果你的组件依赖这个行为(如 [zstd](https://github.com/facebook/zstd)) ,可以仿照下面的例子: + ```go import "github.com/bytedance/sonic" import "github.com/bytedance/sonic/encoder" @@ -188,6 +201,7 @@ err := root.SortKeys() ### HTML 转义 考虑到性能损失(约15%), sonic 默认不会启用这个功能。你可以使用 `encoder.EscapeHTML` 选项来开启(与 `encoding/json.HTMLEscape` 行为一致)。 + ```go import "github.com/bytedance/sonic" @@ -196,11 +210,13 @@ ret, err := Encode(v, EscapeHTML) // ret == `{"\u0026\u0026":{"X":"\u003c\u003e" ``` ### 紧凑格式 + Sonic 默认将基本类型( `struct` , `map` 等)编码为紧凑格式的 JSON ,除非使用 `json.RawMessage` or `json.Marshaler` 进行编码: sonic 确保输出的 JSON 合法,但出于性能考虑,**不会**加工成紧凑格式。我们提供选项 `encoder.CompactMarshaler` 来添加此过程, ### 打印错误 如果输入的 JSON 存在无效的语法,sonic 将返回 `decoder.SyntaxError`,该错误支持错误位置的美化输出。 + ```go import "github.com/bytedance/sonic" import "github.com/bytedance/sonic/decoder" @@ -228,6 +244,7 @@ if err != nil { #### 类型不匹配 [Sonic v1.6.0] 如果给定键中存在**类型不匹配**的值, sonic 会抛出 `decoder.MismatchTypeError` (如果有多个,只会报告最后一个),但仍会跳过错误的值并解码下一个 JSON 。 + ```go import "github.com/bytedance/sonic" import "github.com/bytedance/sonic/decoder" @@ -240,13 +257,15 @@ err := UnmarshalString(`{"A":"1","B":1}`, &data) println(err.Error()) // Mismatch type int with value string "at index 5: mismatched type with value\n\n\t{\"A\":\"1\",\"B\":1}\n\t.....^.........\n" fmt.Printf("%+v", data) // {A:0 B:1} ``` + ### `Ast.Node` -Sonic/ast.Node 是完全独立的 JSON 抽象语法树库。它实现了序列化和反序列化,并提供了获取和修改通用数据的鲁棒的 API。 +Sonic/ast.Node 是完全独立的 JSON 抽象语法树库。它实现了序列化和反序列化,并提供了获取和修改JSON数据的鲁棒的 API。 #### 查找/索引 通过给定的路径搜索 JSON 片段,路径必须为非负整数,字符串或 `nil` 。 + ```go import "github.com/bytedance/sonic" @@ -260,11 +279,29 @@ raw := root.Raw() // == string(input) root, err := sonic.Get(input, "key1", 1, "key2") sub := root.Get("key3").Index(2).Int64() // == 3 ``` + **注意**:由于 `Index()` 使用偏移量来定位数据,比使用扫描的 `Get()` 要快的多,建议尽可能的使用 `Index` 。 Sonic 也提供了另一个 API, `IndexOrGet()` ,以偏移量为基础并且也确保键的匹配。 +#### 查找选项 + +`ast.Searcher`提供了一些选项,以满足用户的不同需求: + +``` +opts:= ast.SearchOption{CopyReturn: true…} +Val, err:= sonic。gettwithoptions (JSON, opts, "key") +``` + +- CopyReturn +指示搜索器复制结果JSON字符串,而不是从输入引用。如果用户缓存结果,这有助于减少内存使用 +- ConcurentRead +因为`ast.Node`使用`Lazy-Load`设计,默认不支持并发读取。如果您想同时读取,请指定它。 +- ValidateJSON +指示搜索器来验证整个JSON。默认情况下启用该选项, 但是对于查找速度有一定影响。 + #### 修改 -使用 ` Set()` / `Unset()` 修改 json 的内容 +使用 `Set()` / `Unset()` 修改 json 的内容 + ```go import "github.com/bytedance/sonic" @@ -281,7 +318,9 @@ println(root.Get("key4").Check()) // "value not exist" ``` #### 序列化 + 要将 `ast.Node` 编码为 json ,使用 `MarshalJson()` 或者 `json.Marshal()` (必须传递指向节点的指针) + ```go import ( "encoding/json" @@ -295,6 +334,7 @@ println(string(buf) == string(exp)) // true ``` #### APIs + - 合法性检查: `Check()`, `Error()`, `Valid()`, `Exist()` - 索引: `Index()`, `Get()`, `IndexPair()`, `IndexOrGet()`, `GetByPath()` - 转换至 go 内置类型: `Int64()`, `Float64()`, `String()`, `Number()`, `Bool()`, `Map[UseNumber|UseNode]()`, `Array[UseNumber|UseNode]()`, `Interface[UseNumber|UseNode]()` @@ -303,7 +343,9 @@ println(string(buf) == string(exp)) // true - 修改: `Set()`, `SetByIndex()`, `Add()` ### `Ast.Visitor` + Sonic 提供了一个高级的 API 用于直接全量解析 JSON 到非标准容器里 (既不是 `struct` 也不是 `map[string]interface{}`) 且不需要借助任何中间表示 (`ast.Node` 或 `interface{}`)。举个例子,你可能定义了下述的类型,它们看起来像 `interface{}`,但实际上并不是: + ```go type UserNode interface {} @@ -318,7 +360,9 @@ type ( UserArray struct{ Value []UserNode } ) ``` + Sonic 提供了下述的 API 来返回 **“对 JSON AST 的前序遍历”**。`ast.Visitor` 是一个 SAX 风格的接口,这在某些 C++ 的 JSON 解析库中被使用到。你需要自己实现一个 `ast.Visitor`,将它传递给 `ast.Preorder()` 方法。在你的实现中你可以使用自定义的类型来表示 JSON 的值。在你的 `ast.Visitor` 中,可能需要有一个 O(n) 空间复杂度的容器(比如说栈)来记录 object / array 的层级。 + ```go func Preorder(str string, visitor Visitor, opts *VisitorOptions) error @@ -335,23 +379,24 @@ type Visitor interface { OnArrayEnd() error } ``` + 详细用法参看 [ast/visitor.go](https://github.com/bytedance/sonic/blob/main/ast/visitor.go),我们还为 `UserNode` 实现了一个示例 `ast.Visitor`,你可以在 [ast/visitor_test.go](https://github.com/bytedance/sonic/blob/main/ast/visitor_test.go) 中找到它。 ## 兼容性 -由于开发高性能代码的困难性, Sonic **不**保证对所有环境的支持。对于在不同环境中使用 Sonic 构建应用程序的开发者,我们有以下建议: -- 在 **Mac M1** 上开发:确保在您的计算机上安装了 Rosetta 2,并在构建时设置 `GOARCH=amd64` 。 Rosetta 2 可以自动将 x86 二进制文件转换为 arm64 二进制文件,并在 Mac M1 上运行 x86 应用程序。 -- 在 **Linux arm64** 上开发:您可以安装 qemu 并使用 `qemu-x86_64 -cpu max` 命令来将 x86 二进制文件转换为 arm64 二进制文件。qemu可以实现与Mac M1上的Rosetta 2类似的转换效果。 +对于想要使用sonic来满足不同场景的开发人员,我们提供了一些集成配置: -对于希望在不使用 qemu 下使用 sonic 的开发者,或者希望处理 JSON 时与 `encoding/JSON` 严格保持一致的开发者,我们在 `sonic.API` 中提供了一些兼容性 API -- `ConfigDefault`: 在支持 sonic 的环境下 sonic 的默认配置(`EscapeHTML=false`,`SortKeys=false`等)。行为与具有相应配置的 `encoding/json` 一致,一些选项,如 `SortKeys=false` 将无效。 -- `ConfigStd`: 在支持 sonic 的环境下与标准库兼容的配置(`EscapeHTML=true`,`SortKeys=true`等)。行为与 `encoding/json` 一致。 -- `ConfigFastest`: 在支持 sonic 的环境下运行最快的配置(`NoQuoteTextMarshaler=true`)。行为与具有相应配置的 `encoding/json` 一致,某些选项将无效。 +- `ConfigDefault`: sonic的默认配置 (`EscapeHTML=false`, `SortKeys=false`…) 保证性能同时兼顾安全性。 +- `ConfigStd`: 与 `encoding/json` 保证完全兼容的配置 +- `ConfigFastest`: 最快的配置(`NoQuoteTextMarshaler=true...`) 保证性能最优但是会缺少一些安全性检查(validate UTF8 等) +Sonic **不**确保支持所有环境,由于开发高性能代码的困难。在不支持声音的环境中,实现将回落到 `encoding/json`。因此上述配置将全部等于`ConfigStd`。 ## 注意事项 ### 预热 + 由于 Sonic 使用 [golang-asm](https://github.com/twitchyliquid64/golang-asm) 作为 JIT 汇编器,这个库并不适用于运行时编译,第一次运行一个大型模式可能会导致请求超时甚至进程内存溢出。为了更好地稳定性,我们建议在运行大型模式或在内存有限的应用中,在使用 `Marshal()/Unmarshal()` 前运行 `Pretouch()`。 + ```go import ( "reflect" @@ -381,16 +426,17 @@ func init() { 当解码 **没有转义字符的字符串**时, sonic 会从原始的 JSON 缓冲区内引用而不是复制到新的一个缓冲区中。这对 CPU 的性能方面很有帮助,但是可能因此在解码后对象仍在使用的时候将整个 JSON 缓冲区保留在内存中。实践中我们发现,通过引用 JSON 缓冲区引入的额外内存通常是解码后对象的 20% 至 80% ,一旦应用长期保留这些对象(如缓存以备重用),服务器所使用的内存可能会增加。我们提供了选项 `decoder.CopyString()` 供用户选择,不引用 JSON 缓冲区。这可能在一定程度上降低 CPU 性能。 ### 传递字符串还是字节数组? + 为了和 `encoding/json` 保持一致,我们提供了传递 `[]byte` 作为参数的 API ,但考虑到安全性,字符串到字节的复制是同时进行的,这在原始 JSON 非常大时可能会导致性能损失。因此,你可以使用 `UnmarshalString()` 和 `GetFromString()` 来传递字符串,只要你的原始数据是字符串,或**零拷贝类型转换**对于你的字节数组是安全的。我们也提供了 `MarshalString()` 的 API ,以便对编码的 JSON 字节数组进行**零拷贝类型转换**,因为 sonic 输出的字节始终是重复并且唯一的,所以这样是安全的。 ### 加速 `encoding.TextMarshaler` 为了保证数据安全性, `sonic.Encoder` 默认会对来自 `encoding.TextMarshaler` 接口的字符串进行引用和转义,如果大部分数据都是这种形式那可能会导致很大的性能损失。我们提供了 `encoder.NoQuoteTextMarshaler` 选项来跳过这些操作,但你**必须**保证他们的输出字符串依照 [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259) 进行了转义和引用。 - ### 泛型的性能优化 在 **完全解析**的场景下, `Unmarshal()` 表现得比 `Get()`+`Node.Interface()` 更好。但是如果你只有特定 JSON 的部分模式,你可以将 `Get()` 和 `Unmarshal()` 结合使用: + ```go import "github.com/bytedance/sonic" @@ -398,7 +444,9 @@ node, err := sonic.GetFromString(_TwitterJson, "statuses", 3, "user") var user User // your partial schema... err = sonic.UnmarshalString(node.Raw(), &user) ``` + 甚至如果你没有任何模式,可以用 `ast.Node` 代替 `map` 或 `interface` 作为泛型的容器: + ```go import "github.com/bytedance/sonic" @@ -409,7 +457,9 @@ err = user.Check() // err = user.LoadAll() // only call this when you want to use 'user' concurrently... go someFunc(user) ``` + 为什么?因为 `ast.Node` 使用 `array` 来存储其子节点: + - 在插入(反序列化)和扫描(序列化)数据时,`Array` 的性能比 `Map` **好得多**; - **哈希**(`map[x]`)的效率不如**索引**(`array[x]`)高效,而 `ast.Node` 可以在数组和对象上使用索引; - 使用 `Interface()` / `Map()` 意味着 sonic 必须解析所有的底层值,而 `ast.Node` 可以**按需解析**它们。 @@ -417,6 +467,7 @@ go someFunc(user) **注意**:由于 `ast.Node` 的惰性加载设计,其**不能**直接保证并发安全性,但你可以调用 `Node.Load()` / `Node.LoadAll()` 来实现并发安全。尽管可能会带来性能损失,但仍比转换成 `map` 或 `interface{}` 更为高效。 ### 使用 `ast.Node` 还是 `ast.Visitor`? + 对于泛型数据的解析,`ast.Node` 在大多数场景上应该能够满足你的需求。 然而,`ast.Node` 是一种针对部分解析 JSON 而设计的泛型容器,它包含一些特殊设计,比如惰性加载,如果你希望像 `Unmarshal()` 那样直接解析整个 JSON,这些设计可能并不合适。尽管 `ast.Node` 相较于 `map` 或 `interface{}` 来说是更好的一种泛型容器,但它毕竟也是一种中间表示,如果你的最终类型是自定义的,你还得在解析完成后将上述类型转化成你自定义的类型。 @@ -425,6 +476,18 @@ go someFunc(user) 但是,`ast.Visitor` 并不是一个很易用的 API。你可能需要写大量的代码去实现自己的 `ast.Visitor`,并且需要在解析过程中仔细维护树的层级。如果你决定要使用这个 API,请先仔细阅读 [ast/visitor.go](https://github.com/bytedance/sonic/blob/main/ast/visitor.go) 中的注释。 +### 缓冲区大小 + +Sonic在许多地方使用内存池,如`encoder.Encode`, `ast.Node.MarshalJSON`等来提高性能,这可能会在服务器负载高时产生更多的内存使用(in-use)。参见[issue 614](https://github.com/bytedance/sonic/issues/614)。因此,我们引入了一些选项来让用户配置内存池的行为。参见[option](https://pkg.go.dev/github.com/bytedance/sonic@v1.11.9/option#pkg-variables)包。 + +### 更快的 JSON Skip + +为了安全起见,在跳过原始JSON 时,sonic decoder 默认使用[FSM](native/skip_one.c)算法扫描来跳过同时校验 JSON。它相比[SIMD-searching-pair](native/skip_one_fast.c)算法跳过要慢得多(1~10倍)。如果用户有很多冗余的JSON值,并且不需要严格验证JSON的正确性,你可以启用以下选项: + +- `Config.NoValidateSkipJSON`: 用于在解码时更快地跳过JSON,例如未知字段,`json.RawMessage`,不匹配的值和冗余的数组元素等 +- `Config.NoValidateJSONMarshaler`: 编码JSON时避免验证JSON。封送拆收器 +- `SearchOption.ValidateJSON`: 指示当`Get`时是否验证定位的JSON值 + ## 社区 Sonic 是 [CloudWeGo](https://www.cloudwego.io/) 下的一个子项目。我们致力于构建云原生生态系统。 diff --git a/vendor/github.com/bytedance/sonic/api.go b/vendor/github.com/bytedance/sonic/api.go index fa738f21d2..af6be70a42 100644 --- a/vendor/github.com/bytedance/sonic/api.go +++ b/vendor/github.com/bytedance/sonic/api.go @@ -20,8 +20,19 @@ import ( `io` `github.com/bytedance/sonic/ast` + `github.com/bytedance/sonic/internal/rt` ) +const ( + // UseStdJSON indicates you are using fallback implementation (encoding/json) + UseStdJSON = iota + // UseSonicJSON indicates you are using real sonic implementation + UseSonicJSON +) + +// APIKind is the kind of API, 0 is std json, 1 is sonic. +const APIKind = apiKind + // Config is a combination of sonic/encoder.Options and sonic/decoder.Options type Config struct { // EscapeHTML indicates encoder to escape all HTML characters @@ -68,11 +79,25 @@ type Config struct { // ValidateString indicates decoder and encoder to valid string values: decoder will return errors // when unescaped control chars(\u0000-\u001f) in the string value of JSON. - ValidateString bool + ValidateString bool + + // NoValidateJSONMarshaler indicates that the encoder should not validate the output string + // after encoding the JSONMarshaler to JSON. + NoValidateJSONMarshaler bool + + // NoValidateJSONSkip indicates the decoder should not validate the JSON value when skipping it, + // such as unknown-fields, mismatched-type, redundant elements.. + NoValidateJSONSkip bool + + // NoEncoderNewline indicates that the encoder should not add a newline after every message + NoEncoderNewline bool + + // Encode Infinity or Nan float into `null`, instead of returning an error. + EncodeNullForInfOrNan bool } var ( - // ConfigDefault is the default config of APIs, aiming at efficiency and safty. + // ConfigDefault is the default config of APIs, aiming at efficiency and safety. ConfigDefault = Config{}.Froze() // ConfigStd is the standard config of APIs, aiming at being compatible with encoding/json. @@ -87,6 +112,8 @@ var ( // ConfigFastest is the fastest config of APIs, aiming at speed. ConfigFastest = Config{ NoQuoteTextMarshaler: true, + NoValidateJSONMarshaler: true, + NoValidateJSONSkip: true, }.Froze() ) @@ -109,7 +136,7 @@ type API interface { NewEncoder(writer io.Writer) Encoder // NewDecoder create a Decoder holding reader NewDecoder(reader io.Reader) Decoder - // Valid validates the JSON-encoded bytes and reportes if it is valid + // Valid validates the JSON-encoded bytes and reports if it is valid Valid(data []byte) bool } @@ -148,6 +175,13 @@ func Marshal(val interface{}) ([]byte, error) { return ConfigDefault.Marshal(val) } +// MarshalIndent is like Marshal but applies Indent to format the output. +// Each JSON element in the output will begin on a new line beginning with prefix +// followed by one or more copies of indent according to the indentation nesting. +func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { + return ConfigDefault.MarshalIndent(v, prefix, indent) +} + // MarshalString returns the JSON encoding string of v. func MarshalString(val interface{}) (string, error) { return ConfigDefault.MarshalToString(val) @@ -165,27 +199,49 @@ func UnmarshalString(buf string, val interface{}) error { return ConfigDefault.UnmarshalFromString(buf, val) } -// Get searches the given path from json, -// and returns its representing ast.Node. +// Get searches and locates the given path from src json, +// and returns a ast.Node representing the partially json. // // Each path arg must be integer or string: // - Integer is target index(>=0), means searching current node as array. // - String is target key, means searching current node as object. // // -// Note, the api expects the json is well-formed at least, -// otherwise it may return unexpected result. +// Notice: It expects the src json is **Well-formed** and **Immutable** when calling, +// otherwise it may return unexpected result. +// Considering memory safety, the returned JSON is **Copied** from the input func Get(src []byte, path ...interface{}) (ast.Node, error) { - return GetFromString(string(src), path...) + return GetCopyFromString(rt.Mem2Str(src), path...) } -// GetFromString is same with Get except src is string, -// which can reduce unnecessary memory copy. +//GetWithOptions searches and locates the given path from src json, +// with specific options of ast.Searcher +func GetWithOptions(src []byte, opts ast.SearchOptions, path ...interface{}) (ast.Node, error) { + s := ast.NewSearcher(rt.Mem2Str(src)) + s.SearchOptions = opts + return s.GetByPath(path...) +} + +// GetFromString is same with Get except src is string. +// +// WARNING: The returned JSON is **Referenced** from the input. +// Caching or long-time holding the returned node may cause OOM. +// If your src is big, consider use GetFromStringCopy(). func GetFromString(src string, path ...interface{}) (ast.Node, error) { return ast.NewSearcher(src).GetByPath(path...) } +// GetCopyFromString is same with Get except src is string +func GetCopyFromString(src string, path ...interface{}) (ast.Node, error) { + return ast.NewSearcher(src).GetByPathCopy(path...) +} + // Valid reports whether data is a valid JSON encoding. func Valid(data []byte) bool { return ConfigDefault.Valid(data) } + +// Valid reports whether data is a valid JSON encoding. +func ValidString(data string) bool { + return ConfigDefault.Valid(rt.Str2Mem(data)) +} diff --git a/vendor/github.com/bytedance/sonic/ast/api.go b/vendor/github.com/bytedance/sonic/ast/api.go new file mode 100644 index 0000000000..7c8253aa11 --- /dev/null +++ b/vendor/github.com/bytedance/sonic/ast/api.go @@ -0,0 +1,135 @@ +//go:build (amd64 && go1.17 && !go1.24) || (arm64 && go1.20 && !go1.24) +// +build amd64,go1.17,!go1.24 arm64,go1.20,!go1.24 + +/* + * Copyright 2022 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ast + +import ( + `runtime` + `unsafe` + + `github.com/bytedance/sonic/encoder` + `github.com/bytedance/sonic/internal/native` + `github.com/bytedance/sonic/internal/native/types` + `github.com/bytedance/sonic/internal/rt` + uq `github.com/bytedance/sonic/unquote` + `github.com/bytedance/sonic/utf8` +) + +var typeByte = rt.UnpackEface(byte(0)).Type + +//go:nocheckptr +func quote(buf *[]byte, val string) { + *buf = append(*buf, '"') + if len(val) == 0 { + *buf = append(*buf, '"') + return + } + + sp := rt.IndexChar(val, 0) + nb := len(val) + b := (*rt.GoSlice)(unsafe.Pointer(buf)) + + // input buffer + for nb > 0 { + // output buffer + dp := unsafe.Pointer(uintptr(b.Ptr) + uintptr(b.Len)) + dn := b.Cap - b.Len + // call native.Quote, dn is byte count it outputs + ret := native.Quote(sp, nb, dp, &dn, 0) + // update *buf length + b.Len += dn + + // no need more output + if ret >= 0 { + break + } + + // double buf size + *b = rt.GrowSlice(typeByte, *b, b.Cap*2) + // ret is the complement of consumed input + ret = ^ret + // update input buffer + nb -= ret + sp = unsafe.Pointer(uintptr(sp) + uintptr(ret)) + } + + runtime.KeepAlive(buf) + runtime.KeepAlive(sp) + *buf = append(*buf, '"') +} + +func unquote(src string) (string, types.ParsingError) { + return uq.String(src) +} + +func (self *Parser) decodeValue() (val types.JsonState) { + sv := (*rt.GoString)(unsafe.Pointer(&self.s)) + flag := types.F_USE_NUMBER + if self.dbuf != nil { + flag = 0 + val.Dbuf = self.dbuf + val.Dcap = types.MaxDigitNums + } + self.p = native.Value(sv.Ptr, sv.Len, self.p, &val, uint64(flag)) + return +} + +func (self *Parser) skip() (int, types.ParsingError) { + fsm := types.NewStateMachine() + start := native.SkipOne(&self.s, &self.p, fsm, 0) + types.FreeStateMachine(fsm) + + if start < 0 { + return self.p, types.ParsingError(-start) + } + return start, 0 +} + +func (self *Node) encodeInterface(buf *[]byte) error { + //WARN: NOT compatible with json.Encoder + return encoder.EncodeInto(buf, self.packAny(), encoder.NoEncoderNewline) +} + +func (self *Parser) skipFast() (int, types.ParsingError) { + start := native.SkipOneFast(&self.s, &self.p) + if start < 0 { + return self.p, types.ParsingError(-start) + } + return start, 0 +} + +func (self *Parser) getByPath(validate bool, path ...interface{}) (int, types.ParsingError) { + var fsm *types.StateMachine + if validate { + fsm = types.NewStateMachine() + } + start := native.GetByPath(&self.s, &self.p, &path, fsm) + if validate { + types.FreeStateMachine(fsm) + } + runtime.KeepAlive(path) + if start < 0 { + return self.p, types.ParsingError(-start) + } + return start, 0 +} + +func validate_utf8(str string) bool { + return utf8.ValidateString(str) +} diff --git a/vendor/github.com/bytedance/sonic/ast/api_amd64.go b/vendor/github.com/bytedance/sonic/ast/api_amd64.go deleted file mode 100644 index da6738efd0..0000000000 --- a/vendor/github.com/bytedance/sonic/ast/api_amd64.go +++ /dev/null @@ -1,157 +0,0 @@ -// +build amd64,go1.16,!go1.22 - -/* - * Copyright 2022 ByteDance Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ast - -import ( - `runtime` - `unsafe` - - `github.com/bytedance/sonic/encoder` - `github.com/bytedance/sonic/internal/native` - `github.com/bytedance/sonic/internal/native/types` - `github.com/bytedance/sonic/internal/rt` - uq `github.com/bytedance/sonic/unquote` - `github.com/chenzhuoyu/base64x` -) - -var typeByte = rt.UnpackEface(byte(0)).Type - -//go:nocheckptr -func quote(buf *[]byte, val string) { - *buf = append(*buf, '"') - if len(val) == 0 { - *buf = append(*buf, '"') - return - } - - sp := rt.IndexChar(val, 0) - nb := len(val) - b := (*rt.GoSlice)(unsafe.Pointer(buf)) - - // input buffer - for nb > 0 { - // output buffer - dp := unsafe.Pointer(uintptr(b.Ptr) + uintptr(b.Len)) - dn := b.Cap - b.Len - // call native.Quote, dn is byte count it outputs - ret := native.Quote(sp, nb, dp, &dn, 0) - // update *buf length - b.Len += dn - - // no need more output - if ret >= 0 { - break - } - - // double buf size - *b = growslice(typeByte, *b, b.Cap*2) - // ret is the complement of consumed input - ret = ^ret - // update input buffer - nb -= ret - sp = unsafe.Pointer(uintptr(sp) + uintptr(ret)) - } - - runtime.KeepAlive(buf) - runtime.KeepAlive(sp) - *buf = append(*buf, '"') -} - -func unquote(src string) (string, types.ParsingError) { - return uq.String(src) -} - -func decodeBase64(src string) ([]byte, error) { - return base64x.StdEncoding.DecodeString(src) -} - -func encodeBase64(src []byte) string { - return base64x.StdEncoding.EncodeToString(src) -} - -func (self *Parser) decodeValue() (val types.JsonState) { - sv := (*rt.GoString)(unsafe.Pointer(&self.s)) - flag := types.F_USE_NUMBER - if self.dbuf != nil { - flag = 0 - val.Dbuf = self.dbuf - val.Dcap = types.MaxDigitNums - } - self.p = native.Value(sv.Ptr, sv.Len, self.p, &val, uint64(flag)) - return -} - -func (self *Parser) skip() (int, types.ParsingError) { - fsm := types.NewStateMachine() - start := native.SkipOne(&self.s, &self.p, fsm, 0) - types.FreeStateMachine(fsm) - - if start < 0 { - return self.p, types.ParsingError(-start) - } - return start, 0 -} - -func (self *Node) encodeInterface(buf *[]byte) error { - //WARN: NOT compatible with json.Encoder - return encoder.EncodeInto(buf, self.packAny(), 0) -} - -func (self *Parser) skipFast() (int, types.ParsingError) { - start := native.SkipOneFast(&self.s, &self.p) - if start < 0 { - return self.p, types.ParsingError(-start) - } - return start, 0 -} - -func (self *Parser) getByPath(path ...interface{}) (int, types.ParsingError) { - fsm := types.NewStateMachine() - start := native.GetByPath(&self.s, &self.p, &path, fsm) - types.FreeStateMachine(fsm) - runtime.KeepAlive(path) - if start < 0 { - return self.p, types.ParsingError(-start) - } - return start, 0 -} - -func (self *Searcher) GetByPath(path ...interface{}) (Node, error) { - var err types.ParsingError - var start int - - self.parser.p = 0 - start, err = self.parser.getByPath(path...) - if err != 0 { - // for compatibility with old version - if err == types.ERR_NOT_FOUND { - return Node{}, ErrNotExist - } - if err == types.ERR_UNSUPPORT_TYPE { - panic("path must be either int(>=0) or string") - } - return Node{}, self.parser.syntaxError(err) - } - - t := switchRawType(self.parser.s[start]) - if t == _V_NONE { - return Node{}, self.parser.ExportError(err) - } - return newRawNode(self.parser.s[start:self.parser.p], t), nil -} diff --git a/vendor/github.com/bytedance/sonic/ast/api_compat.go b/vendor/github.com/bytedance/sonic/ast/api_compat.go index 7b475eb618..a349afc0b9 100644 --- a/vendor/github.com/bytedance/sonic/ast/api_compat.go +++ b/vendor/github.com/bytedance/sonic/ast/api_compat.go @@ -1,40 +1,40 @@ -// +build !amd64 !go1.16 go1.22 +// +build !amd64,!arm64 go1.24 !go1.17 arm64,!go1.20 /* - * Copyright 2022 ByteDance Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Copyright 2022 ByteDance Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ package ast import ( - `encoding/base64` `encoding/json` - `fmt` + `unicode/utf8` `github.com/bytedance/sonic/internal/native/types` `github.com/bytedance/sonic/internal/rt` ) func init() { - println("WARNING: sonic only supports Go1.16~1.20 && CPU amd64, but your environment is not suitable") + println("WARNING:(ast) sonic only supports go1.17~1.23, but your environment is not suitable") } func quote(buf *[]byte, val string) { quoteString(buf, val) } +// unquote unescapes a internal JSON string (it doesn't count quotas at the begining and end) func unquote(src string) (string, types.ParsingError) { sp := rt.IndexChar(src, -1) out, ok := unquoteBytes(rt.BytesFrom(sp, len(src)+2, len(src)+2)) @@ -44,13 +44,6 @@ func unquote(src string) (string, types.ParsingError) { return rt.Mem2Str(out), 0 } -func decodeBase64(src string) ([]byte, error) { - return base64.StdEncoding.DecodeString(src) -} - -func encodeBase64(src []byte) string { - return base64.StdEncoding.EncodeToString(src) -} func (self *Parser) decodeValue() (val types.JsonState) { e, v := decodeValue(self.s, self.p, self.dbuf == nil) @@ -88,37 +81,34 @@ func (self *Node) encodeInterface(buf *[]byte) error { return nil } -func (self *Searcher) GetByPath(path ...interface{}) (Node, error) { - self.parser.p = 0 - - var err types.ParsingError +func (self *Parser) getByPath(validate bool, path ...interface{}) (int, types.ParsingError) { for _, p := range path { if idx, ok := p.(int); ok && idx >= 0 { - if err = self.parser.searchIndex(idx); err != 0 { - return Node{}, self.parser.ExportError(err) + if err := self.searchIndex(idx); err != 0 { + return self.p, err } } else if key, ok := p.(string); ok { - if err = self.parser.searchKey(key); err != 0 { - return Node{}, self.parser.ExportError(err) + if err := self.searchKey(key); err != 0 { + return self.p, err } } else { panic("path must be either int(>=0) or string") } } - var start = self.parser.p - if start, err = self.parser.skip(); err != 0 { - return Node{}, self.parser.ExportError(err) + var start int + var e types.ParsingError + if validate { + start, e = self.skip() + } else { + start, e = self.skipFast() } - ns := len(self.parser.s) - if self.parser.p > ns || start >= ns || start>=self.parser.p { - return Node{}, fmt.Errorf("skip %d char out of json boundary", start) - } - - t := switchRawType(self.parser.s[start]) - if t == _V_NONE { - return Node{}, self.parser.ExportError(err) + if e != 0 { + return self.p, e } + return start, 0 +} - return newRawNode(self.parser.s[start:self.parser.p], t), nil -} \ No newline at end of file +func validate_utf8(str string) bool { + return utf8.ValidString(str) +} diff --git a/vendor/github.com/bytedance/sonic/ast/buffer.go b/vendor/github.com/bytedance/sonic/ast/buffer.go index 93f4ff47a4..04701ef5b3 100644 --- a/vendor/github.com/bytedance/sonic/ast/buffer.go +++ b/vendor/github.com/bytedance/sonic/ast/buffer.go @@ -17,8 +17,10 @@ package ast import ( - `sort` - `unsafe` + "sort" + "unsafe" + + "github.com/bytedance/sonic/internal/caching" ) type nodeChunk [_DEFAULT_NODE_CAP]Node @@ -58,29 +60,82 @@ func (self *linkedNodes) At(i int) (*Node) { return nil } -func (self *linkedNodes) Add(v Node) { - if self.size < _DEFAULT_NODE_CAP { - self.head[self.size] = v - self.size++ +func (self *linkedNodes) MoveOne(source int, target int) { + if source == target { + return + } + if source < 0 || source >= self.size || target < 0 || target >= self.size { + return + } + // reserve source + n := *self.At(source) + if source < target { + // move every element (source,target] one step back + for i:=source; i target; i-- { + *self.At(i) = *self.At(i-1) + } + } + // set target + *self.At(target) = n +} + +func (self *linkedNodes) Pop() { + if self == nil || self.size == 0 { + return + } + self.Set(self.size-1, Node{}) + self.size-- +} + +func (self *linkedNodes) Push(v Node) { + self.Set(self.size, v) +} + + +func (self *linkedNodes) Set(i int, v Node) { + if i < _DEFAULT_NODE_CAP { + self.head[i] = v + if self.size <= i { + self.size = i+1 + } return } + a, b := i/_DEFAULT_NODE_CAP-1, i%_DEFAULT_NODE_CAP + if a < 0 { + self.head[b] = v + } else { + self.growTailLength(a+1) + var n = &self.tail[a] + if *n == nil { + *n = new(nodeChunk) + } + (*n)[b] = v + } + if self.size <= i { + self.size = i+1 + } +} - a, b, c := self.size/_DEFAULT_NODE_CAP-1 , self.size%_DEFAULT_NODE_CAP, cap(self.tail) - if a - c >= 0 { +func (self *linkedNodes) growTailLength(l int) { + if l <= len(self.tail) { + return + } + c := cap(self.tail) + for c < l { c += 1 + c>>_APPEND_GROW_SHIFT - tmp := make([]*nodeChunk, a + 1, c) - copy(tmp, self.tail) - self.tail = tmp - } else if a >= len(self.tail) { - self.tail = self.tail[:a+1] } - - var n = &self.tail[a] - if *n == nil { - *n = new(nodeChunk) + if c == cap(self.tail) { + self.tail = self.tail[:l] + return } - (*n)[b] = v - self.size++ + tmp := make([]*nodeChunk, l, c) + copy(tmp, self.tail) + self.tail = tmp } func (self *linkedNodes) ToSlice(con []Node) { @@ -135,11 +190,22 @@ func (self *linkedNodes) FromSlice(con []Node) { type pairChunk [_DEFAULT_NODE_CAP]Pair type linkedPairs struct { + index map[uint64]int head pairChunk tail []*pairChunk size int } +func (self *linkedPairs) BuildIndex() { + if self.index == nil { + self.index = make(map[uint64]int, self.size) + } + for i:=0; i = 0 { - c += 1 + c>>_APPEND_GROW_SHIFT - tmp := make([]*pairChunk, a + 1, c) - copy(tmp, self.tail) - self.tail = tmp - } else if a >= len(self.tail) { - self.tail = self.tail[:a+1] +func (self *linkedPairs) Unset(i int) { + if self.index != nil { + p := self.At(i) + delete(self.index, p.hash) } + self.set(i, Pair{}) +} - var n = &self.tail[a] - if *n == nil { - *n = new(pairChunk) +func (self *linkedPairs) Set(i int, v Pair) { + if self.index != nil { + h := v.hash + self.index[h] = i } - (*n)[b] = v - self.size++ + self.set(i, v) +} + +func (self *linkedPairs) set(i int, v Pair) { + if i < _DEFAULT_NODE_CAP { + self.head[i] = v + if self.size <= i { + self.size = i+1 + } + return + } + a, b := i/_DEFAULT_NODE_CAP-1, i%_DEFAULT_NODE_CAP + if a < 0 { + self.head[b] = v + } else { + self.growTailLength(a+1) + var n = &self.tail[a] + if *n == nil { + *n = new(pairChunk) + } + (*n)[b] = v + } + if self.size <= i { + self.size = i+1 + } +} + +func (self *linkedPairs) growTailLength(l int) { + if l <= len(self.tail) { + return + } + c := cap(self.tail) + for c < l { + c += 1 + c>>_APPEND_GROW_SHIFT + } + if c == cap(self.tail) { + self.tail = self.tail[:l] + return + } + tmp := make([]*pairChunk, l, c) + copy(tmp, self.tail) + self.tail = tmp } // linear search func (self *linkedPairs) Get(key string) (*Pair, int) { + if self.index != nil { + // fast-path + i, ok := self.index[caching.StrHash(key)] + if ok { + n := self.At(i) + if n.Key == key { + return n, i + } + // hash conflicts + goto linear_search + } else { + return nil, -1 + } + } +linear_search: for i:=0; i len(src) { return -int(types.ERR_EOF) } - if src[pos:ret] == bytesNull { + if src[pos:ret] == strNull { return ret } else { return -int(types.ERR_INVALID_CHAR) @@ -583,3 +586,36 @@ func skipArray(src string, pos int) (ret int, start int) { pos++ } } + +// DecodeString decodes a JSON string from pos and return golang string. +// - needEsc indicates if to unescaped escaping chars +// - hasEsc tells if the returned string has escaping chars +// - validStr enables validating UTF8 charset +// +func _DecodeString(src string, pos int, needEsc bool, validStr bool) (v string, ret int, hasEsc bool) { + p := NewParserObj(src) + p.p = pos + switch val := p.decodeValue(); val.Vt { + case types.V_STRING: + str := p.s[val.Iv : p.p-1] + if validStr && !validate_utf8(str) { + return "", -int(types.ERR_INVALID_UTF8), false + } + /* fast path: no escape sequence */ + if val.Ep == -1 { + return str, p.p, false + } else if !needEsc { + return str, p.p, true + } + /* unquote the string */ + out, err := unquote(str) + /* check for errors */ + if err != 0 { + return "", -int(err), true + } else { + return out, p.p, true + } + default: + return "", -int(_ERR_UNSUPPORT_TYPE), false + } +} diff --git a/vendor/github.com/bytedance/sonic/ast/encode.go b/vendor/github.com/bytedance/sonic/ast/encode.go index 263ae5a9de..eae0bd258b 100644 --- a/vendor/github.com/bytedance/sonic/ast/encode.go +++ b/vendor/github.com/bytedance/sonic/ast/encode.go @@ -17,12 +17,11 @@ package ast import ( - `sync` - `unicode/utf8` -) + "sync" + "unicode/utf8" -const ( - _MaxBuffer = 1024 // 1KB buffer size + "github.com/bytedance/sonic/internal/rt" + "github.com/bytedance/sonic/option" ) func quoteString(e *[]byte, s string) { @@ -30,7 +29,7 @@ func quoteString(e *[]byte, s string) { start := 0 for i := 0; i < len(s); { if b := s[i]; b < utf8.RuneSelf { - if safeSet[b] { + if rt.SafeSet[b] { i++ continue } @@ -54,8 +53,8 @@ func quoteString(e *[]byte, s string) { // user-controlled strings are rendered into JSON // and served to some browsers. *e = append(*e, `u00`...) - *e = append(*e, hex[b>>4]) - *e = append(*e, hex[b&0xF]) + *e = append(*e, rt.Hex[b>>4]) + *e = append(*e, rt.Hex[b&0xF]) } i++ start = i @@ -76,7 +75,7 @@ func quoteString(e *[]byte, s string) { *e = append(*e, s[start:i]...) } *e = append(*e, `\u202`...) - *e = append(*e, hex[c&0xF]) + *e = append(*e, rt.Hex[c&0xF]) i += size start = i continue @@ -92,16 +91,24 @@ func quoteString(e *[]byte, s string) { var bytesPool = sync.Pool{} func (self *Node) MarshalJSON() ([]byte, error) { + if self == nil { + return bytesNull, nil + } + buf := newBuffer() err := self.encode(buf) if err != nil { freeBuffer(buf) return nil, err } - - ret := make([]byte, len(*buf)) - copy(ret, *buf) - freeBuffer(buf) + var ret []byte + if !rt.CanSizeResue(cap(*buf)) { + ret = *buf + } else { + ret = make([]byte, len(*buf)) + copy(ret, *buf) + freeBuffer(buf) + } return ret, err } @@ -109,21 +116,24 @@ func newBuffer() *[]byte { if ret := bytesPool.Get(); ret != nil { return ret.(*[]byte) } else { - buf := make([]byte, 0, _MaxBuffer) + buf := make([]byte, 0, option.DefaultAstBufferSize) return &buf } } func freeBuffer(buf *[]byte) { + if !rt.CanSizeResue(cap(*buf)) { + return + } *buf = (*buf)[:0] bytesPool.Put(buf) } func (self *Node) encode(buf *[]byte) error { - if self.IsRaw() { + if self.isRaw() { return self.encodeRaw(buf) } - switch self.Type() { + switch int(self.itype()) { case V_NONE : return ErrNotExist case V_ERROR : return self.Check() case V_NULL : return self.encodeNull(buf) @@ -139,16 +149,21 @@ func (self *Node) encode(buf *[]byte) error { } func (self *Node) encodeRaw(buf *[]byte) error { - raw, err := self.Raw() - if err != nil { - return err + lock := self.rlock() + if !self.isRaw() { + self.runlock() + return self.encode(buf) + } + raw := self.toString() + if lock { + self.runlock() } *buf = append(*buf, raw...) return nil } func (self *Node) encodeNull(buf *[]byte) error { - *buf = append(*buf, bytesNull...) + *buf = append(*buf, strNull...) return nil } @@ -193,20 +208,9 @@ func (self *Node) encodeArray(buf *[]byte) error { *buf = append(*buf, '[') - var s = (*linkedNodes)(self.p) var started bool - if nb > 0 { - n := s.At(0) - if n.Exists() { - if err := n.encode(buf); err != nil { - return err - } - started = true - } - } - - for i := 1; i < nb; i++ { - n := s.At(i) + for i := 0; i < nb; i++ { + n := self.nodeAt(i) if !n.Exists() { continue } @@ -250,21 +254,10 @@ func (self *Node) encodeObject(buf *[]byte) error { *buf = append(*buf, '{') - var s = (*linkedPairs)(self.p) var started bool - if nb > 0 { - n := s.At(0) - if n.Value.Exists() { - if err := n.encode(buf); err != nil { - return err - } - started = true - } - } - - for i := 1; i < nb; i++ { - n := s.At(i) - if !n.Value.Exists() { + for i := 0; i < nb; i++ { + n := self.pairAt(i) + if n == nil || !n.Value.Exists() { continue } if started { diff --git a/vendor/github.com/bytedance/sonic/ast/error.go b/vendor/github.com/bytedance/sonic/ast/error.go index 00a04468ef..3716e7a91b 100644 --- a/vendor/github.com/bytedance/sonic/ast/error.go +++ b/vendor/github.com/bytedance/sonic/ast/error.go @@ -17,6 +17,10 @@ func newError(err types.ParsingError, msg string) *Node { } } +func newErrorPair(err SyntaxError) *Pair { + return &Pair{0, "", *newSyntaxError(err)} +} + // Error returns error message if the node is invalid func (self Node) Error() string { if self.t != V_ERROR { @@ -79,7 +83,7 @@ func (self SyntaxError) description() string { /* check for empty source */ if self.Src == "" { - return fmt.Sprintf("no sources available: %#v", self) + return fmt.Sprintf("no sources available, the input json is empty: %#v", self) } /* prevent slicing before the beginning */ diff --git a/vendor/github.com/bytedance/sonic/ast/iterator.go b/vendor/github.com/bytedance/sonic/ast/iterator.go index 3c4187a9ca..0766471544 100644 --- a/vendor/github.com/bytedance/sonic/ast/iterator.go +++ b/vendor/github.com/bytedance/sonic/ast/iterator.go @@ -17,30 +17,48 @@ package ast import ( - `fmt` + "fmt" - `github.com/bytedance/sonic/internal/native/types` + "github.com/bytedance/sonic/internal/caching" + "github.com/bytedance/sonic/internal/native/types" ) type Pair struct { + hash uint64 Key string Value Node } +func NewPair(key string, val Node) Pair { + return Pair{ + hash: caching.StrHash(key), + Key: key, + Value: val, + } +} + // Values returns iterator for array's children traversal func (self *Node) Values() (ListIterator, error) { - if err := self.should(types.V_ARRAY, "an array"); err != nil { + if err := self.should(types.V_ARRAY); err != nil { return ListIterator{}, err } - return ListIterator{Iterator{p: self}}, nil + return self.values(), nil +} + +func (self *Node) values() ListIterator { + return ListIterator{Iterator{p: self}} } // Properties returns iterator for object's children traversal func (self *Node) Properties() (ObjectIterator, error) { - if err := self.should(types.V_OBJECT, "an object"); err != nil { + if err := self.should(types.V_OBJECT); err != nil { return ObjectIterator{}, err } - return ObjectIterator{Iterator{p: self}}, nil + return self.properties(), nil +} + +func (self *Node) properties() ObjectIterator { + return ObjectIterator{Iterator{p: self}} } type Iterator struct { @@ -114,7 +132,7 @@ next_start: } else { n := self.p.pairAt(self.i) self.i++ - if !n.Value.Exists() { + if n == nil || !n.Value.Exists() { goto next_start } return n @@ -160,6 +178,9 @@ type Scanner func(path Sequence, node *Node) bool // // NOTICE: A unsetted node WON'T trigger sc, but its index still counts into Path.Index func (self *Node) ForEach(sc Scanner) error { + if err := self.checkRaw(); err != nil { + return err + } switch self.itype() { case types.V_ARRAY: iter, err := self.Values() diff --git a/vendor/github.com/bytedance/sonic/ast/node.go b/vendor/github.com/bytedance/sonic/ast/node.go index 444c8fe42a..0fbcf7835c 100644 --- a/vendor/github.com/bytedance/sonic/ast/node.go +++ b/vendor/github.com/bytedance/sonic/ast/node.go @@ -17,22 +17,15 @@ package ast import ( - `encoding/json` - `fmt` - `strconv` - `unsafe` - `reflect` - - `github.com/bytedance/sonic/internal/native/types` - `github.com/bytedance/sonic/internal/rt` -) - -const ( - _CAP_BITS = 32 - _LEN_MASK = 1 << _CAP_BITS - 1 - - _NODE_SIZE = unsafe.Sizeof(Node{}) - _PAIR_SIZE = unsafe.Sizeof(Pair{}) + "encoding/json" + "fmt" + "strconv" + "sync" + "sync/atomic" + "unsafe" + + "github.com/bytedance/sonic/internal/native/types" + "github.com/bytedance/sonic/internal/rt" ) const ( @@ -45,7 +38,7 @@ const ( _V_ARRAY_LAZY = _V_LAZY | types.V_ARRAY _V_OBJECT_LAZY = _V_LAZY | types.V_OBJECT _MASK_LAZY = _V_LAZY - 1 - _MASK_RAW = _V_RAW - 1 + _MASK_RAW = _V_RAW - 1 ) const ( @@ -61,21 +54,18 @@ const ( V_ANY = int(_V_ANY) ) -var ( - byteType = rt.UnpackType(reflect.TypeOf(byte(0))) -) - type Node struct { t types.ValueType l uint p unsafe.Pointer + m *sync.RWMutex } // UnmarshalJSON is just an adapter to json.Unmarshaler. // If you want better performance, use Searcher.GetByPath() directly func (self *Node) UnmarshalJSON(data []byte) (err error) { - *self, err = NewSearcher(string(data)).GetByPath() - return + *self = NewRaw(string(data)) + return self.Check() } /** Node Type Accessor **/ @@ -92,17 +82,39 @@ func (self *Node) UnmarshalJSON(data []byte) (err error) { // V_STRING = 7 (json value string) // V_NUMBER = 33 (json value number ) // V_ANY = 34 (golang interface{}) +// +// Deprecated: not concurrent safe. Use TypeSafe instead func (self Node) Type() int { return int(self.t & _MASK_LAZY & _MASK_RAW) } -func (self Node) itype() types.ValueType { +// Type concurrently-safe returns json type represented by the node +// It will be one of belows: +// V_NONE = 0 (empty node, key not exists) +// V_ERROR = 1 (error node) +// V_NULL = 2 (json value `null`, key exists) +// V_TRUE = 3 (json value `true`) +// V_FALSE = 4 (json value `false`) +// V_ARRAY = 5 (json value array) +// V_OBJECT = 6 (json value object) +// V_STRING = 7 (json value string) +// V_NUMBER = 33 (json value number ) +// V_ANY = 34 (golang interface{}) +func (self *Node) TypeSafe() int { + return int(self.loadt() & _MASK_LAZY & _MASK_RAW) +} + +func (self *Node) itype() types.ValueType { return self.t & _MASK_LAZY & _MASK_RAW } // Exists returns false only if the self is nil or empty node V_NONE func (self *Node) Exists() bool { - return self.Valid() && self.t != _V_NONE + if self == nil { + return false + } + t := self.loadt() + return t != V_ERROR && t != _V_NONE } // Valid reports if self is NOT V_ERROR or nil @@ -110,7 +122,7 @@ func (self *Node) Valid() bool { if self == nil { return false } - return self.t != V_ERROR + return self.loadt() != V_ERROR } // Check checks if the node itself is valid, and return: @@ -119,45 +131,63 @@ func (self *Node) Valid() bool { func (self *Node) Check() error { if self == nil { return ErrNotExist - } else if self.t != V_ERROR { + } else if self.loadt() != V_ERROR { return nil } else { return self } } -// IsRaw returns true if node's underlying value is raw json +// isRaw returns true if node's underlying value is raw json +// +// Deprecated: not concurent safe func (self Node) IsRaw() bool { - return self.t&_V_RAW != 0 + return self.t & _V_RAW != 0 +} + +// IsRaw returns true if node's underlying value is raw json +func (self *Node) isRaw() bool { + return self.loadt() & _V_RAW != 0 } func (self *Node) isLazy() bool { - return self != nil && self.t&_V_LAZY != 0 + return self != nil && self.t & _V_LAZY != 0 } func (self *Node) isAny() bool { - return self != nil && self.t == _V_ANY + return self != nil && self.loadt() == _V_ANY } /** Simple Value Methods **/ // Raw returns json representation of the node, func (self *Node) Raw() (string, error) { - if !self.IsRaw() { + if self == nil { + return "", ErrNotExist + } + lock := self.rlock() + if !self.isRaw() { + if lock { + self.runlock() + } buf, err := self.MarshalJSON() return rt.Mem2Str(buf), err } - return self.toString(), nil + ret := self.toString() + if lock { + self.runlock() + } + return ret, nil } func (self *Node) checkRaw() error { if err := self.Check(); err != nil { return err } - if self.IsRaw() { + if self.isRaw() { self.parseRaw(false) } - return nil + return self.Check() } // Bool returns bool value represented by this node, @@ -501,7 +531,6 @@ func (self *Node) StrictFloat64() (float64, error) { // Len returns children count of a array|object|string node // WARN: For partially loaded node, it also works but only counts the parsed children -// WARN: For ARRAY|OBJECT nodes which has been conducted `UnsetXX()`, its length WON'T change func (self *Node) Len() (int, error) { if err := self.checkRaw(); err != nil { return 0, err @@ -515,7 +544,7 @@ func (self *Node) Len() (int, error) { } } -func (self Node) len() int { +func (self *Node) len() int { return int(self.l) } @@ -538,14 +567,19 @@ func (self *Node) Cap() (int, error) { // // If self is V_NONE or V_NULL, it becomes V_OBJECT and sets the node at the key. func (self *Node) Set(key string, node Node) (bool, error) { - if self != nil && (self.t == _V_NONE || self.t == types.V_NULL) { - *self = NewObject([]Pair{{key, node}}) - return false, nil + if err := self.checkRaw(); err != nil { + return false, err } - if err := node.Check(); err != nil { return false, err } + + if self.t == _V_NONE || self.t == types.V_NULL { + *self = NewObject([]Pair{NewPair(key, node)}) + return false, nil + } else if self.itype() != types.V_OBJECT { + return false, ErrUnsupportType + } p := self.Get(key) @@ -555,7 +589,7 @@ func (self *Node) Set(key string, node Node) (bool, error) { *self = newObject(new(linkedPairs)) } s := (*linkedPairs)(self.p) - s.Add(Pair{key, node}) + s.Push(NewPair(key, node)) self.l++ return false, nil @@ -572,18 +606,22 @@ func (self *Node) SetAny(key string, val interface{}) (bool, error) { return self.Set(key, NewAny(val)) } -// Unset RESET the node of given key under object parent, and reports if the key has existed. -// WARN: After conducting `UnsetXX()`, the node's length WON'T change +// Unset REMOVE (soft) the node of given key under object parent, and reports if the key has existed. func (self *Node) Unset(key string) (bool, error) { - self.must(types.V_OBJECT, "an object") + if err := self.should(types.V_OBJECT); err != nil { + return false, err + } + // NOTICE: must get acurate length before deduct + if err := self.skipAllKey(); err != nil { + return false, err + } p, i := self.skipKey(key) if !p.Exists() { return false, nil } else if err := p.Check(); err != nil { return false, err } - - self.removePair(i) + self.removePairAt(i) return true, nil } @@ -591,10 +629,18 @@ func (self *Node) Unset(key string) (bool, error) { // // The index must be within self's children. func (self *Node) SetByIndex(index int, node Node) (bool, error) { + if err := self.checkRaw(); err != nil { + return false, err + } if err := node.Check(); err != nil { return false, err } + if index == 0 && (self.t == _V_NONE || self.t == types.V_NULL) { + *self = NewArray([]Node{node}) + return false, nil + } + p := self.Index(index) if !p.Exists() { return false, ErrNotExist @@ -611,18 +657,28 @@ func (self *Node) SetAnyByIndex(index int, val interface{}) (bool, error) { return self.SetByIndex(index, NewAny(val)) } -// UnsetByIndex remove the node of given index -// WARN: After conducting `UnsetXX()`, the node's length WON'T change +// UnsetByIndex REOMVE (softly) the node of given index. +// +// WARN: this will change address of elements, which is a dangerous action. +// Use Unset() for object or Pop() for array instead. func (self *Node) UnsetByIndex(index int) (bool, error) { + if err := self.checkRaw(); err != nil { + return false, err + } + var p *Node it := self.itype() + if it == types.V_ARRAY { - p = self.Index(index) - }else if it == types.V_OBJECT { - if err := self.checkRaw(); err != nil { + if err := self.skipAllIndex(); err != nil { + return false, err + } + p = self.nodeAt(index) + } else if it == types.V_OBJECT { + if err := self.skipAllKey(); err != nil { return false, err } - pr := self.skipIndexPair(index) + pr := self.pairAt(index) if pr == nil { return false, ErrNotExist } @@ -635,6 +691,12 @@ func (self *Node) UnsetByIndex(index int) (bool, error) { return false, ErrNotExist } + // last elem + if index == self.len() - 1 { + return true, self.Pop() + } + + // not last elem, self.len() change but linked-chunk not change if it == types.V_ARRAY { self.removeNode(index) }else if it == types.V_OBJECT { @@ -647,21 +709,112 @@ func (self *Node) UnsetByIndex(index int) (bool, error) { // // If self is V_NONE or V_NULL, it becomes V_ARRAY and sets the node at index 0. func (self *Node) Add(node Node) error { + if err := self.checkRaw(); err != nil { + return err + } + if self != nil && (self.t == _V_NONE || self.t == types.V_NULL) { *self = NewArray([]Node{node}) return nil } + if err := self.should(types.V_ARRAY); err != nil { + return err + } s, err := self.unsafeArray() if err != nil { return err } - s.Add(node) + // Notice: array won't have unset node in tail + s.Push(node) self.l++ return nil } +// Pop remove the last child of the V_Array or V_Object node. +func (self *Node) Pop() error { + if err := self.checkRaw(); err != nil { + return err + } + + if it := self.itype(); it == types.V_ARRAY { + s, err := self.unsafeArray() + if err != nil { + return err + } + // remove tail unset nodes + for i := s.Len()-1; i >= 0; i-- { + if s.At(i).Exists() { + s.Pop() + self.l-- + break + } + s.Pop() + } + + } else if it == types.V_OBJECT { + s, err := self.unsafeMap() + if err != nil { + return err + } + // remove tail unset nodes + for i := s.Len()-1; i >= 0; i-- { + if p := s.At(i); p != nil && p.Value.Exists() { + s.Pop() + self.l-- + break + } + s.Pop() + } + + } else { + return ErrUnsupportType + } + + return nil +} + +// Move moves the child at src index to dst index, +// meanwhile slides sliblings from src+1 to dst. +// +// WARN: this will change address of elements, which is a dangerous action. +func (self *Node) Move(dst, src int) error { + if err := self.should(types.V_ARRAY); err != nil { + return err + } + + s, err := self.unsafeArray() + if err != nil { + return err + } + + // check if any unset node exists + if l := s.Len(); self.len() != l { + di, si := dst, src + // find real pos of src and dst + for i := 0; i < l; i++ { + if s.At(i).Exists() { + di-- + si-- + } + if di == -1 { + dst = i + di-- + } + if si == -1 { + src = i + si-- + } + if di == -2 && si == -2 { + break + } + } + } + + s.MoveOne(src, dst) + return nil +} // SetAny wraps val with V_ANY node, and Add() the node. func (self *Node) AddAny(val interface{}) error { @@ -699,7 +852,7 @@ func (self *Node) GetByPath(path ...interface{}) *Node { // Get loads given key of an object node on demands func (self *Node) Get(key string) *Node { - if err := self.should(types.V_OBJECT, "an object"); err != nil { + if err := self.should(types.V_OBJECT); err != nil { return unwrapError(err) } n, _ := self.skipKey(key) @@ -708,8 +861,6 @@ func (self *Node) Get(key string) *Node { // Index indexies node at given idx, // node type CAN be either V_OBJECT or V_ARRAY -// WARN: After conducting `UnsetXX()`, the node's length WON'T change, -// thus its children's indexing WON'T change too func (self *Node) Index(idx int) *Node { if err := self.checkRaw(); err != nil { return unwrapError(err) @@ -733,28 +884,37 @@ func (self *Node) Index(idx int) *Node { // IndexPair indexies pair at given idx, // node type MUST be either V_OBJECT -// WARN: After conducting `UnsetXX()`, the node's length WON'T change, -// thus its children's indexing WON'T change too func (self *Node) IndexPair(idx int) *Pair { - if err := self.should(types.V_OBJECT, "an object"); err != nil { + if err := self.should(types.V_OBJECT); err != nil { return nil } return self.skipIndexPair(idx) } -// IndexOrGet firstly use idx to index a value and check if its key matches -// If not, then use the key to search value -func (self *Node) IndexOrGet(idx int, key string) *Node { - if err := self.should(types.V_OBJECT, "an object"); err != nil { - return unwrapError(err) +func (self *Node) indexOrGet(idx int, key string) (*Node, int) { + if err := self.should(types.V_OBJECT); err != nil { + return unwrapError(err), idx } pr := self.skipIndexPair(idx) if pr != nil && pr.Key == key { - return &pr.Value + return &pr.Value, idx } - n, _ := self.skipKey(key) - return n + + return self.skipKey(key) +} + +// IndexOrGet firstly use idx to index a value and check if its key matches +// If not, then use the key to search value +func (self *Node) IndexOrGet(idx int, key string) *Node { + node, _ := self.indexOrGet(idx, key) + return node +} + +// IndexOrGetWithIdx attempts to retrieve a node by index and key, returning the node and its correct index. +// If the key does not match at the given index, it searches by key and returns the node with its updated index. +func (self *Node) IndexOrGetWithIdx(idx int, key string) (*Node, int) { + return self.indexOrGet(idx, key) } /** Generic Value Converters **/ @@ -769,10 +929,10 @@ func (self *Node) Map() (map[string]interface{}, error) { return nil, ErrUnsupportType } } - if err := self.should(types.V_OBJECT, "an object"); err != nil { + if err := self.should(types.V_OBJECT); err != nil { return nil, err } - if err := self.loadAllKey(); err != nil { + if err := self.loadAllKey(false); err != nil { return nil, err } return self.toGenericObject() @@ -788,10 +948,10 @@ func (self *Node) MapUseNumber() (map[string]interface{}, error) { return nil, ErrUnsupportType } } - if err := self.should(types.V_OBJECT, "an object"); err != nil { + if err := self.should(types.V_OBJECT); err != nil { return nil, err } - if err := self.loadAllKey(); err != nil { + if err := self.loadAllKey(false); err != nil { return nil, err } return self.toGenericObjectUseNumber() @@ -808,7 +968,7 @@ func (self *Node) MapUseNode() (map[string]Node, error) { return nil, ErrUnsupportType } } - if err := self.should(types.V_OBJECT, "an object"); err != nil { + if err := self.should(types.V_OBJECT); err != nil { return nil, err } if err := self.skipAllKey(); err != nil { @@ -831,10 +991,8 @@ func (self *Node) MapUseNode() (map[string]Node, error) { // return self.toGenericObjectUsePair() // } +//go:nocheckptr func (self *Node) unsafeMap() (*linkedPairs, error) { - if err := self.should(types.V_OBJECT, "an object"); err != nil { - return nil, err - } if err := self.skipAllKey(); err != nil { return nil, err } @@ -846,7 +1004,39 @@ func (self *Node) unsafeMap() (*linkedPairs, error) { // SortKeys sorts children of a V_OBJECT node in ascending key-order. // If recurse is true, it recursively sorts children's children as long as a V_OBJECT node is found. -func (self *Node) SortKeys(recurse bool) (err error) { +func (self *Node) SortKeys(recurse bool) error { + // check raw node first + if err := self.checkRaw(); err != nil { + return err + } + if self.itype() == types.V_OBJECT { + return self.sortKeys(recurse) + } else if self.itype() == types.V_ARRAY { + var err error + err2 := self.ForEach(func(path Sequence, node *Node) bool { + it := node.itype() + if it == types.V_ARRAY || it == types.V_OBJECT { + err = node.SortKeys(recurse) + if err != nil { + return false + } + } + return true + }) + if err != nil { + return err + } + return err2 + } else { + return nil + } +} + +func (self *Node) sortKeys(recurse bool) (err error) { + // check raw node first + if err := self.checkRaw(); err != nil { + return err + } ps, err := self.unsafeMap() if err != nil { return err @@ -856,7 +1046,7 @@ func (self *Node) SortKeys(recurse bool) (err error) { var sc Scanner sc = func(path Sequence, node *Node) bool { if node.itype() == types.V_OBJECT { - if err := node.SortKeys(recurse); err != nil { + if err := node.sortKeys(recurse); err != nil { return false } } @@ -867,7 +1057,9 @@ func (self *Node) SortKeys(recurse bool) (err error) { } return true } - self.ForEach(sc) + if err := self.ForEach(sc); err != nil { + return err + } } return nil } @@ -882,10 +1074,10 @@ func (self *Node) Array() ([]interface{}, error) { return nil, ErrUnsupportType } } - if err := self.should(types.V_ARRAY, "an array"); err != nil { + if err := self.should(types.V_ARRAY); err != nil { return nil, err } - if err := self.loadAllIndex(); err != nil { + if err := self.loadAllIndex(false); err != nil { return nil, err } return self.toGenericArray() @@ -901,10 +1093,10 @@ func (self *Node) ArrayUseNumber() ([]interface{}, error) { return nil, ErrUnsupportType } } - if err := self.should(types.V_ARRAY, "an array"); err != nil { + if err := self.should(types.V_ARRAY); err != nil { return nil, err } - if err := self.loadAllIndex(); err != nil { + if err := self.loadAllIndex(false); err != nil { return nil, err } return self.toGenericArrayUseNumber() @@ -921,7 +1113,7 @@ func (self *Node) ArrayUseNode() ([]Node, error) { return nil, ErrUnsupportType } } - if err := self.should(types.V_ARRAY, "an array"); err != nil { + if err := self.should(types.V_ARRAY); err != nil { return nil, err } if err := self.skipAllIndex(); err != nil { @@ -946,9 +1138,6 @@ func (self *Node) ArrayUseNode() ([]Node, error) { // } func (self *Node) unsafeArray() (*linkedNodes, error) { - if err := self.should(types.V_ARRAY, "an array"); err != nil { - return nil, err - } if err := self.skipAllIndex(); err != nil { return nil, err } @@ -980,12 +1169,12 @@ func (self *Node) Interface() (interface{}, error) { } return v, nil case _V_ARRAY_LAZY : - if err := self.loadAllIndex(); err != nil { + if err := self.loadAllIndex(false); err != nil { return nil, err } return self.toGenericArray() case _V_OBJECT_LAZY : - if err := self.loadAllKey(); err != nil { + if err := self.loadAllKey(false); err != nil { return nil, err } return self.toGenericObject() @@ -1019,12 +1208,12 @@ func (self *Node) InterfaceUseNumber() (interface{}, error) { case types.V_STRING : return self.toString(), nil case _V_NUMBER : return self.toNumber(), nil case _V_ARRAY_LAZY : - if err := self.loadAllIndex(); err != nil { + if err := self.loadAllIndex(false); err != nil { return nil, err } return self.toGenericArrayUseNumber() case _V_OBJECT_LAZY : - if err := self.loadAllKey(); err != nil { + if err := self.loadAllKey(false); err != nil { return nil, err } return self.toGenericObjectUseNumber() @@ -1056,105 +1245,30 @@ func (self *Node) InterfaceUseNode() (interface{}, error) { } } -// LoadAll loads all the node's children and children's children as parsed. -// After calling it, the node can be safely used on concurrency +// LoadAll loads the node's children +// and ensure all its children can be READ concurrently (include its children's children) func (self *Node) LoadAll() error { - if self.IsRaw() { - self.parseRaw(true) - return self.Check() - } - - switch self.itype() { - case types.V_ARRAY: - e := self.len() - if err := self.loadAllIndex(); err != nil { - return err - } - for i := 0; i < e; i++ { - n := self.nodeAt(i) - if n.IsRaw() { - n.parseRaw(true) - } - if err := n.Check(); err != nil { - return err - } - } - return nil - case types.V_OBJECT: - e := self.len() - if err := self.loadAllKey(); err != nil { - return err - } - for i := 0; i < e; i++ { - n := self.pairAt(i) - if n.Value.IsRaw() { - n.Value.parseRaw(true) - } - if err := n.Value.Check(); err != nil { - return err - } - } - return nil - default: - return self.Check() - } + return self.Load() } // Load loads the node's children as parsed. -// After calling it, only the node itself can be used on concurrency (not include its children) +// and ensure all its children can be READ concurrently (include its children's children) func (self *Node) Load() error { - if self.IsRaw() { - self.parseRaw(false) - return self.Load() - } - switch self.t { - case _V_ARRAY_LAZY: - return self.skipAllIndex() - case _V_OBJECT_LAZY: - return self.skipAllKey() - default: - return self.Check() + case _V_ARRAY_LAZY: self.loadAllIndex(true) + case _V_OBJECT_LAZY: self.loadAllKey(true) + case V_ERROR: return self + case V_NONE: return nil + } + if self.m == nil { + self.m = new(sync.RWMutex) } + return self.checkRaw() } /**---------------------------------- Internal Helper Methods ----------------------------------**/ -var ( - _NODE_TYPE = rt.UnpackEface(Node{}).Type - _PAIR_TYPE = rt.UnpackEface(Pair{}).Type -) - -// func (self *Node) setCapAndLen(cap int, len int) { -// if self.t == types.V_ARRAY || self.t == types.V_OBJECT || self.t == _V_ARRAY_LAZY || self.t == _V_OBJECT_LAZY { -// self.l = uint32(len) -// self.c = uint32(cap) -// } else { -// panic("value does not have a length") -// } -// } - -func (self *Node) unsafe_next() *Node { - return (*Node)(unsafe.Pointer(uintptr(unsafe.Pointer(self)) + _NODE_SIZE)) -} - -func (self *Pair) unsafe_next() *Pair { - return (*Pair)(unsafe.Pointer(uintptr(unsafe.Pointer(self)) + _PAIR_SIZE)) -} - -func (self *Node) must(t types.ValueType, s string) { - if err := self.checkRaw(); err != nil { - panic(err) - } - if err := self.Check(); err != nil { - panic(err) - } - if self.itype() != t { - panic("value cannot be represented as " + s) - } -} - -func (self *Node) should(t types.ValueType, s string) error { +func (self *Node) should(t types.ValueType) error { if err := self.checkRaw(); err != nil { return err } @@ -1171,6 +1285,19 @@ func (self *Node) nodeAt(i int) *Node { p = &stack.v } else { p = (*linkedNodes)(self.p) + if l := p.Len(); l != self.len() { + // some nodes got unset, iterate to skip them + for j:=0; j _Threshold_Index { + v.BuildIndex() + } return Node{ t: types.V_OBJECT, l: uint(v.Len()), @@ -1640,53 +1802,42 @@ func newObject(v *linkedPairs) Node { } func (self *Node) setObject(v *linkedPairs) { + if v.size > _Threshold_Index { + v.BuildIndex() + } self.t = types.V_OBJECT self.l = uint(v.Len()) self.p = unsafe.Pointer(v) } -func newRawNode(str string, typ types.ValueType) Node { - return Node{ - t: _V_RAW | typ, - p: rt.StrPtr(str), - l: uint(len(str)), - } -} - func (self *Node) parseRaw(full bool) { + lock := self.lock() + defer self.unlock() + if !self.isRaw() { + return + } raw := self.toString() parser := NewParserObj(raw) + var e types.ParsingError if full { parser.noLazy = true - parser.skipValue = false + *self, e = parser.Parse() + } else if lock { + var n Node + parser.noLazy = true + parser.loadOnce = true + n, e = parser.Parse() + self.assign(n) + } else { + *self, e = parser.Parse() } - var e types.ParsingError - *self, e = parser.Parse() if e != 0 { *self = *newSyntaxError(parser.syntaxError(e)) } } -var typeJumpTable = [256]types.ValueType{ - '"' : types.V_STRING, - '-' : _V_NUMBER, - '0' : _V_NUMBER, - '1' : _V_NUMBER, - '2' : _V_NUMBER, - '3' : _V_NUMBER, - '4' : _V_NUMBER, - '5' : _V_NUMBER, - '6' : _V_NUMBER, - '7' : _V_NUMBER, - '8' : _V_NUMBER, - '9' : _V_NUMBER, - '[' : types.V_ARRAY, - 'f' : types.V_FALSE, - 'n' : types.V_NULL, - 't' : types.V_TRUE, - '{' : types.V_OBJECT, -} - -func switchRawType(c byte) types.ValueType { - return typeJumpTable[c] +func (self *Node) assign(n Node) { + self.l = n.l + self.p = n.p + atomic.StoreInt64(&self.t, n.t) } diff --git a/vendor/github.com/bytedance/sonic/ast/parser.go b/vendor/github.com/bytedance/sonic/ast/parser.go index cb16f20bb0..30bd1f4519 100644 --- a/vendor/github.com/bytedance/sonic/ast/parser.go +++ b/vendor/github.com/bytedance/sonic/ast/parser.go @@ -17,14 +17,16 @@ package ast import ( - `fmt` + "fmt" + "sync" + "sync/atomic" - `github.com/bytedance/sonic/internal/native/types` - `github.com/bytedance/sonic/internal/rt` + "github.com/bytedance/sonic/internal/native/types" + "github.com/bytedance/sonic/internal/rt" ) const ( - _DEFAULT_NODE_CAP int = 8 + _DEFAULT_NODE_CAP int = 16 _APPEND_GROW_SHIFT = 1 ) @@ -45,6 +47,7 @@ type Parser struct { p int s string noLazy bool + loadOnce bool skipValue bool dbuf *byte } @@ -115,6 +118,10 @@ func (self *Parser) lspace(sp int) int { return sp } +func (self *Parser) backward() { + for ; self.p >= 0 && isSpace(self.s[self.p]); self.p-=1 {} +} + func (self *Parser) decodeArray(ret *linkedNodes) (Node, types.ParsingError) { sp := self.p ns := len(self.s) @@ -148,7 +155,7 @@ func (self *Parser) decodeArray(ret *linkedNodes) (Node, types.ParsingError) { if t == _V_NONE { return Node{}, types.ERR_INVALID_CHAR } - val = newRawNode(self.s[start:self.p], t) + val = newRawNode(self.s[start:self.p], t, false) }else{ /* decode the value */ if val, err = self.Parse(); err != 0 { @@ -157,7 +164,7 @@ func (self *Parser) decodeArray(ret *linkedNodes) (Node, types.ParsingError) { } /* add the value to result */ - ret.Add(val) + ret.Push(val) self.p = self.lspace(self.p) /* check for EOF */ @@ -234,7 +241,7 @@ func (self *Parser) decodeObject(ret *linkedPairs) (Node, types.ParsingError) { if t == _V_NONE { return Node{}, types.ERR_INVALID_CHAR } - val = newRawNode(self.s[start:self.p], t) + val = newRawNode(self.s[start:self.p], t, false) } else { /* decode the value */ if val, err = self.Parse(); err != 0 { @@ -244,7 +251,7 @@ func (self *Parser) decodeObject(ret *linkedPairs) (Node, types.ParsingError) { /* add the value to result */ // FIXME: ret's address may change here, thus previous referred node in ret may be invalid !! - ret.Add(Pair{Key: key, Value: val}) + ret.Push(NewPair(key, val)) self.p = self.lspace(self.p) /* check for EOF */ @@ -291,6 +298,10 @@ func (self *Parser) Pos() int { return self.p } + +// Parse returns a ast.Node representing the parser's JSON. +// NOTICE: the specific parsing lazy dependens parser's option +// It only parse first layer and first child for Object or Array be default func (self *Parser) Parse() (Node, types.ParsingError) { switch val := self.decodeValue(); val.Vt { case types.V_EOF : return Node{}, types.ERR_EOF @@ -299,22 +310,48 @@ func (self *Parser) Parse() (Node, types.ParsingError) { case types.V_FALSE : return falseNode, 0 case types.V_STRING : return self.decodeString(val.Iv, val.Ep) case types.V_ARRAY: + s := self.p - 1; if p := skipBlank(self.s, self.p); p >= self.p && self.s[p] == ']' { self.p = p + 1 return Node{t: types.V_ARRAY}, 0 } if self.noLazy { + if self.loadOnce { + self.noLazy = false + } return self.decodeArray(new(linkedNodes)) } + // NOTICE: loadOnce always keep raw json for object or array + if self.loadOnce { + self.p = s + s, e := self.skipFast() + if e != 0 { + return Node{}, e + } + return newRawNode(self.s[s:self.p], types.V_ARRAY, true), 0 + } return newLazyArray(self), 0 case types.V_OBJECT: + s := self.p - 1; if p := skipBlank(self.s, self.p); p >= self.p && self.s[p] == '}' { self.p = p + 1 return Node{t: types.V_OBJECT}, 0 } + // NOTICE: loadOnce always keep raw json for object or array if self.noLazy { + if self.loadOnce { + self.noLazy = false + } return self.decodeObject(new(linkedPairs)) } + if self.loadOnce { + self.p = s + s, e := self.skipFast() + if e != 0 { + return Node{}, e + } + return newRawNode(self.s[s:self.p], types.V_OBJECT, true), 0 + } return newLazyObject(self), 0 case types.V_DOUBLE : return NewNumber(self.s[val.Ep:self.p]), 0 case types.V_INTEGER : return NewNumber(self.s[val.Ep:self.p]), 0 @@ -471,11 +508,11 @@ func (self *Node) skipNextNode() *Node { if t == _V_NONE { return newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR)) } - val = newRawNode(parser.s[start:parser.p], t) + val = newRawNode(parser.s[start:parser.p], t, false) } /* add the value to result */ - ret.Add(val) + ret.Push(val) self.l++ parser.p = parser.lspace(parser.p) @@ -510,7 +547,7 @@ func (self *Node) skipNextPair() (*Pair) { /* check for EOF */ if parser.p = parser.lspace(sp); parser.p >= ns { - return &Pair{"", *newSyntaxError(parser.syntaxError(types.ERR_EOF))} + return newErrorPair(parser.syntaxError(types.ERR_EOF)) } /* check for empty object */ @@ -527,7 +564,7 @@ func (self *Node) skipNextPair() (*Pair) { /* decode the key */ if njs = parser.decodeValue(); njs.Vt != types.V_STRING { - return &Pair{"", *newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR))} + return newErrorPair(parser.syntaxError(types.ERR_INVALID_CHAR)) } /* extract the key */ @@ -537,34 +574,34 @@ func (self *Node) skipNextPair() (*Pair) { /* check for escape sequence */ if njs.Ep != -1 { if key, err = unquote(key); err != 0 { - return &Pair{key, *newSyntaxError(parser.syntaxError(err))} + return newErrorPair(parser.syntaxError(err)) } } /* expect a ':' delimiter */ if err = parser.delim(); err != 0 { - return &Pair{key, *newSyntaxError(parser.syntaxError(err))} + return newErrorPair(parser.syntaxError(err)) } /* skip the value */ if start, err := parser.skipFast(); err != 0 { - return &Pair{key, *newSyntaxError(parser.syntaxError(err))} + return newErrorPair(parser.syntaxError(err)) } else { t := switchRawType(parser.s[start]) if t == _V_NONE { - return &Pair{key, *newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR))} + return newErrorPair(parser.syntaxError(types.ERR_INVALID_CHAR)) } - val = newRawNode(parser.s[start:parser.p], t) + val = newRawNode(parser.s[start:parser.p], t, false) } /* add the value to result */ - ret.Add(Pair{Key: key, Value: val}) + ret.Push(NewPair(key, val)) self.l++ parser.p = parser.lspace(parser.p) /* check for EOF */ if parser.p >= ns { - return &Pair{key, *newSyntaxError(parser.syntaxError(types.ERR_EOF))} + return newErrorPair(parser.syntaxError(types.ERR_EOF)) } /* check for the next character */ @@ -577,7 +614,7 @@ func (self *Node) skipNextPair() (*Pair) { self.setObject(ret) return ret.At(ret.Len()-1) default: - return &Pair{key, *newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR))} + return newErrorPair(parser.syntaxError(types.ERR_INVALID_CHAR)) } } @@ -653,3 +690,77 @@ func (self *Parser) ExportError(err types.ParsingError) error { Code: err, }.Description()) } + +func backward(src string, i int) int { + for ; i>=0 && isSpace(src[i]); i-- {} + return i +} + + +func newRawNode(str string, typ types.ValueType, lock bool) Node { + ret := Node{ + t: typ | _V_RAW, + p: rt.StrPtr(str), + l: uint(len(str)), + } + if lock { + ret.m = new(sync.RWMutex) + } + return ret +} + +var typeJumpTable = [256]types.ValueType{ + '"' : types.V_STRING, + '-' : _V_NUMBER, + '0' : _V_NUMBER, + '1' : _V_NUMBER, + '2' : _V_NUMBER, + '3' : _V_NUMBER, + '4' : _V_NUMBER, + '5' : _V_NUMBER, + '6' : _V_NUMBER, + '7' : _V_NUMBER, + '8' : _V_NUMBER, + '9' : _V_NUMBER, + '[' : types.V_ARRAY, + 'f' : types.V_FALSE, + 'n' : types.V_NULL, + 't' : types.V_TRUE, + '{' : types.V_OBJECT, +} + +func switchRawType(c byte) types.ValueType { + return typeJumpTable[c] +} + +func (self *Node) loadt() types.ValueType { + return (types.ValueType)(atomic.LoadInt64(&self.t)) +} + +func (self *Node) lock() bool { + if m := self.m; m != nil { + m.Lock() + return true + } + return false +} + +func (self *Node) unlock() { + if m := self.m; m != nil { + m.Unlock() + } +} + +func (self *Node) rlock() bool { + if m := self.m; m != nil { + m.RLock() + return true + } + return false +} + +func (self *Node) runlock() { + if m := self.m; m != nil { + m.RUnlock() + } +} diff --git a/vendor/github.com/bytedance/sonic/ast/search.go b/vendor/github.com/bytedance/sonic/ast/search.go index bb6fceaa7c..9a5fb94203 100644 --- a/vendor/github.com/bytedance/sonic/ast/search.go +++ b/vendor/github.com/bytedance/sonic/ast/search.go @@ -16,8 +16,28 @@ package ast +import ( + `github.com/bytedance/sonic/internal/rt` + `github.com/bytedance/sonic/internal/native/types` +) + +// SearchOptions controls Searcher's behavior +type SearchOptions struct { + // ValidateJSON indicates the searcher to validate the entire JSON + ValidateJSON bool + + // CopyReturn indicates the searcher to copy the result JSON instead of refer from the input + // This can help to reduce memory usage if you cache the results + CopyReturn bool + + // ConcurrentRead indicates the searcher to return a concurrently-READ-safe node, + // including: GetByPath/Get/Index/GetOrIndex/Int64/Bool/Float64/String/Number/Interface/Array/Map/Raw/MarshalJSON + ConcurrentRead bool +} + type Searcher struct { parser Parser + SearchOptions } func NewSearcher(str string) *Searcher { @@ -26,5 +46,112 @@ func NewSearcher(str string) *Searcher { s: str, noLazy: false, }, + SearchOptions: SearchOptions{ + ValidateJSON: true, + }, + } +} + +// GetByPathCopy search in depth from top json and returns a **Copied** json node at the path location +func (self *Searcher) GetByPathCopy(path ...interface{}) (Node, error) { + self.CopyReturn = true + return self.getByPath(path...) +} + +// GetByPathNoCopy search in depth from top json and returns a **Referenced** json node at the path location +// +// WARN: this search directly refer partial json from top json, which has faster speed, +// may consumes more memory. +func (self *Searcher) GetByPath(path ...interface{}) (Node, error) { + return self.getByPath(path...) +} + +func (self *Searcher) getByPath(path ...interface{}) (Node, error) { + var err types.ParsingError + var start int + + self.parser.p = 0 + start, err = self.parser.getByPath(self.ValidateJSON, path...) + if err != 0 { + // for compatibility with old version + if err == types.ERR_NOT_FOUND { + return Node{}, ErrNotExist + } + if err == types.ERR_UNSUPPORT_TYPE { + panic("path must be either int(>=0) or string") + } + return Node{}, self.parser.syntaxError(err) + } + + t := switchRawType(self.parser.s[start]) + if t == _V_NONE { + return Node{}, self.parser.ExportError(err) + } + + // copy string to reducing memory usage + var raw string + if self.CopyReturn { + raw = rt.Mem2Str([]byte(self.parser.s[start:self.parser.p])) + } else { + raw = self.parser.s[start:self.parser.p] + } + return newRawNode(raw, t, self.ConcurrentRead), nil +} + +// GetByPath searches a path and returns relaction and types of target +func _GetByPath(src string, path ...interface{}) (start int, end int, typ int, err error) { + p := NewParserObj(src) + s, e := p.getByPath(false, path...) + if e != 0 { + // for compatibility with old version + if e == types.ERR_NOT_FOUND { + return -1, -1, 0, ErrNotExist + } + if e == types.ERR_UNSUPPORT_TYPE { + panic("path must be either int(>=0) or string") + } + return -1, -1, 0, p.syntaxError(e) + } + + t := switchRawType(p.s[s]) + if t == _V_NONE { + return -1, -1, 0, ErrNotExist + } + if t == _V_NUMBER { + p.p = 1 + backward(p.s, p.p-1) + } + return s, p.p, int(t), nil +} + +// ValidSyntax check if a json has a valid JSON syntax, +// while not validate UTF-8 charset +func _ValidSyntax(json string) bool { + p := NewParserObj(json) + _, e := p.skip() + if e != 0 { + return false + } + if skipBlank(p.s, p.p) != -int(types.ERR_EOF) { + return false + } + return true +} + +// SkipFast skip a json value in fast-skip algs, +// while not strictly validate JSON syntax and UTF-8 charset. +func _SkipFast(src string, i int) (int, int, error) { + p := NewParserObj(src) + p.p = i + s, e := p.skipFast() + if e != 0 { + return -1, -1, p.ExportError(e) + } + t := switchRawType(p.s[s]) + if t == _V_NONE { + return -1, -1, ErrNotExist + } + if t == _V_NUMBER { + p.p = 1 + backward(p.s, p.p-1) } + return s, p.p, nil } diff --git a/vendor/github.com/bytedance/sonic/ast/stubs.go b/vendor/github.com/bytedance/sonic/ast/stubs.go new file mode 100644 index 0000000000..53bf3b8aa0 --- /dev/null +++ b/vendor/github.com/bytedance/sonic/ast/stubs.go @@ -0,0 +1,142 @@ +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ast + +import ( + "unicode/utf8" + "unsafe" + + "github.com/bytedance/sonic/internal/rt" +) + +//go:noescape +//go:linkname memmove runtime.memmove +//goland:noinspection GoUnusedParameter +func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) + +//go:linkname unsafe_NewArray reflect.unsafe_NewArray +//goland:noinspection GoUnusedParameter +func unsafe_NewArray(typ *rt.GoType, n int) unsafe.Pointer + +//go:nosplit +func mem2ptr(s []byte) unsafe.Pointer { + return (*rt.GoSlice)(unsafe.Pointer(&s)).Ptr +} + +var safeSet = [utf8.RuneSelf]bool{ + ' ': true, + '!': true, + '"': false, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '(': true, + ')': true, + '*': true, + '+': true, + ',': true, + '-': true, + '.': true, + '/': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + ':': true, + ';': true, + '<': true, + '=': true, + '>': true, + '?': true, + '@': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'V': true, + 'W': true, + 'X': true, + 'Y': true, + 'Z': true, + '[': true, + '\\': false, + ']': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '{': true, + '|': true, + '}': true, + '~': true, + '\u007f': true, +} + +var hex = "0123456789abcdef" + +//go:linkname unquoteBytes encoding/json.unquoteBytes +func unquoteBytes(s []byte) (t []byte, ok bool) diff --git a/vendor/github.com/bytedance/sonic/ast/stubs_go115.go b/vendor/github.com/bytedance/sonic/ast/stubs_go115.go deleted file mode 100644 index 37b9451f08..0000000000 --- a/vendor/github.com/bytedance/sonic/ast/stubs_go115.go +++ /dev/null @@ -1,55 +0,0 @@ -// +build !go1.20 - -/* - * Copyright 2021 ByteDance Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ast - -import ( - `unsafe` - `unicode/utf8` - - `github.com/bytedance/sonic/internal/rt` -) - -//go:noescape -//go:linkname memmove runtime.memmove -//goland:noinspection GoUnusedParameter -func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) - -//go:linkname unsafe_NewArray reflect.unsafe_NewArray -//goland:noinspection GoUnusedParameter -func unsafe_NewArray(typ *rt.GoType, n int) unsafe.Pointer - -//go:linkname growslice runtime.growslice -//goland:noinspection GoUnusedParameter -func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice - -//go:nosplit -func mem2ptr(s []byte) unsafe.Pointer { - return (*rt.GoSlice)(unsafe.Pointer(&s)).Ptr -} - -var ( - //go:linkname safeSet encoding/json.safeSet - safeSet [utf8.RuneSelf]bool - - //go:linkname hex encoding/json.hex - hex string -) - -//go:linkname unquoteBytes encoding/json.unquoteBytes -func unquoteBytes(s []byte) (t []byte, ok bool) \ No newline at end of file diff --git a/vendor/github.com/bytedance/sonic/ast/stubs_go120.go b/vendor/github.com/bytedance/sonic/ast/stubs_go120.go deleted file mode 100644 index 6f830529d2..0000000000 --- a/vendor/github.com/bytedance/sonic/ast/stubs_go120.go +++ /dev/null @@ -1,55 +0,0 @@ -// +build go1.20 - -/* - * Copyright 2021 ByteDance Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ast - -import ( - `unsafe` - `unicode/utf8` - - `github.com/bytedance/sonic/internal/rt` -) - -//go:noescape -//go:linkname memmove runtime.memmove -//goland:noinspection GoUnusedParameter -func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) - -//go:linkname unsafe_NewArray reflect.unsafe_NewArray -//goland:noinspection GoUnusedParameter -func unsafe_NewArray(typ *rt.GoType, n int) unsafe.Pointer - -//go:linkname growslice reflect.growslice -//goland:noinspection GoUnusedParameter -func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice - -//go:nosplit -func mem2ptr(s []byte) unsafe.Pointer { - return (*rt.GoSlice)(unsafe.Pointer(&s)).Ptr -} - -var ( - //go:linkname safeSet encoding/json.safeSet - safeSet [utf8.RuneSelf]bool - - //go:linkname hex encoding/json.hex - hex string -) - -//go:linkname unquoteBytes encoding/json.unquoteBytes -func unquoteBytes(s []byte) (t []byte, ok bool) diff --git a/vendor/github.com/bytedance/sonic/ast/visitor.go b/vendor/github.com/bytedance/sonic/ast/visitor.go index 4019c31a2e..dc04785133 100644 --- a/vendor/github.com/bytedance/sonic/ast/visitor.go +++ b/vendor/github.com/bytedance/sonic/ast/visitor.go @@ -18,6 +18,7 @@ package ast import ( `encoding/json` + `errors` `github.com/bytedance/sonic/internal/native/types` ) @@ -25,7 +26,7 @@ import ( // Visitor handles the callbacks during preorder traversal of a JSON AST. // // According to the JSON RFC8259, a JSON AST can be defined by -// the following rules without seperator / whitespace tokens. +// the following rules without separator / whitespace tokens. // // JSON-AST = value // value = false / null / true / object / array / number / string @@ -174,6 +175,19 @@ func (self *traverser) decodeArray() error { sp := self.parser.p ns := len(self.parser.s) + /* allocate array space and parse every element */ + if err := self.visitor.OnArrayBegin(_DEFAULT_NODE_CAP); err != nil { + if err == VisitOPSkip { + // NOTICE: for user needs to skip entiry object + self.parser.p -= 1 + if _, e := self.parser.skipFast(); e != 0 { + return e + } + return self.visitor.OnArrayEnd() + } + return err + } + /* check for EOF */ self.parser.p = self.parser.lspace(sp) if self.parser.p >= ns { @@ -183,16 +197,9 @@ func (self *traverser) decodeArray() error { /* check for empty array */ if self.parser.s[self.parser.p] == ']' { self.parser.p++ - if err := self.visitor.OnArrayBegin(0); err != nil { - return err - } return self.visitor.OnArrayEnd() } - /* allocate array space and parse every element */ - if err := self.visitor.OnArrayBegin(_DEFAULT_NODE_CAP); err != nil { - return err - } for { /* decode the value */ if err := self.decodeValue(); err != nil { @@ -223,6 +230,19 @@ func (self *traverser) decodeObject() error { sp := self.parser.p ns := len(self.parser.s) + /* allocate object space and decode each pair */ + if err := self.visitor.OnObjectBegin(_DEFAULT_NODE_CAP); err != nil { + if err == VisitOPSkip { + // NOTICE: for user needs to skip entiry object + self.parser.p -= 1 + if _, e := self.parser.skipFast(); e != 0 { + return e + } + return self.visitor.OnObjectEnd() + } + return err + } + /* check for EOF */ self.parser.p = self.parser.lspace(sp) if self.parser.p >= ns { @@ -232,16 +252,9 @@ func (self *traverser) decodeObject() error { /* check for empty object */ if self.parser.s[self.parser.p] == '}' { self.parser.p++ - if err := self.visitor.OnObjectBegin(0); err != nil { - return err - } return self.visitor.OnObjectEnd() } - /* allocate object space and decode each pair */ - if err := self.visitor.OnObjectBegin(_DEFAULT_NODE_CAP); err != nil { - return err - } for { var njs types.JsonState var err types.ParsingError @@ -313,3 +326,7 @@ func (self *traverser) decodeString(iv int64, ep int) error { } return self.visitor.OnString(out) } + +// If visitor return this error on `OnObjectBegin()` or `OnArrayBegin()`, +// the transverer will skip entiry object or array +var VisitOPSkip = errors.New("") diff --git a/vendor/github.com/bytedance/sonic/compat.go b/vendor/github.com/bytedance/sonic/compat.go index ec414c0cf8..b32342a84a 100644 --- a/vendor/github.com/bytedance/sonic/compat.go +++ b/vendor/github.com/bytedance/sonic/compat.go @@ -1,4 +1,4 @@ -// +build !amd64 !go1.16 go1.22 +// +build !amd64,!arm64 go1.24 !go1.17 arm64,!go1.20 /* * Copyright 2021 ByteDance Inc. @@ -27,6 +27,8 @@ import ( `github.com/bytedance/sonic/option` ) +const apiKind = UseStdJSON + type frozenConfig struct { Config } diff --git a/vendor/github.com/bytedance/sonic/decoder/decoder_amd64.go b/vendor/github.com/bytedance/sonic/decoder/decoder_amd64.go deleted file mode 100644 index 7c28455145..0000000000 --- a/vendor/github.com/bytedance/sonic/decoder/decoder_amd64.go +++ /dev/null @@ -1,68 +0,0 @@ -// +build amd64,go1.16,!go1.22 - -/* -* Copyright 2023 ByteDance Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package decoder - -import ( - `github.com/bytedance/sonic/internal/decoder` -) - -// Decoder is the decoder context object -type Decoder = decoder.Decoder - -// SyntaxError represents json syntax error -type SyntaxError = decoder.SyntaxError - -// MismatchTypeError represents dismatching between json and object -type MismatchTypeError = decoder.MismatchTypeError - -// Options for decode. -type Options = decoder.Options - -const ( - OptionUseInt64 Options = decoder.OptionUseInt64 - OptionUseNumber Options = decoder.OptionUseNumber - OptionUseUnicodeErrors Options = decoder.OptionUseUnicodeErrors - OptionDisableUnknown Options = decoder.OptionDisableUnknown - OptionCopyString Options = decoder.OptionCopyString - OptionValidateString Options = decoder.OptionValidateString -) - -// StreamDecoder is the decoder context object for streaming input. -type StreamDecoder = decoder.StreamDecoder - -var ( - // NewDecoder creates a new decoder instance. - NewDecoder = decoder.NewDecoder - - // NewStreamDecoder adapts to encoding/json.NewDecoder API. - // - // NewStreamDecoder returns a new decoder that reads from r. - NewStreamDecoder = decoder.NewStreamDecoder - - // Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in - // order to reduce the first-hit latency. - // - // Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is - // a compile option to set the depth of recursive compile for the nested struct type. - Pretouch = decoder.Pretouch - - // Skip skips only one json value, and returns first non-blank character position and its ending position if it is valid. - // Otherwise, returns negative error code using start and invalid character position using end - Skip = decoder.Skip -) diff --git a/vendor/github.com/bytedance/sonic/decoder/decoder_compat.go b/vendor/github.com/bytedance/sonic/decoder/decoder_compat.go index 466d842e8e..81e1ae4e31 100644 --- a/vendor/github.com/bytedance/sonic/decoder/decoder_compat.go +++ b/vendor/github.com/bytedance/sonic/decoder/decoder_compat.go @@ -1,4 +1,4 @@ -// +build !amd64 !go1.16 go1.22 +// +build !amd64,!arm64 go1.24 !go1.17 arm64,!go1.20 /* * Copyright 2023 ByteDance Inc. @@ -30,7 +30,7 @@ import ( ) func init() { - println("WARNING: sonic only supports Go1.16~1.20 && CPU amd64, but your environment is not suitable") + println("WARNING: sonic/decoder only supports (Go1.17~1.23 && CPU amd64) or (go1.20~1.23 && CPU arm64), but your environment is not suitable") } const ( @@ -42,6 +42,7 @@ const ( _F_use_number = types.B_USE_NUMBER _F_validate_string = types.B_VALIDATE_STRING _F_allow_control = types.B_ALLOW_CONTROL + _F_no_validate_json = types.B_NO_VALIDATE_JSON ) type Options uint64 @@ -53,6 +54,7 @@ const ( OptionDisableUnknown Options = 1 << _F_disable_unknown OptionCopyString Options = 1 << _F_copy_string OptionValidateString Options = 1 << _F_validate_string + OptionNoValidateJSON Options = 1 << _F_no_validate_json ) func (self *Decoder) SetOptions(opts Options) { @@ -112,10 +114,10 @@ func (self *Decoder) CheckTrailings() error { func (self *Decoder) Decode(val interface{}) error { r := bytes.NewBufferString(self.s) dec := json.NewDecoder(r) - if (self.f | uint64(OptionUseNumber)) != 0 { + if (self.f & uint64(OptionUseNumber)) != 0 { dec.UseNumber() } - if (self.f | uint64(OptionDisableUnknown)) != 0 { + if (self.f & uint64(OptionDisableUnknown)) != 0 { dec.DisallowUnknownFields() } return dec.Decode(val) diff --git a/vendor/github.com/bytedance/sonic/decoder/decoder_native.go b/vendor/github.com/bytedance/sonic/decoder/decoder_native.go new file mode 100644 index 0000000000..9317d57f6c --- /dev/null +++ b/vendor/github.com/bytedance/sonic/decoder/decoder_native.go @@ -0,0 +1,71 @@ +//go:build (amd64 && go1.17 && !go1.24) || (arm64 && go1.20 && !go1.24) +// +build amd64,go1.17,!go1.24 arm64,go1.20,!go1.24 + + +/* +* Copyright 2023 ByteDance Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package decoder + +import ( + `github.com/bytedance/sonic/internal/decoder/api` +) + +// Decoder is the decoder context object +type Decoder = api.Decoder + +// SyntaxError represents json syntax error +type SyntaxError = api.SyntaxError + +// MismatchTypeError represents dismatching between json and object +type MismatchTypeError = api.MismatchTypeError + +// Options for decode. +type Options = api.Options + +const ( + OptionUseInt64 Options = api.OptionUseInt64 + OptionUseNumber Options = api.OptionUseNumber + OptionUseUnicodeErrors Options = api.OptionUseUnicodeErrors + OptionDisableUnknown Options = api.OptionDisableUnknown + OptionCopyString Options = api.OptionCopyString + OptionValidateString Options = api.OptionValidateString + OptionNoValidateJSON Options = api.OptionNoValidateJSON +) + +// StreamDecoder is the decoder context object for streaming input. +type StreamDecoder = api.StreamDecoder + +var ( + // NewDecoder creates a new decoder instance. + NewDecoder = api.NewDecoder + + // NewStreamDecoder adapts to encoding/json.NewDecoder API. + // + // NewStreamDecoder returns a new decoder that reads from r. + NewStreamDecoder = api.NewStreamDecoder + + // Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in + // order to reduce the first-hit latency. + // + // Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is + // a compile option to set the depth of recursive compile for the nested struct type. + Pretouch = api.Pretouch + + // Skip skips only one json value, and returns first non-blank character position and its ending position if it is valid. + // Otherwise, returns negative error code using start and invalid character position using end + Skip = api.Skip +) diff --git a/vendor/github.com/bytedance/sonic/encoder/encoder_amd64.go b/vendor/github.com/bytedance/sonic/encoder/encoder_amd64.go deleted file mode 100644 index 0ef336b724..0000000000 --- a/vendor/github.com/bytedance/sonic/encoder/encoder_amd64.go +++ /dev/null @@ -1,108 +0,0 @@ -// +build amd64,go1.16,!go1.22 - -/* - * Copyright 2023 ByteDance Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package encoder - -import ( - `github.com/bytedance/sonic/internal/encoder` -) - - -// Encoder represents a specific set of encoder configurations. -type Encoder = encoder.Encoder - -// StreamEncoder uses io.Writer as input. -type StreamEncoder = encoder.StreamEncoder - -// Options is a set of encoding options. -type Options = encoder.Options - -const ( - // SortMapKeys indicates that the keys of a map needs to be sorted - // before serializing into JSON. - // WARNING: This hurts performance A LOT, USE WITH CARE. - SortMapKeys Options = encoder.SortMapKeys - - // EscapeHTML indicates encoder to escape all HTML characters - // after serializing into JSON (see https://pkg.go.dev/encoding/json#HTMLEscape). - // WARNING: This hurts performance A LOT, USE WITH CARE. - EscapeHTML Options = encoder.EscapeHTML - - // CompactMarshaler indicates that the output JSON from json.Marshaler - // is always compact and needs no validation - CompactMarshaler Options = encoder.CompactMarshaler - - // NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler - // is always escaped string and needs no quoting - NoQuoteTextMarshaler Options = encoder.NoQuoteTextMarshaler - - // NoNullSliceOrMap indicates all empty Array or Object are encoded as '[]' or '{}', - // instead of 'null' - NoNullSliceOrMap Options = encoder.NoNullSliceOrMap - - // ValidateString indicates that encoder should validate the input string - // before encoding it into JSON. - ValidateString Options = encoder.ValidateString - - // CompatibleWithStd is used to be compatible with std encoder. - CompatibleWithStd Options = encoder.CompatibleWithStd -) - - -var ( - // Encode returns the JSON encoding of val, encoded with opts. - Encode = encoder.Encode - - // EncodeInto is like Encode but uses a user-supplied buffer instead of allocating a new one. - EncodeIndented = encoder.EncodeIndented - - // EncodeIndented is like Encode but applies Indent to format the output. - // Each JSON element in the output will begin on a new line beginning with prefix - // followed by one or more copies of indent according to the indentation nesting. - EncodeInto = encoder.EncodeInto - - // HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029 - // characters inside string literals changed to \u003c, \u003e, \u0026, \u2028, \u2029 - // so that the JSON will be safe to embed inside HTML