fix 'decode' tool command range handling

streamingfast · Sep 4, 2024 · b4f4bfc · b4f4bfc
1 parent 74e0072
commit b4f4bfc
Show file tree

Hide file tree

Showing 7 changed files with 78 additions and 48 deletions.
diff --git a/block/range.go b/block/range.go
@@ -8,20 +8,43 @@ import (
 	"go.uber.org/zap/zapcore"
 )
 
-func ParseRange(in string) *Range {
+func MustParseRange(in string) *Range {
+	out, err := ParseRange(in, 0)
+	if err != nil {
+		panic(err)
+	}
+	return out
+}
+
+func ParseRange(in string, defaultSegmentSize uint64) (*Range, error) {
 	if in == "" {
-		return nil
+		return nil, nil
+	}
+	var ch []string
+	switch {
+	case strings.Contains(in, "-"):
+		ch = strings.Split(in, "-")
+	case strings.Contains(in, ":"):
+		ch = strings.Split(in, ":")
+	default:
+		ch = []string{in}
 	}
-	ch := strings.Split(in, "-")
+
 	lo, err := strconv.ParseInt(ch[0], 10, 64)
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
-	hi, err := strconv.ParseInt(ch[1], 10, 64)
-	if err != nil {
-		panic(err)
+
+	if len(ch) == 2 {
+		hi, err := strconv.ParseInt(ch[1], 10, 64)
+		return NewRange(uint64(lo), uint64(hi)), err
 	}
-	return NewRange(uint64(lo), uint64(hi))
+
+	if defaultSegmentSize == 0 {
+		return nil, fmt.Errorf("ParseRange: invalid range format: %q", in)
+	}
+
+	return NewRange(uint64(lo), uint64(lo)+defaultSegmentSize), nil
 }
 
 type Range struct {

diff --git a/block/ranges.go b/block/ranges.go
@@ -5,9 +5,9 @@ import (
 	"strings"
 )
 
-func ParseRanges(in string) (out Ranges) {
+func MustParseRanges(in string) (out Ranges) {
 	for _, e := range strings.Split(in, ",") {
-		newRange := ParseRange(strings.Trim(e, " "))
+		newRange := MustParseRange(strings.Trim(e, " "))
 		if newRange != nil {
 			out = append(out, newRange)
 		}

diff --git a/block/ranges_test.go b/block/ranges_test.go
@@ -7,47 +7,47 @@ import (
 )
 
 func TestRangeMerged(t *testing.T) {
-	assert.Equal(t, ParseRanges("10-40,50-70").String(), ParseRanges("10-20,20-30,30-40,50-60,60-70").Merged().String())
-	assert.Equal(t, ParseRanges("10-40,60-70").String(), ParseRanges("10-20,20-30,30-40,60-70").Merged().String())
-	assert.Equal(t, ParseRanges("10-40").String(), ParseRanges("10-20,20-30,30-40").Merged().String())
-	assert.Equal(t, ParseRanges("1-5,10-12,13-14").String(), ParseRanges("1-2,2-3,3-4,4-5,10-12,13-14").Merged().String())
+	assert.Equal(t, MustParseRanges("10-40,50-70").String(), MustParseRanges("10-20,20-30,30-40,50-60,60-70").Merged().String())
+	assert.Equal(t, MustParseRanges("10-40,60-70").String(), MustParseRanges("10-20,20-30,30-40,60-70").Merged().String())
+	assert.Equal(t, MustParseRanges("10-40").String(), MustParseRanges("10-20,20-30,30-40").Merged().String())
+	assert.Equal(t, MustParseRanges("1-5,10-12,13-14").String(), MustParseRanges("1-2,2-3,3-4,4-5,10-12,13-14").Merged().String())
 }
 
 func TestRangeMergedBuckets(t *testing.T) {
 	assert.Equal(t,
-		ParseRanges("1-10,10-11").String(),
-		ParseRanges("1-10,10-11").MergedBuckets(10).String(),
+		MustParseRanges("1-10,10-11").String(),
+		MustParseRanges("1-10,10-11").MergedBuckets(10).String(),
 	)
 	assert.Equal(t,
-		ParseRanges("1-10,10-12").String(),
-		ParseRanges("1-10,10-12").MergedBuckets(10).String(),
+		MustParseRanges("1-10,10-12").String(),
+		MustParseRanges("1-10,10-12").MergedBuckets(10).String(),
 	)
 	assert.Equal(t,
-		ParseRanges("10-30,30-40,50-70").String(),
-		ParseRanges("10-20,20-30,30-40,50-60,60-70").MergedBuckets(20).String(),
+		MustParseRanges("10-30,30-40,50-70").String(),
+		MustParseRanges("10-20,20-30,30-40,50-60,60-70").MergedBuckets(20).String(),
 	)
 	assert.Equal(t,
-		ParseRanges("10-30,30-50,50-60,80-100").String(),
-		ParseRanges("10-20,20-30,30-40,40-50,50-60,80-90,90-100").MergedBuckets(20).String(),
+		MustParseRanges("10-30,30-50,50-60,80-100").String(),
+		MustParseRanges("10-20,20-30,30-40,40-50,50-60,80-90,90-100").MergedBuckets(20).String(),
 	)
 	assert.Equal(t,
-		ParseRanges("10-20,20-30,30-40").String(),
-		ParseRanges("10-20,20-30,30-40").MergedBuckets(5).String(),
+		MustParseRanges("10-20,20-30,30-40").String(),
+		MustParseRanges("10-20,20-30,30-40").MergedBuckets(5).String(),
 	)
 	assert.Equal(t,
-		ParseRanges("10-20,20-30,30-40,40-50").String(),
-		ParseRanges("10-20,20-30,30-40,40-50").MergedBuckets(11).String(),
+		MustParseRanges("10-20,20-30,30-40,40-50").String(),
+		MustParseRanges("10-20,20-30,30-40,40-50").MergedBuckets(11).String(),
 	)
 	assert.Equal(t,
-		ParseRanges("10-20,20-30,30-40,40-50").String(),
-		ParseRanges("10-20,20-30,30-40,40-50").MergedBuckets(19).String(),
+		MustParseRanges("10-20,20-30,30-40,40-50").String(),
+		MustParseRanges("10-20,20-30,30-40,40-50").MergedBuckets(19).String(),
 	)
 	assert.Equal(t,
-		ParseRanges("10-30,30-50").String(),
-		ParseRanges("10-20,20-30,30-40,40-50").MergedBuckets(20).String(),
+		MustParseRanges("10-30,30-50").String(),
+		MustParseRanges("10-20,20-30,30-40,40-50").MergedBuckets(20).String(),
 	)
 	assert.Equal(t,
-		ParseRanges("1-4,4-5,10-12,13-14").String(),
-		ParseRanges("1-2,2-3,3-4,4-5,10-12,13-14").MergedBuckets(3).String(),
+		MustParseRanges("1-4,4-5,10-12,13-14").String(),
+		MustParseRanges("1-2,2-3,3-4,4-5,10-12,13-14").MergedBuckets(3).String(),
 	)
 }
diff --git a/block/segmenter_test.go b/block/segmenter_test.go
@@ -92,17 +92,17 @@ func TestSegmenter_firstRange(t *testing.T) {
 
 func TestSegmenter_followingRange(t *testing.T) {
 	s := NewSegmenter(10, 1, 100)
-	assert.Equal(t, ParseRange("0-10"), s.followingRange(0))
+	assert.Equal(t, MustParseRange("0-10"), s.followingRange(0))
 	s = NewSegmenter(10, 1, 100)
-	assert.Equal(t, ParseRange("10-20"), s.followingRange(1))
+	assert.Equal(t, MustParseRange("10-20"), s.followingRange(1))
 	s = NewSegmenter(10, 1, 15)
-	assert.Equal(t, ParseRange("10-15"), s.followingRange(1))
+	assert.Equal(t, MustParseRange("10-15"), s.followingRange(1))
 	s = NewSegmenter(10, 1, 25)
-	assert.Equal(t, ParseRange("20-25"), s.followingRange(2))
+	assert.Equal(t, MustParseRange("20-25"), s.followingRange(2))
 	s = NewSegmenter(10, 15, 25)
-	assert.Equal(t, ParseRange("20-25"), s.followingRange(2))
+	assert.Equal(t, MustParseRange("20-25"), s.followingRange(2))
 	s = NewSegmenter(10, 15, 25)
-	assert.Equal(t, ParseRange("10-20"), s.followingRange(1))
+	assert.Equal(t, MustParseRange("10-20"), s.followingRange(1))
 }
 
 func TestSegmenter_Range(t *testing.T) {
@@ -113,15 +113,15 @@ func TestSegmenter_Range(t *testing.T) {
 	assert.Nil(t, s.Range(0))
 	assert.Equal(t, 1, s.FirstIndex())
 	assert.Equal(t, 2, s.LastIndex())
-	assert.Equal(t, ParseRange("15-20"), s.Range(1))
-	assert.Equal(t, ParseRange("20-25"), s.Range(2))
+	assert.Equal(t, MustParseRange("15-20"), s.Range(1))
+	assert.Equal(t, MustParseRange("20-25"), s.Range(2))
 	assert.Nil(t, s.Range(3))
 
 	s = NewSegmenter(10, 1, 99)
 	assert.Equal(t, 10, s.Count())
 	assert.Equal(t, 0, s.FirstIndex())
 	assert.Equal(t, 9, s.LastIndex())
-	assert.Equal(t, ParseRange("90-99"), s.Range(9))
+	assert.Equal(t, MustParseRange("90-99"), s.Range(9))
 	assert.False(t, s.EndsOnInterval(9))
 
 	s = NewSegmenter(10, 1, 15)
@@ -131,7 +131,7 @@ func TestSegmenter_Range(t *testing.T) {
 	assert.Equal(t, 2, s.Count())
 	assert.Equal(t, 0, s.FirstIndex())
 	assert.Equal(t, 1, s.LastIndex())
-	assert.Equal(t, ParseRange("10-20"), s.Range(1))
+	assert.Equal(t, MustParseRange("10-20"), s.Range(1))
 	assert.True(t, s.EndsOnInterval(1))
 
 }
diff --git a/orchestrator/stage/stages_test.go b/orchestrator/stage/stages_test.go
@@ -34,10 +34,10 @@ func TestNewStages(t *testing.T) {
 	assert.Equal(t, 8, stages.storeSegmenter.IndexForStartBlock(80), "index out of range still returned here")
 	assert.Nil(t, stages.storeSegmenter.Range(8), "out of range")
 
-	assert.Equal(t, block.ParseRange("5-10"), stages.storeSegmenter.Range(0))
-	assert.Equal(t, block.ParseRange("10-20"), stages.storeSegmenter.Range(1))
-	assert.Equal(t, block.ParseRange("70-75"), stages.storeSegmenter.Range(7))
-	assert.Equal(t, block.ParseRange("70-75"), stages.globalSegmenter.Range(7))
+	assert.Equal(t, block.MustParseRange("5-10"), stages.storeSegmenter.Range(0))
+	assert.Equal(t, block.MustParseRange("10-20"), stages.storeSegmenter.Range(1))
+	assert.Equal(t, block.MustParseRange("70-75"), stages.storeSegmenter.Range(7))
+	assert.Equal(t, block.MustParseRange("70-75"), stages.globalSegmenter.Range(7))
 }
 
 func unit(seg, stage int) Unit {

diff --git a/storage/store/testing.go b/storage/store/testing.go
@@ -40,7 +40,7 @@ func fileFromRange(kind string, in string, params ...FileInfoParam) *FileInfo {
 }
 
 func fileFromRanges(kind string, in string, params ...FileInfoParam) FileInfos {
-	ranges := block.ParseRanges(in)
+	ranges := block.MustParseRanges(in)
 
 	files := make([]*FileInfo, len(ranges))
 	for i, blockRange := range ranges {

diff --git a/tools/decode.go b/tools/decode.go
@@ -82,6 +82,7 @@ var decodeIndexModuleCmd = &cobra.Command{
 
 func init() {
 	decodeCmd.PersistentFlags().Uint64("save-interval", 1000, "Save interval (segment size)")
+	decodeCmd.PersistentFlags().Bool("use-test-simple-hash", false, "Use the 'simple hashing' function to get module hashes instead of regular hashes, for testing purposes")
 
 	decodeCmd.AddCommand(decodeOutputsModuleCmd)
 	decodeCmd.AddCommand(decodeStatesModuleCmd)
@@ -93,6 +94,7 @@ func init() {
 func runDecodeStatesModuleRunE(cmd *cobra.Command, args []string) error {
 	ctx := cmd.Context()
 	saveInterval := sflags.MustGetUint64(cmd, "save-interval")
+	manifest.TestUseSimpleHash = sflags.MustGetBool(cmd, "use-test-simple-hash")
 
 	manifestPath := ""
 	if len(args) == 5 {
@@ -168,6 +170,7 @@ func runDecodeStatesModuleRunE(cmd *cobra.Command, args []string) error {
 func runDecodeIndexModuleRunE(cmd *cobra.Command, args []string) error {
 	ctx := cmd.Context()
 	saveInterval := sflags.MustGetUint64(cmd, "save-interval")
+	manifest.TestUseSimpleHash = sflags.MustGetBool(cmd, "use-test-simple-hash")
 
 	manifestPath := ""
 	if len(args) == 4 {
@@ -249,6 +252,7 @@ func runDecodeIndexModuleRunE(cmd *cobra.Command, args []string) error {
 func runDecodeOutputsModuleRunE(cmd *cobra.Command, args []string) error {
 	ctx := cmd.Context()
 	saveInterval := sflags.MustGetUint64(cmd, "save-interval")
+	manifest.TestUseSimpleHash = sflags.MustGetBool(cmd, "use-test-simple-hash")
 
 	manifestPath := ""
 	if len(args) == 4 {
@@ -259,7 +263,10 @@ func runDecodeOutputsModuleRunE(cmd *cobra.Command, args []string) error {
 	moduleName := args[0]
 	storeURL := args[1]
 
-	requestedBlocks := block.ParseRange(args[2]) // FIXME: this panics on error :(
+	requestedBlocks, err := block.ParseRange(args[2], saveInterval)
+	if err != nil {
+		return fmt.Errorf("parsing range %q: %w", args[2], err)
+	}
 
 	zlog.Info("decoding module",
 		zap.String("manifest_path", manifestPath),