classical convert

Author: ljcbaby

convert/classical.go

@@ -1 +1,69 @@
 package convert
+
+import (
+	"bufio"
+	"os"
+	"strings"
+
+	"github.com/dn-11/provider2domainset/conf"
+	"github.com/dn-11/provider2domainset/log"
+	"github.com/pkg/errors"
+)
+
+func convertClassical(s *os.File, t *os.File) error {
+	scanner := bufio.NewScanner(s)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		line = strings.Split(line, "#")[0]
+		line = strings.TrimSpace(line)
+
+		if line == "payload:" {
+			continue
+		}
+		line = strings.TrimPrefix(line, "- ")
+
+		if len(line) == 0 {
+			continue
+		}
+
+		args := strings.Split(line, ",")
+		if len(args) < 2 {
+			log.L().Sugar().Warnf("invalid line: %s", line)
+		}
+
+		res := ""
+
+		switch args[0] {
+		case "DOMAIN":
+			res = "full:" + args[1]
+		case "DOMAIN-SUFFIX":
+			res = "domain:" + args[1]
+		case "DOMAIN-KEYWORD":
+			res = "keyword:" + args[1]
+		case "DOMAIN-REGEX":
+			if conf.Convert.EnableRegex {
+				res = "regex:" + args[1]
+			} else {
+				log.L().Sugar().Warnf("regex is disabled: %s", line)
+				continue
+			}
+		default:
+			log.L().Sugar().Warnf("unknown type: %s", line)
+			continue
+		}
+
+		log.L().Sugar().Debugf("convert line: %s -> %s", line, res)
+
+		if _, err := t.WriteString(res + "\n"); err != nil {
+			return errors.Wrap(err, "write target file failed")
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return errors.Wrap(err, "scan source file failed")
+	}
+
+	return nil
+}

convert/classical_test.go

@@ -1 +1,58 @@
 package convert_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/dn-11/provider2domainset/conf"
+	"github.com/dn-11/provider2domainset/convert"
+	"github.com/dn-11/provider2domainset/log"
+	"go.uber.org/zap/zapcore"
+)
+
+func TestConvertClassicalTxt(t *testing.T) {
+	log.L().SetLogLevel(zapcore.DebugLevel)
+
+	pwd, err := os.Getwd()
+	if err != nil {
+		t.Error(err)
+	}
+
+	task := convert.Task{
+		Source: filepath.Join(pwd, "test_case"),
+		Target: filepath.Join(pwd, "test_output"),
+		File: conf.File{
+			Name: "classical.txt",
+			Type: "classical",
+		},
+	}
+
+	err = convert.Convert(task)
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func TestConvertClassicalYaml(t *testing.T) {
+	log.L().SetLogLevel(zapcore.DebugLevel)
+
+	pwd, err := os.Getwd()
+	if err != nil {
+		t.Error(err)
+	}
+
+	task := convert.Task{
+		Source: filepath.Join(pwd, "test_case"),
+		Target: filepath.Join(pwd, "test_output"),
+		File: conf.File{
+			Name: "classical.yaml",
+			Type: "classical",
+		},
+	}
+
+	err = convert.Convert(task)
+	if err != nil {
+		t.Error(err)
+	}
+}

convert/convert.go

@@ -4,14 +4,19 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/dn-11/provider2domainset/conf"
+	"github.com/dn-11/provider2domainset/log"
 	"github.com/pkg/errors"
 )
 
 func Convert(t Task) error {
-	source, err := os.ReadFile(filepath.Join(t.Source, t.Name))
+	log.L().Sugar().Infof("convert %s of %s", t.Name, t.Type)
+
+	source, err := os.Open(filepath.Join(t.Source, t.Name))
 	if err != nil {
 		return errors.Wrap(err, "read source file failed")
 	}
+	defer source.Close()
 
 	err = os.MkdirAll(t.Target, os.ModePerm)
 	if err != nil {
@@ -22,10 +27,18 @@ func Convert(t Task) error {
 	if err != nil {
 		return errors.Wrap(err, "create target file failed")
 	}
+	defer target.Close()
 
-	_, err = target.Write(source)
-	if err != nil {
-		return errors.Wrap(err, "write target file failed")
+	switch t.Type {
+	case conf.TypeClassical:
+		err = convertClassical(source, target)
+		if err != nil {
+			return errors.Wrap(err, "convert classical failed")
+		}
+	// case conf.TypeDomain:
+	// 	err = convertDomain(source, target)
+	default:
+		return errors.New("unknown convert type")
 	}
 
 	return nil

convert/convert_test.go

@@ -35,8 +35,9 @@ func Init() (*Logger, error) {
 			EncodeTime:    zapcore.ISO8601TimeEncoder,
 			EncodeCaller:  zapcore.ShortCallerEncoder,
 		},
-		OutputPaths:      []string{"stdout"},
-		ErrorOutputPaths: []string{"stderr"},
+		OutputPaths:       []string{"stdout"},
+		ErrorOutputPaths:  []string{"stderr"},
+		DisableStacktrace: true,
 	}
 
 	var err error