Merge pull request #22 from onozaty/develop/v1.14.0

Develop v1.14.0
onozaty · Jan 21, 2023 · 0315aaf · 0315aaf
2 parents 9ba3f5d + 0c87a2a
commit 0315aaf
Show file tree

Hide file tree

Showing 27 changed files with 163 additions and 126 deletions.
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -17,7 +17,7 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.16
+        go-version: 1.19
     - name: Run GoReleaser
       uses: goreleaser/goreleaser-action@v2
       with:

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -9,7 +9,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.16.x]
+        go-version: [1.19.x]
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:

diff --git a/README.md b/README.md
@@ -38,7 +38,7 @@ Global Flags:
       --quote string      (optional) CSV quote. The default is '"'
       --sep string        (optional) CSV record separator. The default is CRLF.
       --allquote          (optional) Always quote CSV fields. The default is to quote only the necessary fields.
-      --encoding string   (optional) CSV encoding. The default is utf-8. Supported encodings: utf-8, shift_jis, euc-jp
+      --encoding string   (optional) CSV encoding. The default is utf-8.
       --bom               (optional) CSV with BOM. When reading, the BOM will be automatically removed without this flag.
 ```
 
@@ -48,6 +48,53 @@ For example, when dealing with TSV files, change the delimiter to a tab as shown
 $ csvt count -i INPUT --delim "\t"
 ```
 
+### Supported encodings
+
+The encodings that can be specified with `--encoding` are as follows.
+
+* utf-8
+* ibm866
+* iso-8859-2
+* iso-8859-3
+* iso-8859-4
+* iso-8859-5
+* iso-8859-6
+* iso-8859-7
+* iso-8859-8
+* iso-8859-8-i
+* iso-8859-10
+* iso-8859-13
+* iso-8859-14
+* iso-8859-15
+* iso-8859-16
+* koi8-r
+* koi8-u
+* macintosh
+* windows-874
+* windows-1250
+* windows-1251
+* windows-1252
+* windows-1253
+* windows-1254
+* windows-1255
+* windows-1256
+* windows-1257
+* windows-1258
+* x-mac-cyrillic
+* gbk
+* gb18030
+* big5
+* euc-jp
+* iso-2022-jp
+* shift_jis
+* euc-kr
+* utf-16be
+* utf-16le
+
+It is defined by W3C.
+
+* http://www.w3.org/TR/encoding
+
 ## add
 
 Create a new CSV file by adding column to the input CSV file.
@@ -1176,7 +1223,7 @@ Flags:
       --out-quote string      (optional) Output CSV quote. The default is '"'
       --out-sep string        (optional) Output CSV record separator. The default is CRLF.
       --out-allquote          (optional) Always quote output CSV fields. The default is to quote only the necessary fields.
-      --out-encoding string   (optional) Output CSV encoding. The default is utf-8. Supported encodings: utf-8, shift_jis, euc-jp
+      --out-encoding string   (optional) Output CSV encoding. The default is utf-8.
       --out-bom               (optional) Output CSV with BOM.
   -h, --help                  help for transform
 ```

diff --git a/cmd/add_test.go b/cmd/add_test.go
@@ -347,7 +347,7 @@ func TestAddCmd_templateParseError(t *testing.T) {
 	})
 
 	err := rootCmd.Execute()
-	if err == nil || err.Error() != `--template is invalid: template: template:1: unexpected "}" in operand` {
+	if err == nil || err.Error() != `--template is invalid: template: template:1: bad character U+007D '}'` {
 		t.Fatal("failed test\n", err)
 	}
 }

diff --git a/cmd/choose.go b/cmd/choose.go
@@ -5,9 +5,9 @@ import (
 	"io"
 
 	"github.com/onozaty/csvt/csv"
-	"github.com/onozaty/csvt/util"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"golang.org/x/exp/slices"
 )
 
 func newChooseCmd() *cobra.Command {
@@ -75,7 +75,7 @@ func choose(reader csv.CsvReader, chooseColumnNames []string, writer csv.CsvWrit
 	chooseColumnIndexes := []int{}
 	for _, chooseColumnName := range chooseColumnNames {
 
-		chooseColumnIndex := util.IndexOf(columnNames, chooseColumnName)
+		chooseColumnIndex := slices.Index(columnNames, chooseColumnName)
 		if chooseColumnIndex == -1 {
 			return fmt.Errorf("missing %s in the CSV file", chooseColumnName)
 		}
@@ -90,7 +90,7 @@ func choose(reader csv.CsvReader, chooseColumnNames []string, writer csv.CsvWrit
 
 		for i, item := range row {
 
-			if util.Contains(chooseColumnIndexes, i) {
+			if slices.Contains(chooseColumnIndexes, i) {
 				filtered = append(filtered, item)
 			}
 		}

diff --git a/cmd/common.go b/cmd/common.go
@@ -7,9 +7,9 @@ import (
 	"strings"
 
 	"github.com/onozaty/csvt/csv"
-	"github.com/onozaty/csvt/util"
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
+	"golang.org/x/exp/slices"
 	"golang.org/x/text/encoding"
 )
 
@@ -134,7 +134,7 @@ func getTargetColumnsIndexes(allColumnNames []string, targetColumnNames []string
 
 func getTargetColumnIndex(allColumnNames []string, targetColumnName string) (int, error) {
 
-	targetColumnIndex := util.IndexOf(allColumnNames, targetColumnName)
+	targetColumnIndex := slices.Index(allColumnNames, targetColumnName)
 	if targetColumnIndex == -1 {
 		return -1, fmt.Errorf("missing %s in the CSV file", targetColumnName)
 	}

diff --git a/cmd/count.go b/cmd/count.go
@@ -5,9 +5,9 @@ import (
 	"io"
 
 	"github.com/onozaty/csvt/csv"
-	"github.com/onozaty/csvt/util"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"golang.org/x/exp/slices"
 )
 
 func newCountCmd() *cobra.Command {
@@ -81,7 +81,7 @@ func count(reader csv.CsvReader, options CountOptions) (int, error) {
 
 	targetColumnIndex := -1
 	if options.targetColumnName != "" {
-		targetColumnIndex = util.IndexOf(columnNames, options.targetColumnName)
+		targetColumnIndex = slices.Index(columnNames, options.targetColumnName)
 		if targetColumnIndex == -1 {
 			return 0, fmt.Errorf("missing %s in the CSV file", options.targetColumnName)
 		}

diff --git a/cmd/exclude.go b/cmd/exclude.go
@@ -5,9 +5,9 @@ import (
 	"io"
 
 	"github.com/onozaty/csvt/csv"
-	"github.com/onozaty/csvt/util"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"golang.org/x/exp/slices"
 )
 
 func newExcludeCmd() *cobra.Command {
@@ -94,7 +94,7 @@ func exclude(reader csv.CsvReader, targetColumnName string, anotherReader csv.Cs
 	if err != nil {
 		return errors.Wrap(err, "failed to read the input CSV file")
 	}
-	inputTargetColumnIndex := util.IndexOf(inputColumnNames, inputTargetColumnName)
+	inputTargetColumnIndex := slices.Index(inputColumnNames, inputTargetColumnName)
 	if inputTargetColumnIndex == -1 {
 		return fmt.Errorf("missing %s in the input CSV file", inputTargetColumnName)
 	}

diff --git a/cmd/header_test.go b/cmd/header_test.go
@@ -201,13 +201,13 @@ func TestHeaderCmd_encoding_shift_jis(t *testing.T) {
 	}
 }
 
-func TestHeaderCmd_encoding_eucjp(t *testing.T) {
+func TestHeaderCmd_encoding_euc_jp(t *testing.T) {
 
 	rootCmd := newRootCmd()
 	rootCmd.SetArgs([]string{
 		"header",
 		"-i", "../testdata/users-eucjp.csv",
-		"--encoding", "eucjp",
+		"--encoding", "EUC-JP",
 	})
 
 	buf := new(bytes.Buffer)
@@ -225,13 +225,13 @@ func TestHeaderCmd_encoding_eucjp(t *testing.T) {
 	}
 }
 
-func TestHeaderCmd_encoding_euc_jp(t *testing.T) {
+func TestHeaderCmd_encoding_koi8_r(t *testing.T) {
 
 	rootCmd := newRootCmd()
 	rootCmd.SetArgs([]string{
 		"header",
-		"-i", "../testdata/users-eucjp.csv",
-		"--encoding", "EUC-JP",
+		"-i", "../testdata/users-koi8r.csv",
+		"--encoding", "koi8-r",
 	})
 
 	buf := new(bytes.Buffer)
@@ -244,7 +244,55 @@ func TestHeaderCmd_encoding_euc_jp(t *testing.T) {
 
 	result := buf.String()
 
-	if result != "ID\n名前\n年齢\n" {
+	if result != "ID\nНазовите\nвозраст\n" {
+		t.Fatal("failed test\n", result)
+	}
+}
+
+func TestHeaderCmd_encoding_euc_kr(t *testing.T) {
+
+	rootCmd := newRootCmd()
+	rootCmd.SetArgs([]string{
+		"header",
+		"-i", "../testdata/users-euckr.csv",
+		"--encoding", "euc-kr",
+	})
+
+	buf := new(bytes.Buffer)
+	rootCmd.SetOutput(buf)
+
+	err := rootCmd.Execute()
+	if err != nil {
+		t.Fatal("failed test\n", err)
+	}
+
+	result := buf.String()
+
+	if result != "ID\n이름\n나이\n" {
+		t.Fatal("failed test\n", result)
+	}
+}
+
+func TestHeaderCmd_encoding_big5(t *testing.T) {
+
+	rootCmd := newRootCmd()
+	rootCmd.SetArgs([]string{
+		"header",
+		"-i", "../testdata/users-big5.csv",
+		"--encoding", "big5",
+	})
+
+	buf := new(bytes.Buffer)
+	rootCmd.SetOutput(buf)
+
+	err := rootCmd.Execute()
+	if err != nil {
+		t.Fatal("failed test\n", err)
+	}
+
+	result := buf.String()
+
+	if result != "ID\n名稱\n年齡\n" {
 		t.Fatal("failed test\n", result)
 	}
 }

diff --git a/cmd/include.go b/cmd/include.go
@@ -5,9 +5,9 @@ import (
 	"io"
 
 	"github.com/onozaty/csvt/csv"
-	"github.com/onozaty/csvt/util"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"golang.org/x/exp/slices"
 )
 
 func newIncludeCmd() *cobra.Command {
@@ -94,7 +94,7 @@ func include(reader csv.CsvReader, targetColumnName string, anotherReader csv.Cs
 	if err != nil {
 		return errors.Wrap(err, "failed to read the input CSV file")
 	}
-	inputTargetColumnIndex := util.IndexOf(inputColumnNames, inputTargetColumnName)
+	inputTargetColumnIndex := slices.Index(inputColumnNames, inputTargetColumnName)
 	if inputTargetColumnIndex == -1 {
 		return fmt.Errorf("missing %s in the input CSV file", inputTargetColumnName)
 	}

diff --git a/cmd/join.go b/cmd/join.go
@@ -8,6 +8,7 @@ import (
 	"github.com/onozaty/csvt/util"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"golang.org/x/exp/slices"
 )
 
 func newJoinCmd() *cobra.Command {
@@ -117,7 +118,7 @@ func join(first csv.CsvReader, second csv.CsvReader, joinColumnName string, writ
 	if err != nil {
 		return errors.Wrap(err, "failed to read the first CSV file")
 	}
-	firstJoinColumnIndex := util.IndexOf(firstColumnNames, firstJoinColumnName)
+	firstJoinColumnIndex := slices.Index(firstColumnNames, firstJoinColumnName)
 	if firstJoinColumnIndex == -1 {
 		return fmt.Errorf("missing %s in the first CSV file", firstJoinColumnName)
 	}

diff --git a/cmd/remove.go b/cmd/remove.go
@@ -5,9 +5,9 @@ import (
 	"io"
 
 	"github.com/onozaty/csvt/csv"
-	"github.com/onozaty/csvt/util"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"golang.org/x/exp/slices"
 )
 
 func newRemoveCmd() *cobra.Command {
@@ -75,7 +75,7 @@ func remove(reader csv.CsvReader, removeColumnNames []string, writer csv.CsvWrit
 	removeColumnIndexes := []int{}
 	for _, removeColumnName := range removeColumnNames {
 
-		removeColumnIndex := util.IndexOf(columnNames, removeColumnName)
+		removeColumnIndex := slices.Index(columnNames, removeColumnName)
 		if removeColumnIndex == -1 {
 			return fmt.Errorf("missing %s in the CSV file", removeColumnName)
 		}
@@ -90,7 +90,7 @@ func remove(reader csv.CsvReader, removeColumnNames []string, writer csv.CsvWrit
 
 		for i, item := range row {
 
-			if !util.Contains(removeColumnIndexes, i) {
+			if !slices.Contains(removeColumnIndexes, i) {
 				filtered = append(filtered, item)
 			}
 		}

diff --git a/cmd/rename.go b/cmd/rename.go
@@ -5,9 +5,9 @@ import (
 	"io"
 
 	"github.com/onozaty/csvt/csv"
-	"github.com/onozaty/csvt/util"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"golang.org/x/exp/slices"
 )
 
 func newRenameCmd() *cobra.Command {
@@ -85,7 +85,7 @@ func rename(reader csv.CsvReader, targetColumnNames []string, afterColumnNames [
 	targetColumnIndexes := []int{}
 	for _, targetColumnName := range targetColumnNames {
 
-		targetColumnIndex := util.IndexOf(columnNames, targetColumnName)
+		targetColumnIndex := slices.Index(columnNames, targetColumnName)
 		if targetColumnIndex == -1 {
 			return fmt.Errorf("missing %s in the CSV file", targetColumnName)
 		}

diff --git a/cmd/root.go b/cmd/root.go
@@ -20,7 +20,7 @@ func newRootCmd() *cobra.Command {
 	rootCmd.PersistentFlags().StringP("quote", "", "", "(optional) CSV quote. The default is '\"'")
 	rootCmd.PersistentFlags().StringP("sep", "", "", "(optional) CSV record separator. The default is CRLF.")
 	rootCmd.PersistentFlags().BoolP("allquote", "", false, "(optional) Always quote CSV fields. The default is to quote only the necessary fields.")
-	rootCmd.PersistentFlags().StringP("encoding", "", "", "(optional) CSV encoding. The default is utf-8. Supported encodings: utf-8, shift_jis, euc-jp")
+	rootCmd.PersistentFlags().StringP("encoding", "", "", "(optional) CSV encoding. The default is utf-8.")
 	rootCmd.PersistentFlags().BoolP("bom", "", false, "(optional) CSV with BOM. When reading, the BOM will be automatically removed without this flag.")
 	rootCmd.PersistentFlags().SortFlags = false
 	rootCmd.Flags().SortFlags = false

diff --git a/cmd/transform.go b/cmd/transform.go
@@ -45,7 +45,7 @@ func newTransformCmd() *cobra.Command {
 	transformCmd.Flags().StringP("out-quote", "", "", "(optional) Output CSV quote. The default is '\"'")
 	transformCmd.Flags().StringP("out-sep", "", "", "(optional) Output CSV record separator. The default is CRLF.")
 	transformCmd.Flags().BoolP("out-allquote", "", false, "(optional) Always quote output CSV fields. The default is to quote only the necessary fields.")
-	transformCmd.Flags().StringP("out-encoding", "", "", "(optional) Output CSV encoding. The default is utf-8. Supported encodings: utf-8, shift_jis, euc-jp")
+	transformCmd.Flags().StringP("out-encoding", "", "", "(optional) Output CSV encoding. The default is utf-8.")
 	transformCmd.Flags().BoolP("out-bom", "", false, "(optional) Output CSV with BOM.")
 
 	return transformCmd