Skip to content

Commit

Permalink
fix: make some changes under the review of yuki.
Browse files Browse the repository at this point in the history
Signed-off-by: Electronic-Waste <[email protected]>
  • Loading branch information
Electronic-Waste committed Apr 3, 2024
1 parent 9f7d255 commit da94a7d
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -36,26 +36,26 @@ import (
)

var (
ErrFileFormat = fmt.Errorf("format must be set %v or %v", commonv1beta1.TextFormat, commonv1beta1.JsonFormat)
ErrOpenFile = errors.New("failed to open the file")
ErrReadFile = errors.New("failed to read the file")
ErrParseJson = errors.New("failed to parse the json object")
errFileFormat = fmt.Errorf("format must be set %v or %v", commonv1beta1.TextFormat, commonv1beta1.JsonFormat)
errOpenFile = errors.New("failed to open the file")
errReadFile = errors.New("failed to read the file")
errParseJson = errors.New("failed to parse the json object")
)

func CollectObservationLog(fileName string, metrics []string, filters []string, fileFormat commonv1beta1.FileFormat) (*v1beta1.ObservationLog, error) {
// we should check fileFormat first in case of opening an invalid file
if fileFormat != commonv1beta1.JsonFormat && fileFormat != commonv1beta1.TextFormat {
return nil, ErrFileFormat
return nil, errFileFormat
}

file, err := os.Open(fileName)
if err != nil {
return nil, fmt.Errorf("%w: %s", ErrOpenFile, err.Error())
return nil, fmt.Errorf("%w: %s", errOpenFile, err.Error())
}
defer file.Close()
content, err := io.ReadAll(file)
if err != nil {
return nil, fmt.Errorf("%w: %s", ErrReadFile, err.Error())
return nil, fmt.Errorf("%w: %s", errReadFile, err.Error())
}
logs := string(content)

Expand All @@ -64,8 +64,9 @@ func CollectObservationLog(fileName string, metrics []string, filters []string,
return parseLogsInTextFormat(strings.Split(logs, "\n"), metrics, filters)
case commonv1beta1.JsonFormat:
return parseLogsInJsonFormat(strings.Split(logs, "\n"), metrics)
default:
return nil, nil
}
return nil, nil
}

func parseLogsInTextFormat(logs []string, metrics []string, filters []string) (*v1beta1.ObservationLog, error) {
Expand Down Expand Up @@ -133,7 +134,7 @@ func parseLogsInJsonFormat(logs []string, metrics []string) (*v1beta1.Observatio
}
var jsonObj map[string]interface{}
if err := json.Unmarshal([]byte(logline), &jsonObj); err != nil {
return nil, fmt.Errorf("%w: %s", ErrParseJson, err.Error())
return nil, fmt.Errorf("%w: %s", errParseJson, err.Error())
}

timestamp := time.Time{}.UTC().Format(time.RFC3339)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -185,19 +185,19 @@ func TestCollectObservationLog(t *testing.T) {
"Invalid file name": {
fileName: "invalid",
fileFormat: commonv1beta1.JsonFormat,
wantError: ErrOpenFile,
wantError: errOpenFile,
},
"Invalid file format": {
fileName: "good.log",
fileFormat: "invalid",
wantError: ErrFileFormat,
wantError: errFileFormat,
},
"Invalid formatted file for logs in JSON format": {
fileName: "invalid-format.json",
testData: `"checkpoint_path": "", "global_step": "0", "loss": "0.22082142531871796", "timestamp": 1638422847.28721, "trial": "0"
{"acc": "0.9349666833877563", "checkpoint_path": "", "global_step": "0", "timestamp": 1638422847.287801, "trial": "0`,
fileFormat: commonv1beta1.JsonFormat,
wantError: ErrParseJson,
wantError: errParseJson,
},
"Invalid formatted file for logs in TEXT format": {
fileName: "invalid-format.log",
Expand Down Expand Up @@ -313,7 +313,7 @@ invalid INFO {metricName: loss, metricValue: 0.3634}`,
t.Run(name, func(t *testing.T) {
if test.testData != "" {
if err := os.WriteFile(filepath.Join(tmpDir, test.fileName), []byte(test.testData), 0600); err != nil {
t.Fatal(err)
t.Fatalf("failed to write test data: %v", err)
}
}
actual, err := CollectObservationLog(filepath.Join(tmpDir, test.fileName), test.metrics, test.filters, test.fileFormat)
Expand Down

0 comments on commit da94a7d

Please sign in to comment.