Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

initial work for fs api forwarding #49

Merged
merged 5 commits into from
Sep 30, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions filesys/fd.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
//go:build darwin || linux

package filesys

func FdType(fd int) int {
return fd
}
7 changes: 7 additions & 0 deletions filesys/fd_windows.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package filesys

import "syscall"

func FdType(fd int) syscall.Handle {
return syscall.Handle(fd)
}
300 changes: 300 additions & 0 deletions filesys/handler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,300 @@
package filesys

import (
"encoding/base64"
"encoding/json"
"log"
"net/http"
"os"
"strings"
"syscall"
)

// Handler translates json payload data to and from system calls like syscall.Stat
type Handler struct {
debug bool
securityToken string
logger *log.Logger
}

func NewHandler(securityToken string, logger *log.Logger) *Handler {
return &Handler{
debug: false,
securityToken: securityToken,
logger: logger,
}
}

func (fa *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("WBT-Token") != fa.securityToken {
fa.doError("not implemented", "ENOSYS", w)
return
}
switch r.URL.Path {
case "/fs/stat":
fa.handle(&Stat{}, w, r)
case "/fs/fstat":
fa.handle(&Fstat{}, w, r)
case "/fs/open":
fa.handle(&Open{}, w, r)
case "/fs/write":
fa.handle(&Write{}, w, r)
case "/fs/close":
fa.handle(&Close{}, w, r)
case "/fs/rename":
fa.handle(&Rename{}, w, r)
case "/fs/readdir":
fa.handle(&Readdir{}, w, r)
case "/fs/lstat":
fa.handle(&Lstat{}, w, r)
case "/fs/read":
fa.handle(&Read{}, w, r)
case "/fs/mkdir":
fa.handle(&Mkdir{}, w, r)
case "/fs/unlink":
fa.handle(&Unlink{}, w, r)
case "/fs/rmdir":
fa.handle(&Rmdir{}, w, r)
default:
fa.doError("not implemented", "ENOSYS", w)
}
}

type Responder interface {
WriteResponse(fa *Handler, w http.ResponseWriter)
}

func (fa *Handler) handle(responder Responder, w http.ResponseWriter, r *http.Request) {
if err := json.NewDecoder(r.Body).Decode(responder); err != nil {
fa.logger.Printf("ERROR handle : %s\n", err)
w.WriteHeader(http.StatusBadRequest)
return
}
if fa.debug {
fa.logger.Printf("handle %s %+v\n", r.URL.Path, responder)
}
responder.WriteResponse(fa, w)
}

type ErrorCode struct {
Error string `json:"error"`
Code string `json:"code"`
}

func (fa *Handler) doError(msg, code string, w http.ResponseWriter) {
if fa.debug {
fa.logger.Printf("doError %s : %s\n", msg, code)
}
e := &ErrorCode{Error: msg, Code: code}

w.WriteHeader(http.StatusBadRequest)
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(e); err != nil {
fa.logger.Println("doError json error :", err)
mlctrez marked this conversation as resolved.
Show resolved Hide resolved
}
}

func (fa *Handler) okResponse(data any, w http.ResponseWriter) {
if marshal, err := json.Marshal(data); err != nil {
fa.logger.Println("okResponse json error:", err)
w.WriteHeader(http.StatusInternalServerError)
mlctrez marked this conversation as resolved.
Show resolved Hide resolved
} else {
if fa.debug {
fa.logger.Printf("okResponse %s\n", string(marshal))
}

w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(marshal)
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Missing error handling

}
}

func fixPath(path string) string {
return strings.TrimPrefix(path, "/fs/")
}

type Stat struct {
Path string `json:"path,omitempty"`
}

type Open struct {
Path string `json:"path"`
Flags int `json:"flags"`
Mode uint32 `json:"mode"`
}

func (o *Open) WriteResponse(fa *Handler, w http.ResponseWriter) {
fd, err := syscall.Open(fixPath(o.Path), o.Flags, o.Mode)
if fa.handleError(w, err, true) {
return
}
response := map[string]any{"fd": fd}
fa.okResponse(response, w)
}

type Fstat struct {
Fd int `json:"fd"`
}

type Write struct {
Fd int `json:"fd"`
Buffer string `json:"buffer"`
Offset int `json:"offset"`
Length int `json:"length"`
Position *int `json:"position,omitempty"`
}

func (wr *Write) WriteResponse(fa *Handler, w http.ResponseWriter) {

if wr.Position != nil || wr.Offset != 0 {
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we plan to implement those in a future PR? I do see a syscall.Seek for the Read endpoint for position.

Copy link
Contributor Author

@mlctrez mlctrez Sep 24, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was mainly a carry over from the original wasm_exec.js write

https://github.com/golang/go/blob/451e4727ec825a7ce6f6e6f82761ff90c33fec83/misc/wasm/wasm_exec.js#L27-L34

The read endpoint seek was implemented due to the coverage tool performing reads with offsets.
i.e. coverage metadata generation was broke without it.

I'll look and see how feasible it is to implement it here also.

syscall.Seek is now also used on the Write api.

fa.doError("not implemented", "ENOSYS", w)
return
}

bytes, err := base64.StdEncoding.DecodeString(wr.Buffer)
if err != nil {
fa.doError("not implemented", "ENOSYS", w)
return
}

var written int
written, err = syscall.Write(FdType(wr.Fd), bytes)
if err != nil {
agnivade marked this conversation as resolved.
Show resolved Hide resolved
fa.doError("not implemented", "ENOSYS", w)
return
}

fa.okResponse(map[string]any{"written": written}, w)
}

type Close struct {
Fd int `json:"fd"`
}

func (c *Close) WriteResponse(fa *Handler, w http.ResponseWriter) {
err := syscall.Close(FdType(c.Fd))
if err != nil {
fa.doError(syscall.ENOSYS.Error(), "ENOSYS", w)
return
}
fa.okResponse(map[string]any{}, w)
}

type Rename struct {
From string `json:"from"`
To string `json:"to"`
}

func (r *Rename) WriteResponse(fa *Handler, w http.ResponseWriter) {
err := syscall.Rename(fixPath(r.From), fixPath(r.To))
if fa.handleError(w, err, true) {
return
}
fa.okResponse(map[string]any{}, w)
}

type Readdir struct {
Path string `json:"path"`
}

func (r *Readdir) WriteResponse(fa *Handler, w http.ResponseWriter) {
entries, err := os.ReadDir(fixPath(r.Path))
if err != nil {
fa.doError(syscall.ENOSYS.Error(), "ENOSYS", w)
return
}
stringNames := make([]string, 0)
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: we can do make([]string, 0, len(entries)) as a further optimization.

for _, entry := range entries {
stringNames = append(stringNames, entry.Name())
}
fa.okResponse(map[string]any{"entries": stringNames}, w)
}

type Lstat struct {
Path string `json:"path"`
}

type Read struct {
Fd int `json:"fd"`
Offset int `json:"offset"`
Length int `json:"length"`
Position *int `json:"position,omitempty"`
}

func (r *Read) WriteResponse(fa *Handler, w http.ResponseWriter) {
if r.Offset != 0 {
fa.doError("not implemented", "ENOSYS", w)
return
}
if r.Position != nil {
_, err := syscall.Seek(FdType(r.Fd), int64(*r.Position), 0)
if err != nil {
fa.doError("not implemented", "ENOSYS", w)
return
}
}

buffer := make([]byte, r.Length)
read, err := syscall.Read(FdType(r.Fd), buffer)
if err != nil {
fa.doError("not implemented", "ENOSYS", w)
return
}
response := map[string]any{
"read": read,
"buffer": base64.StdEncoding.EncodeToString(buffer[:read]),
}
fa.okResponse(response, w)

}

type Mkdir struct {
Path string `json:"path"`
Perm uint32 `json:"perm"`
}

func (m *Mkdir) WriteResponse(fa *Handler, w http.ResponseWriter) {
err := syscall.Mkdir(fixPath(m.Path), m.Perm)
if err != nil {
fa.doError("not implemented", "ENOSYS", w)
return
}
fa.okResponse(map[string]any{}, w)
}

type Unlink struct {
Path string `json:"path"`
}

func (u *Unlink) WriteResponse(fa *Handler, w http.ResponseWriter) {
err := syscall.Unlink(fixPath(u.Path))
if err != nil {
fa.doError("not implemented", "ENOSYS", w)
return
}
fa.okResponse(map[string]any{}, w)
}

type Rmdir struct {
Path string `json:"path"`
}

func (r *Rmdir) WriteResponse(fa *Handler, w http.ResponseWriter) {
err := syscall.Rmdir(fixPath(r.Path))
if fa.handleError(w, err, true) {
return
}
fa.okResponse(map[string]any{}, w)
}

func (fa *Handler) handleError(w http.ResponseWriter, err error, noEnt bool) bool {
if err == nil {
return false
}
if noEnt && os.IsNotExist(err) {
fa.doError(syscall.ENOENT.Error(), "ENOENT", w)
} else {
fa.doError(syscall.ENOSYS.Error(), "ENOSYS", w)
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any reason to be ENOSYS? Also, we seem to be completely gobbling up the actual err. Could we pass that down or log that somewhere?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ENOSYS was the error code previously used in the wasm_exec.js for all unimplemented file system calls.

I think it still makes sense to use it as a generic "not implemented" error with the following rationale:

For each of the different platforms: linux, mac, and windows, the various syscall.Xxxx are handled differently, have different error codes, and for windows, are not POSIX at all. It would be quite a task to sort out what the appropriate error code would be based on the current running platform. ENOENT ( effectively file not found ) was the most important one to get right.

I'll see about adding logging for the err in doError and updating all callers to pass in the error for logging.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good, thanks.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good, thanks.

}
return true
}
Loading