From dcff43e593c327c7a91b4e93d65cb7da6c4c6449 Mon Sep 17 00:00:00 2001 From: zyxkad Date: Tue, 24 Oct 2023 10:38:32 -0600 Subject: [PATCH] now script will load a packed zip plugin --- script/README.MD | 34 ++++++-- script/manager.go | 125 ++++++++++----------------- script/require.go | 214 ++++++++++++++++++++++++++++++++++++++++++++++ script/script.go | 79 ++++++++++------- 4 files changed, 329 insertions(+), 123 deletions(-) create mode 100644 script/require.go diff --git a/script/README.MD b/script/README.MD index 41a99f2..2e62f53 100644 --- a/script/README.MD +++ b/script/README.MD @@ -8,25 +8,41 @@ We use [goja](https://github.com/dop251/goja) as our javascript engine. Goja support full ES5, but it **DOES NOT** support full ES6 yet. Please read their [Features](https://github.com/dop251/goja#features) and [Known incompatibilities](https://github.com/dop251/goja#known-incompatibilities-and-caveats) to see what you can use and what you cannot -# TS support +# Setup -1. Install the package below: +## Lazy setup + +```sh +npm init glp@latest "" +``` + +Note: Typescript will be automaticly configured. + +## Manually setup + +You may want to configure your project manually, then you have to do the commands below + +1. Install [go-liter-plugin](https://www.npmjs.com/package/go-liter-plugin): ```sh - npm install --save-dev go-liter-plugin + npm install go-liter-plugin ``` 2. Add the following element to your `tsconfig.json` ```js { - "compilerOptions": { - "typeRoots": [ - "node_modules/@types", - "node_modules/go-liter-plugin" - ] - } + "compilerOptions": { + "typeRoots": [ + "node_modules/@types", + "node_modules/go-liter-plugin" + ] + } } ``` +# Typescript + +Check files under [./types/lib/](./types/lib/) for more information + # Events TODO diff --git a/script/manager.go b/script/manager.go index c9c956d..27efc42 100644 --- a/script/manager.go +++ b/script/manager.go @@ -2,8 +2,10 @@ package script import ( + "archive/zip" "context" "errors" + "fmt" "os" "path/filepath" "regexp" @@ -11,25 +13,37 @@ import ( "sync" "github.com/dop251/goja" - "github.com/dop251/goja/ast" - "github.com/dop251/goja/file" "github.com/dop251/goja_nodejs/eventloop" "github.com/dop251/goja_nodejs/require" "github.com/kmcsr/go-logger" "github.com/kmcsr/go-logger/logrus" "github.com/kmcsr/go-liter" - "github.com/kmcsr/go-liter/script/console" ) var ( fileNameRe = regexp.MustCompile(`^([a-z_][0-9a-z_]{0,31})(?:@(\d+(?:\.\d+)*))?(?:-.+)?\..+$`) ) -var ( - ErrFileNameInvalid = errors.New("The plugin's filename is invalid") - ErrPluginLoaded = errors.New("The plugin is already loaded") - ErrScriptInvalid = errors.New("Invalid script") -) +type PluginIdLoadedError struct { + Id string +} + +func (e *PluginIdLoadedError)Error()(string){ + return fmt.Sprintf("Plugin %s is already loaded", e.Id) +} + +type PluginLoadError struct { + Path string + Origin error +} + +func (e *PluginLoadError)Unwrap()(error){ + return e.Origin +} + +func (e *PluginLoadError)Error()(string){ + return fmt.Sprintf("Error when loading %q: %v", e.Path, e.Origin) +} type Manager struct { logger logger.Logger @@ -59,7 +73,7 @@ func (m *Manager)SetLogger(loger logger.Logger){ defer m.scriptMux.Unlock() m.logger = loger for _, s := range m.scripts { - s.console.SetLogger(setPrefixLogger(loger, s.id)) + s.console.SetLogger(setPrefixLogger(loger, s.Id)) } } @@ -86,87 +100,33 @@ func (m *Manager)Load(path string)(script *Script, err error){ return m.LoadWithContext(context.Background(), path) } -// LoadWithContext will load a script plugin use the given filepath. -// Script's filename must match `^([a-z_][0-9a-z_]{0,31})(?:@(\d+(?:\.\d+)*)(?:-.+)?)?\..+$` +// LoadWithContext will load a plugin packet use the given filepath. // The first capture group will be the script's ID. The second capture group is the script's version // If the script's ID is already loaded, then an error will be returned func (m *Manager)LoadWithContext(ctx context.Context, path string)(script *Script, err error){ - name := filepath.Base(path) - matches := fileNameRe.FindStringSubmatch(name) - if matches == nil { - return nil, ErrFileNameInvalid - } - id := matches[1] - version := matches[2] - - m.scriptMux.Lock() - if _, ok := m.scripts[id]; ok { - m.scriptMux.Unlock() - return nil, ErrPluginLoaded - } - m.scripts[id] = nil // reserve the slot - m.scriptMux.Unlock() - - data, err := os.ReadFile(path) - if err != nil { + var packet *zip.ReadCloser + if packet, err = zip.OpenReader(path); err != nil { return } - parsed, err := goja.Parse(path, (string)(data)) + meta, err := loadScriptMeta(packet) if err != nil { return } - // wrap code as `function($, console){ ... }` - parsed.Body = []ast.Statement{&ast.ExpressionStatement{ - Expression: &ast.FunctionLiteral{ - Function: -1, - Name: nil, - ParameterList: &ast.ParameterList{ - Opening: -1, - List: []*ast.Binding{ - { Target: &ast.Identifier{ Name:"$", Idx: -1 } }, - { Target: &ast.Identifier{ Name:"console", Idx: -1 } }, - }, - Closing: -1, - }, - Body: &ast.BlockStatement{ - LeftBrace: 0, - List: parsed.Body, - RightBrace: (file.Idx)(len(data) - 1), - }, - DeclarationList: parsed.DeclarationList, - }, - }} - prog, err := goja.CompileAST(parsed, true) - if err != nil { - return + m.scriptMux.Lock() + if _, ok := m.scripts[meta.Id]; ok { + m.scriptMux.Unlock() + return nil, &PluginIdLoadedError{ Id: meta.Id } } - - script = newScript(id, version, path, prog, m.loop) + m.scripts[meta.Id] = nil // reserve the slot + m.scriptMux.Unlock() errCh := make(chan error, 1) m.loop.RunOnLoop(func(vm *goja.Runtime){ - defer close(errCh) - if ctx.Err() != nil { - return - } var err error - var res goja.Value - if res, err = vm.RunProgram(prog); err != nil { - errCh <- err - return - } - cb, ok := goja.AssertFunction(res) - if !ok { - errCh <- ErrScriptInvalid - return - } - if ctx.Err() != nil { - return - } - script.init(vm) - script.console = console.NewConsole(vm, setPrefixLogger(m.logger, id)) - if _, err = cb(nil, script.doll, script.console.Exports()); err != nil { + defer func(){ errCh <- err + }() + if script, err = loadScript(packet, meta, m.logger, vm, m.loop); err != nil { return } }) @@ -180,7 +140,7 @@ func (m *Manager)LoadWithContext(ctx context.Context, path string)(script *Scrip } m.scriptMux.Lock() defer m.scriptMux.Unlock() - m.scripts[id] = script + m.scripts[script.Id] = script return } @@ -232,9 +192,9 @@ func (m *Manager)UnloadAll()(scripts []*Script){ return } -// LoadFromDir will load all plugins under the path which have the ext `.js`, +// LoadFromDir will load all plugins under the path which have the ext `.zip`, // and return all successfully loaded plugins with a possible error -// If the target path is not exists, LoadFromDir will do nothing and return no error +// If the target path is not exists, LoadFromDir will do nothing and return without error // If there are errors during load any plugin, the errors will be wrapped use `errors.Join`, // and other plugins will continue to be load. func (m *Manager)LoadFromDir(path string)(scripts []*Script, err error){ @@ -249,10 +209,11 @@ func (m *Manager)LoadFromDir(path string)(scripts []*Script, err error){ for _, e := range entries { name := e.Name() if !e.IsDir() { - if strings.HasSuffix(name, ".js") { - s, err := m.Load(filepath.Join(path, name)) + if strings.HasSuffix(name, ".zip") { + p := filepath.Join(path, name) + s, err := m.Load(p) if err != nil { - errs = append(errs, err) + errs = append(errs, &PluginLoadError{ Path: p, Origin: err }) continue } scripts = append(scripts, s) diff --git a/script/require.go b/script/require.go new file mode 100644 index 0000000..e7a64cf --- /dev/null +++ b/script/require.go @@ -0,0 +1,214 @@ + +package script + +import ( + "errors" + "io" + "io/fs" + stdpath "path" + + "github.com/dop251/goja" + "github.com/dop251/goja/ast" + "github.com/dop251/goja/file" + "github.com/dop251/goja/parser" + "github.com/dop251/goja/unistring" +) + +var ( + ErrScriptInvalid = errors.New("Invalid script") + ErrCircularImport = errors.New("Circular import") + ErrIsDir = errors.New("Target file is a directory") +) + +type addonVar struct { + name string + val goja.Value +} + +type moduleLoader struct { + vm *goja.Runtime + packet fs.FS + loading map[string]struct{} + loaded map[string]*goja.Object + addonVars []goja.Value + paramAst []*ast.Binding +} + +func newModuleLoader(packet fs.FS, vm *goja.Runtime, vars []addonVar)(r *moduleLoader){ + paramAst := make([]*ast.Binding, 3 + len(vars)) + paramAst[0] = &ast.Binding{ Target: &ast.Identifier{ Name:"require", Idx: -1 } } + paramAst[1] = &ast.Binding{ Target: &ast.Identifier{ Name:"module", Idx: -1 } } + paramAst[2] = &ast.Binding{ Target: &ast.Identifier{ Name:"exports", Idx: -1 } } + addonVars := make([]goja.Value, len(vars)) + for i, v := range vars { + paramAst[i + 3] = &ast.Binding{ + Target: &ast.Identifier{ Name: unistring.NewFromString(v.name), Idx: -1 }, + } + addonVars[i] = v.val + } + return &moduleLoader{ + vm: vm, + packet: packet, + loading: make(map[string]struct{}, 3), + loaded: make(map[string]*goja.Object, 3), + addonVars: addonVars, + paramAst: paramAst, + } +} + +func (r *moduleLoader)makeRequire(base string)(goja.Value){ + return r.vm.ToValue(func(call goja.FunctionCall)(goja.Value){ + path := call.Argument(0).String() + res, err := r.load(path, base) + if err != nil { + panic(wrap2JsErr(r.vm, err)) + } + return res + }) +} + +func (r *moduleLoader)load(path string, base string)(exports *goja.Object, err error){ + fd, path, err := r.resolveAndOpen(path, base) + if err != nil { + return + } + defer fd.Close() + exports, ok := r.loaded[path] + if ok { + return + } + if _, ok = r.loading[path]; ok { + return nil, ErrCircularImport + } + r.loading[path] = struct{}{} + defer delete(r.loading, path) + + data, err := io.ReadAll(fd) + if err != nil { + return + } + parsed, err := goja.Parse(path, (string)(data), parser.WithSourceMapLoader(func(name string)(data []byte, err error){ + fd, err := r.packet.Open(name) + if err != nil { + return + } + defer fd.Close() + return io.ReadAll(fd) + })) + if err != nil { + return + } + + parsed.Body = []ast.Statement{&ast.ExpressionStatement{ + Expression: &ast.FunctionLiteral{ + Function: -1, + Name: nil, + ParameterList: &ast.ParameterList{ + Opening: -1, + List: r.paramAst, + Closing: -1, + }, + Body: &ast.BlockStatement{ + LeftBrace: 0, + List: parsed.Body, + RightBrace: (file.Idx)(len(parsed.File.Source())), + }, + DeclarationList: parsed.DeclarationList, + }, + }} + prog, err := goja.CompileAST(parsed, true) + if err != nil { + return + } + var cb0 goja.Value + if cb0, err = r.vm.RunProgram(prog); err != nil { + return + } + cb, ok := goja.AssertFunction(cb0) + if !ok { + return nil, ErrScriptInvalid + } + + module := r.vm.NewObject() + module.Set("exports", r.vm.NewObject()) + + args := make([]goja.Value, 3 + len(r.addonVars)) + args[0] = r.makeRequire(stdpath.Dir(path)) + args[1] = module + args[2] = module.Get("exports") + copy(args[3:], r.addonVars) + if _, err = cb(nil, args...); err != nil { + return + } + exports = module.Get("exports").ToObject(r.vm) + r.loaded[path] = exports + return +} + +func openFileStat(fs fs.FS, path string)(fd fs.File, info fs.FileInfo, err error){ + if fd, err = fs.Open(path); err != nil { + return + } + if info, err = fd.Stat(); err != nil { + fd.Close() + return + } + return +} + +func (r *moduleLoader)resolveAndOpen(path string, base string)(fd fs.File, _ string, err error){ + path = stdpath.Join(base, path) + var info fs.FileInfo + var er error + if fd, info, err = openFileStat(r.packet, path); err == nil { + if !info.IsDir() { + return fd, path, nil + } + p := path + "index.js" + if fd, info, err = openFileStat(r.packet, p); err == nil { + if info.IsDir() { + err = ErrIsDir + return + } + return fd, p, nil + } + p = path + "index.json" + if fd, info, err = openFileStat(r.packet, p); err == nil { + if info.IsDir() { + err = ErrIsDir + return + } + return fd, p, nil + } + return + } + if !errors.Is(err, fs.ErrNotExist) { + return + } + if stdpath.Ext(path) != ".js" { + p := path + ".js" + if fd, info, er = openFileStat(r.packet, p); er == nil { + if info.IsDir() { + return + } + return fd, p, nil + } + } + if stdpath.Ext(path) != ".json" { + p := path + ".js" + if fd, info, er = openFileStat(r.packet, p); er == nil { + if info.IsDir() { + return + } + return fd, p, nil + } + } + return +} + +func wrap2JsErr(vm *goja.Runtime, err error)(any){ + if _, ok := err.(*goja.Exception); ok { + return err + } + return vm.NewGoError(err) +} diff --git a/script/script.go b/script/script.go index 51901d6..131e9fa 100644 --- a/script/script.go +++ b/script/script.go @@ -2,45 +2,76 @@ package script import ( + "io/fs" + "encoding/json" + "github.com/dop251/goja" "github.com/dop251/goja_nodejs/eventloop" "github.com/kmcsr/go-logger" "github.com/kmcsr/go-liter/script/console" ) +type ScriptMeta struct { + Id string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` +} + type Script struct { - id string - version string - path string + ScriptMeta + + loader *moduleLoader prog *goja.Program vm *goja.Runtime loop *eventloop.EventLoop - // exports *goja.Object - doll *goja.Object + exports *goja.Object console *console.Console storage *MemoryStorage emitter *EventEmitter + + packet fs.FS + modules map[string]*goja.Object } -func newScript(id string, version string, path string, prog *goja.Program, loop *eventloop.EventLoop)(*Script){ - return &Script{ - id: id, - version: version, - path: path, - prog: prog, - loop: loop, +func loadScriptMeta(packet fs.FS)(meta ScriptMeta, err error){ + fd, err := packet.Open("plugin.meta.json") + if err != nil { + return + } + defer fd.Close() + if err = json.NewDecoder(fd).Decode(&meta); err != nil { + return } + return } -func (s *Script)Id()(string){ - return s.id +func loadScript(packet fs.FS, meta ScriptMeta, loger logger.Logger, vm *goja.Runtime, loop *eventloop.EventLoop)(s *Script, err error){ + s = &Script{ + ScriptMeta: meta, + vm: vm, + loop: loop, + emitter: NewEventEmitter(vm, loop), + } + + doll := vm.NewObject() + doll.Set("ID", s.Id) + doll.Set("VERSION", s.Version) + s.emitter.ExportTo(doll) + s.console = console.NewConsole(vm, setPrefixLogger(loger, s.Id)) + s.loader = newModuleLoader(packet, vm, []addonVar{ + { name: "$", val: doll }, + { name: "console", val: s.console.Exports() }, + }) + if s.exports, err = s.loader.load("index.js", "."); err != nil { + return + } + return } -// TODO: Exports return a possible exports object from the script -// it's always nil for now, maybe it will be used later +// Exports return the module.exports from index.js func (s *Script)Exports()(*goja.Object){ // return s.exports return nil @@ -50,22 +81,6 @@ func (s *Script)Logger()(logger.Logger){ return s.console.Logger() } -func (s *Script)init(vm *goja.Runtime){ - if s.vm != nil { - panic("liter: script: Script already initialized") - } - s.vm = vm - s.storage = NewMemoryStorage(vm) - s.emitter = NewEventEmitter(vm, s.loop) - - o := vm.NewObject() - o.Set("ID", s.id) - o.Set("VERSION", s.version) - o.Set("storage", s.storage.Exports()) - s.emitter.ExportTo(o) - s.doll = o -} - func (s *Script)On(name string, listener goja.Callable){ s.emitter.OnAsync(name, listener) }