reorganize into multiple packages

Most Core proto implementations have moved into package internal; some have moved to subpackages under coreext. Package iolang itself proxies important internal types and constants for external use. This change makes documentation much easier to read and allows embedders to be more selective about what features are available.

Tests pass, but it is possible that dependencies were missed, so some methods might be broken. Some individual methods of core protos can also still be moved to external packages. Documentation on almost everything in package iolang will need to be fixed.

addons/Range/Range/main.go

@@ -1,11 +1,11 @@
 package main
 
 import (
-	"github.com/zephyrtronium/iolang"
+	"github.com/zephyrtronium/iolang/coreext/addon"
 	"github.com/zephyrtronium/iolang/addons/Range"
 )
 
 // IoAddon returns an object to load the addon.
-func IoAddon() iolang.Addon {
+func IoAddon() addon.Interface {
 	return Range.IoAddon()
 }

cmd/gencore/gencore.go

@@ -11,20 +11,20 @@ import (
 )
 
 func main() {
-	if len(os.Args) < 3 {
-		fmt.Fprintln(os.Stderr, os.Args[0], "output.go iofiles/ iofiles/ ...")
+	if len(os.Args) < 4 {
+		fmt.Fprintln(os.Stderr, os.Args[0], "output.go package iofiles/ iofiles/ ...")
 		os.Exit(1)
 	}
 	out, err := os.Create(os.Args[1])
 	if err != nil {
 		panic(err)
 	}
 	defer out.Close()
-	if _, err = out.WriteString("package iolang\n\n// Code generated by gencore; DO NOT EDIT\n\nvar coreIo = []string{\n"); err != nil {
+	if _, err = out.WriteString("package " + os.Args[2] + "\n\n// Code generated by gencore; DO NOT EDIT\n\nvar coreIo = []string{\n"); err != nil {
 		panic(err)
 	}
 	var paths []string
-	for _, dir := range os.Args[2:] {
+	for _, dir := range os.Args[3:] {
 		fis, err := ioutil.ReadDir(dir)
 		if err != nil {
 			panic(err)

cmd/io/main.go

@@ -6,6 +6,8 @@ import (
 	"os"
 
 	"github.com/zephyrtronium/iolang"
+	// import for side effects
+	_ "github.com/zephyrtronium/iolang/coreext"
 
 	"runtime"
 	"runtime/pprof"

cmd/mkaddon/mkaddon.go

@@ -163,10 +163,11 @@ import (
 	"compress/zlib"
 
 	"github.com/zephyrtronium/iolang"
+	"github.com/zephyrtronium/iolang/coreext/addon"
 )
 
 // IoAddon returns a loader for the {{.Addon}} addon.
-func IoAddon() iolang.Addon {
+func IoAddon() addon.Interface {
 	return addon{{.Addon}}{}
 }
 
@@ -224,12 +225,12 @@ var plugin = template.Must(template.New("plugin").Parse(pluginsource))
 const pluginsource = `package main
 
 import (
-	"github.com/zephyrtronium/iolang"
+	"github.com/zephyrtronium/iolang/coreext/addon"
 	{{printf "%q" .Import}}
 )
 
 // IoAddon returns an object to load the addon.
-func IoAddon() iolang.Addon {
+func IoAddon() addon.Interface {
 	return {{.Addon}}.IoAddon()
 }
 `

coreext/README.md

@@ -0,0 +1,5 @@
+Package coreext sets up "optional" components of the iolang core.
+
+These packages may be imported for side effects only in order to make their functionality available on all VMs. Many, but not all, provide additional functionality (e.g. type definitions and constructors) that may be useful for adding values when embedding or for developing addons.
+
+Importing package coreext directly transitively sets up all core extensions.

coreext/addon/addon.go

@@ -0,0 +1,117 @@
+//go:generate go run ../../cmd/gencore addon_init.go addon ./io
+//go:generate gofmt -s -w addon_init.go
+
+package addon
+
+import (
+	"os"
+	"plugin"
+	"sync"
+
+	"github.com/zephyrtronium/iolang"
+	"github.com/zephyrtronium/iolang/internal"
+
+	// importing for side effects
+	_ "github.com/zephyrtronium/iolang/coreext/directory"
+)
+
+// Interface is the interface via which an addon is loaded into a VM.
+//
+// Addons in iolang are separate packages, which may be linked dynamically (on
+// platforms supporting -buildmode=plugin) or statically. The addon is loaded
+// by calling its IoAddon function, which must be of type func() Interface. This
+// function will be called no more than once per interpreter. The plugin itself
+// is opened only once per program, so its init functions may run less than
+// once per time it is added to an interpreter.
+//
+// For dynamically loaded addons, the Io program's Importer uses a CFunction to
+// lookup the IoAddon function in the plugin when needed. For statically
+// linked addons, however, the program which creates the VM must manually load
+// all addons using the VM's LoadAddon method.
+type Interface = internal.Addon
+
+func init() {
+	internal.Register(initAddon)
+}
+
+func initAddon(vm *iolang.VM) {
+	addonOnce.Do(initAddonOnce)
+	internal.InitAddon(vm)
+	slots := iolang.Slots{
+		"havePlugins":      vm.IoBool(havePlugins),
+		"open":             vm.NewCFunction(open, nil),
+		"scanForNewAddons": vm.NewCFunction(scanForNewAddons, nil),
+		"type":             vm.NewString("Addon"),
+	}
+	internal.CoreInstall(vm, "Addon", slots, nil, nil)
+	internal.Ioz(vm, coreIo, coreFiles)
+}
+
+// addonOpen is an Addon method.
+//
+// open loads the addon at the receiver's path and returns the addon's object.
+func open(vm *iolang.VM, target, locals *iolang.Object, msg *iolang.Message) *iolang.Object {
+	p, proto := vm.GetSlot(target, "path")
+	if proto == nil {
+		return vm.RaiseExceptionf("addon path unset")
+	}
+	p.Lock()
+	path, ok := p.Value.(iolang.Sequence)
+	if !ok {
+		p.Unlock()
+		return vm.RaiseExceptionf("addon path must be Sequence, not %s", vm.TypeName(p))
+	}
+	plug, err := plugin.Open(path.String())
+	p.Unlock()
+	if err != nil {
+		return vm.IoError(err)
+	}
+	open, err := plug.Lookup("IoAddon")
+	if err != nil {
+		return vm.IoError(err)
+	}
+	f, ok := open.(func() Interface)
+	if !ok {
+		return vm.RaiseExceptionf("%s is not an iolang addon", path)
+	}
+	<-vm.LoadAddon(f())
+	return target
+}
+
+// addonScanForNewAddons is an Addon method.
+//
+// scanForNewAddons marks a directory to be scanned for new addons.
+func scanForNewAddons(vm *iolang.VM, target, locals *iolang.Object, msg *iolang.Message) *iolang.Object {
+	path, exc, stop := msg.StringArgAt(vm, locals, 0)
+	if stop != iolang.NoStop {
+		return vm.Stop(exc, stop)
+	}
+	file, err := os.Open(path)
+	if err != nil {
+		return vm.IoError(err)
+	}
+	fi, err := file.Stat()
+	if err != nil {
+		return vm.IoError(err)
+	}
+	if !fi.IsDir() {
+		return vm.RaiseExceptionf("%s is not a directory", path)
+	}
+	internal.ScanForAddons(vm, file)
+	return target
+}
+
+// havePlugins indicates whether Go's plugin system is available on the current
+// system. Currently this should become true on Linux or Darwin with cgo
+// enabled, but the cgo requirement might drop in the future (unlikely) and more
+// platforms might be added (likely).
+var havePlugins = false
+
+func initAddonOnce() {
+	_, err := plugin.Open("/dev/null")
+	if err == nil || err.Error() != "plugin: not implemented" {
+		havePlugins = true
+	}
+}
+
+var addonOnce sync.Once

collector.go → coreext/collector/collector.go

@@ -1,31 +1,38 @@
-package iolang
+package collector
 
 import (
 	"fmt"
 	"runtime"
 	"time"
+
+	"github.com/zephyrtronium/iolang"
+	"github.com/zephyrtronium/iolang/internal"
 )
 
 // Io has its own mark-and-sweep garbage collector that has more features than
 // Go's. Because of this, the Collector interface is very different.
 
-func (vm *VM) initCollector() {
-	slots := Slots{
-		"collect":   vm.NewCFunction(CollectorCollect, nil),
-		"showStats": vm.NewCFunction(CollectorShowStats, nil),
-		"timeUsed":  vm.NewCFunction(CollectorTimeUsed, nil),
+func init() {
+	internal.Register(initCollector)
+}
+
+func initCollector(vm *iolang.VM) {
+	slots := iolang.Slots{
+		"collect":   vm.NewCFunction(collectorCollect, nil),
+		"showStats": vm.NewCFunction(collectorShowStats, nil),
+		"timeUsed":  vm.NewCFunction(collectorTimeUsed, nil),
 		"type":      vm.NewString("Collector"),
 	}
-	vm.coreInstall("Collector", slots, nil, nil)
+	internal.CoreInstall(vm, "Collector", slots, nil, nil)
 }
 
-// CollectorCollect is a Collector method.
+// collectorCollect is a Collector method.
 //
 // collect triggers a garbage collection cycle and returns the number of
 // objects collected program-wide (not only in the Io VM). This is much slower
 // than allowing collection to happen automatically, as the GC statistics must
 // be recorded twice to retrieve the freed object count.
-func CollectorCollect(vm *VM, target, locals *Object, msg *Message) *Object {
+func collectorCollect(vm *iolang.VM, target, locals *iolang.Object, msg *iolang.Message) *iolang.Object {
 	var stats runtime.MemStats
 	runtime.ReadMemStats(&stats)
 	old := stats.Frees
@@ -34,10 +41,10 @@ func CollectorCollect(vm *VM, target, locals *Object, msg *Message) *Object {
 	return vm.NewNumber(float64(stats.Frees - old))
 }
 
-// CollectorShowStats is a Collector method.
+// collectorShowStats is a Collector method.
 //
 // showStats prints detailed garbage collector information to standard output.
-func CollectorShowStats(vm *VM, target, locals *Object, msg *Message) *Object {
+func collectorShowStats(vm *iolang.VM, target, locals *iolang.Object, msg *iolang.Message) *iolang.Object {
 	var s runtime.MemStats
 	runtime.ReadMemStats(&s)
 	if s.NumGC > 0 {
@@ -61,11 +68,11 @@ func CollectorShowStats(vm *VM, target, locals *Object, msg *Message) *Object {
 	return target
 }
 
-// CollectorTimeUsed is a Collector method.
+// collectorTimeUsed is a Collector method.
 //
 // timeUsed reports the number of seconds spent in stop-the-world garbage
 // collection.
-func CollectorTimeUsed(vm *VM, target, locals *Object, msg *Message) *Object {
+func collectorTimeUsed(vm *iolang.VM, target, locals *iolang.Object, msg *iolang.Message) *iolang.Object {
 	var stats runtime.MemStats
 	runtime.ReadMemStats(&stats)
 	return vm.NewNumber(float64(stats.PauseTotalNs) / 1e9)

coreext/coreext.go

@@ -0,0 +1,16 @@
+package coreext
+
+import (
+	// importing for side effects
+	_ "github.com/zephyrtronium/iolang/coreext/addon"
+	_ "github.com/zephyrtronium/iolang/coreext/collector"
+	_ "github.com/zephyrtronium/iolang/coreext/coroutine"
+	_ "github.com/zephyrtronium/iolang/coreext/date"
+	_ "github.com/zephyrtronium/iolang/coreext/debugger"
+	_ "github.com/zephyrtronium/iolang/coreext/directory"
+	_ "github.com/zephyrtronium/iolang/coreext/duration"
+	_ "github.com/zephyrtronium/iolang/coreext/file"
+	_ "github.com/zephyrtronium/iolang/coreext/future"
+	_ "github.com/zephyrtronium/iolang/coreext/path"
+	_ "github.com/zephyrtronium/iolang/coreext/unittest"
+)