From 9fa946682e2524102cd9148f96c9ee9b135b3426 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Mon, 4 Mar 2024 23:55:15 +0100
Subject: [PATCH 01/72] readme: fix wrong code block

---
 docs/examples/generic_union.md | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/docs/examples/generic_union.md b/docs/examples/generic_union.md
index 35d17f9b..010a3401 100644
--- a/docs/examples/generic_union.md
+++ b/docs/examples/generic_union.md
@@ -78,11 +78,7 @@ func ExampleTreeSumValues() {
 
 You can also reduce tree to complex structure, for example to keep track of order of values in the tree, along with sum of all values in the tree.
 
-```go title="example/tree.go"
-
 ```go title="example/tree_test.go"
-
-```go
 func ExampleTreeCustomReduction() {
 	tree := &Branch[int]{
 		L: &Leaf[int]{Value: 1},

From b3b87064b9c7c1c8461569e28970fabdab805035 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 00:17:23 +0100
Subject: [PATCH 02/72] x/machine: better testing suite usability

---
 x/machine/infer_state_machine.go              |  26 +-
 x/machine/infer_state_machine_test.go         |   1 -
 x/machine/machine.go                          |  36 +-
 x/machine/test_suite.go                       | 373 ++++++++++--------
 x/machine/test_suite_test.go                  |  16 +-
 .../test_suite_test.go.state_diagram.mmd      |   3 -
 ...uite_test.go.state_diagram_with_errors.mmd |   5 +-
 7 files changed, 267 insertions(+), 193 deletions(-)

diff --git a/x/machine/infer_state_machine.go b/x/machine/infer_state_machine.go
index 0b6e859d..9bf4d368 100644
--- a/x/machine/infer_state_machine.go
+++ b/x/machine/infer_state_machine.go
@@ -3,6 +3,7 @@ package machine
 import (
 	"fmt"
 	"reflect"
+	"sort"
 	"strings"
 )
 
@@ -71,6 +72,11 @@ func (t *InferTransition[Transition, State]) Record(tr Transition, prev, curr St
 		err = errAfterTransition.Error()
 	}
 
+	// map only transitions with names
+	if transitionName == "" {
+		return
+	}
+
 	tt := transition{
 		transitionName,
 		prevStateName,
@@ -78,6 +84,8 @@ func (t *InferTransition[Transition, State]) Record(tr Transition, prev, curr St
 		err,
 	}
 
+	name := tt.String()
+	_ = name
 	if t.exists[tt.String()] {
 		return
 	}
@@ -90,6 +98,12 @@ func (t *InferTransition[Transition, State]) Record(tr Transition, prev, curr St
 // https://mermaid-js.github.io/mermaid/#/stateDiagram
 func (t *InferTransition[Transition, State]) ToMermaid() string {
 	result := &strings.Builder{}
+
+	// sort transitions by name
+	sort.SliceStable(t.transitions, func(i, j int) bool {
+		return t.transitions[i].String() < t.transitions[j].String()
+	})
+
 	if t.name != "" {
 		fmt.Fprintf(result, "---\ntitle: %s\n---\n", t.name)
 	}
@@ -111,12 +125,12 @@ func (t *InferTransition[Transition, State]) ToMermaid() string {
 
 		name := tt.name()
 		if tt.err() != "" {
-			fmt.Fprintf(result, " %%%% error=%s \n", strings.TrimSpace(strings.ReplaceAll(tt.err(), "\n", " ")))
-			name = fmt.Sprintf("❌%s", name)
-		}
-
-		if tt.err() != "" && !t.showErrorTransitions {
-			continue
+			if t.showErrorTransitions {
+				fmt.Fprintf(result, " %%%% error=%s \n", strings.TrimSpace(strings.ReplaceAll(tt.err(), "\n", " ")))
+				name = fmt.Sprintf("❌%s", name)
+			} else {
+				continue
+			}
 		}
 
 		fmt.Fprintf(result, "\t"+`%s --> %s: "%s"`+"\n", prev, curr, name)
diff --git a/x/machine/infer_state_machine_test.go b/x/machine/infer_state_machine_test.go
index 79f521ef..aedd47a7 100644
--- a/x/machine/infer_state_machine_test.go
+++ b/x/machine/infer_state_machine_test.go
@@ -15,6 +15,5 @@ func TestInferStateMachine(t *testing.T) {
 
 	assert.Equal(t, `stateDiagram
 	"int" --> "int": "string"
- %% error=unknown cmd: unknown 
 `, result)
 }
diff --git a/x/machine/machine.go b/x/machine/machine.go
index 406d5407..4d1175c1 100644
--- a/x/machine/machine.go
+++ b/x/machine/machine.go
@@ -1,24 +1,36 @@
 package machine
 
-func NewSimpleMachine[C, S any](f func(C, S) (S, error)) *Machine[C, S] {
+func NewMachine[D, C, S any](d D, f func(D, C, S) (S, error), state S) *Machine[D, C, S] {
+	return &Machine[D, C, S]{
+		di:     d,
+		handle: f,
+		state:  state,
+	}
+}
+
+func NewSimpleMachine[C, S any](f func(C, S) (S, error)) *Machine[any, C, S] {
 	var s S
 	return NewSimpleMachineWithState(f, s)
 }
 
-func NewSimpleMachineWithState[C, S any](f func(C, S) (S, error), state S) *Machine[C, S] {
-	return &Machine[C, S]{
-		handle: f,
-		state:  state,
+func NewSimpleMachineWithState[C, S any](f func(C, S) (S, error), state S) *Machine[any, C, S] {
+	return &Machine[any, C, S]{
+		di: nil,
+		handle: func(a any, c C, s S) (S, error) {
+			return f(c, s)
+		},
+		state: state,
 	}
 }
 
-type Machine[C, S any] struct {
+type Machine[D, C, S any] struct {
+	di     D
 	state  S
-	handle func(C, S) (S, error)
+	handle func(D, C, S) (S, error)
 }
 
-func (o *Machine[C, S]) Handle(cmd C) error {
-	state, err := o.handle(cmd, o.state)
+func (o *Machine[D, C, S]) Handle(cmd C) error {
+	state, err := o.handle(o.di, cmd, o.state)
 	if err != nil {
 		return err
 	}
@@ -27,6 +39,10 @@ func (o *Machine[C, S]) Handle(cmd C) error {
 	return nil
 }
 
-func (o *Machine[C, S]) State() S {
+func (o *Machine[D, C, S]) State() S {
 	return o.state
 }
+
+func (o *Machine[D, C, S]) Dep() D {
+	return o.di
+}
diff --git a/x/machine/test_suite.go b/x/machine/test_suite.go
index c041a2a5..077347f3 100644
--- a/x/machine/test_suite.go
+++ b/x/machine/test_suite.go
@@ -1,135 +1,119 @@
 package machine
 
 import (
+	"errors"
 	"fmt"
+	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
+	"github.com/widmogrod/mkunion/x/shared"
 	"math/rand"
 	"os"
+	"reflect"
 	"testing"
 )
 
-func NewTestSuite[TCommand, TState any](mkMachine func() *Machine[TCommand, TState]) *Suite[TCommand, TState] {
-	infer := NewInferTransition[TCommand, TState]()
-	return &Suite[TCommand, TState]{
+func NewTestSuite[D, C, S any](dep D, mkMachine func(dep D, init S) *Machine[D, C, S]) *Suite[D, C, S] {
+	infer := NewInferTransition[C, S]()
+	return &Suite[D, C, S]{
+		dep:       dep,
 		mkMachine: mkMachine,
 		infer:     infer,
 	}
 }
 
-type Suite[TCommand, TState any] struct {
-	mkMachine func() *Machine[TCommand, TState]
-	infer     *InferTransition[TCommand, TState]
-	then      []*Case[TCommand, TState]
+type Suite[D, C, S any] struct {
+	dep       D
+	mkMachine func(dep D, init S) *Machine[D, C, S]
+	infer     *InferTransition[C, S]
+	cases     []*Case[D, C, S]
 }
 
-func (suite *Suite[TCommand, TState]) Case(name string, definition func(c *Case[TCommand, TState])) {
-	useCase := &Case[TCommand, TState]{
-		name: name,
-	}
-	definition(useCase)
-
-	suite.then = append(suite.then, useCase)
-}
-
-// Run runs all test then that describe state machine transitions
-func (suite *Suite[TCommand, TState]) Run(t *testing.T) {
+func (suite *Suite[D, C, S]) Case(t *testing.T, name string, f func(t *testing.T, c *Case[D, C, S])) *Suite[D, C, S] {
 	t.Helper()
-	for _, c := range suite.then {
-		m := suite.mkMachine()
-		suite.assert(t, c, m)
+	c := &Case[D, C, S]{
+		suit: suite,
+		step: Step[D, C, S]{
+			Name: name,
+		},
 	}
-}
+	f(t, c)
 
-func (suite *Suite[TCommand, TState]) assert(t *testing.T, c *Case[TCommand, TState], m *Machine[TCommand, TState]) bool {
-	return t.Run(c.name, func(t *testing.T) {
-		for idx, cmd := range c.command {
-			state := m.State()
-			t.Run(fmt.Sprintf("Apply(cmd=%T, state=%T)", cmd, state), func(t *testing.T) {
-				c.commandOption[idx].before()
-				err := m.Handle(cmd)
-				c.commandOption[idx].after()
-
-				newState := m.State()
-
-				if c.err[idx] == nil {
-					assert.NoError(t, err)
-				} else {
-					assert.ErrorIs(t, err, c.err[idx])
-				}
-
-				assert.Equal(t, c.state[idx], newState)
-
-				suite.infer.Record(cmd, state, newState, err)
-
-				if len(c.then[idx]) > 0 {
-					for _, c2 := range c.then[idx] {
-						m2 := *m
-						suite.assert(t, c2, &m2)
-					}
-				}
-			})
-		}
-	})
+	suite.cases = append(suite.cases, c)
+	return suite
 }
 
-// Fuzzy takes commands and states from recorded transitions and tries to find all possible combinations of commands and states.
-// This can help complete state diagrams with missing transitions, or find errors in state machine that haven't been tested yet.
-// It's useful when connected with AssertSelfDocumentStateDiagram, to automatically update state diagram.
-func (suite *Suite[TCommand, TState]) Fuzzy(t *testing.T) {
-	t.Helper()
-
+func (suite *Suite[D, C, S]) fuzzy() {
 	r := rand.New(rand.NewSource(0))
 
-	m := suite.mkMachine()
-	var states []TState
-	var commands []TCommand
-	var commandOptions []*caseOption
+	// Some commands or states can be more popular
+	// when we randomly select them, we can increase the chance of selecting them, and skip less popular ones, which is not desired
+	// we want to have a good coverage of all commands and states
+	// to achieve this, we will group commands and states, and randomly select group, and then randomly select command or state from this group
 
-	then := suite.then
-	for len(then) > 0 {
-		c := then[0]
-		then = then[1:]
+	states := make(map[string][]Step[D, C, S])
+	commands := make(map[string][]Step[D, C, S])
 
-		for _, opt := range c.commandOption {
-			commandOptions = append(commandOptions, opt)
+	for _, c := range suite.cases {
+		if any(c.step.ExpectedState) != nil {
+			stateName := reflect.TypeOf(c.step.ExpectedState).String()
+			states[stateName] = append(states[stateName], c.step)
 		}
-		for _, cmd := range c.command {
-			commands = append(commands, cmd)
+
+		if any(c.step.GivenCommand) != nil {
+			commandName := reflect.TypeOf(c.step.GivenCommand).String()
+			commands[commandName] = append(commands[commandName], c.step)
 		}
-		for _, state := range c.state {
-			states = append(states, state)
+
+		if any(c.step.InitState) != nil {
+			stateName := reflect.TypeOf(c.step.InitState).String()
+			states[stateName] = append(states[stateName], c.step)
 		}
-		for _, t := range c.then {
-			for _, tt := range t {
-				then = append(then, tt)
+	}
+
+	for _, seed := range rand.Perm(len(suite.cases) * 100) {
+		r.Seed(int64(seed))
+
+		var step Step[D, C, S]
+		var state S
+
+		// randomly select step with command
+		for _, steps := range commands {
+			if r.Float64() < 0.1 {
+				step = steps[r.Intn(len(steps))]
+				break
 			}
 		}
-	}
 
-	for _, seed := range rand.Perm(len(states) * len(commands) * 10) {
-		//r.Seed(int64(seed))
-		_ = seed
-		// randomly select command and state
-		idx := r.Intn(len(commands))
-		commandOptions[idx].before()
-		cmd := commands[idx]
-		commandOptions[idx].after()
+		// randomly select step with state
+		for _, steps := range states {
+			if r.Float64() < 0.1 {
+				state = steps[r.Intn(len(steps))].ExpectedState
+				break
+			}
+		}
 
 		// with some chance keep previous state, or randomly select new state
 		// this helps to generate new states, that can succeed after applying command
 		prob := r.Float64()
 		if prob < 0.3 {
-			m.state = states[r.Intn(len(states))]
+			state = suite.cases[r.Intn(len(suite.cases))].step.ExpectedState
 		} else if prob < 0.6 {
 			// explore also initial states
-			var zeroState TState
-			m.state = zeroState
+			var zeroState S
+			state = zeroState
 		}
 
-		state := m.State()
-		err := m.Handle(cmd)
+		m := suite.mkMachine(suite.dep, state)
+		// Before and After commands can have assertions, when we fuzzing we don't want to run them
+		if step.BeforeCommand != nil {
+			step.BeforeCommand(zeroT, suite.dep)
+		}
+		if step.AfterCommand != nil {
+			step.AfterCommand(zeroT, suite.dep)
+		}
+		err := m.Handle(step.GivenCommand)
 		newState := m.State()
-		suite.infer.Record(cmd, state, newState, err)
+		suite.infer.Record(step.GivenCommand, state, newState, err)
 	}
 }
 
@@ -140,10 +124,12 @@ func (suite *Suite[TCommand, TState]) Fuzzy(t *testing.T) {
 //
 // If file does not exist, function will return true, to indicate that file should be created.
 // For this purpose call SelfDocumentStateDiagram.
-func (suite *Suite[TCommand, TState]) AssertSelfDocumentStateDiagram(t *testing.T, baseFileName string) (shouldSelfDocument bool) {
+func (suite *Suite[D, C, S]) AssertSelfDocumentStateDiagram(t *testing.T, filename string) bool {
+	suite.fuzzy()
+
 	// extract fine name from file, if there is extension remove it
-	fileName := baseFileName + ".state_diagram.mmd"
-	fileNameWithErrorTransitions := baseFileName + ".state_diagram_with_errors.mmd"
+	fileName := filename + ".state_diagram.mmd"
+	fileNameWithErrorTransitions := filename + ".state_diagram_with_errors.mmd"
 
 	for _, f := range []struct {
 		filename  string
@@ -166,7 +152,10 @@ func (suite *Suite[TCommand, TState]) AssertSelfDocumentStateDiagram(t *testing.
 		}
 
 		// if stored content is not equal, fail assertion
-		assert.Equalf(t, string(date), mermaidDiagram, "state diagram is not equal to stored in file %s", f.filename)
+		if diff := cmp.Diff(string(date), mermaidDiagram); diff != "" {
+			t.Fatalf("unexpected state diagram (-want +got):\n%s", diff)
+			return false
+		}
 	}
 
 	return false
@@ -174,10 +163,12 @@ func (suite *Suite[TCommand, TState]) AssertSelfDocumentStateDiagram(t *testing.
 
 // SelfDocumentStateDiagram help to self document state machine transitions, just by running tests.
 // It will always overwrite stored state diagram files, useful in TDD loop, when tests are being written.
-func (suite *Suite[TCommand, TState]) SelfDocumentStateDiagram(t *testing.T, baseFileName string) {
+func (suite *Suite[D, C, S]) SelfDocumentStateDiagram(t *testing.T, filename string) {
+	suite.fuzzy()
+
 	// extract fine name from file, if there is extension remove it
-	fileName := baseFileName + ".state_diagram.mmd"
-	fileNameWithErrorTransitions := baseFileName + ".state_diagram_with_errors.mmd"
+	fileName := filename + ".state_diagram.mmd"
+	fileNameWithErrorTransitions := filename + ".state_diagram_with_errors.mmd"
 
 	for _, f := range []struct {
 		filename  string
@@ -195,94 +186,154 @@ func (suite *Suite[TCommand, TState]) SelfDocumentStateDiagram(t *testing.T, bas
 	}
 }
 
-func (suite *Suite[TCommand, TState]) SelfDocumentTitle(title string) {
-	suite.infer.WithTitle(title)
-}
+type Case[D, C, S any] struct {
+	suit *Suite[D, C, S]
 
-type caseOption struct {
-	before func()
-	after  func()
-}
+	step Step[D, C, S]
 
-var zeroCaseOption caseOption = caseOption{
-	before: func() {},
-	after:  func() {},
+	process     bool
+	resultErr   error
+	resultState S
 }
 
-type InitCaseOptions func(o *caseOption)
+// GivenCommand starts building assertion that when command is applied to machine, it will result in given state or error.
+func (suitcase *Case[D, C, S]) GivenCommand(c C) *Case[D, C, S] {
+	suitcase.step.GivenCommand = c
+	return suitcase
+}
 
-func WithBefore(f func()) InitCaseOptions {
-	return func(o *caseOption) {
-		o.before = f
-	}
+// BeforeCommand is optional, if provided it will be called before command is executed
+// useful when you want to prepare some data before command is executed,
+// like change dependency to return error, or change some state
+func (suitcase *Case[D, C, S]) BeforeCommand(f func(testing.TB, D)) *Case[D, C, S] {
+	suitcase.step.BeforeCommand = f
+	return suitcase
 }
 
-func WithAfter(f func()) InitCaseOptions {
-	return func(o *caseOption) {
-		o.after = f
-	}
+// AfterCommand is optional, if provided it will be called after command is executed
+// useful when you want to assert some data after command is executed,
+// like what function were called, and with what arguments
+func (suitcase *Case[D, C, S]) AfterCommand(f func(testing.TB, D)) *Case[D, C, S] {
+	suitcase.step.AfterCommand = f
+	return suitcase
 }
 
-type Case[TCommand, TState any] struct {
-	name          string
-	command       []TCommand
-	commandOption []*caseOption
-	state         []TState
-	err           []error
-	then          [][]*Case[TCommand, TState]
+// ThenState asserts that command applied to machine will result in given state
+// implicitly assumes that error is nil
+func (suitcase *Case[D, C, S]) ThenState(t *testing.T, o S) *Case[D, C, S] {
+	t.Helper()
+
+	suitcase.step.ExpectedState = o
+	suitcase.step.ExpectedErr = nil
+	suitcase.run(t)
+
+	return suitcase
 }
 
-func (c *Case[TCommand, TState]) next() {
-	var zeroCmd TCommand
-	var zeroState TState
-	var zeroErr error
+// ThenStateAndError asserts that command applied to machine will result in given state and error
+// state is required because we want to know what is the expected state after command fails to be applied, and return error.
+// state most of the time shouldn't be modified, and explicit definition of state help to make this behaviour explicit.
+func (suitcase *Case[D, C, S]) ThenStateAndError(t *testing.T, state S, err error) *Case[D, C, S] {
+	t.Helper()
+	suitcase.step.ExpectedState = state
+	suitcase.step.ExpectedErr = err
+	suitcase.run(t)
 
-	c.commandOption = append(c.commandOption, &zeroCaseOption)
-	c.command = append(c.command, zeroCmd)
-	c.state = append(c.state, zeroState)
-	c.err = append(c.err, zeroErr)
-	c.then = append(c.then, nil)
+	return suitcase
 }
 
-func (c *Case[TCommand, TState]) index() int {
-	return len(c.command) - 1
+// ForkCase takes previous state of machine and allows to apply another case from this point onward
+// it's useful when you want to test multiple scenarios from one state
+func (suitcase *Case[D, C, S]) ForkCase(t *testing.T, name string, f func(t *testing.T, c *Case[D, C, S])) *Case[D, C, S] {
+	t.Helper()
+
+	// We have to run the current test case,
+	// if we want to have state to form from
+	suitcase.run(t)
+
+	newState := suitcase.deepCopy(suitcase.resultState)
+
+	newCase := &Case[D, C, S]{
+		suit: suitcase.suit,
+		step: Step[D, C, S]{
+			Name:      name,
+			InitState: newState,
+		},
+	}
+
+	f(t, newCase)
+
+	suitcase.suit.cases = append(suitcase.suit.cases, newCase)
+	return suitcase
 }
 
-// GivenCommand starts building assertion that when command is applied to machine, it will result in given state or error.
-// Use this method always with ThenState or ThenStateAndError
-func (c *Case[TCommand, TState]) GivenCommand(cmd TCommand, opts ...InitCaseOptions) *Case[TCommand, TState] {
-	c.next()
+func (suitcase *Case[D, C, S]) run(t *testing.T) {
+	if suitcase.process {
+		return
+	}
+	suitcase.process = true
 
-	option := &caseOption{
-		before: func() {},
-		after:  func() {},
+	t.Helper()
+	machine := suitcase.suit.mkMachine(suitcase.suit.dep, suitcase.step.InitState)
+	if suitcase.step.BeforeCommand != nil {
+		suitcase.step.BeforeCommand(t, suitcase.suit.dep)
 	}
-	for _, o := range opts {
-		o(option)
+
+	err := machine.Handle(suitcase.step.GivenCommand)
+	suitcase.resultErr = err
+	suitcase.resultState = machine.State()
+
+	if suitcase.step.AfterCommand != nil {
+		suitcase.step.AfterCommand(t, suitcase.suit.dep)
+	}
+
+	suitcase.suit.infer.Record(suitcase.step.GivenCommand, suitcase.step.InitState, suitcase.resultState, err)
+
+	if !errors.Is(err, suitcase.step.ExpectedErr) {
+		t.Fatalf("unexpected error \n  expect: %v \n     got: %v\n", suitcase.step.ExpectedErr, err)
 	}
 
-	c.commandOption[c.index()] = option
-	c.command[c.index()] = cmd
-	return c
+	if diff := cmp.Diff(suitcase.step.ExpectedState, suitcase.resultState); diff != "" {
+		t.Fatalf("unexpected state (-want +got):\n%suitcase", diff)
+	}
 }
 
-// ThenState asserts that command applied to machine will result in given state
-func (c *Case[TCommand, TState]) ThenState(state TState) *Case[TCommand, TState] {
-	c.state[c.index()] = state
-	c.err[c.index()] = nil
-	return c
+func (suitcase *Case[D, C, S]) deepCopy(state S) S {
+	data, err := shared.JSONMarshal[S](state)
+	if err != nil {
+		panic(fmt.Errorf("failed deep copying state %T, reason: %w", state, err))
+	}
+	result, err := shared.JSONUnmarshal[S](data)
+	if err != nil {
+		panic(fmt.Errorf("failed deep copying state %T, reason: %w", state, err))
+	}
+	return result
 }
 
-// ForkCase takes previous state of machine and allows to apply another case from this point onward
-// there can be many forks from one state
-func (c *Case[TCommand, TState]) ForkCase(name string, definition func(c *Case[TCommand, TState])) *Case[TCommand, TState] {
-	useCase := &Case[TCommand, TState]{name: name}
-	definition(useCase)
-	c.then[c.index()] = append(c.then[c.index()], useCase)
-	return c
+type TestingT interface {
+	Errorf(format string, args ...interface{})
 }
 
-func (c *Case[TCommand, TState]) ThenStateAndError(state TState, err error) {
-	c.state[c.index()] = state
-	c.err[c.index()] = err
+// Step is a single test case that describe state machine transition
+type Step[D, C, S any] struct {
+	// Name human readable description of the test case. It's required
+	Name string
+
+	// InitState is optional, if not provided it will be nil
+	// and when step is part of sequence, then state will be inherited from previous step
+	InitState S
+
+	// GivenCommand is the command that will be applied to the machine. It's required
+	GivenCommand C
+	// BeforeCommand is optional, if provided it will be called before command is executed
+	BeforeCommand func(t testing.TB, x D)
+	// AfterCommand is optional, if provided it will be called after command is executed
+	AfterCommand func(t testing.TB, x D)
+
+	// ExpectedState is the expected state after command is executed. It's required, but can be nil
+	ExpectedState S
+	// ExpectedErr is the expected error after command is executed. It's required, but can be nil
+	ExpectedErr error
 }
+
+var zeroT testing.TB = &testing.T{}
diff --git a/x/machine/test_suite_test.go b/x/machine/test_suite_test.go
index bfc00981..5e9ab653 100644
--- a/x/machine/test_suite_test.go
+++ b/x/machine/test_suite_test.go
@@ -17,17 +17,15 @@ func TestSuite_Run(t *testing.T) {
 		}
 	}, 10)
 
-	suite := NewTestSuite(func() *Machine[string, int] { return m })
-	suite.Case("inc", func(c *Case[string, int]) {
-		c.GivenCommand("inc").ThenState(11)
-		c.GivenCommand("inc").ThenState(12)
-		c.GivenCommand("inc").ThenState(13)
+	suite := NewTestSuite(nil, func(dep any, init int) *Machine[any, string, int] {
+		return m
+	})
+	suite.Case(t, "inc", func(t *testing.T, c *Case[any, string, int]) {
+		c.GivenCommand("inc").ThenState(t, 11)
+		c.GivenCommand("inc").ThenState(t, 12)
+		c.GivenCommand("inc").ThenState(t, 13)
 	})
 
-	suite.Run(t)
-	suite.Fuzzy(t)
-
-	suite.SelfDocumentTitle("SimpleMachine")
 	if suite.AssertSelfDocumentStateDiagram(t, "test_suite_test.go") {
 		suite.SelfDocumentStateDiagram(t, "test_suite_test.go")
 	}
diff --git a/x/machine/test_suite_test.go.state_diagram.mmd b/x/machine/test_suite_test.go.state_diagram.mmd
index a7134016..bc915161 100644
--- a/x/machine/test_suite_test.go.state_diagram.mmd
+++ b/x/machine/test_suite_test.go.state_diagram.mmd
@@ -1,5 +1,2 @@
----
-title: SimpleMachine
----
 stateDiagram
 	"int" --> "int": "string"
diff --git a/x/machine/test_suite_test.go.state_diagram_with_errors.mmd b/x/machine/test_suite_test.go.state_diagram_with_errors.mmd
index a7134016..81d6a82f 100644
--- a/x/machine/test_suite_test.go.state_diagram_with_errors.mmd
+++ b/x/machine/test_suite_test.go.state_diagram_with_errors.mmd
@@ -1,5 +1,4 @@
----
-title: SimpleMachine
----
 stateDiagram
 	"int" --> "int": "string"
+ %% error=unknown cmd: 
+	"int" --> "int": "❌string"

From 149944ddf05bc04f19518fadb4f18934306e83dd Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 00:18:57 +0100
Subject: [PATCH 03/72] x/workflow: refactor to new testing suite

---
 x/workflow/machine.state_diagram.mmd          |  39 +-----
 .../machine.state_diagram_with_errors.mmd     |  80 ++++++------
 x/workflow/workflow_machine_test.go           | 118 +++++++++---------
 x/workflow/workflow_transition.go             |   8 +-
 4 files changed, 106 insertions(+), 139 deletions(-)

diff --git a/x/workflow/machine.state_diagram.mmd b/x/workflow/machine.state_diagram.mmd
index 227f9f65..f671677c 100644
--- a/x/workflow/machine.state_diagram.mmd
+++ b/x/workflow/machine.state_diagram.mmd
@@ -1,38 +1,11 @@
 stateDiagram
-	[*] --> "*workflow.Done": "*workflow.Run"
-	[*] --> "*workflow.Scheduled": "*workflow.Run"
-	"*workflow.Scheduled" --> "*workflow.Done": "*workflow.Run"
-	"*workflow.Scheduled" --> "*workflow.ScheduleStopped": "*workflow.StopSchedule"
+	"*workflow.Await" --> "*workflow.Done": "*workflow.Callback"
 	"*workflow.ScheduleStopped" --> "*workflow.Scheduled": "*workflow.ResumeSchedule"
+	"*workflow.Scheduled" --> "*workflow.Done": "*workflow.Run"
 	[*] --> "*workflow.Await": "*workflow.Run"
-	"*workflow.Await" --> "*workflow.Done": "*workflow.Callback"
- %% error=callback not match 
+	[*] --> "*workflow.Done": "*workflow.Run"
 	[*] --> "*workflow.Error": "*workflow.Run"
- %% error=failed to find workflow hello_world_flow_non_existing: flow hello_world_flow_non_existing not found; flow not found 
-	"*workflow.Error" --> "*workflow.Error": "*workflow.TryRecover"
- %% error=invalid state transition 
- %% error=invalid state transition 
- %% error=invalid state transition 
- %% error=cannot apply commands, when workflow is completed 
- %% error=cannot apply commands, when workflow is completed 
- %% error=invalid state transition 
- %% error=flow not set 
- %% error=invalid state transition 
- %% error=invalid state transition 
- %% error=invalid state transition 
- %% error=invalid state transition 
- %% error=cannot apply commands, when workflow is completed 
- %% error=invalid state transition 
- %% error=cannot apply commands, when workflow is completed 
- %% error=invalid state transition 
- %% error=invalid state transition 
- %% error=invalid state transition 
- %% error=invalid state transition 
- %% error=invalid state transition 
- %% error=cannot apply commands, when workflow is completed 
- %% error=invalid state transition 
- %% error=invalid state transition 
- %% error=invalid state transition 
- %% error=invalid state transition 
- %% error=invalid state transition 
+	[*] --> "*workflow.Scheduled": "*workflow.Run"
+	"*workflow.Scheduled" --> "*workflow.ScheduleStopped": "*workflow.StopSchedule"
 	"*workflow.Error" --> "*workflow.Done": "*workflow.TryRecover"
+	"*workflow.Error" --> "*workflow.Error": "*workflow.TryRecover"
diff --git a/x/workflow/machine.state_diagram_with_errors.mmd b/x/workflow/machine.state_diagram_with_errors.mmd
index e3de7fd2..c94b4b87 100644
--- a/x/workflow/machine.state_diagram_with_errors.mmd
+++ b/x/workflow/machine.state_diagram_with_errors.mmd
@@ -1,65 +1,65 @@
 stateDiagram
-	[*] --> "*workflow.Done": "*workflow.Run"
-	[*] --> "*workflow.Scheduled": "*workflow.Run"
-	"*workflow.Scheduled" --> "*workflow.Done": "*workflow.Run"
-	"*workflow.Scheduled" --> "*workflow.ScheduleStopped": "*workflow.StopSchedule"
-	"*workflow.ScheduleStopped" --> "*workflow.Scheduled": "*workflow.ResumeSchedule"
-	[*] --> "*workflow.Await": "*workflow.Run"
-	"*workflow.Await" --> "*workflow.Done": "*workflow.Callback"
  %% error=callback not match 
 	"*workflow.Await" --> "*workflow.Await": "❌*workflow.Callback"
-	[*] --> "*workflow.Error": "*workflow.Run"
- %% error=failed to find workflow hello_world_flow_non_existing: flow hello_world_flow_non_existing not found; flow not found 
-	[*] --> [*]: "❌*workflow.Run"
-	"*workflow.Error" --> "*workflow.Error": "*workflow.TryRecover"
- %% error=invalid state transition 
-	"*workflow.ScheduleStopped" --> "*workflow.ScheduleStopped": "❌*workflow.Run"
- %% error=invalid state transition 
-	"*workflow.Error" --> "*workflow.Error": "❌*workflow.Run"
- %% error=invalid state transition 
-	[*] --> [*]: "❌*workflow.ResumeSchedule"
- %% error=cannot apply commands, when workflow is completed 
-	"*workflow.Done" --> "*workflow.Done": "❌*workflow.Run"
+	"*workflow.Await" --> "*workflow.Done": "*workflow.Callback"
  %% error=cannot apply commands, when workflow is completed 
 	"*workflow.Done" --> "*workflow.Done": "❌*workflow.Callback"
  %% error=invalid state transition 
-	[*] --> [*]: "❌*workflow.Callback"
- %% error=flow not set 
-	[*] --> [*]: "❌*workflow.Run"
- %% error=invalid state transition 
-	"*workflow.ScheduleStopped" --> "*workflow.ScheduleStopped": "❌*workflow.StopSchedule"
+	"*workflow.Error" --> "*workflow.Error": "❌*workflow.Callback"
  %% error=invalid state transition 
-	"*workflow.ScheduleStopped" --> "*workflow.ScheduleStopped": "❌*workflow.TryRecover"
+	"*workflow.ScheduleStopped" --> "*workflow.ScheduleStopped": "❌*workflow.Callback"
  %% error=invalid state transition 
-	[*] --> [*]: "❌*workflow.StopSchedule"
+	"*workflow.Scheduled" --> "*workflow.Scheduled": "❌*workflow.Callback"
  %% error=invalid state transition 
-	[*] --> [*]: "❌*workflow.TryRecover"
- %% error=cannot apply commands, when workflow is completed 
-	"*workflow.Done" --> "*workflow.Done": "❌*workflow.StopSchedule"
+	[*] --> [*]: "❌*workflow.Callback"
  %% error=invalid state transition 
-	"*workflow.Error" --> "*workflow.Error": "❌*workflow.StopSchedule"
+	"*workflow.Await" --> "*workflow.Await": "❌*workflow.ResumeSchedule"
  %% error=cannot apply commands, when workflow is completed 
 	"*workflow.Done" --> "*workflow.Done": "❌*workflow.ResumeSchedule"
  %% error=invalid state transition 
-	"*workflow.Await" --> "*workflow.Await": "❌*workflow.Run"
+	"*workflow.Error" --> "*workflow.Error": "❌*workflow.ResumeSchedule"
+	"*workflow.ScheduleStopped" --> "*workflow.Scheduled": "*workflow.ResumeSchedule"
  %% error=invalid state transition 
-	"*workflow.Error" --> "*workflow.Error": "❌*workflow.Callback"
+	"*workflow.Scheduled" --> "*workflow.Scheduled": "❌*workflow.ResumeSchedule"
  %% error=invalid state transition 
-	"*workflow.Scheduled" --> "*workflow.Scheduled": "❌*workflow.TryRecover"
+	[*] --> [*]: "❌*workflow.ResumeSchedule"
  %% error=invalid state transition 
-	"*workflow.Await" --> "*workflow.Await": "❌*workflow.ResumeSchedule"
+	"*workflow.Await" --> "*workflow.Await": "❌*workflow.Run"
+ %% error=cannot apply commands, when workflow is completed 
+	"*workflow.Done" --> "*workflow.Done": "❌*workflow.Run"
+ %% error=invalid state transition 
+	"*workflow.Error" --> "*workflow.Error": "❌*workflow.Run"
+ %% error=invalid state transition 
+	"*workflow.ScheduleStopped" --> "*workflow.ScheduleStopped": "❌*workflow.Run"
+	"*workflow.Scheduled" --> "*workflow.Done": "*workflow.Run"
+	[*] --> "*workflow.Await": "*workflow.Run"
+	[*] --> "*workflow.Done": "*workflow.Run"
+	[*] --> "*workflow.Error": "*workflow.Run"
+	[*] --> "*workflow.Scheduled": "*workflow.Run"
+ %% error=failed to find workflow hello_world_flow_non_existing: flow hello_world_flow_non_existing not found; flow not found 
+	[*] --> [*]: "❌*workflow.Run"
+ %% error=flow not set 
+	[*] --> [*]: "❌*workflow.Run"
  %% error=invalid state transition 
 	"*workflow.Await" --> "*workflow.Await": "❌*workflow.StopSchedule"
  %% error=cannot apply commands, when workflow is completed 
-	"*workflow.Done" --> "*workflow.Done": "❌*workflow.TryRecover"
- %% error=invalid state transition 
-	"*workflow.ScheduleStopped" --> "*workflow.ScheduleStopped": "❌*workflow.Callback"
+	"*workflow.Done" --> "*workflow.Done": "❌*workflow.StopSchedule"
  %% error=invalid state transition 
-	"*workflow.Scheduled" --> "*workflow.Scheduled": "❌*workflow.ResumeSchedule"
+	"*workflow.Error" --> "*workflow.Error": "❌*workflow.StopSchedule"
  %% error=invalid state transition 
-	"*workflow.Error" --> "*workflow.Error": "❌*workflow.ResumeSchedule"
+	"*workflow.ScheduleStopped" --> "*workflow.ScheduleStopped": "❌*workflow.StopSchedule"
+	"*workflow.Scheduled" --> "*workflow.ScheduleStopped": "*workflow.StopSchedule"
  %% error=invalid state transition 
-	"*workflow.Scheduled" --> "*workflow.Scheduled": "❌*workflow.Callback"
+	[*] --> [*]: "❌*workflow.StopSchedule"
  %% error=invalid state transition 
 	"*workflow.Await" --> "*workflow.Await": "❌*workflow.TryRecover"
+ %% error=cannot apply commands, when workflow is completed 
+	"*workflow.Done" --> "*workflow.Done": "❌*workflow.TryRecover"
 	"*workflow.Error" --> "*workflow.Done": "*workflow.TryRecover"
+	"*workflow.Error" --> "*workflow.Error": "*workflow.TryRecover"
+ %% error=invalid state transition 
+	"*workflow.ScheduleStopped" --> "*workflow.ScheduleStopped": "❌*workflow.TryRecover"
+ %% error=invalid state transition 
+	"*workflow.Scheduled" --> "*workflow.Scheduled": "❌*workflow.TryRecover"
+ %% error=invalid state transition 
+	[*] --> [*]: "❌*workflow.TryRecover"
diff --git a/x/workflow/workflow_machine_test.go b/x/workflow/workflow_machine_test.go
index 60be851f..48891468 100644
--- a/x/workflow/workflow_machine_test.go
+++ b/x/workflow/workflow_machine_test.go
@@ -221,17 +221,15 @@ func TestMachine(t *testing.T) {
 		MockTimeNow: &timeNow,
 	}
 
-	suite := machine.NewTestSuite(func() *machine.Machine[Command, State] {
-		return NewMachine(di, nil)
-	})
+	suite := machine.NewTestSuite[Dependency](di, NewMachine)
 
-	suite.Case("start execution", func(c *machine.Case[Command, State]) {
+	suite.Case(t, "start execution", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 		c.
 			GivenCommand(&Run{
 				Flow:  &FlowRef{FlowID: "hello_world_flow"},
 				Input: schema.MkString("world"),
 			}).
-			ThenState(&Done{
+			ThenState(t, &Done{
 				Result: schema.MkString("hello world"),
 				BaseState: BaseState{
 					RunID:  runID,
@@ -246,7 +244,7 @@ func TestMachine(t *testing.T) {
 				},
 			})
 	})
-	suite.Case("start scheduled execution delay 10s", func(c *machine.Case[Command, State]) {
+	suite.Case(t, "start scheduled execution delay 10s", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 		c.
 			GivenCommand(&Run{
 				Flow:  &FlowRef{FlowID: "hello_world_flow"},
@@ -255,7 +253,7 @@ func TestMachine(t *testing.T) {
 					DelayBySeconds: 10,
 				},
 			}).
-			ThenState(&Scheduled{
+			ThenState(t, &Scheduled{
 				ExpectedRunTimestamp: di.TimeNow().Add(time.Duration(10) * time.Second).Unix(),
 				BaseState: BaseState{
 					RunID:  runID,
@@ -271,10 +269,10 @@ func TestMachine(t *testing.T) {
 					},
 				},
 			}).
-			ForkCase("resume execution", func(c *machine.Case[Command, State]) {
+			ForkCase(t, "resume execution", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 				c.
 					GivenCommand(&Run{}).
-					ThenState(&Done{
+					ThenState(t, &Done{
 						Result: schema.MkString("hello world"),
 						BaseState: BaseState{
 							RunID:  runID,
@@ -292,12 +290,12 @@ func TestMachine(t *testing.T) {
 						},
 					})
 			}).
-			ForkCase("stop execution", func(c *machine.Case[Command, State]) {
+			ForkCase(t, "stop execution", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 				c.
 					GivenCommand(&StopSchedule{
 						ParentRunID: runID,
 					}).
-					ThenState(&ScheduleStopped{
+					ThenState(t, &ScheduleStopped{
 						BaseState: BaseState{
 							RunID:  runID,
 							StepID: "",
@@ -315,7 +313,7 @@ func TestMachine(t *testing.T) {
 					GivenCommand(&ResumeSchedule{
 						ParentRunID: runID,
 					}).
-					ThenState(&Scheduled{
+					ThenState(t, &Scheduled{
 						ExpectedRunTimestamp: di.TimeNow().Add(time.Duration(10) * time.Second).Unix(),
 						BaseState: BaseState{
 							RunID:  runID,
@@ -333,13 +331,13 @@ func TestMachine(t *testing.T) {
 					})
 			})
 	})
-	suite.Case("start execution that awaits for callback", func(c *machine.Case[Command, State]) {
+	suite.Case(t, "start execution that awaits for callback", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 		c.
 			GivenCommand(&Run{
 				Flow:  &FlowRef{FlowID: "hello_world_flow_await"},
 				Input: schema.MkString("world"),
 			}).
-			ThenState(&Await{
+			ThenState(t, &Await{
 				Timeout:    int64(10 * time.Second),
 				CallbackID: callbackID,
 				BaseState: BaseState{
@@ -353,14 +351,14 @@ func TestMachine(t *testing.T) {
 					DefaultMaxRetries: 3,
 				},
 			}).
-			ForkCase("callback received", func(c *machine.Case[Command, State]) {
+			ForkCase(t, "callback received", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 				// Assuming that callback is received before timeout.
 				c.
 					GivenCommand(&Callback{
 						CallbackID: callbackID,
 						Result:     schema.MkString("hello + world"),
 					}).
-					ThenState(&Done{
+					ThenState(t, &Done{
 						Result: schema.MkString("hello + world"),
 						BaseState: BaseState{
 							RunID:  runID,
@@ -377,13 +375,13 @@ func TestMachine(t *testing.T) {
 						},
 					})
 			}).
-			ForkCase("received invalid callbackID", func(c *machine.Case[Command, State]) {
+			ForkCase(t, "received invalid callbackID", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 				c.
 					GivenCommand(&Callback{
 						CallbackID: "invalid_callback_id",
 						Result:     schema.MkString("hello + world"),
 					}).
-					ThenStateAndError(&Await{
+					ThenStateAndError(t, &Await{
 						Timeout:    int64(10 * time.Second),
 						CallbackID: callbackID,
 						BaseState: BaseState{
@@ -399,12 +397,12 @@ func TestMachine(t *testing.T) {
 					}, ErrCallbackNotMatch)
 			})
 	})
-	suite.Case("start execution no input variable", func(c *machine.Case[Command, State]) {
+	suite.Case(t, "start execution no input variable", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 		c.
 			GivenCommand(&Run{
 				Flow: &FlowRef{FlowID: "hello_world_flow"},
 			}).
-			ThenState(&Error{
+			ThenState(t, &Error{
 				Code:   "function-execution",
 				Reason: "function concat() returned error: expected string, got <nil>",
 				BaseState: BaseState{
@@ -419,33 +417,35 @@ func TestMachine(t *testing.T) {
 				},
 			})
 	})
-	suite.Case("start execution fails on non existing flowID", func(c *machine.Case[Command, State]) {
+	suite.Case(t, "start execution fails on non existing flowID", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 		c.
 			GivenCommand(&Run{
 				Flow:  &FlowRef{FlowID: "hello_world_flow_non_existing"},
 				Input: schema.MkString("world"),
 			}).
-			ThenStateAndError(nil, ErrFlowNotFound)
+			ThenStateAndError(t, nil, ErrFlowNotFound)
 	})
-	suite.Case("start execution fails on function retrival", func(c *machine.Case[Command, State]) {
+	suite.Case(t, "start execution fails on function retrival", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 		c.
 			GivenCommand(&Run{
 				Flow:  &FlowRef{FlowID: "hello_world_flow"},
 				Input: schema.MkString("world"),
-			}, machine.WithBefore(func() {
-				di.FindFunctionF = func(funcID string) (Function, error) {
+			}).
+			BeforeCommand(func(t testing.TB, di Dependency) {
+				di.(*DI).FindFunctionF = func(funcID string) (Function, error) {
 					return nil, fmt.Errorf("function funcID='%s' not found", funcID)
 				}
-			}), machine.WithAfter(func() {
-				di.FindFunctionF = func(funcID string) (Function, error) {
+			}).
+			AfterCommand(func(t testing.TB, di Dependency) {
+				di.(*DI).FindFunctionF = func(funcID string) (Function, error) {
 					if fn, ok := functions[funcID]; ok {
 						return fn, nil
 					}
 
 					return nil, fmt.Errorf("function %s not found", funcID)
 				}
-			})).
-			ThenState(&Error{
+			}).
+			ThenState(t, &Error{
 				Code:   "function-missing",
 				Reason: "function concat() not found, details: function funcID='concat' not found",
 				BaseState: BaseState{
@@ -459,26 +459,26 @@ func TestMachine(t *testing.T) {
 					DefaultMaxRetries: 3,
 				},
 			}).
-			ForkCase("retry execution", func(c *machine.Case[Command, State]) {
+			ForkCase(t, "retry execution", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 				c.
 					GivenCommand(&TryRecover{
 						RunID: runID,
-					},
-						machine.WithBefore(func() {
-							di.FindFunctionF = func(funcID string) (Function, error) {
-								return nil, fmt.Errorf("function funcID='%s' not found", funcID)
+					}).
+					BeforeCommand(func(t testing.TB, di Dependency) {
+						di.(*DI).FindFunctionF = func(funcID string) (Function, error) {
+							return nil, fmt.Errorf("function funcID='%s' not found", funcID)
+						}
+					}).
+					AfterCommand(func(t testing.TB, di Dependency) {
+						di.(*DI).FindFunctionF = func(funcID string) (Function, error) {
+							if fn, ok := functions[funcID]; ok {
+								return fn, nil
 							}
-						}), machine.WithAfter(func() {
-							di.FindFunctionF = func(funcID string) (Function, error) {
-								if fn, ok := functions[funcID]; ok {
-									return fn, nil
-								}
 
-								return nil, fmt.Errorf("function %s not found", funcID)
-							}
-						}),
-					).
-					ThenState(&Error{
+							return nil, fmt.Errorf("function %s not found", funcID)
+						}
+					}).
+					ThenState(t, &Error{
 						Code:    "function-missing",
 						Reason:  "function concat() not found, details: function funcID='concat' not found",
 						Retried: 1,
@@ -495,13 +495,13 @@ func TestMachine(t *testing.T) {
 					})
 			})
 	})
-	suite.Case("execute function with if statement", func(c *machine.Case[Command, State]) {
+	suite.Case(t, "execute function with if statement", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 		c.
 			GivenCommand(&Run{
 				Flow:  &FlowRef{FlowID: "hello_world_flow_if"},
 				Input: schema.MkString("El Mundo"),
 			}).
-			ThenState(&Done{
+			ThenState(t, &Done{
 				Result: schema.MkString("only Spanish will work!"),
 				BaseState: BaseState{
 					RunID:  runID,
@@ -516,7 +516,7 @@ func TestMachine(t *testing.T) {
 				},
 			})
 	})
-	suite.Case("scheduled run", func(c *machine.Case[Command, State]) {
+	suite.Case(t, "scheduled run", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 		c.
 			GivenCommand(&Run{
 				Flow:  &FlowRef{FlowID: "hello_world_flow_if"},
@@ -525,7 +525,7 @@ func TestMachine(t *testing.T) {
 					Interval: "@every 1s",
 				},
 			}).
-			ThenState(&Scheduled{
+			ThenState(t, &Scheduled{
 				ExpectedRunTimestamp: di.TimeNow().Add(time.Duration(1) * time.Second).Unix(),
 				BaseState: BaseState{
 					RunID:  runID,
@@ -542,10 +542,10 @@ func TestMachine(t *testing.T) {
 					},
 				},
 			}).
-			ForkCase("run scheduled run", func(c *machine.Case[Command, State]) {
+			ForkCase(t, "run scheduled run", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 				c.
 					GivenCommand(&Run{}).
-					ThenState(&Done{
+					ThenState(t, &Done{
 						Result: schema.MkString("only Spanish will work!"),
 						BaseState: BaseState{
 							RunID:  runID,
@@ -564,12 +564,12 @@ func TestMachine(t *testing.T) {
 						},
 					})
 			}).
-			ForkCase("stop scheduled run", func(c *machine.Case[Command, State]) {
+			ForkCase(t, "stop scheduled run", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 				c.
 					GivenCommand(&StopSchedule{
 						ParentRunID: runID,
 					}).
-					ThenState(&ScheduleStopped{
+					ThenState(t, &ScheduleStopped{
 						BaseState: BaseState{
 							RunID:  runID,
 							StepID: "",
@@ -585,10 +585,10 @@ func TestMachine(t *testing.T) {
 							},
 						},
 					}).
-					ForkCase("run stopped", func(c *machine.Case[Command, State]) {
+					ForkCase(t, "run stopped", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 						c.
 							GivenCommand(&Run{}).
-							ThenStateAndError(&ScheduleStopped{
+							ThenStateAndError(t, &ScheduleStopped{
 								BaseState: BaseState{
 									RunID:  runID,
 									StepID: "",
@@ -605,12 +605,12 @@ func TestMachine(t *testing.T) {
 								},
 							}, ErrInvalidStateTransition)
 					}).
-					ForkCase("resume scheduled run", func(c *machine.Case[Command, State]) {
+					ForkCase(t, "resume scheduled run", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
 						c.
 							GivenCommand(&ResumeSchedule{
 								ParentRunID: runID,
 							}).
-							ThenState(&Scheduled{
+							ThenState(t, &Scheduled{
 								ExpectedRunTimestamp: di.TimeNow().Add(time.Duration(1) * time.Second).Unix(),
 								BaseState: BaseState{
 									RunID:  runID,
@@ -628,14 +628,10 @@ func TestMachine(t *testing.T) {
 								},
 							})
 					})
-
 			})
 	})
 
-	suite.Run(t)
-	suite.Fuzzy(t)
-
-	if true || suite.AssertSelfDocumentStateDiagram(t, "machine") {
+	if suite.AssertSelfDocumentStateDiagram(t, "machine") {
 		suite.SelfDocumentStateDiagram(t, "machine")
 	}
 }
diff --git a/x/workflow/workflow_transition.go b/x/workflow/workflow_transition.go
index e0412aa6..c6332bec 100644
--- a/x/workflow/workflow_transition.go
+++ b/x/workflow/workflow_transition.go
@@ -31,13 +31,11 @@ type Dependency interface {
 	TimeNow() time.Time
 }
 
-func NewMachine(di Dependency, state State) *machine.Machine[Command, State] {
-	return machine.NewSimpleMachineWithState(func(cmd Command, state State) (State, error) {
-		return Transition(cmd, state, di)
-	}, state)
+func NewMachine(di Dependency, state State) *machine.Machine[Dependency, Command, State] {
+	return machine.NewMachine(di, Transition, state)
 }
 
-func Transition(cmd Command, state State, dep Dependency) (State, error) {
+func Transition(dep Dependency, cmd Command, state State) (State, error) {
 	switch state.(type) {
 	case *Done:
 		return nil, ErrStateReachEnd

From 173f5190bf30fe62e0cda6db3cb5cd50dfb4b407 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 00:19:34 +0100
Subject: [PATCH 04/72] example/state: refactor to order service and use new
 testing suite

---
 example/state/machine.go                      | 187 ++++++++++
 example/state/machine_test.go                 | 337 ++++++++++++++++++
 .../state/machine_test.go.state_diagram.mmd   |   7 +
 ...hine_test.go.state_diagram_with_errors.mmd |  57 +++
 example/state/model.go                        |  97 +++++
 example/state/simple_machine.go               |  62 ----
 example/state/simple_machine_test.go          | 171 ---------
 .../simple_machine_test.go.state_diagram.mmd  |  21 --
 ...hine_test.go.state_diagram_with_errors.mmd |  37 --
 example/state/simple_state_transitions.go     |  34 --
 10 files changed, 685 insertions(+), 325 deletions(-)
 create mode 100644 example/state/machine.go
 create mode 100644 example/state/machine_test.go
 create mode 100644 example/state/machine_test.go.state_diagram.mmd
 create mode 100644 example/state/machine_test.go.state_diagram_with_errors.mmd
 create mode 100644 example/state/model.go
 delete mode 100644 example/state/simple_machine.go
 delete mode 100644 example/state/simple_machine_test.go
 delete mode 100644 example/state/simple_machine_test.go.state_diagram.mmd
 delete mode 100644 example/state/simple_machine_test.go.state_diagram_with_errors.mmd
 delete mode 100644 example/state/simple_state_transitions.go

diff --git a/example/state/machine.go b/example/state/machine.go
new file mode 100644
index 00000000..2b99baa3
--- /dev/null
+++ b/example/state/machine.go
@@ -0,0 +1,187 @@
+package state
+
+import (
+	"fmt"
+	"github.com/widmogrod/mkunion/x/machine"
+	"time"
+)
+
+func NewMachine(di Dependency, init State) *machine.Machine[Dependency, Command, State] {
+	return machine.NewMachine(di, Transition, init)
+}
+
+var (
+	ErrInvalidTransition                = fmt.Errorf("invalid transition")
+	ErrOrderAlreadyExist                = fmt.Errorf("cannot attemp order creation, order exists: %w", ErrInvalidTransition)
+	ErrCannotCancelNonProcessingOrder   = fmt.Errorf("cannot cancel order, order must be processing to cancel it; %w", ErrInvalidTransition)
+	ErrCannotCompleteNonProcessingOrder = fmt.Errorf("cannot mark order as complete, order is not being process; %w", ErrInvalidTransition)
+	ErrCannotRecoverNonErrorState       = fmt.Errorf("cannot recover from non error state; %w", ErrInvalidTransition)
+)
+
+var (
+	ErrValidationFailed = fmt.Errorf("validation failed")
+
+	ErrOrderIDRequired = fmt.Errorf("order ID is required; %w", ErrValidationFailed)
+	ErrOrderIDMismatch = fmt.Errorf("order ID mismatch; %w", ErrValidationFailed)
+
+	ErrWorkerIDRequired = fmt.Errorf("worker ID required; %w", ErrValidationFailed)
+)
+
+type Dependency interface {
+	TimeNow() *time.Time
+	WarehouseRemoveStock(quantity Quantity) error
+	PaymentCharge(price Price) error
+}
+
+func Transition(di Dependency, cmd Command, state State) (State, error) {
+	return MatchCommandR2(
+		cmd,
+		func(x *CreateOrderCMD) (State, error) {
+			if x.OrderID == "" {
+				return nil, ErrOrderIDRequired
+			}
+
+			switch state.(type) {
+			case nil:
+				o := Order{
+					ID:        x.OrderID,
+					OrderAttr: x.Attr,
+				}
+				return &OrderPending{
+					Order: o,
+				}, nil
+			}
+
+			return nil, ErrOrderAlreadyExist
+		},
+		func(x *MarkAsProcessingCMD) (State, error) {
+			if x.OrderID == "" {
+				return nil, ErrOrderIDRequired
+			}
+			if x.WorkerID == "" {
+				return nil, ErrWorkerIDRequired
+			}
+
+			switch s := state.(type) {
+			case *OrderPending:
+				if s.Order.ID != x.OrderID {
+					return nil, ErrOrderIDMismatch
+				}
+
+				o := s.Order
+				o.WorkerID = x.WorkerID
+
+				return &OrderProcessing{
+					Order: o,
+				}, nil
+			}
+
+			return nil, ErrInvalidTransition
+
+		},
+		func(x *CancelOrderCMD) (State, error) {
+			if x.OrderID == "" {
+				return nil, ErrOrderIDRequired
+			}
+
+			switch s := state.(type) {
+			case *OrderProcessing:
+				o := s.Order
+				o.CancelledAt = di.TimeNow()
+				o.CancelledReason = x.Reason
+
+				return &OrderCancelled{
+					Order: o,
+				}, nil
+			}
+
+			return nil, ErrCannotCancelNonProcessingOrder
+		},
+		func(x *MarkOrderCompleteCMD) (State, error) {
+			if x.OrderID == "" {
+				return nil, ErrOrderIDRequired
+			}
+
+			switch s := state.(type) {
+			case *OrderProcessing:
+				if s.Order.StockRemovedAt == nil {
+					// we need to remove stock first
+					// we can retry this operation (if warehouse is idempotent)
+					// OrderID could be used to deduplicate operation
+					// it's not required in this example
+					err := di.WarehouseRemoveStock(s.Order.OrderAttr.Quantity)
+					if err != nil {
+						return &OrderError{
+							ProblemCode:    ProblemWarehouseAPIUnreachable,
+							ProblemCommand: x,
+							ProblemState:   s,
+						}, nil
+					}
+
+					s.Order.StockRemovedAt = di.TimeNow()
+				}
+
+				if s.Order.PaymentChargedAt == nil {
+					// we need to charge payment first
+					// we can retry this operation (if payment gateway is idempotent)
+					// OrderID could be used to deduplicate operation
+					// it's not required in this example
+					err := di.PaymentCharge(s.Order.OrderAttr.Price)
+					if err != nil {
+						return &OrderError{
+							ProblemCode:    ProblemPaymentAPIUnreachable,
+							ProblemCommand: x,
+							ProblemState:   s,
+						}, nil
+					}
+
+					s.Order.PaymentChargedAt = di.TimeNow()
+				}
+
+				s.Order.DeliveredAt = di.TimeNow()
+
+				return &OrderCompleted{
+					Order: s.Order,
+				}, nil
+			}
+
+			return nil, ErrCannotCompleteNonProcessingOrder
+		},
+		func(x *TryRecoverErrorCMD) (State, error) {
+			if x.OrderID == "" {
+				return nil, ErrOrderIDRequired
+			}
+
+			switch s := state.(type) {
+			case *OrderError:
+				s.Retried += 1
+				s.RetriedAt = di.TimeNow()
+
+				switch s.ProblemCode {
+				case ProblemWarehouseAPIUnreachable,
+					ProblemPaymentAPIUnreachable:
+					// we can retry this operation
+					newState, err := Transition(di, s.ProblemCommand, s.ProblemState)
+					if err != nil {
+						return s, err
+					}
+
+					// make sure that error retries are preserved
+					if es, ok := newState.(*OrderError); ok {
+						es.Retried = s.Retried
+						es.RetriedAt = s.RetriedAt
+						return es, nil
+					}
+
+					return newState, nil
+
+				default:
+					// we don't know what to do, return to previous state
+					return s, nil
+				}
+			}
+
+			return nil, ErrCannotRecoverNonErrorState
+		},
+	)
+}
diff --git a/example/state/machine_test.go b/example/state/machine_test.go
new file mode 100644
index 00000000..ca21a707
--- /dev/null
+++ b/example/state/machine_test.go
@@ -0,0 +1,337 @@
+package state
+
+import (
+	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/assert"
+	"github.com/widmogrod/mkunion/x/machine"
+	"testing"
+	"time"
+)
+
+//go:generate moq -with-resets -stub -out machine_mock_test.go . Dependency
+
+func TestSuite(t *testing.T) {
+	now := time.Now()
+	var di Dependency = &DependencyMock{
+		TimeNowFunc: func() *time.Time {
+			return &now
+		},
+	}
+
+	order := OrderAttr{
+		Price:    100,
+		Quantity: 3,
+	}
+
+	suite := machine.NewTestSuite(di, NewMachine)
+	suite.Case(t, "happy path of order state transition",
+		func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
+			c.
+				GivenCommand(&CreateOrderCMD{OrderID: "123", Attr: order}).
+				ThenState(t, &OrderPending{
+					Order: Order{
+						ID:        "123",
+						OrderAttr: order,
+					},
+				}).
+				ForkCase(t, "start processing order", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
+					c.
+						GivenCommand(&MarkAsProcessingCMD{
+							OrderID:  "123",
+							WorkerID: "worker-1",
+						}).
+						ThenState(t, &OrderProcessing{
+							Order: Order{
+								ID:        "123",
+								OrderAttr: order,
+								WorkerID:  "worker-1",
+							},
+						}).
+						ForkCase(t, "mark order as completed", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
+							c.
+								GivenCommand(&MarkOrderCompleteCMD{
+									OrderID: "123",
+								}).
+								ThenState(t, &OrderCompleted{
+									Order: Order{
+										ID:               "123",
+										OrderAttr:        order,
+										WorkerID:         "worker-1",
+										DeliveredAt:      &now,
+										StockRemovedAt:   &now,
+										PaymentChargedAt: &now,
+									},
+								})
+						}).
+						ForkCase(t, "cancel order", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
+							c.
+								GivenCommand(&CancelOrderCMD{
+									OrderID: "123",
+									Reason:  "out of stock",
+								}).
+								ThenState(t, &OrderCancelled{
+									Order: Order{
+										ID:              "123",
+										OrderAttr:       order,
+										WorkerID:        "worker-1",
+										CancelledAt:     &now,
+										CancelledReason: "out of stock",
+									},
+								})
+						}).
+						ForkCase(t, "try complete order but removing products from stock fails", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
+							c.
+								GivenCommand(&MarkOrderCompleteCMD{
+									OrderID: "123",
+								}).
+								BeforeCommand(func(t testing.TB, di Dependency) {
+									di.(*DependencyMock).ResetCalls()
+									di.(*DependencyMock).WarehouseRemoveStockFunc = func(quantity int) error {
+										return fmt.Errorf("warehouse api unreachable")
+									}
+								}).
+								AfterCommand(func(t testing.TB, di Dependency) {
+									dep := di.(*DependencyMock)
+									dep.WarehouseRemoveStockFunc = nil
+									if assert.Len(t, dep.WarehouseRemoveStockCalls(), 1) {
+										assert.Equal(t, order.Quantity, dep.WarehouseRemoveStockCalls()[0].Quantity)
+									}
+
+									assert.Len(t, dep.PaymentChargeCalls(), 0)
+								}).
+								ThenState(t, &OrderError{
+									Retried:        0,
+									RetriedAt:      nil,
+									ProblemCode:    ProblemWarehouseAPIUnreachable,
+									ProblemCommand: &MarkOrderCompleteCMD{OrderID: "123"},
+									ProblemState: &OrderProcessing{
+										Order: Order{
+											ID:        "123",
+											OrderAttr: order,
+											WorkerID:  "worker-1",
+										},
+									},
+								}).
+								ForkCase(t, "successfully recover", func(t *testing.T, c *machine.Case[Dependency, Command, State]) {
+									c.
+										GivenCommand(&TryRecoverErrorCMD{OrderID: "123"}).
+										BeforeCommand(func(t testing.TB, di Dependency) {
+											di.(*DependencyMock).ResetCalls()
+										}).
+										AfterCommand(func(t testing.TB, di Dependency) {
+											dep := di.(*DependencyMock)
+											if assert.Len(t, dep.WarehouseRemoveStockCalls(), 1) {
+												assert.Equal(t, order.Quantity, dep.WarehouseRemoveStockCalls()[0].Quantity)
+											}
+											if assert.Len(t, dep.PaymentChargeCalls(), 1) {
+												assert.Equal(t, order.Price, dep.PaymentChargeCalls()[0].Price)
+											}
+										}).
+										ThenState(t, &OrderCompleted{
+											Order: Order{
+												ID:               "123",
+												OrderAttr:        order,
+												WorkerID:         "worker-1",
+												DeliveredAt:      &now,
+												StockRemovedAt:   &now,
+												PaymentChargedAt: &now,
+											},
+										})
+								})
+						})
+				})
+		},
+	)
+
+	if suite.AssertSelfDocumentStateDiagram(t, "machine_test.go") {
+		suite.SelfDocumentStateDiagram(t, "machine_test.go")
+	}
+}
+
+func TestStateTransition_UsingTableTests(t *testing.T) {
+	now := time.Now()
+	dep := &DependencyMock{
+		TimeNowFunc: func() *time.Time {
+			return &now
+		},
+	}
+
+	order := OrderAttr{
+		Price:    100,
+		Quantity: 3,
+	}
+
+	steps := []machine.Step[Dependency, Command, State]{
+		{
+			Name:          "create order without order ID is not allowed",
+			GivenCommand:  &CreateOrderCMD{OrderID: ""},
+			ExpectedState: nil,
+			ExpectedErr:   ErrOrderIDRequired,
+		},
+		{
+			Name:         "create order with valid data",
+			GivenCommand: &CreateOrderCMD{OrderID: "123", Attr: order},
+			ExpectedState: &OrderPending{
+				Order: Order{ID: "123", OrderAttr: order},
+			},
+		},
+		{
+			Name:          "double order creation is not allowed",
+			GivenCommand:  &CreateOrderCMD{OrderID: "123", Attr: order},
+			ExpectedState: &OrderPending{Order: Order{ID: "123", OrderAttr: order}},
+			ExpectedErr:   ErrOrderAlreadyExist,
+		},
+		{
+			Name:          "mark order as processing without order ID must return validation error and not change state",
+			GivenCommand:  &MarkAsProcessingCMD{},
+			ExpectedState: &OrderPending{Order: Order{ID: "123", OrderAttr: order}},
+			ExpectedErr:   ErrOrderIDRequired,
+		},
+		{
+			Name:          "mark order as processing without worker ID must return validation error and not change state",
+			GivenCommand:  &MarkAsProcessingCMD{OrderID: "123"},
+			ExpectedState: &OrderPending{Order: Order{ID: "123", OrderAttr: order}},
+			ExpectedErr:   ErrWorkerIDRequired,
+		},
+		{
+			Name:          "mark order as with not matching order ID must return validation error and not change state",
+			GivenCommand:  &MarkAsProcessingCMD{OrderID: "xxx", WorkerID: "worker-1"},
+			ExpectedState: &OrderPending{Order: Order{ID: "123", OrderAttr: order}},
+			ExpectedErr:   ErrOrderIDMismatch,
+		},
+		{
+			Name:         "mark order as processing with valid data",
+			GivenCommand: &MarkAsProcessingCMD{OrderID: "123", WorkerID: "worker-1"},
+			ExpectedState: &OrderProcessing{
+				Order: Order{
+					ID:        "123",
+					OrderAttr: order,
+					WorkerID:  "worker-1",
+				},
+			},
+		},
+		{
+			Name:          "complete order without order ID must return validation error and not change state",
+			GivenCommand:  &MarkOrderCompleteCMD{},
+			ExpectedState: &OrderProcessing{Order: Order{ID: "123", OrderAttr: order, WorkerID: "worker-1"}},
+			ExpectedErr:   ErrOrderIDRequired,
+		},
+		{
+			Name:         "complete order but removing products from stock fails",
+			GivenCommand: &MarkOrderCompleteCMD{OrderID: "123"},
+			BeforeCommand: func(t testing.TB, di Dependency) {
+				di.(*DependencyMock).ResetCalls()
+				di.(*DependencyMock).WarehouseRemoveStockFunc = func(quantity int) error {
+					return fmt.Errorf("warehouse api unreachable")
+				}
+			},
+			AfterCommand: func(t testing.TB, di Dependency) {
+				dep := di.(*DependencyMock)
+				dep.WarehouseRemoveStockFunc = nil
+				if assert.Len(t, dep.WarehouseRemoveStockCalls(), 1) {
+					assert.Equal(t, order.Quantity, dep.WarehouseRemoveStockCalls()[0].Quantity)
+				}
+
+				assert.Len(t, dep.PaymentChargeCalls(), 0)
+			},
+			ExpectedState: &OrderError{
+				Retried:        0,
+				RetriedAt:      nil,
+				ProblemCode:    ProblemWarehouseAPIUnreachable,
+				ProblemCommand: &MarkOrderCompleteCMD{OrderID: "123"},
+				ProblemState:   &OrderProcessing{Order: Order{ID: "123", OrderAttr: order, WorkerID: "worker-1"}},
+			},
+		},
+		{
+			Name:         "attempt and fail recover error",
+			GivenCommand: &TryRecoverErrorCMD{OrderID: "123"},
+			BeforeCommand: func(t testing.TB, di Dependency) {
+				di.(*DependencyMock).ResetCalls()
+				di.(*DependencyMock).WarehouseRemoveStockFunc = func(quantity int) error {
+					return fmt.Errorf("warehouse api unreachable")
+				}
+			},
+			AfterCommand: func(t testing.TB, di Dependency) {
+				dep := di.(*DependencyMock)
+				dep.WarehouseRemoveStockFunc = nil
+				if assert.Len(t, dep.WarehouseRemoveStockCalls(), 1) {
+					assert.Equal(t, order.Quantity, dep.WarehouseRemoveStockCalls()[0].Quantity)
+				}
+
+				assert.Len(t, dep.PaymentChargeCalls(), 0)
+			},
+			ExpectedState: &OrderError{
+				Retried:        1,
+				RetriedAt:      &now,
+				ProblemCode:    ProblemWarehouseAPIUnreachable,
+				ProblemCommand: &MarkOrderCompleteCMD{OrderID: "123"},
+				ProblemState:   &OrderProcessing{Order: Order{ID: "123", OrderAttr: order, WorkerID: "worker-1"}},
+			},
+		},
+		{
+			Name:         "successful recover from warehouse api unreachable error, and complete order",
+			GivenCommand: &TryRecoverErrorCMD{OrderID: "123"},
+			BeforeCommand: func(t testing.TB, di Dependency) {
+				di.(*DependencyMock).ResetCalls()
+			},
+			AfterCommand: func(t testing.TB, di Dependency) {
+				dep := di.(*DependencyMock)
+				if assert.Len(t, dep.WarehouseRemoveStockCalls(), 1) {
+					assert.Equal(t, order.Quantity, dep.WarehouseRemoveStockCalls()[0].Quantity)
+				}
+				if assert.Len(t, dep.PaymentChargeCalls(), 1) {
+					assert.Equal(t, order.Price, dep.PaymentChargeCalls()[0].Price)
+				}
+			},
+			ExpectedState: &OrderCompleted{
+				Order: Order{
+					ID:               "123",
+					OrderAttr:        order,
+					WorkerID:         "worker-1",
+					DeliveredAt:      &now,
+					StockRemovedAt:   &now,
+					PaymentChargedAt: &now,
+				},
+			},
+		},
+	}
+
+	AssertScenario[Dependency](t, dep, NewMachine, steps)
+}
+
+func AssertScenario[D, C, S any](
+	t *testing.T,
+	dep D,
+	newMachine func(dep D, init S) *machine.Machine[D, C, S],
+	steps []machine.Step[D, C, S],
+) {
+	var prev S
+	for _, step := range steps {
+		t.Run(step.Name, func(t *testing.T) {
+			if any(step.InitState) != nil {
+				prev = step.InitState
+			}
+
+			m := newMachine(dep, prev)
+			if step.BeforeCommand != nil {
+				step.BeforeCommand(t, m.Dep())
+			}
+
+			err := m.Handle(step.GivenCommand)
+
+			if step.AfterCommand != nil {
+				step.AfterCommand(t, m.Dep())
+			}
+
+			assert.ErrorIs(t, step.ExpectedErr, err, step.Name)
+			if diff := cmp.Diff(step.ExpectedState, m.State()); diff != "" {
+				assert.Fail(t, "unexpected state (-want +got):\n%s", diff)
+			}
+
+			prev = m.State()
+
+			//infer.Record(step.GivenCommand, m.State(), step.ExpectedState, err)
+		})
+	}
+}
diff --git a/example/state/machine_test.go.state_diagram.mmd b/example/state/machine_test.go.state_diagram.mmd
new file mode 100644
index 00000000..8d44923d
--- /dev/null
+++ b/example/state/machine_test.go.state_diagram.mmd
@@ -0,0 +1,7 @@
+stateDiagram
+	"*state.OrderProcessing" --> "*state.OrderCancelled": "*state.CancelOrderCMD"
+	[*] --> "*state.OrderPending": "*state.CreateOrderCMD"
+	"*state.OrderPending" --> "*state.OrderProcessing": "*state.MarkAsProcessingCMD"
+	"*state.OrderProcessing" --> "*state.OrderCompleted": "*state.MarkOrderCompleteCMD"
+	"*state.OrderProcessing" --> "*state.OrderError": "*state.MarkOrderCompleteCMD"
+	"*state.OrderError" --> "*state.OrderCompleted": "*state.TryRecoverErrorCMD"
diff --git a/example/state/machine_test.go.state_diagram_with_errors.mmd b/example/state/machine_test.go.state_diagram_with_errors.mmd
new file mode 100644
index 00000000..8a55cc70
--- /dev/null
+++ b/example/state/machine_test.go.state_diagram_with_errors.mmd
@@ -0,0 +1,57 @@
+stateDiagram
+ %% error=cannot cancel order, order must be processing to cancel it; invalid transition 
+	"*state.OrderCancelled" --> "*state.OrderCancelled": "❌*state.CancelOrderCMD"
+ %% error=cannot cancel order, order must be processing to cancel it; invalid transition 
+	"*state.OrderCompleted" --> "*state.OrderCompleted": "❌*state.CancelOrderCMD"
+ %% error=cannot cancel order, order must be processing to cancel it; invalid transition 
+	"*state.OrderError" --> "*state.OrderError": "❌*state.CancelOrderCMD"
+ %% error=cannot cancel order, order must be processing to cancel it; invalid transition 
+	"*state.OrderPending" --> "*state.OrderPending": "❌*state.CancelOrderCMD"
+	"*state.OrderProcessing" --> "*state.OrderCancelled": "*state.CancelOrderCMD"
+ %% error=cannot cancel order, order must be processing to cancel it; invalid transition 
+	[*] --> [*]: "❌*state.CancelOrderCMD"
+ %% error=cannot attemp order creation, order exists: invalid transition 
+	"*state.OrderCancelled" --> "*state.OrderCancelled": "❌*state.CreateOrderCMD"
+ %% error=cannot attemp order creation, order exists: invalid transition 
+	"*state.OrderCompleted" --> "*state.OrderCompleted": "❌*state.CreateOrderCMD"
+ %% error=cannot attemp order creation, order exists: invalid transition 
+	"*state.OrderError" --> "*state.OrderError": "❌*state.CreateOrderCMD"
+ %% error=cannot attemp order creation, order exists: invalid transition 
+	"*state.OrderPending" --> "*state.OrderPending": "❌*state.CreateOrderCMD"
+ %% error=cannot attemp order creation, order exists: invalid transition 
+	"*state.OrderProcessing" --> "*state.OrderProcessing": "❌*state.CreateOrderCMD"
+	[*] --> "*state.OrderPending": "*state.CreateOrderCMD"
+ %% error=invalid transition 
+	"*state.OrderCancelled" --> "*state.OrderCancelled": "❌*state.MarkAsProcessingCMD"
+ %% error=invalid transition 
+	"*state.OrderCompleted" --> "*state.OrderCompleted": "❌*state.MarkAsProcessingCMD"
+ %% error=invalid transition 
+	"*state.OrderError" --> "*state.OrderError": "❌*state.MarkAsProcessingCMD"
+	"*state.OrderPending" --> "*state.OrderProcessing": "*state.MarkAsProcessingCMD"
+ %% error=invalid transition 
+	"*state.OrderProcessing" --> "*state.OrderProcessing": "❌*state.MarkAsProcessingCMD"
+ %% error=invalid transition 
+	[*] --> [*]: "❌*state.MarkAsProcessingCMD"
+ %% error=cannot mark order as complete, order is not being process; invalid transition 
+	"*state.OrderCancelled" --> "*state.OrderCancelled": "❌*state.MarkOrderCompleteCMD"
+ %% error=cannot mark order as complete, order is not being process; invalid transition 
+	"*state.OrderCompleted" --> "*state.OrderCompleted": "❌*state.MarkOrderCompleteCMD"
+ %% error=cannot mark order as complete, order is not being process; invalid transition 
+	"*state.OrderError" --> "*state.OrderError": "❌*state.MarkOrderCompleteCMD"
+ %% error=cannot mark order as complete, order is not being process; invalid transition 
+	"*state.OrderPending" --> "*state.OrderPending": "❌*state.MarkOrderCompleteCMD"
+	"*state.OrderProcessing" --> "*state.OrderCompleted": "*state.MarkOrderCompleteCMD"
+	"*state.OrderProcessing" --> "*state.OrderError": "*state.MarkOrderCompleteCMD"
+ %% error=cannot mark order as complete, order is not being process; invalid transition 
+	[*] --> [*]: "❌*state.MarkOrderCompleteCMD"
+ %% error=cannot recover from non error state; invalid transition 
+	"*state.OrderCancelled" --> "*state.OrderCancelled": "❌*state.TryRecoverErrorCMD"
+ %% error=cannot recover from non error state; invalid transition 
+	"*state.OrderCompleted" --> "*state.OrderCompleted": "❌*state.TryRecoverErrorCMD"
+	"*state.OrderError" --> "*state.OrderCompleted": "*state.TryRecoverErrorCMD"
+ %% error=cannot recover from non error state; invalid transition 
+	"*state.OrderPending" --> "*state.OrderPending": "❌*state.TryRecoverErrorCMD"
+ %% error=cannot recover from non error state; invalid transition 
+	"*state.OrderProcessing" --> "*state.OrderProcessing": "❌*state.TryRecoverErrorCMD"
+ %% error=cannot recover from non error state; invalid transition 
+	[*] --> [*]: "❌*state.TryRecoverErrorCMD"
diff --git a/example/state/model.go b/example/state/model.go
new file mode 100644
index 00000000..c100eafd
--- /dev/null
+++ b/example/state/model.go
@@ -0,0 +1,97 @@
+package state
+
+import "time"
+
+//go:generate go run ../../cmd/mkunion/main.go
+
+type (
+	// OrderID Price, Quantity are placeholders for value objects, to ensure better data semantic and type safety
+	OrderID  = string
+	Price    = float64
+	Quantity = int
+
+	OrderAttr struct {
+		// placeholder for order attributes
+		// like customer name, address, etc.
+		// like product name, price, etc.
+		// for simplicity we only have Price and Quantity
+		Price    Price
+		Quantity Quantity
+	}
+
+	// WorkerID represent human that process the order
+	WorkerID = string
+
+	// Order everything we know about order
+	Order struct {
+		ID               OrderID
+		OrderAttr        OrderAttr
+		WorkerID         WorkerID
+		StockRemovedAt   *time.Time
+		PaymentChargedAt *time.Time
+		DeliveredAt      *time.Time
+		CancelledAt      *time.Time
+		CancelledReason  string
+	}
+)
+
+type ProblemCode int
+
+const (
+	ProblemWarehouseAPIUnreachable ProblemCode = iota
+	ProblemPaymentAPIUnreachable
+)
+
+//go:tag mkunion:"Command"
+type (
+	CreateOrderCMD struct {
+		OrderID OrderID
+		Attr    OrderAttr
+	}
+	MarkAsProcessingCMD struct {
+		OrderID  OrderID
+		WorkerID WorkerID
+	}
+	CancelOrderCMD struct {
+		OrderID OrderID
+		Reason  string
+	}
+	MarkOrderCompleteCMD struct {
+		OrderID OrderID
+	}
+	// TryRecoverErrorCMD is a special command that can be used to recover from error state
+	// you can have different "self-healing" rules based on the error code or even return to previous healthy state
+	TryRecoverErrorCMD struct {
+		OrderID OrderID
+	}
+)
+
+//go:tag mkunion:"State"
+type (
+	OrderPending struct {
+		Order Order
+	}
+	OrderProcessing struct {
+		Order Order
+	}
+	OrderCompleted struct {
+		Order Order
+	}
+	OrderCancelled struct {
+		Order Order
+	}
+	// OrderError is a special state that represent an error
+	// during order processing, you can have different "self-healing jobs" based on the error code
+	// like retrying the order, cancel the order, etc.
+	// treating error as state is a good practice in state machine, it allow you to centralise the error handling
+	OrderError struct {
+		// error information
+		Retried   int
+		RetriedAt *time.Time
+
+		ProblemCode ProblemCode
+
+		ProblemCommand Command
+		ProblemState   State
+	}
+)
diff --git a/example/state/simple_machine.go b/example/state/simple_machine.go
deleted file mode 100644
index d1d0d08d..00000000
--- a/example/state/simple_machine.go
+++ /dev/null
@@ -1,62 +0,0 @@
-package state
-
-import (
-	"fmt"
-	"github.com/widmogrod/mkunion/x/machine"
-)
-
-var (
-	ErrInvalidTransition = fmt.Errorf("invalid cmds")
-)
-
-func NewMachine() *machine.Machine[Command, State] {
-	return machine.NewSimpleMachine(Transition)
-}
-
-func Transition(cmd Command, state State) (State, error) {
-	return MatchCommandR2(
-		cmd,
-		func(x *CreateCandidateCMD) (State, error) {
-			if state != nil {
-				return nil, fmt.Errorf("candidate already created, state: %T; %w", state, ErrInvalidTransition)
-			}
-
-			newState := &Candidate{
-				ID:         x.ID,
-				Attributes: nil,
-			}
-
-			return newState, nil
-		},
-		func(x *MarkAsCanonicalCMD) (State, error) {
-			stateCandidate, ok := state.(*Candidate)
-			if !ok {
-				return nil, fmt.Errorf("state is not candidate, state: %T; %w", state, ErrInvalidTransition)
-			}
-
-			return &Canonical{
-				ID: stateCandidate.ID,
-			}, nil
-		},
-		func(x *MarkAsDuplicateCMD) (State, error) {
-			stateCandidate, ok := state.(*Candidate)
-			if !ok {
-				return nil, fmt.Errorf("state is not candidate, state: %T; %w", state, ErrInvalidTransition)
-			}
-
-			return &Duplicate{
-				ID:          stateCandidate.ID,
-				CanonicalID: x.CanonicalID,
-			}, nil
-		},
-		func(x *MarkAsUniqueCMD) (State, error) {
-			stateCandidate, ok := state.(*Candidate)
-			if !ok {
-				return nil, fmt.Errorf("state is not candidate, state: %T; %w", state, ErrInvalidTransition)
-			}
-			return &Unique{
-				ID: stateCandidate.ID,
-			}, nil
-		},
-	)
-}
diff --git a/example/state/simple_machine_test.go b/example/state/simple_machine_test.go
deleted file mode 100644
index 03770c55..00000000
--- a/example/state/simple_machine_test.go
+++ /dev/null
@@ -1,171 +0,0 @@
-package state
-
-import (
-	"fmt"
-	"github.com/stretchr/testify/assert"
-	"github.com/widmogrod/mkunion/x/machine"
-	"testing"
-)
-
-func TestSuite(t *testing.T) {
-	suite := machine.NewTestSuite(NewMachine)
-	suite.Case(
-		"happy path of transitions",
-		func(c *machine.Case[Command, State]) {
-			c.GivenCommand(&CreateCandidateCMD{ID: "123"}).
-				ThenState(&Candidate{ID: "123"}).
-				ForkCase("can mark as canonical", func(c *machine.Case[Command, State]) {
-					c.GivenCommand(&MarkAsCanonicalCMD{}).
-						ThenState(&Canonical{ID: "123"})
-				}).
-				ForkCase("can mark as duplicate", func(c *machine.Case[Command, State]) {
-					c.GivenCommand(&MarkAsDuplicateCMD{CanonicalID: "456"}).
-						ThenState(&Duplicate{ID: "123", CanonicalID: "456"})
-				}).
-				ForkCase("can mark as unique", func(c *machine.Case[Command, State]) {
-					c.GivenCommand(&MarkAsUniqueCMD{}).
-						ThenState(&Unique{ID: "123"})
-				})
-		},
-	)
-	suite.Run(t)
-	suite.Fuzzy(t)
-
-	if suite.AssertSelfDocumentStateDiagram(t, "simple_machine_test.go") {
-		suite.SelfDocumentStateDiagram(t, "simple_machine_test.go")
-	}
-}
-
-func TestStateTransition(t *testing.T) {
-	useCases := []struct {
-		name   string
-		cmds   []Command
-		state  []State
-		errors []error
-	}{
-		{
-			name: "create candidate (valid)",
-			cmds: []Command{
-				&CreateCandidateCMD{ID: "123"},
-			},
-			state: []State{
-				&Candidate{ID: "123"},
-			},
-			errors: []error{
-				nil,
-			},
-		},
-		{
-			name: "candidate state and transit to duplicate  (valid)",
-			cmds: []Command{
-				&CreateCandidateCMD{ID: "123"},
-				&MarkAsDuplicateCMD{CanonicalID: "456"},
-			},
-			state: []State{
-				&Candidate{ID: "123"},
-				&Duplicate{ID: "123", CanonicalID: "456"},
-			},
-			errors: []error{
-				nil,
-				nil,
-			},
-		},
-		{
-			name: "candidate state and transit to canonical  (valid)",
-			cmds: []Command{
-				&CreateCandidateCMD{ID: "123"},
-				&MarkAsCanonicalCMD{},
-			},
-			state: []State{
-				&Candidate{ID: "123"},
-				&Canonical{ID: "123"},
-			},
-			errors: []error{
-				nil,
-				nil,
-			},
-		},
-		{
-			name: "candidate state and transit to unique  (valid)",
-			cmds: []Command{
-				&CreateCandidateCMD{ID: "123"},
-				&MarkAsUniqueCMD{},
-			},
-			state: []State{
-				&Candidate{ID: "123"},
-				&Unique{ID: "123"},
-			},
-			errors: []error{
-				nil,
-				nil,
-			},
-		},
-		{
-			name: "initial state cannot be market as duplicate (invalid)",
-			cmds: []Command{
-				&MarkAsDuplicateCMD{CanonicalID: "456"},
-			},
-			state: []State{
-				nil,
-			},
-			errors: []error{
-				ErrInvalidTransition,
-			},
-		},
-		{
-			name: "candidate state and transit to canonical and duplicate  (invalid)",
-			cmds: []Command{
-				&CreateCandidateCMD{ID: "123"},
-				&MarkAsCanonicalCMD{},
-				&MarkAsDuplicateCMD{CanonicalID: "456"},
-			},
-			state: []State{
-				&Candidate{ID: "123"},
-				&Canonical{ID: "123"},
-				&Canonical{ID: "123"},
-			},
-			errors: []error{
-				nil,
-				nil,
-				ErrInvalidTransition,
-			},
-		},
-	}
-
-	infer := machine.NewInferTransition[Command, State]()
-	infer.WithTitle("Canonical question transition")
-
-	for _, uc := range useCases {
-		t.Run(uc.name, func(t *testing.T) {
-			m := NewMachine()
-			for i, tr := range uc.cmds {
-				prev := m.State()
-				err := m.Handle(tr)
-				if uc.errors[i] == nil {
-					assert.NoError(t, err)
-				} else {
-					assert.Error(t, uc.errors[i], err)
-				}
-				assert.Equal(t, uc.state[i], m.State())
-				infer.Record(tr, prev, m.State(), err)
-			}
-		})
-	}
-
-	infer.WithErrorTransitions(true)
-	result := infer.ToMermaid()
-	fmt.Println(result)
-	assert.Equal(t, `---
-title: Canonical question transition
----
-stateDiagram
-	[*] --> "*state.Candidate": "*state.CreateCandidateCMD"
-	"*state.Candidate" --> "*state.Duplicate": "*state.MarkAsDuplicateCMD"
-	"*state.Candidate" --> "*state.Canonical": "*state.MarkAsCanonicalCMD"
-	"*state.Candidate" --> "*state.Unique": "*state.MarkAsUniqueCMD"
- %% error=state is not candidate, state: <nil>; invalid cmds 
-	[*] --> [*]: "❌*state.MarkAsDuplicateCMD"
- %% error=state is not candidate, state: *state.Canonical; invalid cmds 
-	"*state.Canonical" --> "*state.Canonical": "❌*state.MarkAsDuplicateCMD"
-`, result)
-}
diff --git a/example/state/simple_machine_test.go.state_diagram.mmd b/example/state/simple_machine_test.go.state_diagram.mmd
deleted file mode 100644
index 7e5f0770..00000000
--- a/example/state/simple_machine_test.go.state_diagram.mmd
+++ /dev/null
@@ -1,21 +0,0 @@
-stateDiagram
-	[*] --> "*state.Candidate": "*state.CreateCandidateCMD"
-	"*state.Candidate" --> "*state.Canonical": "*state.MarkAsCanonicalCMD"
-	"*state.Candidate" --> "*state.Duplicate": "*state.MarkAsDuplicateCMD"
-	"*state.Candidate" --> "*state.Unique": "*state.MarkAsUniqueCMD"
- %% error=state is not candidate, state: *state.Canonical; invalid cmds 
- %% error=state is not candidate, state: <nil>; invalid cmds 
- %% error=candidate already created, state: *state.Canonical; invalid cmds 
- %% error=candidate already created, state: *state.Candidate; invalid cmds 
- %% error=candidate already created, state: *state.Unique; invalid cmds 
- %% error=state is not candidate, state: *state.Unique; invalid cmds 
- %% error=candidate already created, state: *state.Duplicate; invalid cmds 
- %% error=state is not candidate, state: *state.Unique; invalid cmds 
- %% error=state is not candidate, state: *state.Unique; invalid cmds 
- %% error=state is not candidate, state: *state.Canonical; invalid cmds 
- %% error=state is not candidate, state: *state.Canonical; invalid cmds 
- %% error=state is not candidate, state: *state.Duplicate; invalid cmds 
- %% error=state is not candidate, state: <nil>; invalid cmds 
- %% error=state is not candidate, state: <nil>; invalid cmds 
- %% error=state is not candidate, state: *state.Duplicate; invalid cmds 
- %% error=state is not candidate, state: *state.Duplicate; invalid cmds 
diff --git a/example/state/simple_machine_test.go.state_diagram_with_errors.mmd b/example/state/simple_machine_test.go.state_diagram_with_errors.mmd
deleted file mode 100644
index c9e2a37b..00000000
--- a/example/state/simple_machine_test.go.state_diagram_with_errors.mmd
+++ /dev/null
@@ -1,37 +0,0 @@
-stateDiagram
-	[*] --> "*state.Candidate": "*state.CreateCandidateCMD"
-	"*state.Candidate" --> "*state.Canonical": "*state.MarkAsCanonicalCMD"
-	"*state.Candidate" --> "*state.Duplicate": "*state.MarkAsDuplicateCMD"
-	"*state.Candidate" --> "*state.Unique": "*state.MarkAsUniqueCMD"
- %% error=state is not candidate, state: *state.Canonical; invalid cmds 
-	"*state.Canonical" --> "*state.Canonical": "❌*state.MarkAsDuplicateCMD"
- %% error=state is not candidate, state: <nil>; invalid cmds 
-	[*] --> [*]: "❌*state.MarkAsDuplicateCMD"
- %% error=candidate already created, state: *state.Canonical; invalid cmds 
-	"*state.Canonical" --> "*state.Canonical": "❌*state.CreateCandidateCMD"
- %% error=candidate already created, state: *state.Candidate; invalid cmds 
-	"*state.Candidate" --> "*state.Candidate": "❌*state.CreateCandidateCMD"
- %% error=candidate already created, state: *state.Unique; invalid cmds 
-	"*state.Unique" --> "*state.Unique": "❌*state.CreateCandidateCMD"
- %% error=state is not candidate, state: *state.Unique; invalid cmds 
-	"*state.Unique" --> "*state.Unique": "❌*state.MarkAsDuplicateCMD"
- %% error=candidate already created, state: *state.Duplicate; invalid cmds 
-	"*state.Duplicate" --> "*state.Duplicate": "❌*state.CreateCandidateCMD"
- %% error=state is not candidate, state: *state.Unique; invalid cmds 
-	"*state.Unique" --> "*state.Unique": "❌*state.MarkAsCanonicalCMD"
- %% error=state is not candidate, state: *state.Unique; invalid cmds 
-	"*state.Unique" --> "*state.Unique": "❌*state.MarkAsUniqueCMD"
- %% error=state is not candidate, state: *state.Canonical; invalid cmds 
-	"*state.Canonical" --> "*state.Canonical": "❌*state.MarkAsUniqueCMD"
- %% error=state is not candidate, state: *state.Canonical; invalid cmds 
-	"*state.Canonical" --> "*state.Canonical": "❌*state.MarkAsCanonicalCMD"
- %% error=state is not candidate, state: *state.Duplicate; invalid cmds 
-	"*state.Duplicate" --> "*state.Duplicate": "❌*state.MarkAsCanonicalCMD"
- %% error=state is not candidate, state: <nil>; invalid cmds 
-	[*] --> [*]: "❌*state.MarkAsUniqueCMD"
- %% error=state is not candidate, state: <nil>; invalid cmds 
-	[*] --> [*]: "❌*state.MarkAsCanonicalCMD"
- %% error=state is not candidate, state: *state.Duplicate; invalid cmds 
-	"*state.Duplicate" --> "*state.Duplicate": "❌*state.MarkAsDuplicateCMD"
- %% error=state is not candidate, state: *state.Duplicate; invalid cmds 
-	"*state.Duplicate" --> "*state.Duplicate": "❌*state.MarkAsUniqueCMD"
diff --git a/example/state/simple_state_transitions.go b/example/state/simple_state_transitions.go
deleted file mode 100644
index 109b705f..00000000
--- a/example/state/simple_state_transitions.go
+++ /dev/null
@@ -1,34 +0,0 @@
-package state
-
-type (
-	ID   = string
-	Attr = map[string]any
-)
-
-//go:generate go run ../../cmd/mkunion/main.go --name=Command
-type (
-	CreateCandidateCMD struct {
-		ID ID
-	}
-	MarkAsCanonicalCMD struct{}
-	MarkAsDuplicateCMD struct{ CanonicalID ID }
-	MarkAsUniqueCMD    struct{}
-)
-
-//go:generate go run ../../cmd/mkunion/main.go --name=State
-type (
-	Candidate struct {
-		ID         ID
-		Attributes Attr
-	}
-	Canonical struct {
-		ID ID
-	}
-	Duplicate struct {
-		ID          ID
-		CanonicalID ID
-	}
-	Unique struct {
-		ID ID
-	}
-)

From 57f610fd686cf280b04db911ed1130b4e172002e Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 00:20:06 +0100
Subject: [PATCH 05/72] dev: introduce moq

---
 dev/bootstrap.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/dev/bootstrap.sh b/dev/bootstrap.sh
index 16af449c..ebf4abf8 100755
--- a/dev/bootstrap.sh
+++ b/dev/bootstrap.sh
@@ -6,6 +6,7 @@ project_root=$(dirname "$cwd")
 envrc_file=$project_root/.envrc
 
 echo "Check if necessary tools are installed"
+command -v go >/dev/null 2>&1 || { echo >&2 "golang is not installed. Aborting."; exit 1; }
 command -v docker >/dev/null 2>&1 || { echo >&2 "docker is not installed. Aborting."; exit 1; }
 command -v docker-compose >/dev/null 2>&1 || { echo >&2 "docker-compose is not installed. Aborting."; exit 1; }
 command -v awslocal >/dev/null 2>&1 || { echo >&2 "awslocal is not installed. Aborting. Please run
@@ -14,6 +15,9 @@ command -v awslocal >/dev/null 2>&1 || { echo >&2 "awslocal is not installed. Ab
 echo "Creating volume directory"
 mkdir -p $cwd/_volume
 
+echo "Install moq"
+go get github.com/matryer/moq@latest
+
 echo "Starting localstack"
 docker compose -f $cwd/compose.yml up -d
 # trap exit and stop docker compose

From 884a8666d2dc247ec7c6c3786b02953bc8d93808 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 00:20:21 +0100
Subject: [PATCH 06/72] dev: expose different port for documentation

---
 dev/docs.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dev/docs.sh b/dev/docs.sh
index d55008bf..603d78a5 100755
--- a/dev/docs.sh
+++ b/dev/docs.sh
@@ -5,7 +5,8 @@ cwd=$(dirname "$0")
 project_root=$(dirname "$cwd")
 
 if [ "$1" == "run" ]; then
-  docker run --rm -it -p 8000:8000 -v ${project_root}:/docs squidfunk/mkdocs-material
+  echo "Serving documentation at http://localhost:8088"
+  docker run --rm -it -p 8088:8000 -v ${project_root}:/docs squidfunk/mkdocs-material
 elif [ "$1" == "build" ]; then
   docker run --rm -it -v ${project_root}:/docs squidfunk/mkdocs-material build
 else

From a23dac3f917d3f12f5144c581ed7f096078e3523 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 00:21:45 +0100
Subject: [PATCH 07/72] docs: description of state machine

---
 docs/examples/state_machine.md | 370 +++++++++++++++++++++++++++++++++
 1 file changed, 370 insertions(+)
 create mode 100644 docs/examples/state_machine.md

diff --git a/docs/examples/state_machine.md b/docs/examples/state_machine.md
new file mode 100644
index 00000000..d6139cca
--- /dev/null
+++ b/docs/examples/state_machine.md
@@ -0,0 +1,370 @@
+---
+title: State machines and unions
+---
+# MkUnion and state machines in golang
+
+This document will show how to use `mkunion` to manage application state on example of an order service. 
+You will learn:
+
+- how to model state machines in golang, and find similarities to "__clean architecture__"
+- How to **test state machines** (with fuzzing), and as a bonus you will get mermaid diagrams for free
+- How to **persist state in database** and how optimistic concurrency helps __resolve concurrency conflicts__
+- How to **handle errors** in state machines, and build foundations for __self-healing__ systems
+
+
+## Working example
+
+As an driving example, we will use an order service that can be in one of the following states:
+
+- `Pending` - order is created, and is waiting for someone to process it
+- `Processing` - order is being processed, an human is going to pick up items from warehouse and pack them
+- `Cancelled` - order was cancelled, there can be many reason, one of them is that warehouse is out of stock.
+- `Completed` - order is completed, and can be shipped to customer.
+
+Such states, have rules that govern **transitions**, like order cannot be cancelled if it's already completed, and so on.
+
+And we need to have also to trigger changes in state, like create order that pending for processing, or cancel order. We will call those triggers **commands**.
+
+Some of those rules could change in future, and we want to be able to change them without rewriting whole application.
+This also informs us that our design should be open for extensions.
+
+Site note, if you want go strait to final code product, then into [example/state/](example/state/) directory and have fun exploring.
+
+## Modeling commands and states
+
+Our example can be represented as state machine that looks like this:
+[simple_machine_test.go.state_diagram.mmd](example/state/simple_machine_test.go.state_diagram.mmd)
+```mermaid
+--8<-- "example/state/machine_test.go.state_diagram.mmd"
+```
+
+In this diagram, we can see that we have 4 states, and 4 commands that can trigger transitions between states shown as arrows.
+
+Because this diagram is generated from code, it has names that represent types in golang that we use. 
+
+For example `*state.CreateOrderCMD`:
+
+- `state` it's a package name
+- `CreateOrderCMD` is a struct name in that package.
+- `CMD` suffix it's naming convention, that it's optional, but I find it to make code more readable.
+
+
+Below is a code snippet that demonstrate complete model of **state** and **commands** of order service, that we talked about.
+
+**Notice** that we use `mkunion` to group commands and states. (Look for `//go:tag mkunion:"Command"`)
+
+This is one example how union types can be used in golang. 
+Historically in golang it would be very hard to achieve such thing, and it would require a lot of boilerplate code.
+Here interface that group those types is generated automatically.
+
+```go
+--8<-- "example/state/model.go"
+```
+
+## Modeling transitions
+One thing that is missing is implementation of transitions between states. 
+There are few ways to do it, today I will show you how to do it using functional approach.
+
+You can think about this function like a `reduce` or `map` function, that you may be familiar.
+
+
+Let's name this function `Transition` that has following signature (name is arbitrary, you can name it as you like):
+Our function takes current state and command, and returns new state, and as you can see it.
+
+```go
+func Transition(dep Dependencies, cmd Command, state State) (State, error)
+```
+
+Let's break down this signature.
+
+Input arguments are:
+
+- `dep` encapsulates dependencies like API clients, database connection, configuration, context etc.
+   everything that is needed for complete production implementation.
+- `cmd` it's a command that we want to apply to state, 
+   and it has `Command` interface, that was generate by `mkunion` when it was used to group commands.
+- `state` it's a state that we want to apply our command to and change it, 
+   and it has `State` interface, that was generate similarly to `Command` interface.
+
+
+Output value is either new state, or error if something went wrong. This is very important assumption, 
+that will come very handy when we will talk about self-healing processes.
+
+Below is snippet of implementation of `Transition` function for our order service:
+
+```go
+--8<-- "example/state/machine.go:30:81"
+// ...
+// rest remove for brevity 
+// ...
+```
+
+You can notice few patterns in this snippet:
+
+- `Dependency` interface help us to keep, well  dependencies - well defined, which helps greatly in testability and readability of the code. 
+- Use of generated function `MatchCommandR2` to exhaustively match all commands. 
+  This is powerful, when new command is added, you can be sure that you will get compile time error, if you don't handle it.
+- Validation of commands in done in transition function. Current implementation is simple, but you can use go-validate or other libraries to make it more robust.
+- Each command check state to which is being applied using `switch` statement, it ignore states that it does not care about. 
+  Which means as implementation you have to focus only on small bit of the picture, and not worry about rest of the states. 
+  This is also example where non-exhaustive use of `switch` statement is welcome.
+
+Simple, isn't it? Simplicity also comes from fact that we don't have to worry about marshalling/unmarshalling data, working with database, those are things that will be done in other parts of the application, keeping this part clean and focused on business logic.
+
+Note: Implementation for educational purposes is kept in one big function, 
+but for large projects it may be better to split it into smaller functions, 
+or use visitor pattern interface, that is also generated for you.
+
+
+## Testing state machines & self-documenting
+Before we go further, let's talk about testing our implementation.
+
+Testing will help us not only ensure that our implementation is correct, but also will help us to document our state machine, 
+and discover transition that we didn't think about, that should or shouldn't be possible.
+
+Here is how you can test state machine, in declarative way, using `mkunion/x/machine` package:
+
+```go
+--8<-- "example/state/machine_test.go:12:86"
+```
+Few things to notice in this test:
+
+- We use of `moq` to generate mocks for dependencies
+- We use standard go testing
+- We use `machine.NewTestSuite` as an standard way to test state machines
+- We start with describing **happy path**, and use `suite.Case` to define test case.
+- But most importantly, we define test cases using `GivenCommand` and `ThenState` functions, that help in making test more readable.
+- You can see use of `ForkCase` command, that allow you to take a definition of a state declared in `ThenState` command, and apply new command to it, and expect new state.
+
+I know it's subjective, but I find it very readable, and easy to understand, even for non-programmers.
+
+## Generating state diagram from tests
+Last bit is this line at the bottom:
+
+```go
+if suite.AssertSelfDocumentStateDiagram(t, "machine_test.go") {
+   suite.SelfDocumentStateDiagram(t, "machine_test.go")
+}
+```
+
+This code takes all inputs provided in test suit and fuzzy them, apply commands to random states, and records result of those transitions.
+
+ - `SelfDocumentStateDiagram` - produce two `mermaid` diagrams, that show all possible transitions that are possible in our state machine.
+ - `AssertSelfDocumentStateDiagram` can be used to compare new generated diagrams to diagrams committed in repository, and fail test if they are different.
+   You don't have to use it, but it's good practice to ensure that your state machine is well tested and don't regress without you noticing.
+
+
+There are two diagrams that are generated.
+
+One is a diagram of ONLY successful transitions, that you saw at the beginning of this post.
+
+```mermaid
+--8<-- "example/state/machine_test.go.state_diagram.mmd"
+```
+
+Second is a diagram that includes commands that resulted in an errors:
+```mermaid
+--8<-- "example/state/machine_test.go.state_diagram_with_errors.mmd"
+```
+
+Those diagrams are stored in the same directory as test file, and are prefixed with name used in `AssertSelfDocumentStateDiagram` function.
+```
+machine_test.go
+machine_test.go.state_diagram.mmd
+machine_test.go.state_diagram_with_errors.mmd
+```
+
+
+## Persisting state in database
+
+```mermaid
+sequenceDiagram
+    participant R as Request
+    participant Store as Store
+
+    activate R
+    R->>R: Validate(request) -> error
+    
+    R->>Store: Load state from database by request.ObjectId
+    activate Store
+    Store->>R: Ok(State)
+    deactivate Store
+    
+    R->>R: Create machine with state
+    R->>R: Apply command on a state
+    
+    R->>Store: Save state in database under request.ObjectId
+    activate Store
+    Store->>R: Ok()
+    deactivate Store
+    
+    deactivate R
+```
+
+Example implementation of such sequence diagram:
+
+```go
+func Handle(rq Request, response Resopnse) {
+	ctx := rq.Context()
+	
+	// extract objectId and command from request + do some validation
+    id := rq.ObjectId
+	command := rq.Command
+	
+    // Load state from store
+    state, err := store.Find(ctx, id)
+	if err != nil { /*handle error*/ }
+
+    machine := NewSimpleMachineWithState(Transition, state)
+    newState, err := machine.Apply(cmd, state)
+    if err != nil { /*handle error*/ }
+	
+    err := store.Save(ctx, newState)
+    if err != nil { /*handle error*/ }
+	
+	// serialize response
+	response.Write(newState)
+}
+```
+
+## Error as state. Self-healing systems.
+In request-response situation, handing errors is easy, but what if in some long-lived process something goes wrong?
+How to handle errors in such situation? Without making what we learn about state machines useless or hard to use?
+
+One solution is to treat errors as state.
+In such case, our state machines will never return error, but instead will return new state, that will represent error.
+
+When we introduce explicit command responsible for correcting RecoverableError, we can create self-healing systems.
+Thanks to that, even in situation when errors are unknown, we can retroactivly introduce self-healing logic that correct states.
+
+Because there is always there is only one error state, it makes such state machines easy to reason about.
+
+```go
+//go:generate mkunion -name State
+type (
+    // ...
+    RecoverableError struct {
+        ErrCode int
+        PrevState State
+        RetryCount int
+    }
+)
+
+//go:generate mkunion -name Command
+type (
+    // ...
+    CorrectStateCMD struct {}
+)
+```
+
+Now, we have to implement recoverable logic in our state machine.
+We show example above how to do it in `Transition` function.
+
+Here is example implementation of such transition function:
+
+```go
+func Transition(cmd Command, state State) (State, error) {
+return MustMatchCommandR2(
+    cmd,
+    /* ... */
+    func(cmd *CorrectStateCMD) (State, error) {
+        switch state := state.(type) {
+        case *RecoverableError:
+            state.RetryCount = state.RetryCount + 1
+			
+            // here we can do some self-healing logic
+            if state.ErrCode == DuplicateServiceUnavailable {
+                newState, err := Transition(&MarkAsDuplicateCMD{}, state.PrevState)
+                 if err != nil {
+                    // we failed to correct error, so we return error state 
+                     return &RecoverableError{
+                        ErrCode: err,
+                        PrevState: state.PrevState,
+                        RetryCount: state.RetryCount,
+                    }, nil
+                }
+				
+                 // we manage to fix state, so we return new state
+                 return newState, nil
+             } else {
+                 // log information that we have new code, that we don't know how to handle
+             }
+			
+            // try to correct error in next iteration
+            return state, nil
+        }
+    }
+}
+```
+
+Now, to correct states we have to select from database all states that are in error state.
+It can be use in many ways, example below use a abstraction called `TaskQueue` that is responsible for running tasks in background.
+
+This abstraction guaranties that all records (historical and new ones) will be processed.
+You can think about it, as a queue that is populated by records from database, that meet SQL query criteria.
+
+You can use CRON job and pull database.
+
+```go
+//go:generate mms deployyml -type=TaskQueue -name=CorrectMSPErrors -autoscale=1,10 -memory=128Mi -cpu=100m -timeout=10s -schedule="0 0 * * *"
+func main()
+    sql := "SELECT * FROM ObjectState WHERE RecoverableError.RetryCount < 3"
+    store := datalayer.DefaultStore()
+    queue := TaskQueueFrom("correct-msp-errors", sql, store)
+    queue.OnTask(func (ctx context.Context, task Task) error {
+        state := task.State()
+        cmd := &CorrectStateCMD{}
+        machine := NewSimpleMachineWithState(Transition, state)
+        newState, err := machine.Apply(cmd, state)
+        if err != nil {
+            return err
+        }
+        return task.Save(ctx, newState)
+    })
+    err := queue.Run(ctx)
+    if err != nil {
+        log.Panic(err)
+    }
+}
+```
+
+
+## State machines and command queues and workflows
+What if command would initiate state "to process" and save it in db
+What if task queue would take such state and process it
+Woudn't this be something like command queue?
+
+When to make a list of background processes that transition such states?
+
+### processors per state
+It's like micromanage TaskQueue, where each state has it's own state, and it knows what command to apply to given state
+This could be good starting point, when there is not a lot of good tooling
+
+### processor for state machine
+With good tooling, transition of states can be declared in one place,
+and deployment to task queue could be done automatically.
+
+Note, that only some of the transitions needs to happen in background, other can be done in request-response manner.
+
+### processor for state machine with workflow
+State machine could be generalized to workflow.
+We can think about it as set of generic Command and State (like a turing machine).
+
+States like Pending, Completed, Failed
+Commands like Process, Retry, Cancel
+
+And workflow DSL with commands like: Invoke, Choose, Assign
+Where function is some ID string, and functions needs to be either
+pulled from registry, or called remotely (InvokeRemote).
+some operations would require callback (InvokeAndAwait)
+
+Then background processor would be responsible for executing such workflow (using task queue)
+Program would be responsible for defining workflow, and registering functions.
+
+Such programs could be also optimised for deployment,
+if some function would be better to run on same machine that do RPC call
+like function doing RPC call to database, and caching result in memory or in cache cluster dedicated to specific BFF
+
+
+
+

From f4d73b592df6029082c82406a33f2bc0808b0d64 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 00:22:46 +0100
Subject: [PATCH 08/72] docs: small fixes

---
 README.md                       |  8 ++++----
 docs/development/development.md | 11 ++++++-----
 docs/examples/generic_union.md  |  2 +-
 docs/getting_started.md         |  4 ++--
 docs/index.md                   | 12 ++++++------
 docs/roadmap.md                 | 17 ++++++++---------
 mkdocs.yml                      |  3 ++-
 7 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/README.md b/README.md
index cc2bf9b2..eed847eb 100644
--- a/README.md
+++ b/README.md
@@ -5,10 +5,10 @@
 
 
 ## About
-Strongly typed **union type** in golang.
+Strongly typed **union type** in golang with generics*.
 
-* with full _pattern matching_ support
-* with full _json marshalling_ support
+* with exhaustive _pattern matching_ support
+* with _json marshalling_ including generics
 * and as a bonus, can generate compatible typescript types for end-to-end type safety in your application
 
 ## Why
@@ -19,7 +19,7 @@ Visitor pattern requires a lot of boiler plate code and hand crafting of the `Ac
 
 On top of that, any data marshalling like to/from JSON requires additional, hand crafted code, to make it work.
 
-MkUnion solves all of those problems, by generating opinionated and strongly typed mindful code for you.
+MkUnion solves all of those problems, by generating opinionated and strongly typed meaningful code for you.
 
 ## Example
 
diff --git a/docs/development/development.md b/docs/development/development.md
index f7bc9d31..2660adfa 100644
--- a/docs/development/development.md
+++ b/docs/development/development.md
@@ -3,6 +3,12 @@ title: Contributing and development
 ---
 
 # Contributing and development
+## Contributing
+
+If you want to contribute to `mkunion` project, please open issue first to discuss your idea.
+
+I have opinions about how `mkunion` should work, how I want to evolve it, and I want to make sure that your idea fits into the project.
+
 ## Development
 
 Checkout repo and run:
@@ -28,8 +34,3 @@ To preview documentation run:
 ```
  ./dev/docs.sh run
 ```
-
-## Contributing
-
-If you want to contribute to `mkunion` project, please open issue first to discuss your idea.
-I have opinions about how `mkunion` should work, how I want to evolve it, and I want to make sure that your idea fits into the project.
diff --git a/docs/examples/generic_union.md b/docs/examples/generic_union.md
index 010a3401..c807b27c 100644
--- a/docs/examples/generic_union.md
+++ b/docs/examples/generic_union.md
@@ -27,7 +27,7 @@ type (
 )
 ```
 
-After you run generation (as described in [getting started](/getting_started.md)), 
+After you run generation (as described in [getting started](../getting_started.md)), 
 you have access to the same features as with non-generic unions.
 
 ## Matching function
diff --git a/docs/getting_started.md b/docs/getting_started.md
index 12e16afe..d4ff38f9 100644
--- a/docs/getting_started.md
+++ b/docs/getting_started.md
@@ -77,7 +77,7 @@ In feature we plan to add `mkununion watch ./...` command that will watch for ch
 This will allow you to remove `//go:generate` directive from your code, and have faster feedback loop.
 
 ### Match over union type
-When you run `mkunion` command, it will generate file alongside your original file with `union_gen.go` suffix (example [vehicle_union_gen.go](..%2Fexample%2Fvehicle_union_gen.go))
+When you run `mkunion` command, it will generate file alongside your original file with `union_gen.go` suffix (example [vehicle_union_gen.go](../example/vehicle_union_gen.go))
 
 You can use those function to do exhaustive matching on your union type.
 
@@ -148,4 +148,4 @@ func ExampleVehicleToJSON() {
 You can notice that it has opinionated way of marshalling and unmarshalling your union type.
 It uses `$type` field to store type information, and then store actual data in separate field, with corresponding name.
 
-You can read more about it in [Marshaling union in JSON](./example/json.md) section.
\ No newline at end of file
+You can read more about it in [Marshaling union in JSON](./examples/json.md) section.
\ No newline at end of file
diff --git a/docs/index.md b/docs/index.md
index 1a3b2d33..61dd7fb6 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -9,10 +9,10 @@ title: Introduction
 
 
 ## About
-Strongly typed **union type** in golang.
+Strongly typed **union type** in golang that supports generics*
 
-* with full _pattern matching_ support
-* with full _json marshalling_ support
+* with exhaustive _pattern matching_ support
+* with _json marshalling_ including generics
 * and as a bonus, can generate compatible typescript types for end-to-end type safety in your application
 
 ## Why
@@ -23,7 +23,7 @@ Visitor pattern requires a lot of boiler plate code and hand crafting of the `Ac
 
 On top of that, any data marshalling like to/from JSON requires additional, hand crafted code, to make it work.
 
-MkUnion solves all of those problems, by generating opinionated and strongly typed mindful code for you.
+MkUnion solves all of those problems, by generating opinionated and strongly typed meaningful code for you.
 
 ## Example
 
@@ -86,5 +86,5 @@ func ExampleFromJSON() {
 
 ## Next
 
-- Read [getting started](docs/getting_started.md)  to learn more.
-- Or to understand better concepts jump and read [value proposition](docs/value_proposition.md)
\ No newline at end of file
+- Read [getting started](./getting_started.md)  to learn more.
+- Or to understand better concepts jump and read [value proposition](./value_proposition.md)
\ No newline at end of file
diff --git a/docs/roadmap.md b/docs/roadmap.md
index 794a9042..061dc926 100644
--- a/docs/roadmap.md
+++ b/docs/roadmap.md
@@ -1,15 +1,14 @@
 # Roadmap
-
 ## Learning and adoption
 
-- [_] feature: `mkunion watch ./...` command that watches for changes in files and runs faster than go generate
-- [_] docs: document simple state machine and how to use `mkunion` for it
-- [_] docs: document other packages in `x/` directory
-- [_] docs: document typescript types generation and end-to-end typs concepts (from backend to frontend)
-- [_] feature: expose functions to extract `go:tag` metadata
-- [_] docs: describe philosophy of "data as resource" and how it translates to some of library concepts
+- [ ] **docs**: document simple state machine and how to use `mkunion` for it
+- [ ] **feature**: `mkunion watch ./...` command that watches for changes in files and runs faster than `go generate ./...` that executes each go:generate separately
+- [ ] **docs**: document other packages in `x/` directory
+- [ ] **docs**: document typescript types generation and end-to-end typs concepts (from backend to frontend)
+- [ ] **feature**: expose functions to extract `go:tag` metadata
+- [ ] **docs**: describe philosophy of "data as resource" and how it translates to some of library concepts
 
 ## Long tern experiments and prototypes
 
-- [_] experiment: generate other serialization formats (e.g. grpc)
-- [_] prototype: http & gRPC client for end-to-end types. 
\ No newline at end of file
+- [ ] **experiment**: generate other (de)serialization formats (e.g. grpc, sql, graphql)
+- [ ] **prototype**: http & gRPC client for end-to-end types. 
\ No newline at end of file
diff --git a/mkdocs.yml b/mkdocs.yml
index 42e6efca..cc319a02 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -23,4 +23,5 @@ markdown_extensions:
         - name: mermaid
           class: mermaid
           format: !!python/name:pymdownx.superfences.fence_code_format
-
+  - pymdownx.tasklist:
+      custom_checkbox: true
\ No newline at end of file

From 1f3fb67d7a2def03790a899ea6aeaa38eac12c1e Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 00:23:29 +0100
Subject: [PATCH 09/72] example: tic tac toe machine use new testing suite

---
 example/tic_tac_toe_machine/machine.go | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/example/tic_tac_toe_machine/machine.go b/example/tic_tac_toe_machine/machine.go
index d531ee27..578eabe8 100644
--- a/example/tic_tac_toe_machine/machine.go
+++ b/example/tic_tac_toe_machine/machine.go
@@ -146,14 +146,10 @@ func Transition(cmd Command, state State) (State, error) {
 	)
 }
 
-func NewMachine() *machine.Machine[Command, State] {
+func NewMachine() *machine.Machine[any, Command, State] {
 	return machine.NewSimpleMachine(Transition)
 }
 
-func NewMachineWithState(s State) *machine.Machine[Command, State] {
-	return machine.NewSimpleMachineWithState(Transition, s)
-}
-
 func ParsePosition(position Move, boardRows int, boardCols int) (Move, error) {
 	var r, c int
 	_, err := fmt.Sscanf(position, "%d.%d", &r, &c)

From 4a7b9b1f49fca0e6d053e6588efaaf840d46dab6 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 00:27:47 +0100
Subject: [PATCH 10/72] go.mod update

---
 go.mod | 4 +++-
 go.sum | 6 ++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 7d05aae7..7b8a1eeb 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/urfave/cli/v2 v2.27.1
 	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
-	golang.org/x/mod v0.12.0
+	golang.org/x/mod v0.14.0
 )
 
 require (
@@ -42,11 +42,13 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
+	github.com/matryer/moq v0.3.4 // indirect
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/tools v0.17.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
diff --git a/go.sum b/go.sum
index 1dccc5ec..a601a2dc 100644
--- a/go.sum
+++ b/go.sum
@@ -106,6 +106,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
 github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
+github.com/matryer/moq v0.3.4 h1:czCFIos9rI2tyOehN9ktc/6bQ76N9J4xQ2n3dk063ac=
+github.com/matryer/moq v0.3.4/go.mod h1:wqm9QObyoMuUtH81zFfs3EK6mXEcByy+TjvSROOXJ2U=
 github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs=
 github.com/moby/sys/mount v0.3.3/go.mod h1:PBaEorSNTLG5t/+4EgukEQVlAvVEc6ZjTySwKdqp5K0=
 github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
@@ -161,6 +163,8 @@ golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMe
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
@@ -192,6 +196,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
+golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 h1:0BOZf6qNozI3pkN3fJLwNubheHJYHhMh91GRFOWWK08=
 google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=

From c825ebfa3ca7fa28475b67e82823b33c5d29ee0d Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 12:59:01 +0200
Subject: [PATCH 11/72] my-app: few adjustments to code

---
 example/my-app/go.mod                         |  3 +-
 example/my-app/server.go                      | 28 +++++++++----------
 example/my-app/src/App.tsx                    | 14 +++++++---
 ...b_com_widmogrod_mkunion_exammple_my-app.ts |  2 +-
 4 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/example/my-app/go.mod b/example/my-app/go.mod
index 66f2672d..a6c79405 100644
--- a/example/my-app/go.mod
+++ b/example/my-app/go.mod
@@ -27,6 +27,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fatih/structtag v1.2.0 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/labstack/gommon v0.4.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -39,7 +40,7 @@ require (
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
 	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
-	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
diff --git a/example/my-app/server.go b/example/my-app/server.go
index 2291359e..6625ea62 100644
--- a/example/my-app/server.go
+++ b/example/my-app/server.go
@@ -63,12 +63,12 @@ type (
 //go:tag mkunion:"ChatResult"
 type (
 	SystemResponse struct {
-		//ID 	  string
+		//OrderID 	  string
 		Message   string
 		ToolCalls []openai.ToolCall
 	}
 	UserResponse struct {
-		//ID 	  string
+		//OrderID 	  string
 		Message string
 	}
 	ChatResponses struct {
@@ -154,10 +154,10 @@ func main() {
 
 	oaic := openai.NewClient(os.Getenv("OPENAI_API_KEY"))
 
-	srv := NewService[workflow.Command, workflow.State](
+	srv := NewService[workflow.Dependency, workflow.Command, workflow.State](
 		"process",
 		statesRepo,
-		func(state workflow.State) *machine.Machine[workflow.Command, workflow.State] {
+		func(state workflow.State) *machine.Machine[workflow.Dependency, workflow.Command, workflow.State] {
 			return workflow.NewMachine(di, state)
 		},
 		func(cmd workflow.Command) (*predicate.WherePredicates, bool) {
@@ -271,7 +271,7 @@ func main() {
 						return nil, err
 					}
 
-					result, err := shared.JSONMarshal[schemaless.FindingRecords[schemaless.Record[workflow.State]]](records)
+					result, err := shared.JSONMarshal[schemaless.PageResult[schemaless.Record[workflow.State]]](records)
 					if err != nil {
 						log.Errorf("failed to convert to json: %v", err)
 						return nil, err
@@ -342,7 +342,7 @@ func main() {
 			return err
 		}
 
-		resultJSON, err := shared.JSONMarshal[workflow.FunctionOutput](result)
+		resultJSON, err := shared.JSONMarshal[*workflow.FunctionOutput](result)
 		if err != nil {
 			log.Errorf("failed to convert to json: %v", err)
 			return err
@@ -380,7 +380,7 @@ func main() {
 			return err
 		}
 
-		result, err := shared.JSONMarshal[workflow.Workflow](record.Data)
+		result, err := shared.JSONMarshal[workflow.Flow](record.Data)
 		if err != nil {
 			if errors.Is(err, schemaless.ErrNotFound) {
 				return c.JSONBlob(http.StatusNotFound, []byte(`{"error": "not found"}`))
@@ -725,14 +725,14 @@ func TypedJSONRequest[A, B any](handle func(x A) (B, error)) func(c echo.Context
 	}
 }
 
-func NewService[CMD any, State any](
+func NewService[Dep any, CMD any, State any](
 	recordType string,
 	statesRepo *typedful.TypedRepoWithAggregator[State, any],
-	newMachine func(state State) *machine.Machine[CMD, State],
+	newMachine func(state State) *machine.Machine[Dep, CMD, State],
 	extractWhere func(CMD) (*predicate.WherePredicates, bool),
 	extractIDFromState func(State) (string, bool),
-) *Service[CMD, State] {
-	return &Service[CMD, State]{
+) *Service[Dep, CMD, State] {
+	return &Service[Dep, CMD, State]{
 		repo:                     statesRepo,
 		extractWhereFromCommandF: extractWhere,
 		recordType:               recordType,
@@ -741,15 +741,15 @@ func NewService[CMD any, State any](
 	}
 }
 
-type Service[CMD any, State any] struct {
+type Service[Dep any, CMD any, State any] struct {
 	repo                     *typedful.TypedRepoWithAggregator[State, any]
 	extractWhereFromCommandF func(CMD) (*predicate.WherePredicates, bool)
 	extractIDFromStateF      func(State) (string, bool)
 	recordType               string
-	newMachine               func(state State) *machine.Machine[CMD, State]
+	newMachine               func(state State) *machine.Machine[Dep, CMD, State]
 }
 
-func (service *Service[CMD, State]) CreateOrUpdate(cmd CMD) (res State, err error) {
+func (service *Service[Dep, CMD, State]) CreateOrUpdate(cmd CMD) (res State, err error) {
 	version := uint16(0)
 	recordID := ""
 	where, foundAndUpdate := service.extractWhereFromCommandF(cmd)
diff --git a/example/my-app/src/App.tsx b/example/my-app/src/App.tsx
index 20ad6b25..5b885c73 100644
--- a/example/my-app/src/App.tsx
+++ b/example/my-app/src/App.tsx
@@ -682,7 +682,7 @@ function App() {
                     </button>
                 </form>
 
-                <form className={"action-section"}>
+                <div className={"action-section"}>
                     <h2>Chat</h2>
 
                     <Chat
@@ -717,7 +717,7 @@ function App() {
                             }
                         }}
                     />
-                </form>
+                </div>
             </aside>
             <main>
                 <table>
@@ -1426,12 +1426,18 @@ function HelloWorldDemo() {
     }
 
 
-    return <div
+    return <form
         className={"action-section"}>
         <h2>Hello world demo</h2>
         <input type="text"
                placeholder="Enter your name"
                value={state.input}
+               onChange={(e) => {
+                   setState({
+                       ...state,
+                       input: e.currentTarget.value,
+                   })
+               }}
         />
         <button onClick={() => dispatch({type: "run_hello_world"})}>
             Run hello world workflow
@@ -1440,6 +1446,6 @@ function HelloWorldDemo() {
             Run hello world workflow with error
         </button>
         {state.loading && <div>Loading...</div>}
-    </div>
+    </form>
 }
 
diff --git a/example/my-app/src/workflow/github_com_widmogrod_mkunion_exammple_my-app.ts b/example/my-app/src/workflow/github_com_widmogrod_mkunion_exammple_my-app.ts
index e26c0307..a8f888b1 100644
--- a/example/my-app/src/workflow/github_com_widmogrod_mkunion_exammple_my-app.ts
+++ b/example/my-app/src/workflow/github_com_widmogrod_mkunion_exammple_my-app.ts
@@ -65,7 +65,7 @@ export type Reshaper = workflow.Reshaper
 
 export type Schema = schema.Schema
 
-export type Service<CMD, State> = {}
+export type Service<Dep, CMD, State> = {}
 
 export type State = workflow.State
 

From dcb4437f1a949030f2ca29d2757c8e13c546f2c1 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 13:04:24 +0200
Subject: [PATCH 12/72] example/state: moq generation needs to be in the same
 file as mkunion

---
 example/state/machine_test.go | 2 --
 example/state/model.go        | 1 +
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/example/state/machine_test.go b/example/state/machine_test.go
index ca21a707..d1ae3f72 100644
--- a/example/state/machine_test.go
+++ b/example/state/machine_test.go
@@ -9,8 +9,6 @@ import (
 	"time"
 )
 
-//go:generate moq -with-resets -stub -out machine_mock_test.go . Dependency
-
 func TestSuite(t *testing.T) {
 	now := time.Now()
 	var di Dependency = &DependencyMock{
diff --git a/example/state/model.go b/example/state/model.go
index c100eafd..9bc2d5fe 100644
--- a/example/state/model.go
+++ b/example/state/model.go
@@ -3,6 +3,7 @@ package state
 import "time"
 
 //go:generate go run ../../cmd/mkunion/main.go
+//go:generate moq -with-resets -stub -out machine_mock_test.go . Dependency
 
 type (
 	// OrderID Price, Quantity are placeholders for value objects, to ensure better data semantic and type safety

From 1c5e6ed5dd92299f71c7355d11501c3bc6a3955f Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 13:11:38 +0200
Subject: [PATCH 13/72] update ci.yml with go moq and deps bump

---
 .github/workflows/ci.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index aceb10aa..eee41d4d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -6,8 +6,8 @@ jobs:
     name: Test implementation in Golang
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: ^1.21
         id: go
@@ -15,6 +15,8 @@ jobs:
       - run: cd cmd/mkunion; go get -v -t -d ./...
       - run: cd cmd/mkunion; go build -o mkunion
 
+      - run: go get github.com/matryer/moq@latest
+
       - run: go get -v -t -d ./...
       - run: go generate ./...
 

From 03cc0349e6988027000dbc0eb3119ac93a3613f0 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 13:14:29 +0200
Subject: [PATCH 14/72] update ci.yml with go moq and deps bump

---
 .github/workflows/ci.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index eee41d4d..70065b87 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -15,7 +15,9 @@ jobs:
       - run: cd cmd/mkunion; go get -v -t -d ./...
       - run: cd cmd/mkunion; go build -o mkunion
 
-      - run: go get github.com/matryer/moq@latest
+      - run: | 
+          go get github.com/matryer/moq@latest
+          echo "${{ github.workspace }}/go/bin" >> $GITHUB_PATH
 
       - run: go get -v -t -d ./...
       - run: go generate ./...

From fe52a7d14da535d419b5d3094c1ed7740507a08f Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 13:15:24 +0200
Subject: [PATCH 15/72] update ci.yml GOPATH

---
 .github/workflows/ci.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 70065b87..e73302bd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -17,7 +17,9 @@ jobs:
 
       - run: | 
           go get github.com/matryer/moq@latest
-          echo "${{ github.workspace }}/go/bin" >> $GITHUB_PATH
+
+      - name: Add $GOPATH/bin to $PATH
+        run: echo "${{ env.GOPATH }}/bin" >> $GITHUB_PATH
 
       - run: go get -v -t -d ./...
       - run: go generate ./...

From 826d26c43bf8960d2b675a2ac9f697d6be230678 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 13:19:41 +0200
Subject: [PATCH 16/72] update ci.yml GOPATH

---
 .github/workflows/ci.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e73302bd..cdc11591 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,15 +12,17 @@ jobs:
           go-version: ^1.21
         id: go
 
+      - name: Add $GOPATH/bin to $PATH
+        run: |
+          echo "${{ env.GOPATH }}/bin" >> $GITHUB_PATH
+          echo "${{ env.GOPATH }}/bin" >> $PATH
+
       - run: cd cmd/mkunion; go get -v -t -d ./...
       - run: cd cmd/mkunion; go build -o mkunion
 
       - run: | 
           go get github.com/matryer/moq@latest
 
-      - name: Add $GOPATH/bin to $PATH
-        run: echo "${{ env.GOPATH }}/bin" >> $GITHUB_PATH
-
       - run: go get -v -t -d ./...
       - run: go generate ./...
 

From 72f3f893faa3db37445d11ac80dae971cae2933e Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 13:20:07 +0200
Subject: [PATCH 17/72] update ci.yml GOPATH

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cdc11591..7f90c733 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -16,6 +16,7 @@ jobs:
         run: |
           echo "${{ env.GOPATH }}/bin" >> $GITHUB_PATH
           echo "${{ env.GOPATH }}/bin" >> $PATH
+          export PATH=$PATH:$(go env GOPATH)/bin
 
       - run: cd cmd/mkunion; go get -v -t -d ./...
       - run: cd cmd/mkunion; go build -o mkunion

From 1560c2128b88d2fdb3bb4af13395bc9fbe2e248d Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 13:21:24 +0200
Subject: [PATCH 18/72] update ci.yml GOPATH

---
 .github/workflows/ci.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7f90c733..98a716ba 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -14,8 +14,6 @@ jobs:
 
       - name: Add $GOPATH/bin to $PATH
         run: |
-          echo "${{ env.GOPATH }}/bin" >> $GITHUB_PATH
-          echo "${{ env.GOPATH }}/bin" >> $PATH
           export PATH=$PATH:$(go env GOPATH)/bin
 
       - run: cd cmd/mkunion; go get -v -t -d ./...

From a7c9c86f5d99fe184b03dd613e0915b7da6c5761 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 22:31:58 +0200
Subject: [PATCH 19/72] update ci.yml GOPATH

---
 .github/workflows/ci.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 98a716ba..525fb62b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -14,13 +14,17 @@ jobs:
 
       - name: Add $GOPATH/bin to $PATH
         run: |
-          export PATH=$PATH:$(go env GOPATH)/bin
+          echo "${{ env.GOPATH }}/bin" >> $GITHUB_PATH
+          echo $GITHUB_PATH
+          echo "asd"
+          echo $(go env GOPATH)/bin
 
       - run: cd cmd/mkunion; go get -v -t -d ./...
       - run: cd cmd/mkunion; go build -o mkunion
 
       - run: | 
-          go get github.com/matryer/moq@latest
+          go install github.com/matryer/moq@latest
+          which moq
 
       - run: go get -v -t -d ./...
       - run: go generate ./...

From 8e6a8c836f703fd70a9d74476d826ae6947b9831 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 31 Mar 2024 22:43:20 +0200
Subject: [PATCH 20/72] update dev loop

---
 .github/workflows/ci.yml | 15 ++-------------
 dev/bootstrap.sh         | 10 ++++++++--
 2 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 525fb62b..47d61744 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,26 +12,15 @@ jobs:
           go-version: ^1.21
         id: go
 
-      - name: Add $GOPATH/bin to $PATH
-        run: |
-          echo "${{ env.GOPATH }}/bin" >> $GITHUB_PATH
-          echo $GITHUB_PATH
-          echo "asd"
-          echo $(go env GOPATH)/bin
-
       - run: cd cmd/mkunion; go get -v -t -d ./...
       - run: cd cmd/mkunion; go build -o mkunion
 
-      - run: | 
-          go install github.com/matryer/moq@latest
-          which moq
+      - run: go install github.com/matryer/moq@latest
 
       - run: go get -v -t -d ./...
       - run: go generate ./...
 
-      # initiate docker-compose services
-      - run: |
-          pip install awscli-local
+      - run: pip install awscli-local
       - run: dev/bootstrap.sh -nologs
 
       # run tests
diff --git a/dev/bootstrap.sh b/dev/bootstrap.sh
index ebf4abf8..f29f2094 100755
--- a/dev/bootstrap.sh
+++ b/dev/bootstrap.sh
@@ -12,11 +12,17 @@ command -v docker-compose >/dev/null 2>&1 || { echo >&2 "docker-compose is not i
 command -v awslocal >/dev/null 2>&1 || { echo >&2 "awslocal is not installed. Aborting. Please run
   pip install awscli-local  "; exit 1; }
 
+# check for moq
+command -v moq >/dev/null 2>&1 || { echo >&2 "moq is not installed. Aborting please run
+   go install github.com/matryer/moq@latest"; exit 1; }
+
 echo "Creating volume directory"
 mkdir -p $cwd/_volume
 
-echo "Install moq"
-go get github.com/matryer/moq@latest
+if [ "$1" == "-install-only" ]; then
+  trap - EXIT
+  exit 0
+fi
 
 echo "Starting localstack"
 docker compose -f $cwd/compose.yml up -d

From 2e1c2732bc626a04efc89d37d07d4d10ae6c1b30 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Fri, 3 May 2024 19:09:25 +0200
Subject: [PATCH 21/72] docs: clean and expand on generic unions

---
 docs/examples/generic_union.md | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/docs/examples/generic_union.md b/docs/examples/generic_union.md
index c807b27c..3a1709b0 100644
--- a/docs/examples/generic_union.md
+++ b/docs/examples/generic_union.md
@@ -32,13 +32,11 @@ you have access to the same features as with non-generic unions.
 
 ## Matching function
 
-Let's define higher order function `ReduceTree` that will travers leaves in tree and produce a single value.
+Let's define higher order function `ReduceTree` that will travers leaves in `Tree` and produce a single value.
 
 This function uses `MatchTreeR1` function that is generated automatically for you.
 
 ```go title="example/tree.go"
-
-```go
 func ReduceTree[A, B any](x Tree[A], f func(A, B) B, init B) B {
 	return MatchTreeR1(
 		x,
@@ -150,4 +148,11 @@ func MapOption[A, B any](x Option[A], f func(A) B) Option[B] {
 		},
 	)
 }
-```
\ No newline at end of file
+```
+
+In above example, we define `MapEither` and `MapOption` functions that will apply function `f` to value inside `Either` or `Option` type.
+
+It would be much better to have only one `Map` definition, but due to limitations of Go type system, we need to define separate functions for each type.
+
+I'm considering adding code generation for such behaviours in the future. Not yet due to focus on validating core concepts.
+

From 2b6d15d4d755f415b1784bb09d5aa1197ef87521 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Fri, 3 May 2024 19:09:43 +0200
Subject: [PATCH 22/72] docs: clean and fix typos in getting started

---
 docs/getting_started.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/getting_started.md b/docs/getting_started.md
index d4ff38f9..cea7545c 100644
--- a/docs/getting_started.md
+++ b/docs/getting_started.md
@@ -52,13 +52,13 @@ type User struct {
 
 Unfortunately Golang don't extend this feature to other parts of the language.
 
-MkUnion defines `//go:tag` comment, following other idiomatic definitions `go:generate`, `go:embed` to allow to add metadata to types.
-And use it heavily to offer way of adding new behaviour to go types.
+MkUnion defines `//go:tag` comment, following other idiomatic definitions `go:generate`, `go:embed` to allow to add metadata to struct type.
+And MkUnion use it heavily to offer way of adding new behaviour to go types.
 
-#### `type ()` convention
+#### `type (...)` convention
 
 Union type is defined as a set of types in a single type declaration. You can think of it as "one of" type.
-To make it more readable, as convention I decided to use `type ()` declaration block, instead of individual `type` declaration.
+To make it more readable, as convention I decided to use `type (...)` declaration block, instead of individual `type` declaration.
 
 ### Generate code
 In IDEs like Goland run `Option + Command + G` for fast code generation
@@ -73,7 +73,7 @@ Alternatively you can run `mkunion` command directly
 mkunion -i example/vehicle.go
 ```
 
-In feature we plan to add `mkununion watch ./...` command that will watch for changes in your code and automatically generate union types for you.
+In future I plan to add `mkununion watch ./...` command that will watch for changes in your code and automatically generate union types for you.
 This will allow you to remove `//go:generate` directive from your code, and have faster feedback loop.
 
 ### Match over union type

From 7dd388489957453e6ded8f9116df83d2020fb7d9 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Fri, 3 May 2024 19:09:57 +0200
Subject: [PATCH 23/72] docs: introduce roadmap idea

---
 docs/roadmap.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/roadmap.md b/docs/roadmap.md
index 061dc926..214d99f0 100644
--- a/docs/roadmap.md
+++ b/docs/roadmap.md
@@ -11,4 +11,5 @@
 ## Long tern experiments and prototypes
 
 - [ ] **experiment**: generate other (de)serialization formats (e.g. grpc, sql, graphql)
-- [ ] **prototype**: http & gRPC client for end-to-end types. 
\ No newline at end of file
+- [ ] **prototype**: http & gRPC client for end-to-end types. 
+- [ ] **experiment**: allow to derive behaviour for types, like derive(Map), would generated union type with Map() method  
\ No newline at end of file

From 7f6c3c3ca64079cbf52611ce6c43823d2362bae9 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Fri, 3 May 2024 19:12:35 +0200
Subject: [PATCH 24/72] docs: expand state machine documentation

---
 docs/examples/state_machine.md | 75 ++++++++++++++++++++++------------
 1 file changed, 50 insertions(+), 25 deletions(-)

diff --git a/docs/examples/state_machine.md b/docs/examples/state_machine.md
index d6139cca..0892bc16 100644
--- a/docs/examples/state_machine.md
+++ b/docs/examples/state_machine.md
@@ -3,7 +3,7 @@ title: State machines and unions
 ---
 # MkUnion and state machines in golang
 
-This document will show how to use `mkunion` to manage application state on example of an order service. 
+This document will show how to use `mkunion` to manage application state on example of an Order Service. 
 You will learn:
 
 - how to model state machines in golang, and find similarities to "__clean architecture__"
@@ -14,7 +14,7 @@ You will learn:
 
 ## Working example
 
-As an driving example, we will use an order service that can be in one of the following states:
+As an driving example, we will use e-commerce inspired Order Service that can be in one of the following states:
 
 - `Pending` - order is created, and is waiting for someone to process it
 - `Processing` - order is being processed, an human is going to pick up items from warehouse and pack them
@@ -28,7 +28,7 @@ And we need to have also to trigger changes in state, like create order that pen
 Some of those rules could change in future, and we want to be able to change them without rewriting whole application.
 This also informs us that our design should be open for extensions.
 
-Site note, if you want go strait to final code product, then into [example/state/](example/state/) directory and have fun exploring.
+Side note, if you want go strait to final code product, then into [example/state/](example/state/) directory and have fun exploring.
 
 ## Modeling commands and states
 
@@ -38,18 +38,18 @@ Our example can be represented as state machine that looks like this:
 --8<-- "example/state/machine_test.go.state_diagram.mmd"
 ```
 
-In this diagram, we can see that we have 4 states, and 4 commands that can trigger transitions between states shown as arrows.
+In this diagram, we can see that we have 5 states, and 6 commands that can trigger transitions between states shown as arrows.
 
-Because this diagram is generated from code, it has names that represent types in golang that we use. 
+Because this diagram is generated from code, it has names that represent types in golang that we use in implementation. 
 
 For example `*state.CreateOrderCMD`:
 
 - `state` it's a package name
 - `CreateOrderCMD` is a struct name in that package.
-- `CMD` suffix it's naming convention, that it's optional, but I find it to make code more readable.
+- `CMD` suffix it's naming convention, that it's optional, but I find it makes code more readable.
 
 
-Below is a code snippet that demonstrate complete model of **state** and **commands** of order service, that we talked about.
+Below is a code snippet that demonstrate complete model of **state** and **commands** of Order Service, that we talked about.
 
 **Notice** that we use `mkunion` to group commands and states. (Look for `//go:tag mkunion:"Command"`)
 
@@ -63,21 +63,15 @@ Here interface that group those types is generated automatically.
 
 ## Modeling transitions
 One thing that is missing is implementation of transitions between states. 
-There are few ways to do it, today I will show you how to do it using functional approach.
+There are few ways to do it. I will show you how to do it using functional approach (think  `reduce` or `map` function).
 
-You can think about this function like a `reduce` or `map` function, that you may be familiar.
-
-
-Let's name this function `Transition` that has following signature (name is arbitrary, you can name it as you like):
-Our function takes current state and command, and returns new state, and as you can see it.
+Let's name function that we will build `Transition` and define it as:
 
 ```go
 func Transition(dep Dependencies, cmd Command, state State) (State, error)
 ```
 
-Let's break down this signature.
-
-Input arguments are:
+Our function has few arguments, let's break them down:
 
 - `dep` encapsulates dependencies like API clients, database connection, configuration, context etc.
    everything that is needed for complete production implementation.
@@ -87,10 +81,9 @@ Input arguments are:
    and it has `State` interface, that was generate similarly to `Command` interface.
 
 
-Output value is either new state, or error if something went wrong. This is very important assumption, 
-that will come very handy when we will talk about self-healing processes.
+Our function must return either new state, or error when something went wrong during transition, like network error, or validation error.
 
-Below is snippet of implementation of `Transition` function for our order service:
+Below is snippet of implementation of `Transition` function for our Order Service:
 
 ```go
 --8<-- "example/state/machine.go:30:81"
@@ -104,7 +97,7 @@ You can notice few patterns in this snippet:
 - `Dependency` interface help us to keep, well  dependencies - well defined, which helps greatly in testability and readability of the code. 
 - Use of generated function `MatchCommandR2` to exhaustively match all commands. 
   This is powerful, when new command is added, you can be sure that you will get compile time error, if you don't handle it.
-- Validation of commands in done in transition function. Current implementation is simple, but you can use go-validate or other libraries to make it more robust.
+- Validation of commands in done in transition function. Current implementation is simple, but you can use go-validate to make it more robust, or refactor code and introduce domain helper functions or methods to the types.
 - Each command check state to which is being applied using `switch` statement, it ignore states that it does not care about. 
   Which means as implementation you have to focus only on small bit of the picture, and not worry about rest of the states. 
   This is also example where non-exhaustive use of `switch` statement is welcome.
@@ -113,8 +106,11 @@ Simple, isn't it? Simplicity also comes from fact that we don't have to worry ab
 
 Note: Implementation for educational purposes is kept in one big function, 
 but for large projects it may be better to split it into smaller functions, 
-or use visitor pattern interface, that is also generated for you.
+or define OrderService struct that conforms to visitor pattern interface, that was also generated for you:
 
+```go
+--8<-- "example/state/model_union_gen.go:11:17"
+```
 
 ## Testing state machines & self-documenting
 Before we go further, let's talk about testing our implementation.
@@ -125,16 +121,16 @@ and discover transition that we didn't think about, that should or shouldn't be
 Here is how you can test state machine, in declarative way, using `mkunion/x/machine` package:
 
 ```go
---8<-- "example/state/machine_test.go:12:86"
+--8<-- "example/state/machine_test.go:15:151"
 ```
 Few things to notice in this test:
 
-- We use of `moq` to generate mocks for dependencies
 - We use standard go testing
 - We use `machine.NewTestSuite` as an standard way to test state machines
 - We start with describing **happy path**, and use `suite.Case` to define test case.
-- But most importantly, we define test cases using `GivenCommand` and `ThenState` functions, that help in making test more readable.
+- But most importantly, we define test cases using `GivenCommand` and `ThenState` functions, that help in making test more readable, and hopefully self-documenting.
 - You can see use of `ForkCase` command, that allow you to take a definition of a state declared in `ThenState` command, and apply new command to it, and expect new state.
+- Less visible is use of `moq` to generate `DependencyMock` for dependencies, but still important to write more concise code.
 
 I know it's subjective, but I find it very readable, and easy to understand, even for non-programmers.
 
@@ -169,14 +165,43 @@ Second is a diagram that includes commands that resulted in an errors:
 
 Those diagrams are stored in the same directory as test file, and are prefixed with name used in `AssertSelfDocumentStateDiagram` function.
 ```
-machine_test.go
 machine_test.go.state_diagram.mmd
 machine_test.go.state_diagram_with_errors.mmd
 ```
 
+## State machines builder
+
+Now, since we have Transition function, and we have tests, let's standardize how it can be use in different parts of the application.
+
+```go
+--8<-- "example/state/machine.go:9:11"
+```
+
+Simple, isn't it?
+
+MkUnion also provide `*machine.Machine[Dependency, Command, State]` struct that wires Transition, and standardise how dependencies, and commands are applied.
+
+To see how to use it, let's move to the next section.
 
 ## Persisting state in database
 
+At this point of time, we have implemented and tested Order Service state machine.
+
+Next thing that we need to address in our road to the production is to persist state in database.
+
+MkUnion aims to support you in this task, by providing you `x/storage/schemaless` package that will take care of: 
+- mapping  golang objets to database representation and back.
+- handling optimistic concurrency conflicts
+- providing you with simple API to work with database
+- and more
+
+Below is test case that demonstrate complete example of initializing database, 
+building an state using `NewMachine` , and saving and loading state from database.
+
+```go
+--8<-- "example/state/machine_test.go:153:227"
+```
+
 ```mermaid
 sequenceDiagram
     participant R as Request

From f9635dd8ffc00bc3a1694a091d32a42b42b28f52 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Fri, 3 May 2024 19:13:14 +0200
Subject: [PATCH 25/72] example: reorder model.go file

---
 example/state/model.go | 78 +++++++++++++++++++++---------------------
 1 file changed, 39 insertions(+), 39 deletions(-)

diff --git a/example/state/model.go b/example/state/model.go
index 9bc2d5fe..35b5fe17 100644
--- a/example/state/model.go
+++ b/example/state/model.go
@@ -3,45 +3,6 @@ package state
 import "time"
 
 //go:generate go run ../../cmd/mkunion/main.go
-//go:generate moq -with-resets -stub -out machine_mock_test.go . Dependency
-
-type (
-	// OrderID Price, Quantity are placeholders for value objects, to ensure better data semantic and type safety
-	OrderID  = string
-	Price    = float64
-	Quantity = int
-
-	OrderAttr struct {
-		// placeholder for order attributes
-		// like customer name, address, etc.
-		// like product name, price, etc.
-		// for simplicity we only have Price and Quantity
-		Price    Price
-		Quantity Quantity
-	}
-
-	// WorkerID represent human that process the order
-	WorkerID = string
-
-	// Order everything we know about order
-	Order struct {
-		ID               OrderID
-		OrderAttr        OrderAttr
-		WorkerID         WorkerID
-		StockRemovedAt   *time.Time
-		PaymentChargedAt *time.Time
-		DeliveredAt      *time.Time
-		CancelledAt      *time.Time
-		CancelledReason  string
-	}
-)
-
-type ProblemCode int
-
-const (
-	ProblemWarehouseAPIUnreachable ProblemCode = iota
-	ProblemPaymentAPIUnreachable
-)
 
 //go:tag mkunion:"Command"
 type (
@@ -96,3 +57,42 @@ type (
 		ProblemState   State
 	}
 )
+
+//go:generate moq -with-resets -stub -out machine_mock_test.go . Dependency
+type (
+	// OrderID Price, Quantity are placeholders for value objects, to ensure better data semantic and type safety
+	OrderID  = string
+	Price    = float64
+	Quantity = int
+
+	OrderAttr struct {
+		// placeholder for order attributes
+		// like customer name, address, etc.
+		// like product name, price, etc.
+		// for simplicity we only have Price and Quantity
+		Price    Price
+		Quantity Quantity
+	}
+
+	// WorkerID represent human that process the order
+	WorkerID = string
+
+	// Order everything we know about order
+	Order struct {
+		ID               OrderID
+		OrderAttr        OrderAttr
+		WorkerID         WorkerID
+		StockRemovedAt   *time.Time
+		PaymentChargedAt *time.Time
+		DeliveredAt      *time.Time
+		CancelledAt      *time.Time
+		CancelledReason  string
+	}
+)
+
+type ProblemCode int
+
+const (
+	ProblemWarehouseAPIUnreachable ProblemCode = iota
+	ProblemPaymentAPIUnreachable
+)

From 9893242cf3fa880ba6d9cfa1cd3048cfeed2f46f Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 4 May 2024 14:12:25 +0200
Subject: [PATCH 26/72] docs: refine state machine description

---
 docs/examples/state_machine.md | 42 +++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/docs/examples/state_machine.md b/docs/examples/state_machine.md
index 0892bc16..04f55193 100644
--- a/docs/examples/state_machine.md
+++ b/docs/examples/state_machine.md
@@ -57,7 +57,7 @@ This is one example how union types can be used in golang.
 Historically in golang it would be very hard to achieve such thing, and it would require a lot of boilerplate code.
 Here interface that group those types is generated automatically.
 
-```go
+```go title="example/state/model.go"
 --8<-- "example/state/model.go"
 ```
 
@@ -85,7 +85,7 @@ Our function must return either new state, or error when something went wrong du
 
 Below is snippet of implementation of `Transition` function for our Order Service:
 
-```go
+```go title="example/state/machine.go"
 --8<-- "example/state/machine.go:30:81"
 // ...
 // rest remove for brevity 
@@ -108,7 +108,7 @@ Note: Implementation for educational purposes is kept in one big function,
 but for large projects it may be better to split it into smaller functions, 
 or define OrderService struct that conforms to visitor pattern interface, that was also generated for you:
 
-```go
+```go  title="example/state/model_union_gen.go"
 --8<-- "example/state/model_union_gen.go:11:17"
 ```
 
@@ -120,7 +120,7 @@ and discover transition that we didn't think about, that should or shouldn't be
 
 Here is how you can test state machine, in declarative way, using `mkunion/x/machine` package:
 
-```go
+```go title="example/state/machine_test.go"
 --8<-- "example/state/machine_test.go:15:151"
 ```
 Few things to notice in this test:
@@ -137,7 +137,7 @@ I know it's subjective, but I find it very readable, and easy to understand, eve
 ## Generating state diagram from tests
 Last bit is this line at the bottom:
 
-```go
+```go title="example/state/machine_test.go"
 if suite.AssertSelfDocumentStateDiagram(t, "machine_test.go") {
    suite.SelfDocumentStateDiagram(t, "machine_test.go")
 }
@@ -154,12 +154,12 @@ There are two diagrams that are generated.
 
 One is a diagram of ONLY successful transitions, that you saw at the beginning of this post.
 
-```mermaid
+```mermaid 
 --8<-- "example/state/machine_test.go.state_diagram.mmd"
 ```
 
 Second is a diagram that includes commands that resulted in an errors:
-```mermaid
+```mermaid 
 --8<-- "example/state/machine_test.go.state_diagram_with_errors.mmd"
 ```
 
@@ -171,34 +171,40 @@ machine_test.go.state_diagram_with_errors.mmd
 
 ## State machines builder
 
-Now, since we have Transition function, and we have tests, let's standardize how it can be use in different parts of the application.
+MkUnion provide `*machine.Machine[Dependency, Command, State]` struct that wires Transition, dependencies and state together.
+It provide methods like:
 
-```go
---8<-- "example/state/machine.go:9:11"
-```
+- `Handle(cmd C) error` that apply command to state, and return error if something went wrong during transition.
+- `State() S` that return current state of the machine
+- `Dep() D` that return dependencies that machine was build with.
 
-Simple, isn't it?
 
-MkUnion also provide `*machine.Machine[Dependency, Command, State]` struct that wires Transition, and standardise how dependencies, and commands are applied.
+This standard helps build on top of it, for example testing library that we use in [Testing state machines & self-documenting](#testing-state-machines-self-documenting) leverage it.
 
-To see how to use it, let's move to the next section.
+Another good practice is that every package that defines state machine in the way described here, 
+should provide `NewMachine` function that will return bootstrapped machine with package types, like so:
+
+```go title="example/state/machine.go"
+--8<-- "example/state/machine.go:9:11"
+```
 
 ## Persisting state in database
 
 At this point of time, we have implemented and tested Order Service state machine.
 
-Next thing that we need to address in our road to the production is to persist state in database.
+Next thing that we need to address in our road to the production is to persist state in database and learn how to deal with runtime errors like network failure.
 
 MkUnion aims to support you in this task, by providing you `x/storage/schemaless` package that will take care of: 
-- mapping  golang objets to database representation and back.
-- handling optimistic concurrency conflicts
+
+- mapping  golang structs to database representation and back from database to struct.
+- leveraging optimistic concurrency control to resolve conflicts
 - providing you with simple API to work with database
 - and more
 
 Below is test case that demonstrate complete example of initializing database, 
 building an state using `NewMachine` , and saving and loading state from database.
 
-```go
+```go title="example/state/machine_test.go"
 --8<-- "example/state/machine_test.go:153:227"
 ```
 

From 80e8e24940ec2e42fded50b23bd16f3690002816 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 19:54:59 +0200
Subject: [PATCH 27/72] docs: Marshaling union as JSON

---
 docs/examples/json.md | 81 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100644 docs/examples/json.md

diff --git a/docs/examples/json.md b/docs/examples/json.md
new file mode 100644
index 00000000..df92c830
--- /dev/null
+++ b/docs/examples/json.md
@@ -0,0 +1,81 @@
+---
+title: Marshaling union as JSON
+---
+
+# Marshaling union as JSON
+
+MkUnion provides you with utility function that allows you to marshal and unmarshal union types to JSON, 
+reducing burden of writing custom marshaling and unmarshaling functions for union types.
+
+- `shared.JSONMarshal[A any](in A) ([]byte, error)`
+- `shared.JSONUnmarshal[A any](data []byte) (A, error)`
+
+Below is an example of how to use those functions and how the output JSON looks like.
+
+
+```go title="example/tree_json_test.go"
+import (
+    "github.com/widmogrod/mkunion/x/shared"
+)
+
+--8<-- "example/tree_json_test.go:8:30"
+```
+
+Formated JSON output of the example above:
+```json
+{
+  "$type": "example.Branch",
+  "example.Branch": {
+    "L": {
+      "$type": "example.Leaf",
+      "example.Leaf": {
+        "Value": 1
+      }
+    },
+    "R": {
+      "$type": "example.Branch",
+      "example.Branch": {
+        "L": {
+          "$type": "example.Branch",
+          "example.Branch": {
+            "L": {
+              "$type": "example.Leaf",
+              "example.Leaf": {
+                "Value": 2
+              }
+            },
+            "R": {
+              "$type": "example.Leaf",
+              "example.Leaf": {
+                "Value": 3
+              }
+            }
+          }
+        },
+        "R": {
+          "$type": "example.Leaf",
+          "example.Leaf": {
+            "Value": 4
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+
+There are few things that you can notice in this example:
+
+- Each union type discriminator field `$type` field that holds the type name, and corresponding key with the name of the type, that holds value of union variant.
+    - This is opinionated way, and library don't allow to change it.
+      I was experimenting with making this behaviour customizable, but it make code and API mode complex, and I prefer to keep it simple, and increase interoperability between different libraries and applications, that way.
+
+- Recursive union types are supported, and they are marshaled as nested JSON objects.]
+
+- `$type` don't have to have full package import name, nor type parameter,
+  mostly because in `shared.JSONUnmarshal[Tree[int]](json)` you hint that your code accepts `Tree[int]`.
+    - I'm considering adding explicit type discriminators like `example.Branch[int]` or `example.Leaf[int]`.
+      It could increase type strictness on client side, BUT it makes generating TypeScript types more complex, and I'm not sure if it's worth it.
+
+- It's not shown on this example, but you can also reference types and union types from other packages, and serialization will work as expected.
\ No newline at end of file

From 182d567ff31a0f72a8dd56dfb02a3593f17217f7 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 19:55:52 +0200
Subject: [PATCH 28/72] docs: Improve title and wording

---
 docs/examples/generic_union.md |   4 +-
 docs/examples/state_machine.md | 215 +--------------------------------
 2 files changed, 5 insertions(+), 214 deletions(-)

diff --git a/docs/examples/generic_union.md b/docs/examples/generic_union.md
index 3a1709b0..61498f96 100644
--- a/docs/examples/generic_union.md
+++ b/docs/examples/generic_union.md
@@ -1,7 +1,7 @@
 ---
-title: Generic unions
+title: Union and generic types
 ---
-# Generic unions
+# Union and generic types
 MkUnion will generate generic unions for you.
 
 You only need to declare each variant type of the union with a type parameter,
diff --git a/docs/examples/state_machine.md b/docs/examples/state_machine.md
index 04f55193..875e989f 100644
--- a/docs/examples/state_machine.md
+++ b/docs/examples/state_machine.md
@@ -68,11 +68,12 @@ There are few ways to do it. I will show you how to do it using functional appro
 Let's name function that we will build `Transition` and define it as:
 
 ```go
-func Transition(dep Dependencies, cmd Command, state State) (State, error)
+func Transition(ctx context.Context, dep Dependencies, cmd Command, state State) (State, error)
 ```
 
 Our function has few arguments, let's break them down:
 
+- `ctx` standard golang context, that is used to pass deadlines, and cancelation signals, etc.
 - `dep` encapsulates dependencies like API clients, database connection, configuration, context etc.
    everything that is needed for complete production implementation.
 - `cmd` it's a command that we want to apply to state, 
@@ -174,7 +175,7 @@ machine_test.go.state_diagram_with_errors.mmd
 MkUnion provide `*machine.Machine[Dependency, Command, State]` struct that wires Transition, dependencies and state together.
 It provide methods like:
 
-- `Handle(cmd C) error` that apply command to state, and return error if something went wrong during transition.
+- `Handle(ctx context.Context, cmd C) error` that apply command to state, and return error if something went wrong during transition.
 - `State() S` that return current state of the machine
 - `Dep() D` that return dependencies that machine was build with.
 
@@ -188,214 +189,4 @@ should provide `NewMachine` function that will return bootstrapped machine with
 --8<-- "example/state/machine.go:9:11"
 ```
 
-## Persisting state in database
-
-At this point of time, we have implemented and tested Order Service state machine.
-
-Next thing that we need to address in our road to the production is to persist state in database and learn how to deal with runtime errors like network failure.
-
-MkUnion aims to support you in this task, by providing you `x/storage/schemaless` package that will take care of: 
-
-- mapping  golang structs to database representation and back from database to struct.
-- leveraging optimistic concurrency control to resolve conflicts
-- providing you with simple API to work with database
-- and more
-
-Below is test case that demonstrate complete example of initializing database, 
-building an state using `NewMachine` , and saving and loading state from database.
-
-```go title="example/state/machine_test.go"
---8<-- "example/state/machine_test.go:153:227"
-```
-
-```mermaid
-sequenceDiagram
-    participant R as Request
-    participant Store as Store
-
-    activate R
-    R->>R: Validate(request) -> error
-    
-    R->>Store: Load state from database by request.ObjectId
-    activate Store
-    Store->>R: Ok(State)
-    deactivate Store
-    
-    R->>R: Create machine with state
-    R->>R: Apply command on a state
-    
-    R->>Store: Save state in database under request.ObjectId
-    activate Store
-    Store->>R: Ok()
-    deactivate Store
-    
-    deactivate R
-```
-
-Example implementation of such sequence diagram:
-
-```go
-func Handle(rq Request, response Resopnse) {
-	ctx := rq.Context()
-	
-	// extract objectId and command from request + do some validation
-    id := rq.ObjectId
-	command := rq.Command
-	
-    // Load state from store
-    state, err := store.Find(ctx, id)
-	if err != nil { /*handle error*/ }
-
-    machine := NewSimpleMachineWithState(Transition, state)
-    newState, err := machine.Apply(cmd, state)
-    if err != nil { /*handle error*/ }
-	
-    err := store.Save(ctx, newState)
-    if err != nil { /*handle error*/ }
-	
-	// serialize response
-	response.Write(newState)
-}
-```
-
-## Error as state. Self-healing systems.
-In request-response situation, handing errors is easy, but what if in some long-lived process something goes wrong?
-How to handle errors in such situation? Without making what we learn about state machines useless or hard to use?
-
-One solution is to treat errors as state.
-In such case, our state machines will never return error, but instead will return new state, that will represent error.
-
-When we introduce explicit command responsible for correcting RecoverableError, we can create self-healing systems.
-Thanks to that, even in situation when errors are unknown, we can retroactivly introduce self-healing logic that correct states.
-
-Because there is always there is only one error state, it makes such state machines easy to reason about.
-
-```go
-//go:generate mkunion -name State
-type (
-    // ...
-    RecoverableError struct {
-        ErrCode int
-        PrevState State
-        RetryCount int
-    }
-)
-
-//go:generate mkunion -name Command
-type (
-    // ...
-    CorrectStateCMD struct {}
-)
-```
-
-Now, we have to implement recoverable logic in our state machine.
-We show example above how to do it in `Transition` function.
-
-Here is example implementation of such transition function:
-
-```go
-func Transition(cmd Command, state State) (State, error) {
-return MustMatchCommandR2(
-    cmd,
-    /* ... */
-    func(cmd *CorrectStateCMD) (State, error) {
-        switch state := state.(type) {
-        case *RecoverableError:
-            state.RetryCount = state.RetryCount + 1
-			
-            // here we can do some self-healing logic
-            if state.ErrCode == DuplicateServiceUnavailable {
-                newState, err := Transition(&MarkAsDuplicateCMD{}, state.PrevState)
-                 if err != nil {
-                    // we failed to correct error, so we return error state 
-                     return &RecoverableError{
-                        ErrCode: err,
-                        PrevState: state.PrevState,
-                        RetryCount: state.RetryCount,
-                    }, nil
-                }
-				
-                 // we manage to fix state, so we return new state
-                 return newState, nil
-             } else {
-                 // log information that we have new code, that we don't know how to handle
-             }
-			
-            // try to correct error in next iteration
-            return state, nil
-        }
-    }
-}
-```
-
-Now, to correct states we have to select from database all states that are in error state.
-It can be use in many ways, example below use a abstraction called `TaskQueue` that is responsible for running tasks in background.
-
-This abstraction guaranties that all records (historical and new ones) will be processed.
-You can think about it, as a queue that is populated by records from database, that meet SQL query criteria.
-
-You can use CRON job and pull database.
-
-```go
-//go:generate mms deployyml -type=TaskQueue -name=CorrectMSPErrors -autoscale=1,10 -memory=128Mi -cpu=100m -timeout=10s -schedule="0 0 * * *"
-func main()
-    sql := "SELECT * FROM ObjectState WHERE RecoverableError.RetryCount < 3"
-    store := datalayer.DefaultStore()
-    queue := TaskQueueFrom("correct-msp-errors", sql, store)
-    queue.OnTask(func (ctx context.Context, task Task) error {
-        state := task.State()
-        cmd := &CorrectStateCMD{}
-        machine := NewSimpleMachineWithState(Transition, state)
-        newState, err := machine.Apply(cmd, state)
-        if err != nil {
-            return err
-        }
-        return task.Save(ctx, newState)
-    })
-    err := queue.Run(ctx)
-    if err != nil {
-        log.Panic(err)
-    }
-}
-```
-
-
-## State machines and command queues and workflows
-What if command would initiate state "to process" and save it in db
-What if task queue would take such state and process it
-Woudn't this be something like command queue?
-
-When to make a list of background processes that transition such states?
-
-### processors per state
-It's like micromanage TaskQueue, where each state has it's own state, and it knows what command to apply to given state
-This could be good starting point, when there is not a lot of good tooling
-
-### processor for state machine
-With good tooling, transition of states can be declared in one place,
-and deployment to task queue could be done automatically.
-
-Note, that only some of the transitions needs to happen in background, other can be done in request-response manner.
-
-### processor for state machine with workflow
-State machine could be generalized to workflow.
-We can think about it as set of generic Command and State (like a turing machine).
-
-States like Pending, Completed, Failed
-Commands like Process, Retry, Cancel
-
-And workflow DSL with commands like: Invoke, Choose, Assign
-Where function is some ID string, and functions needs to be either
-pulled from registry, or called remotely (InvokeRemote).
-some operations would require callback (InvokeAndAwait)
-
-Then background processor would be responsible for executing such workflow (using task queue)
-Program would be responsible for defining workflow, and registering functions.
-
-Such programs could be also optimised for deployment,
-if some function would be better to run on same machine that do RPC call
-like function doing RPC call to database, and caching result in memory or in cache cluster dedicated to specific BFF
-
-
-
 

From 753969bbdf53258947a113ac6e2de7c5d8f8fd95 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 19:57:21 +0200
Subject: [PATCH 29/72] docs: Add feature to the roadmap

---
 docs/roadmap.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/roadmap.md b/docs/roadmap.md
index 214d99f0..f3f71663 100644
--- a/docs/roadmap.md
+++ b/docs/roadmap.md
@@ -11,5 +11,6 @@
 ## Long tern experiments and prototypes
 
 - [ ] **experiment**: generate other (de)serialization formats (e.g. grpc, sql, graphql)
-- [ ] **prototype**: http & gRPC client for end-to-end types. 
-- [ ] **experiment**: allow to derive behaviour for types, like derive(Map), would generated union type with Map() method  
\ No newline at end of file
+- [ ] **prototype**: http & gRPC client for end-to-end types.
+- [ ] **experiment**: allow to derive behaviour for types, like derive(Map), would generated union type with Map() method
+- [ ] **experiment**: consider adding explicit discriminator type names like `example.Branch[int]` instead of `example.Branch`. This may complicate TypeScript codegen but it could increase end-to-end type safety.

From 80954d2ef719cba2bd42bfdf884102822d3a2880 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 19:58:54 +0200
Subject: [PATCH 30/72] example: Add TreeJson serialisation

---
 example/tree_json_test.go | 45 +++++++++++++++++++++++++++++++++++++++
 example/tree_test.go      | 16 --------------
 2 files changed, 45 insertions(+), 16 deletions(-)
 create mode 100644 example/tree_json_test.go

diff --git a/example/tree_json_test.go b/example/tree_json_test.go
new file mode 100644
index 00000000..335f5f77
--- /dev/null
+++ b/example/tree_json_test.go
@@ -0,0 +1,45 @@
+package example
+
+import (
+	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/widmogrod/mkunion/x/shared"
+)
+
+func ExampleTreeJson() {
+	tree := &Branch[int]{
+		L: &Leaf[int]{Value: 1},
+		R: &Branch[int]{
+			L: &Branch[int]{
+				L: &Leaf[int]{Value: 2},
+				R: &Leaf[int]{Value: 3},
+			},
+			R: &Leaf[int]{Value: 4},
+		},
+	}
+
+	json, _ := shared.JSONMarshal[Tree[int]](tree)
+	result, _ := shared.JSONUnmarshal[Tree[int]](json)
+
+	fmt.Println(string(json))
+	if diff := cmp.Diff(tree, result); diff != "" {
+		fmt.Println("expected tree and result to be equal, but got diff:", diff)
+	}
+	//Output: {"$type":"example.Branch","example.Branch":{"L":{"$type":"example.Leaf","example.Leaf":{"Value":1}},"R":{"$type":"example.Branch","example.Branch":{"L":{"$type":"example.Branch","example.Branch":{"L":{"$type":"example.Leaf","example.Leaf":{"Value":2}},"R":{"$type":"example.Leaf","example.Leaf":{"Value":3}}}},"R":{"$type":"example.Leaf","example.Leaf":{"Value":4}}}}}}
+}
+
+//func TestMyTriesMatchR0(t *testing.T) {
+//	MyTriesMatchR0(
+//		&Leaf{Value: 1}, &Leaf{Value: 3},
+//		func(x *Leaf, y *Leaf) {
+//			assert.Equal(t, x.Value, 1)
+//			assert.Equal(t, y.Value, 3)
+//		},
+//		func(x0 *Branch, x1 any) {
+//			assert.Fail(t, "should not match")
+//		},
+//		func(x0 any, x1 any) {
+//			assert.Fail(t, "should not match")
+//		},
+//	)
+//}
diff --git a/example/tree_test.go b/example/tree_test.go
index 89620226..04d3a948 100644
--- a/example/tree_test.go
+++ b/example/tree_test.go
@@ -75,19 +75,3 @@ func TestTreeSchema(t *testing.T) {
 	result := schema.ToGo[Tree[int]](sch)
 	assert.Equal(t, tree, result)
 }
-
-//func TestMyTriesMatchR0(t *testing.T) {
-//	MyTriesMatchR0(
-//		&Leaf{Value: 1}, &Leaf{Value: 3},
-//		func(x *Leaf, y *Leaf) {
-//			assert.Equal(t, x.Value, 1)
-//			assert.Equal(t, y.Value, 3)
-//		},
-//		func(x0 *Branch, x1 any) {
-//			assert.Fail(t, "should not match")
-//		},
-//		func(x0 any, x1 any) {
-//			assert.Fail(t, "should not match")
-//		},
-//	)
-//}

From 7d3fe4dde95a5d6844ac4e9c935c50962b3165f4 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 20:00:38 +0200
Subject: [PATCH 31/72] cmd: type-registry now registers json marshaler

---
 x/generators/serde_json_union.go | 34 +++++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/x/generators/serde_json_union.go b/x/generators/serde_json_union.go
index 592e1878..8eabef89 100644
--- a/x/generators/serde_json_union.go
+++ b/x/generators/serde_json_union.go
@@ -172,11 +172,7 @@ func (g *SerdeJSONUnion) GenerateInitFunc(init []string) (string, error) {
 
 func (g *SerdeJSONUnion) ExtractImportFuncs(s shape.Shape) []string {
 	result := []string{
-		fmt.Sprintf("shared.JSONMarshallerRegister(%q, %s, %s)",
-			shape.ToGoTypeName(s, shape.WithPkgImportName(), shape.WithInstantiation()),
-			g.FuncNameFromJSONInstantiated(s),
-			g.FuncNameToSONInstantiated(s),
-		),
+		StrRegisterUnionFuncName(g.union.PkgName, s),
 	}
 
 	switch x := s.(type) {
@@ -189,15 +185,35 @@ func (g *SerdeJSONUnion) ExtractImportFuncs(s shape.Shape) []string {
 	return result
 }
 
+func StrRegisterUnionFuncName(rootPkgName string, x shape.Shape) string {
+	return fmt.Sprintf("shared.JSONMarshallerRegister(%q, %s, %s)",
+		shape.ToGoTypeName(x, shape.WithPkgImportName(), shape.WithInstantiation()),
+		StrFuncNameFromJSONInstantiated(rootPkgName, x),
+		StrFuncNameToJSONInstantiated(rootPkgName, x),
+	)
+}
+
 func (g *SerdeJSONUnion) FuncNameFromJSONInstantiated(x shape.Shape) string {
-	return g.instantiatef(x, "%sFromJSON")
+	return StrFuncNameFromJSONInstantiated(g.union.PkgName, x)
 }
 
-func (g *SerdeJSONUnion) FuncNameToSONInstantiated(x shape.Shape) string {
-	return g.instantiatef(x, "%sToJSON")
+func (g *SerdeJSONUnion) FuncNameToJSONInstantiated(x shape.Shape) string {
+	return StrFuncNameToJSONInstantiated(g.union.PkgName, x)
 }
 
 func (g *SerdeJSONUnion) instantiatef(x shape.Shape, template string) string {
+	return StrInstantiatef(g.union.PkgName, x, template)
+}
+
+func StrFuncNameFromJSONInstantiated(rootPkgName string, x shape.Shape) string {
+	return StrInstantiatef(rootPkgName, x, "%sFromJSON")
+}
+
+func StrFuncNameToJSONInstantiated(rootPkgName string, x shape.Shape) string {
+	return StrInstantiatef(rootPkgName, x, "%sToJSON")
+}
+
+func StrInstantiatef(pkgName string, x shape.Shape, template string) string {
 	typeParamTypes := shape.ToGoTypeParamsTypes(x)
 	typeName := fmt.Sprintf(template, shape.Name(x))
 	if len(typeParamTypes) == 0 {
@@ -207,7 +223,7 @@ func (g *SerdeJSONUnion) instantiatef(x shape.Shape, template string) string {
 	instantiatedNames := make([]string, len(typeParamTypes))
 	for i, t := range typeParamTypes {
 		instantiatedNames[i] = shape.ToGoTypeName(t,
-			shape.WithRootPackage(g.union.PkgName),
+			shape.WithRootPackage(pkgName),
 			shape.WithInstantiation(),
 		)
 	}

From bdbab97ccb56b52aef9c7ed8d7f1bfed91d6809c Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 20:01:31 +0200
Subject: [PATCH 32/72] shared/json: add method description

---
 x/shared/json.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/x/shared/json.go b/x/shared/json.go
index 02954fe7..53dbf16a 100644
--- a/x/shared/json.go
+++ b/x/shared/json.go
@@ -61,6 +61,8 @@ func JSONMarshallerRegister[A any](
 	})
 }
 
+// JSONUnmarshal is a generic function to unmarshal json data into destination type
+// that supports union types and fallback to native json.Unmarshal when available.
 func JSONUnmarshal[A any](data []byte) (A, error) {
 	var destinationTypePtr *A = new(A)
 	var destinationType A = *destinationTypePtr
@@ -111,6 +113,8 @@ func JSONUnmarshal[A any](data []byte) (A, error) {
 	return result.(A), nil
 }
 
+// JSONMarshal is a generic function to marshal destination type into json data
+// that supports union types and fallback to native json.Marshal when available
 func JSONMarshal[A any](in A) ([]byte, error) {
 	x := any(in)
 	if x == nil {

From 87b6196d76e846d24e3017970b4362f1eaf34e7b Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 20:02:22 +0200
Subject: [PATCH 33/72] cmd/mkunion: add registering JSONMarshaler during
 generation

---
 cmd/mkunion/main.go | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/cmd/mkunion/main.go b/cmd/mkunion/main.go
index 610cb0db..015ab9f5 100644
--- a/cmd/mkunion/main.go
+++ b/cmd/mkunion/main.go
@@ -563,7 +563,20 @@ func GenerateTypeRegistry(inferred *shape.IndexedTypeWalker) (bytes.Buffer, erro
 			shape.WithPkgImportName(),
 		)
 
+		// Register go type
 		contents.WriteString(fmt.Sprintf("\tshared.TypeRegistryStore[%s](%q)\n", instantiatedTypeName, fullTypeName))
+
+		// Try to register type JSON marshaller
+		if ref, ok := inst.(*shape.RefName); ok {
+			some, found := shape.LookupShapeOnDisk(ref)
+			if !found {
+				continue
+			}
+			some = shape.IndexWith(some, ref)
+			if shape.IsUnion(some) {
+				contents.WriteString(fmt.Sprintf("\t%s\n", generators.StrRegisterUnionFuncName(shape.ToGoPkgName(some), some)))
+			}
+		}
 	}
 
 	contents.WriteString("}\n")

From ba2187635d0a39dee7378bb3b560fe0b80636646 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 20:03:36 +0200
Subject: [PATCH 34/72] x/machine: add context to machine.Handle(ctx, ...)

---
 x/machine/machine.go      | 12 +++++++-----
 x/machine/machine_test.go |  6 +++---
 x/machine/test_suite.go   |  5 +++--
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/x/machine/machine.go b/x/machine/machine.go
index 4d1175c1..fb1519d6 100644
--- a/x/machine/machine.go
+++ b/x/machine/machine.go
@@ -1,6 +1,8 @@
 package machine
 
-func NewMachine[D, C, S any](d D, f func(D, C, S) (S, error), state S) *Machine[D, C, S] {
+import "context"
+
+func NewMachine[D, C, S any](d D, f func(context.Context, D, C, S) (S, error), state S) *Machine[D, C, S] {
 	return &Machine[D, C, S]{
 		di:     d,
 		handle: f,
@@ -16,7 +18,7 @@ func NewSimpleMachine[C, S any](f func(C, S) (S, error)) *Machine[any, C, S] {
 func NewSimpleMachineWithState[C, S any](f func(C, S) (S, error), state S) *Machine[any, C, S] {
 	return &Machine[any, C, S]{
 		di: nil,
-		handle: func(a any, c C, s S) (S, error) {
+		handle: func(ctx context.Context, a any, c C, s S) (S, error) {
 			return f(c, s)
 		},
 		state: state,
@@ -26,11 +28,11 @@ func NewSimpleMachineWithState[C, S any](f func(C, S) (S, error), state S) *Mach
 type Machine[D, C, S any] struct {
 	di     D
 	state  S
-	handle func(D, C, S) (S, error)
+	handle func(context.Context, D, C, S) (S, error)
 }
 
-func (o *Machine[D, C, S]) Handle(cmd C) error {
-	state, err := o.handle(o.di, cmd, o.state)
+func (o *Machine[D, C, S]) Handle(ctx context.Context, cmd C) error {
+	state, err := o.handle(ctx, o.di, cmd, o.state)
 	if err != nil {
 		return err
 	}
diff --git a/x/machine/machine_test.go b/x/machine/machine_test.go
index d30afdf5..98d4f485 100644
--- a/x/machine/machine_test.go
+++ b/x/machine/machine_test.go
@@ -20,15 +20,15 @@ func TestMachine(t *testing.T) {
 
 	assert.Equal(t, 10, m.State())
 
-	err := m.Handle("inc")
+	err := m.Handle(nil, "inc")
 	assert.NoError(t, err)
 	assert.Equal(t, 11, m.State())
 
-	err = m.Handle("dec")
+	err = m.Handle(nil, "dec")
 	assert.NoError(t, err)
 	assert.Equal(t, 10, m.State())
 
-	err = m.Handle("unknown")
+	err = m.Handle(nil, "unknown")
 	assert.Error(t, err)
 	assert.Equal(t, 10, m.State())
 }
diff --git a/x/machine/test_suite.go b/x/machine/test_suite.go
index 077347f3..eb9fee62 100644
--- a/x/machine/test_suite.go
+++ b/x/machine/test_suite.go
@@ -1,6 +1,7 @@
 package machine
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"github.com/google/go-cmp/cmp"
@@ -111,7 +112,7 @@ func (suite *Suite[D, C, S]) fuzzy() {
 		if step.AfterCommand != nil {
 			step.AfterCommand(zeroT, suite.dep)
 		}
-		err := m.Handle(step.GivenCommand)
+		err := m.Handle(context.Background(), step.GivenCommand)
 		newState := m.State()
 		suite.infer.Record(step.GivenCommand, state, newState, err)
 	}
@@ -279,7 +280,7 @@ func (suitcase *Case[D, C, S]) run(t *testing.T) {
 		suitcase.step.BeforeCommand(t, suitcase.suit.dep)
 	}
 
-	err := machine.Handle(suitcase.step.GivenCommand)
+	err := machine.Handle(context.Background(), suitcase.step.GivenCommand)
 	suitcase.resultErr = err
 	suitcase.resultState = machine.State()
 

From c34ff6cc8e0dc59a3f66ce2ebb18b30ac05a11d9 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 20:04:21 +0200
Subject: [PATCH 35/72] x/workflow: update Transition function to support
 context

---
 x/workflow/workflow_machine_test.go | 4 ++--
 x/workflow/workflow_transition.go   | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/x/workflow/workflow_machine_test.go b/x/workflow/workflow_machine_test.go
index 48891468..e12b50e2 100644
--- a/x/workflow/workflow_machine_test.go
+++ b/x/workflow/workflow_machine_test.go
@@ -75,7 +75,7 @@ func TestExecution(t *testing.T) {
 	assert.ErrorIs(t, err, schemaless.ErrNotFound)
 
 	work := NewMachine(di, state.Data)
-	err = work.Handle(&Run{
+	err = work.Handle(nil, &Run{
 		Flow:  &FlowRef{FlowID: "hello_world_flow"},
 		Input: schema.MkString("world"),
 	})
@@ -108,7 +108,7 @@ func TestExecution(t *testing.T) {
 	assert.NoError(t, err)
 
 	work = NewMachine(di, state.Data)
-	err = work.Handle(&Run{
+	err = work.Handle(nil, &Run{
 		Flow:  &FlowRef{FlowID: "hello_world_flow"},
 		Input: schema.MkString("world"),
 	})
diff --git a/x/workflow/workflow_transition.go b/x/workflow/workflow_transition.go
index c6332bec..7cc2ae11 100644
--- a/x/workflow/workflow_transition.go
+++ b/x/workflow/workflow_transition.go
@@ -1,6 +1,7 @@
 package workflow
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"github.com/robfig/cron/v3"
@@ -35,7 +36,7 @@ func NewMachine(di Dependency, state State) *machine.Machine[Dependency, Command
 	return machine.NewMachine(di, Transition, state)
 }
 
-func Transition(dep Dependency, cmd Command, state State) (State, error) {
+func Transition(ctx context.Context, dep Dependency, cmd Command, state State) (State, error) {
 	switch state.(type) {
 	case *Done:
 		return nil, ErrStateReachEnd

From 9f99887804a4986f2cd2f6a9a2555a16899c0363 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 20:05:03 +0200
Subject: [PATCH 36/72] x/taskqueue: change signature machine.Handle(ctx,...)

---
 x/taskqueue/taskqueue_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/x/taskqueue/taskqueue_test.go b/x/taskqueue/taskqueue_test.go
index e58c5d7a..460033b7 100644
--- a/x/taskqueue/taskqueue_test.go
+++ b/x/taskqueue/taskqueue_test.go
@@ -99,7 +99,7 @@ AND Data["workflow.Scheduled"].ExpectedRunTimestamp > 0`,
 			t.Logf("data id: %s \n", task.Data.ID)
 			t.Logf("version: %d \n", task.Data.Version)
 			work := workflow.NewMachine(di, task.Data.Data)
-			err := work.Handle(&workflow.Run{})
+			err := work.Handle(nil, &workflow.Run{})
 			//err := work.Handle(&workflow.TryRecover{})
 			if err != nil {
 				t.Logf("err: %s", err)
@@ -123,7 +123,7 @@ AND Data["workflow.Scheduled"].ExpectedRunTimestamp > 0`,
 				//d, _ := schema.ToJSON(schema.FromPrimitiveGo(next))
 				//t.Logf("next: %s", string(d))
 				work := workflow.NewMachine(di, nil)
-				err := work.Handle(next)
+				err := work.Handle(nil, next)
 				if err != nil {
 					t.Logf("err: %s", err)
 					return
@@ -203,7 +203,7 @@ To run this test, please set AWS_SQS_QUEUE_URL to the address of your AWS SQS in
 	}()
 
 	work := workflow.NewMachine(di, nil)
-	err := work.Handle(&workflow.Run{
+	err := work.Handle(nil, &workflow.Run{
 		//RunOption: &workflow.DelayRun{
 		//	DelayBySeconds: int64(1 * time.Second),
 		//},

From a146d53e425127ef5c84092310f07ba6ff4ca1e6 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 20:15:53 +0200
Subject: [PATCH 37/72] x/taskqueue: change signature machine.Handle(ctx,...)
 to have context support

---
 example/state/machine.go      | 6 ++++--
 example/state/machine_test.go | 3 ++-
 example/state/model.go        | 3 +--
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/example/state/machine.go b/example/state/machine.go
index 2b99baa3..5de2f381 100644
--- a/example/state/machine.go
+++ b/example/state/machine.go
@@ -1,6 +1,7 @@
 package state
 
 import (
+	"context"
 	"fmt"
 	"github.com/widmogrod/mkunion/x/machine"
 	"time"
@@ -27,13 +28,14 @@ var (
 	ErrWorkerIDRequired = fmt.Errorf("worker ID required; %w", ErrValidationFailed)
 )
 
+//go:generate moq -with-resets -stub -out machine_mock_test.go . Dependency
 type Dependency interface {
 	TimeNow() *time.Time
 	WarehouseRemoveStock(quantity Quantity) error
 	PaymentCharge(price Price) error
 }
 
-func Transition(di Dependency, cmd Command, state State) (State, error) {
+func Transition(ctx context.Context, di Dependency, cmd Command, state State) (State, error) {
 	return MatchCommandR2(
 		cmd,
 		func(x *CreateOrderCMD) (State, error) {
@@ -161,7 +163,7 @@ func Transition(di Dependency, cmd Command, state State) (State, error) {
 				case ProblemWarehouseAPIUnreachable,
 					ProblemPaymentAPIUnreachable:
 					// we can retry this operation
-					newState, err := Transition(di, s.ProblemCommand, s.ProblemState)
+					newState, err := Transition(ctx, di, s.ProblemCommand, s.ProblemState)
 					if err != nil {
 						return s, err
 					}
diff --git a/example/state/machine_test.go b/example/state/machine_test.go
index d1ae3f72..51fb0d79 100644
--- a/example/state/machine_test.go
+++ b/example/state/machine_test.go
@@ -1,6 +1,7 @@
 package state
 
 import (
+	"context"
 	"fmt"
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
@@ -316,7 +317,7 @@ func AssertScenario[D, C, S any](
 				step.BeforeCommand(t, m.Dep())
 			}
 
-			err := m.Handle(step.GivenCommand)
+			err := m.Handle(context.TODO(), step.GivenCommand)
 
 			if step.AfterCommand != nil {
 				step.AfterCommand(t, m.Dep())
diff --git a/example/state/model.go b/example/state/model.go
index 35b5fe17..328751df 100644
--- a/example/state/model.go
+++ b/example/state/model.go
@@ -2,7 +2,7 @@ package state
 
 import "time"
 
-//go:generate go run ../../cmd/mkunion/main.go
+//go:generate go run ../../cmd/mkunion/main.go --type-registry
 
 //go:tag mkunion:"Command"
 type (
@@ -58,7 +58,6 @@ type (
 	}
 )
 
-//go:generate moq -with-resets -stub -out machine_mock_test.go . Dependency
 type (
 	// OrderID Price, Quantity are placeholders for value objects, to ensure better data semantic and type safety
 	OrderID  = string

From bceca05aa58dbc358c5ee1e927e693774e252ddc Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 20:16:23 +0200
Subject: [PATCH 38/72] x/state: Introduce storage example and tests

---
 example/state/machine_database_test.go | 129 +++++++++++++++++++++++++
 1 file changed, 129 insertions(+)
 create mode 100644 example/state/machine_database_test.go

diff --git a/example/state/machine_database_test.go b/example/state/machine_database_test.go
new file mode 100644
index 00000000..23f9491e
--- /dev/null
+++ b/example/state/machine_database_test.go
@@ -0,0 +1,129 @@
+package state
+
+import (
+	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/assert"
+	"github.com/widmogrod/mkunion/x/schema"
+	"github.com/widmogrod/mkunion/x/shared"
+	"github.com/widmogrod/mkunion/x/storage/predicate"
+	"github.com/widmogrod/mkunion/x/storage/schemaless"
+	"testing"
+	"time"
+)
+
+// StoreStateInDatabase is an example how to store state in database
+func ExampleStoreStateInDatabase() {
+	now := time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)
+
+	// example state
+	state := &OrderCompleted{
+		Order: Order{
+			ID:          "123",
+			OrderAttr:   OrderAttr{Price: 100, Quantity: 3},
+			DeliveredAt: &now,
+		},
+	}
+
+	// let's use in memory storage for storing State union
+	storage := schemaless.NewInMemoryRepository[State]()
+
+	// let's save it to storage
+	_, err := storage.UpdateRecords(schemaless.Save(schemaless.Record[State]{
+		ID:   state.Order.ID,
+		Type: "orders",
+		Data: state,
+	}))
+
+	records, err := storage.FindingRecords(schemaless.FindingRecords[schemaless.Record[State]]{
+		RecordType: "orders",
+	})
+
+	fmt.Println(err)
+	fmt.Printf("%+#v\n", *records.Items[0].Data.(*OrderCompleted))
+	//Output: <nil>
+	//state.OrderCompleted{Order:state.Order{ID:"123", OrderAttr:state.OrderAttr{Price:100, Quantity:3}, WorkerID:"", StockRemovedAt:<nil>, PaymentChargedAt:<nil>, DeliveredAt:time.Date(2021, time.January, 1, 0, 0, 0, 0, time.UTC), CancelledAt:<nil>, CancelledReason:""}}
+}
+
+func TestPersistMachine(t *testing.T) {
+	orderId := "123"
+	recordType := "orders"
+
+	// let's use in memory storage for storing State union
+	storage := schemaless.NewInMemoryRepository[State]()
+
+	// before we will save our state to storage, let's check if orderId it's not there already
+	records, err := storage.FindingRecords(schemaless.FindingRecords[schemaless.Record[State]]{
+		RecordType: recordType,
+		Where: predicate.MustWhere("ID = :id", predicate.ParamBinds{
+			":id": schema.MkString(orderId),
+		}),
+	})
+
+	assert.NoError(t, err)
+	assert.Len(t, records.Items, 0)
+
+	// let's simulate order processing
+	now := time.Now()
+	dep := &DependencyMock{
+		TimeNowFunc: func() *time.Time {
+			return &now
+		},
+	}
+
+	order := OrderAttr{
+		Price:    100,
+		Quantity: 3,
+	}
+
+	m := NewMachine(dep, nil)
+	err = m.Handle(nil, &CreateOrderCMD{OrderID: "123", Attr: order})
+	assert.NoError(t, err)
+
+	err = m.Handle(nil, &MarkAsProcessingCMD{OrderID: "123", WorkerID: "worker-1"})
+	assert.NoError(t, err)
+
+	err = m.Handle(nil, &MarkOrderCompleteCMD{OrderID: "123"})
+	assert.NoError(t, err)
+
+	state := m.State()
+	assert.Equal(t, &OrderCompleted{
+		Order: Order{
+			ID:               "123",
+			OrderAttr:        order,
+			WorkerID:         "worker-1",
+			DeliveredAt:      &now,
+			StockRemovedAt:   &now,
+			PaymentChargedAt: &now,
+		},
+	}, state)
+
+	res, err := shared.JSONMarshal[State](state)
+	assert.NoError(t, err)
+	t.Log(string(res))
+
+	schemed := schema.FromGo[State](state)
+	t.Logf("%+v", schemed)
+
+	// we have correct state, let's save it to storage
+	_, err = storage.UpdateRecords(schemaless.Save(schemaless.Record[State]{
+		ID:   "123",
+		Data: state,
+		Type: recordType,
+	}))
+	assert.NoError(t, err)
+
+	// let's check if we can load state from storage
+	records, err = storage.FindingRecords(schemaless.FindingRecords[schemaless.Record[State]]{
+		RecordType: recordType,
+		Where: predicate.MustWhere("ID = :id", predicate.ParamBinds{
+			":id": schema.MkString(orderId),
+		}),
+	})
+	assert.NoError(t, err)
+	if assert.Len(t, records.Items, 1) {
+		if diff := cmp.Diff(state, records.Items[0].Data); diff != "" {
+			assert.Fail(t, "unexpected state (-want +got):\n%s", diff)
+		}
+	}
+}

From d48af06e4bc1cc7743d9a04dab5fde035442f957 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 20:17:12 +0200
Subject: [PATCH 39/72] example/tic_tac_toe: change signature
 machine.Handle(ctx,...) to have context support

---
 example/tic_tac_toe_machine/machine_test.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/example/tic_tac_toe_machine/machine_test.go b/example/tic_tac_toe_machine/machine_test.go
index 33ebfd03..050ecb80 100644
--- a/example/tic_tac_toe_machine/machine_test.go
+++ b/example/tic_tac_toe_machine/machine_test.go
@@ -1,6 +1,7 @@
 package tictacstatemachine
 
 import (
+	"context"
 	"github.com/stretchr/testify/assert"
 	"testing"
 )
@@ -690,7 +691,7 @@ func TestNewMachine(t *testing.T) {
 		t.Run(name, func(t *testing.T) {
 			m := NewMachine()
 			for i, cmd := range uc.commands {
-				err := m.Handle(cmd)
+				err := m.Handle(context.TODO(), cmd)
 				assert.Equal(t, uc.states[i], m.State(), "state at index: %d", i)
 				assert.ErrorIs(t, err, uc.err[i], "error at index: %d", i)
 			}

From e4bde3d896f41aace6a6c4295afea85a0c404ba5 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 20:18:31 +0200
Subject: [PATCH 40/72] f: generic union example with JSON serialisation

---
 f/datas.go      |  4 ++--
 f/datas_test.go | 13 +++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 f/datas_test.go

diff --git a/f/datas.go b/f/datas.go
index de331213..509ca02f 100644
--- a/f/datas.go
+++ b/f/datas.go
@@ -1,8 +1,8 @@
 package f
 
-//go:generate go run ../cmd/mkunion
+//go:generate go run ../cmd/mkunion --type-registry
 
-//go:tag mkunion:"Either"
+//go:tag mkunion:"Either,serde"
 type (
 	Left[A, B any]  struct{ Value A }
 	Right[A, B any] struct{ Value B }
diff --git a/f/datas_test.go b/f/datas_test.go
new file mode 100644
index 00000000..c18d1311
--- /dev/null
+++ b/f/datas_test.go
@@ -0,0 +1,13 @@
+package f
+
+import (
+	"fmt"
+	"github.com/widmogrod/mkunion/x/shared"
+)
+
+func ExampleEitherToJSON() {
+	var either Either[int, string] = &Right[int, string]{Value: "hello"}
+	result, _ := shared.JSONMarshal(either)
+	fmt.Println(string(result))
+	// Output: {"$type":"f.Right","f.Right":{"Value":"hello"}}
+}

From ab496adec4949a6cb0f08c9f1d6e31312cf6ea10 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 20:21:19 +0200
Subject: [PATCH 41/72] my-app: change signature machine.Handle(ctx,...)

---
 example/my-app/server.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/example/my-app/server.go b/example/my-app/server.go
index 6625ea62..755079a2 100644
--- a/example/my-app/server.go
+++ b/example/my-app/server.go
@@ -551,7 +551,7 @@ func main() {
 
 			// apply command
 			work := workflow.NewMachine(di, state.Data)
-			err = work.Handle(cmd)
+			err = work.Handle(context.TODO(), cmd)
 			if err != nil {
 				log.Errorf("failed to handle command: %v", err)
 				return nil, err
@@ -576,7 +576,7 @@ func main() {
 	proc := &taskqueue.FunctionProcessor[schemaless.Record[workflow.State]]{
 		F: func(task taskqueue.Task[schemaless.Record[workflow.State]]) {
 			work := workflow.NewMachine(di, task.Data.Data)
-			err := work.Handle(&workflow.Run{})
+			err := work.Handle(context.TODO(), &workflow.Run{})
 			if err != nil {
 				log.Errorf("err: %s", err)
 				return
@@ -596,7 +596,7 @@ func main() {
 
 			if next := workflow.ScheduleNext(newState, di); next != nil {
 				work := workflow.NewMachine(di, nil)
-				err := work.Handle(next)
+				err := work.Handle(context.TODO(), next)
 				if err != nil {
 					log.Infof("err: %s", err)
 					return
@@ -773,7 +773,7 @@ func (service *Service[Dep, CMD, State]) CreateOrUpdate(cmd CMD) (res State, err
 	}
 
 	work := service.newMachine(res)
-	err = work.Handle(cmd)
+	err = work.Handle(context.TODO(), cmd)
 	if err != nil {
 		log.Errorf("failed to handle command: %v", err)
 		return res, err

From abe5d83e6449902e114fc487c6579d8e22006264 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 23:44:30 +0200
Subject: [PATCH 42/72] x/schema: test fallback conversions

---
 x/schema/go.go      | 141 ++++++++++++++++++++++++++++--------------
 x/schema/go_test.go | 147 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 242 insertions(+), 46 deletions(-)
 create mode 100644 x/schema/go_test.go

diff --git a/x/schema/go.go b/x/schema/go.go
index 03b29474..e42ab001 100644
--- a/x/schema/go.go
+++ b/x/schema/go.go
@@ -147,15 +147,15 @@ func ToGoPrimitive(x Schema) (any, error) {
 }
 
 func ToGoG[A any](x Schema) (res A, err error) {
-	//defer func() {
-	//	if r := recover(); r != nil {
-	//		if e, ok := r.(error); ok {
-	//			err = fmt.Errorf("schema.ToGoG: panic recover; %w", e)
-	//		} else {
-	//			err = fmt.Errorf("schema.ToGoG: panic recover; %#v", e)
-	//		}
-	//	}
-	//}()
+	defer func() {
+		if r := recover(); r != nil {
+			if e, ok := r.(error); ok {
+				err = fmt.Errorf("schema.ToGoG: panic recover; %w", e)
+			} else {
+				err = fmt.Errorf("schema.ToGoG: panic recover; %#v", e)
+			}
+		}
+	}()
 
 	res = ToGo[A](x)
 	return
@@ -174,48 +174,77 @@ func ToGo[A any](x Schema) A {
 			switch any(result).(type) {
 			case int:
 				return any(int(y)).(A)
-			//case int8:
-			//	return any(int8(y)).(A)
-			//case int16:
-			//	return any(int16(y)).(A)
-			//case int32:
-			//	return any(int32(y)).(A)
-			//case int64:
-			//	return any(int64(y)).(A)
-			//case uint:
-			//	return any(uint(y)).(A)
-			//case uint8:
-			//	return any(uint8(y)).(A)
-			//case uint16:
-			//	return any(uint16(y)).(A)
-			//case uint32:
-			//	return any(uint32(y)).(A)
-			//case uint64:
-			//	return any(uint64(y)).(A)
-			//case float32:
-			//	return any(float32(y)).(A)
+			case int8:
+				return any(int8(y)).(A)
+			case int16:
+				return any(int16(y)).(A)
+			case int32:
+				return any(int32(y)).(A)
+			case int64:
+				return any(int64(y)).(A)
+			case uint:
+				return any(uint(y)).(A)
+			case uint8:
+				return any(uint8(y)).(A)
+			case uint16:
+				return any(uint16(y)).(A)
+			case uint32:
+				return any(uint32(y)).(A)
+			case uint64:
+				return any(uint64(y)).(A)
+			case float32:
+				return any(float32(y)).(A)
 			case float64:
 				return any(float64(y)).(A)
 			}
 		}
+
+		return value.(A)
 	}
 
 	v := reflect.TypeOf(new(A)).Elem()
 	original := shape.MkRefNameFromReflect(v)
 
 	s, found := shape.LookupShape(original)
-	if !found {
-		panic(fmt.Errorf("schema.FromGo: shape.RefName not found %s; %w", v.String(), shape.ErrShapeNotFound))
+	if found {
+		s = shape.IndexWith(s, original)
+
+		value, err := ToGoReflect(s, x, v)
+		if err != nil {
+			panic(fmt.Errorf("schema.ToGo: %w", err))
+		}
+
+		return value.Interface().(A)
 	}
 
-	s = shape.IndexWith(s, original)
+	str, ok := x.(*String)
+	if ok {
+		// to properly fallback, type needs to have MarshalJSON/UnmarshalJSON methods
+		res := unmarshalFallback(reflect.ValueOf(new(A)), str, *new(A))
+		val := res.(*A)
+		return *val
+	}
+
+	panic(fmt.Errorf("schema.ToGo: cannot build type %T", *new(A)))
+}
 
-	value, err := ToGoReflect(s, x, v)
-	if err != nil {
-		panic(fmt.Errorf("schema.ToGo: %w", err))
+func unmarshalFallback(ref reflect.Value, str *String, typ any) any {
+	marshal := ref.MethodByName("MarshalJSON")
+	unmarshal := ref.MethodByName("UnmarshalJSON")
+	if marshal.IsZero() && unmarshal.IsZero() {
+		panic(fmt.Errorf("schema.ToGo: shape.RefName not found for %T", typ))
 	}
 
-	return value.Interface().(A)
+	res := unmarshal.Call([]reflect.Value{reflect.ValueOf([]byte(*str))})
+	if len(res) != 1 {
+		panic(fmt.Errorf("schema.ToGo: %T.UnmarshalJSON() expected 1 return value, got %d", typ, len(res)))
+	}
+
+	if res[0].IsZero() {
+		return ref.Interface()
+	}
+
+	panic(fmt.Errorf("schema.ToGo: %T.UnmarshalJSON() error: %w", typ, res[0].Interface().(error)))
 }
 
 func FromGo[A any](x A) Schema {
@@ -224,11 +253,30 @@ func FromGo[A any](x A) Schema {
 	}
 
 	s, found := shape.LookupShapeReflectAndIndex[A]()
-	if !found {
-		panic(fmt.Errorf("schema.FromGo: shape.RefName not found for %T; %w", *new(A), shape.ErrShapeNotFound))
+	if found {
+		return FromGoReflect(s, reflect.ValueOf(x))
+	}
+
+	return marshalFallback(reflect.ValueOf(x), *new(A))
+}
+
+func marshalFallback(ref reflect.Value, typ any) Schema {
+	marshal := ref.MethodByName("MarshalJSON")
+	unmarshal := ref.MethodByName("UnmarshalJSON")
+	if marshal.IsZero() && unmarshal.IsZero() {
+		panic(fmt.Errorf("schema.FromGo: shape.RefName not found for %T; %w", typ, shape.ErrShapeNotFound))
+	}
+
+	res := marshal.Call(nil)
+	if len(res) != 2 {
+		panic(fmt.Errorf("schema.FromGo: %T.MarshalJSON() expected 2 return values, got %d", typ, len(res)))
+	}
+
+	if res[1].IsZero() {
+		return MkString(string(res[0].Bytes()))
 	}
 
-	return FromGoReflect(s, reflect.ValueOf(x))
+	panic(fmt.Errorf("schema.FromGo: %T.MarshalJSON() error: %w", typ, res[1].Interface().(error)))
 }
 
 func FromGoReflect(xschema shape.Shape, yreflect reflect.Value) Schema {
@@ -239,15 +287,16 @@ func FromGoReflect(xschema shape.Shape, yreflect reflect.Value) Schema {
 		},
 		func(x *shape.RefName) Schema {
 			y, found := shape.LookupShape(x)
-			if !found {
-				panic(fmt.Errorf("schema.FromGoReflect: shape.RefName not found %s; %w",
-					shape.ToGoTypeName(x, shape.WithPkgImportName()),
-					shape.ErrShapeNotFound))
-			}
+			if found {
+				y = shape.IndexWith(y, x)
 
-			y = shape.IndexWith(y, x)
+				return FromGoReflect(y, yreflect)
+			}
 
-			return FromGoReflect(y, yreflect)
+			// Convert types that are not registered in shape registry, or don't have schema mapping, like time.Time, etc.
+			// to String, but only when they have MarshalJSON/UnmarshalJSON methods.
+			// Because JSON is quite popular format, this should cover most of the cases.
+			return marshalFallback(yreflect, shape.ToGoTypeName(x, shape.WithPkgImportName()))
 		},
 		func(x *shape.PointerLike) Schema {
 			if yreflect.IsNil() {
diff --git a/x/schema/go_test.go b/x/schema/go_test.go
new file mode 100644
index 00000000..a46ecbe6
--- /dev/null
+++ b/x/schema/go_test.go
@@ -0,0 +1,147 @@
+package schema
+
+import (
+	"encoding/json"
+	"github.com/google/go-cmp/cmp"
+	"testing"
+	"testing/quick"
+)
+
+func TestNative(t *testing.T) {
+	t.Run("int", func(t *testing.T) {
+		assertTypeConversion(t, 1)
+	})
+	t.Run("int8", func(t *testing.T) {
+		if err := quick.Check(func(x int8) bool {
+			assertTypeConversion(t, x)
+			return true
+		}, nil); err != nil {
+			t.Error(err)
+		}
+	})
+	t.Run("int16", func(t *testing.T) {
+		if err := quick.Check(func(x int16) bool {
+			assertTypeConversion(t, x)
+			return true
+		}, nil); err != nil {
+			t.Error(err)
+		}
+	})
+	t.Run("int32", func(t *testing.T) {
+		if err := quick.Check(func(x int32) bool {
+			assertTypeConversion(t, x)
+			return true
+		}, nil); err != nil {
+			t.Error(err)
+		}
+	})
+	t.Run("int64", func(t *testing.T) {
+		t.Skip("boundary conversion issue because *Number is float64")
+		if err := quick.Check(func(x int64) bool {
+			assertTypeConversion(t, x)
+			return true
+		}, nil); err != nil {
+			t.Error(err)
+		}
+	})
+	t.Run("uint", func(t *testing.T) {
+		t.Skip("boundary conversion issue because *Number is float64")
+		if err := quick.Check(func(x uint) bool {
+			assertTypeConversion(t, x)
+			return true
+		}, nil); err != nil {
+			t.Error(err)
+		}
+	})
+
+	t.Run("uint8", func(t *testing.T) {
+		if err := quick.Check(func(x uint8) bool {
+			assertTypeConversion(t, x)
+			return true
+		}, nil); err != nil {
+			t.Error(err)
+		}
+	})
+	t.Run("uint16", func(t *testing.T) {
+		if err := quick.Check(func(x uint16) bool {
+			assertTypeConversion(t, x)
+			return true
+		}, nil); err != nil {
+			t.Error(err)
+		}
+
+	})
+	t.Run("uint32", func(t *testing.T) {
+		if err := quick.Check(func(x uint32) bool {
+			assertTypeConversion(t, x)
+			return true
+		}, nil); err != nil {
+			t.Error(err)
+		}
+	})
+	t.Run("uint64", func(t *testing.T) {
+		t.Skip("boundary conversion issue because *Number is float64")
+		if err := quick.Check(func(x uint64) bool {
+			assertTypeConversion(t, x)
+			return true
+		}, nil); err != nil {
+			t.Error(err)
+		}
+	})
+	t.Run("float32", func(t *testing.T) {
+		if err := quick.Check(func(x float32) bool {
+			assertTypeConversion(t, x)
+			return true
+		}, nil); err != nil {
+			t.Error(err)
+		}
+	})
+	t.Run("float64", func(t *testing.T) {
+		if err := quick.Check(func(x float64) bool {
+			assertTypeConversion(t, x)
+			return true
+		}, nil); err != nil {
+			t.Error(err)
+		}
+	})
+	t.Run("string", func(t *testing.T) {
+		if err := quick.Check(func(x string) bool {
+			assertTypeConversion(t, x)
+			return true
+		}, nil); err != nil {
+			t.Error(err)
+		}
+	})
+	t.Run("[]byte", func(t *testing.T) {
+		if err := quick.Check(func(x []byte) bool {
+			assertTypeConversion(t, x)
+			return true
+		}, nil); err != nil {
+			t.Error(err)
+		}
+	})
+}
+
+func TestNonNative(t *testing.T) {
+	t.Run("json.RawMessage", func(t *testing.T) {
+		assertTypeConversion(t, json.RawMessage(`{"hello": "world"}`))
+	})
+	t.Run("time.Time", func(t *testing.T) {
+		assertTypeConversion(t, "2021-01-01T00:00:00Z")
+	})
+}
+
+func assertTypeConversion[A any](t *testing.T, value A) {
+	expected := value
+	t.Logf("expected = %+#v", expected)
+
+	schemed := FromGo[A](expected)
+	t.Logf("  FromGo = %+#v", schemed)
+
+	result := ToGo[A](schemed)
+	t.Logf("    ToGo = %+#v", result)
+
+	if diff := cmp.Diff(expected, result); diff != "" {
+		t.Error(diff)
+	}
+}

From 45fb01d732d69f99e0c7e1535d6ad3f1e20f28a0 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 23:52:10 +0200
Subject: [PATCH 43/72] update comments

---
 example/my-app/server.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/example/my-app/server.go b/example/my-app/server.go
index 755079a2..b915af00 100644
--- a/example/my-app/server.go
+++ b/example/my-app/server.go
@@ -37,7 +37,7 @@ import (
 // this command make sure that all types that are imported will have generated typescript mapping
 //go:generate ../../cmd/mkunion/mkunion shape-export --language=typescript -o ./src/workflow
 
-// this lines defines all types that should have typescript mapping
+// this lines defines all types that should have typescript mapping generated by above command
 type (
 	Workflow       = workflow.Workflow
 	State          = workflow.State

From b6af66fca055c51a7dab0fde871972ecc33b6df6 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sat, 11 May 2024 23:52:28 +0200
Subject: [PATCH 44/72] docs: introduce incomplete documentations

---
 docs/examples/state_storage.md | 215 +++++++++++++++++++++++++++++++++
 docs/examples/type_script.md   |  11 ++
 2 files changed, 226 insertions(+)
 create mode 100644 docs/examples/state_storage.md
 create mode 100644 docs/examples/type_script.md

diff --git a/docs/examples/state_storage.md b/docs/examples/state_storage.md
new file mode 100644
index 00000000..28fedb3b
--- /dev/null
+++ b/docs/examples/state_storage.md
@@ -0,0 +1,215 @@
+---
+title: Persisting union in database
+---
+## Persisting state in database
+
+TODO complete description!
+
+At this point of time, we have implemented and tested Order Service state machine.
+
+Next thing that we need to address in our road to the production is to persist state in database.
+
+MkUnion aims to support you in this task, by providing you `x/storage/schemaless` package that will take care of:
+
+- mapping  golang structs to database representation and back from database to struct.
+- leveraging optimistic concurrency control to resolve conflicts
+- providing you with simple API to work with database
+- and more
+
+Below is test case that demonstrate complete example of initializing database,
+building an state using `NewMachine` , and saving and loading state from database.
+
+```go title="example/state/machine_database_test.go"
+--8<-- "example/state/machine_database_test.go:16:46"
+```
+
+```mermaid
+sequenceDiagram
+    participant R as Request
+    participant Store as Store
+
+    activate R
+    R->>R: Validate(request) -> error
+    
+    R->>Store: Load state from database by request.ObjectId
+    activate Store
+    Store->>R: Ok(State)
+    deactivate Store
+    
+    R->>R: Create machine with state
+    R->>R: Apply command on a state
+    
+    R->>Store: Save state in database under request.ObjectId
+    activate Store
+    Store->>R: Ok()
+    deactivate Store
+    
+    deactivate R
+```
+
+Example implementation of such sequence diagram:
+
+```go
+func Handle(rq Request, response Resopnse) {
+	ctx := rq.Context()
+	
+	// extract objectId and command from request + do some validation
+    id := rq.ObjectId
+	command := rq.Command
+	
+    // Load state from store
+    state, err := store.Find(ctx, id)
+	if err != nil { /*handle error*/ }
+
+    machine := NewSimpleMachineWithState(Transition, state)
+    newState, err := machine.Apply(cmd, state)
+    if err != nil { /*handle error*/ }
+	
+    err := store.Save(ctx, newState)
+    if err != nil { /*handle error*/ }
+	
+	// serialize response
+	response.Write(newState)
+}
+```
+
+## Error as state. Self-healing systems.
+In request-response situation, handing errors is easy, but what if in some long-lived process something goes wrong?
+How to handle errors in such situation? Without making what we learn about state machines useless or hard to use?
+
+One solution is to treat errors as state.
+In such case, our state machines will never return error, but instead will return new state, that will represent error.
+
+When we introduce explicit command responsible for correcting RecoverableError, we can create self-healing systems.
+Thanks to that, even in situation when errors are unknown, we can retroactivly introduce self-healing logic that correct states.
+
+Because there is always there is only one error state, it makes such state machines easy to reason about.
+
+```go
+//go:generate mkunion -name State
+type (
+    // ...
+    RecoverableError struct {
+        ErrCode int
+        PrevState State
+        RetryCount int
+    }
+)
+
+//go:generate mkunion -name Command
+type (
+    // ...
+    CorrectStateCMD struct {}
+)
+```
+
+Now, we have to implement recoverable logic in our state machine.
+We show example above how to do it in `Transition` function.
+
+Here is example implementation of such transition function:
+
+```go
+func Transition(cmd Command, state State) (State, error) {
+return MustMatchCommandR2(
+    cmd,
+    /* ... */
+    func(cmd *CorrectStateCMD) (State, error) {
+        switch state := state.(type) {
+        case *RecoverableError:
+            state.RetryCount = state.RetryCount + 1
+			
+            // here we can do some self-healing logic
+            if state.ErrCode == DuplicateServiceUnavailable {
+                newState, err := Transition(&MarkAsDuplicateCMD{}, state.PrevState)
+                 if err != nil {
+                    // we failed to correct error, so we return error state 
+                     return &RecoverableError{
+                        ErrCode: err,
+                        PrevState: state.PrevState,
+                        RetryCount: state.RetryCount,
+                    }, nil
+                }
+				
+                 // we manage to fix state, so we return new state
+                 return newState, nil
+             } else {
+                 // log information that we have new code, that we don't know how to handle
+             }
+			
+            // try to correct error in next iteration
+            return state, nil
+        }
+    }
+}
+```
+
+Now, to correct states we have to select from database all states that are in error state.
+It can be use in many ways, example below use a abstraction called `TaskQueue` that is responsible for running tasks in background.
+
+This abstraction guaranties that all records (historical and new ones) will be processed.
+You can think about it, as a queue that is populated by records from database, that meet SQL query criteria.
+
+You can use CRON job and pull database.
+
+```go
+//go:generate mms deployyml -type=TaskQueue -name=CorrectMSPErrors -autoscale=1,10 -memory=128Mi -cpu=100m -timeout=10s -schedule="0 0 * * *"
+func main()
+    sql := "SELECT * FROM ObjectState WHERE RecoverableError.RetryCount < 3"
+    store := datalayer.DefaultStore()
+    queue := TaskQueueFrom("correct-msp-errors", sql, store)
+    queue.OnTask(func (ctx context.Context, task Task) error {
+        state := task.State()
+        cmd := &CorrectStateCMD{}
+        machine := NewSimpleMachineWithState(Transition, state)
+        newState, err := machine.Apply(cmd, state)
+        if err != nil {
+            return err
+        }
+        return task.Save(ctx, newState)
+    })
+    err := queue.Run(ctx)
+    if err != nil {
+        log.Panic(err)
+    }
+}
+```
+
+
+## State machines and command queues and workflows
+What if command would initiate state "to process" and save it in db
+What if task queue would take such state and process it
+Woudn't this be something like command queue?
+
+When to make a list of background processes that transition such states?
+
+### processors per state
+It's like micromanage TaskQueue, where each state has it's own state, and it knows what command to apply to given state
+This could be good starting point, when there is not a lot of good tooling
+
+### processor for state machine
+With good tooling, transition of states can be declared in one place,
+and deployment to task queue could be done automatically.
+
+Note, that only some of the transitions needs to happen in background, other can be done in request-response manner.
+
+### processor for state machine with workflow
+State machine could be generalized to workflow.
+We can think about it as set of generic Command and State (like a turing machine).
+
+States like Pending, Completed, Failed
+Commands like Process, Retry, Cancel
+
+And workflow DSL with commands like: Invoke, Choose, Assign
+Where function is some ID string, and functions needs to be either
+pulled from registry, or called remotely (InvokeRemote).
+some operations would require callback (InvokeAndAwait)
+
+Then background processor would be responsible for executing such workflow (using task queue)
+Program would be responsible for defining workflow, and registering functions.
+
+Such programs could be also optimised for deployment,
+if some function would be better to run on same machine that do RPC call
+like function doing RPC call to database, and caching result in memory or in cache cluster dedicated to specific BFF
+
+
+
diff --git a/docs/examples/type_script.md b/docs/examples/type_script.md
new file mode 100644
index 00000000..3a680462
--- /dev/null
+++ b/docs/examples/type_script.md
@@ -0,0 +1,11 @@
+---
+title: End-to-End types between Go and TypeScript
+---
+
+# End-to-End types between Go and TypeScript
+
+TODO description of generating TypeScript Definitions from using MkUnion
+
+```go title="example/my-app/server.go"
+--8<-- "example/my-app/server.go:37:55"
+```
\ No newline at end of file

From b818e4338bd63a10b20a519e4bf86058d1568b75 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 00:10:12 +0200
Subject: [PATCH 45/72] feature: make type-registry ON by default

---
 cmd/mkunion/main.go                  |   2 +-
 example/state/model.go               |   2 +-
 example/tree.go                      |   2 +-
 f/datas.go                           |   2 +-
 x/projection/projection.go           |   6 +-
 x/projection/projection_union_gen.go | 503 +++++++++++++++++++++++++++
 x/projection/types_reg_gen.go        |   8 +
 x/shape/testasset/type_example.go    |   2 +-
 x/shape/testasset/type_other.go      |   2 +-
 9 files changed, 520 insertions(+), 9 deletions(-)

diff --git a/cmd/mkunion/main.go b/cmd/mkunion/main.go
index 015ab9f5..42a82ba1 100644
--- a/cmd/mkunion/main.go
+++ b/cmd/mkunion/main.go
@@ -50,7 +50,7 @@ func main() {
 			},
 			&cli.BoolFlag{
 				Name:  "type-registry",
-				Value: false,
+				Value: true,
 			},
 		},
 		Action: func(c *cli.Context) error {
diff --git a/example/state/model.go b/example/state/model.go
index 328751df..0655d1d5 100644
--- a/example/state/model.go
+++ b/example/state/model.go
@@ -2,7 +2,7 @@ package state
 
 import "time"
 
-//go:generate go run ../../cmd/mkunion/main.go --type-registry
+//go:generate go run ../../cmd/mkunion/main.go
 
 //go:tag mkunion:"Command"
 type (
diff --git a/example/tree.go b/example/tree.go
index f487bf65..38fe2586 100644
--- a/example/tree.go
+++ b/example/tree.go
@@ -1,6 +1,6 @@
 package example
 
-//go:generate go run ../cmd/mkunion/main.go --type-registry
+//go:generate go run ../cmd/mkunion/main.go
 
 //go:tag mkunion:"Tree"
 type (
diff --git a/f/datas.go b/f/datas.go
index 509ca02f..e38d9f83 100644
--- a/f/datas.go
+++ b/f/datas.go
@@ -1,6 +1,6 @@
 package f
 
-//go:generate go run ../cmd/mkunion --type-registry
+//go:generate go run ../cmd/mkunion
 
 //go:tag mkunion:"Either,serde"
 type (
diff --git a/x/projection/projection.go b/x/projection/projection.go
index 8504ca8f..f9e84c28 100644
--- a/x/projection/projection.go
+++ b/x/projection/projection.go
@@ -10,7 +10,7 @@ import (
 	"time"
 )
 
-//go:generate go run ../../cmd/mkunion/main.go -v -type-registry
+//go:generate go run ../../cmd/mkunion/main.go -v
 
 var (
 	ErrStateAckNilOffset    = errors.New("cannot acknowledge nil offset")
@@ -21,7 +21,7 @@ const (
 	KeySystemWatermark = "watermark"
 )
 
-//go:tag mkunion:"Data,noserde"
+//go:tag mkunion:"Data"
 type (
 	Record[A any] struct {
 		Key       string
@@ -559,7 +559,7 @@ func DoSink[A any](ctx PushAndPull[A, any], f func(*Record[A]) error) error {
 	}
 }
 
-//go:tag mkunion:"Either,noserde"
+//go:tag mkunion:"Either"
 type (
 	Left[A, B any] struct {
 		Left A
diff --git a/x/projection/projection_union_gen.go b/x/projection/projection_union_gen.go
index 683f87a0..5392337e 100644
--- a/x/projection/projection_union_gen.go
+++ b/x/projection/projection_union_gen.go
@@ -1,6 +1,12 @@
 // Code generated by mkunion. DO NOT EDIT.
 package projection
 
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/widmogrod/mkunion/x/shared"
+)
+
 type DataVisitor[A any] interface {
 	VisitRecord(v *Record[A]) any
 	VisitWatermark(v *Watermark[A]) any
@@ -78,6 +84,281 @@ func MatchDataR0[A any](
 		f2(v)
 	}
 }
+func init() {
+	shared.JSONMarshallerRegister("github.com/widmogrod/mkunion/x/projection.Data[any]", DataFromJSON[any], DataToJSON[any])
+	shared.JSONMarshallerRegister("github.com/widmogrod/mkunion/x/projection.Record[any]", RecordFromJSON[any], RecordToJSON[any])
+	shared.JSONMarshallerRegister("github.com/widmogrod/mkunion/x/projection.Watermark[any]", WatermarkFromJSON[any], WatermarkToJSON[any])
+}
+
+type DataUnionJSON[A any] struct {
+	Type      string          `json:"$type,omitempty"`
+	Record    json.RawMessage `json:"projection.Record,omitempty"`
+	Watermark json.RawMessage `json:"projection.Watermark,omitempty"`
+}
+
+func DataFromJSON[A any](x []byte) (Data[A], error) {
+	if x == nil || len(x) == 0 {
+		return nil, nil
+	}
+	if string(x[:4]) == "null" {
+		return nil, nil
+	}
+	var data DataUnionJSON[A]
+	err := json.Unmarshal(x, &data)
+	if err != nil {
+		return nil, fmt.Errorf("projection.DataFromJSON[A]: %w", err)
+	}
+
+	switch data.Type {
+	case "projection.Record":
+		return RecordFromJSON[A](data.Record)
+	case "projection.Watermark":
+		return WatermarkFromJSON[A](data.Watermark)
+	}
+
+	if data.Record != nil {
+		return RecordFromJSON[A](data.Record)
+	} else if data.Watermark != nil {
+		return WatermarkFromJSON[A](data.Watermark)
+	}
+	return nil, fmt.Errorf("projection.DataFromJSON[A]: unknown type: %s", data.Type)
+}
+
+func DataToJSON[A any](x Data[A]) ([]byte, error) {
+	if x == nil {
+		return []byte(`null`), nil
+	}
+	return MatchDataR2(
+		x,
+		func(y *Record[A]) ([]byte, error) {
+			body, err := RecordToJSON[A](y)
+			if err != nil {
+				return nil, fmt.Errorf("projection.DataToJSON[A]: %w", err)
+			}
+			return json.Marshal(DataUnionJSON[A]{
+				Type:   "projection.Record",
+				Record: body,
+			})
+		},
+		func(y *Watermark[A]) ([]byte, error) {
+			body, err := WatermarkToJSON[A](y)
+			if err != nil {
+				return nil, fmt.Errorf("projection.DataToJSON[A]: %w", err)
+			}
+			return json.Marshal(DataUnionJSON[A]{
+				Type:      "projection.Watermark",
+				Watermark: body,
+			})
+		},
+	)
+}
+
+func RecordFromJSON[A any](x []byte) (*Record[A], error) {
+	result := new(Record[A])
+	err := result.UnmarshalJSON(x)
+	if err != nil {
+		return nil, fmt.Errorf("projection.RecordFromJSON[A]: %w", err)
+	}
+	return result, nil
+}
+
+func RecordToJSON[A any](x *Record[A]) ([]byte, error) {
+	return x.MarshalJSON()
+}
+
+var (
+	_ json.Unmarshaler = (*Record[any])(nil)
+	_ json.Marshaler   = (*Record[any])(nil)
+)
+
+func (r *Record[A]) MarshalJSON() ([]byte, error) {
+	if r == nil {
+		return nil, nil
+	}
+	return r._marshalJSONRecordLb_A_bL(*r)
+}
+func (r *Record[A]) _marshalJSONRecordLb_A_bL(x Record[A]) ([]byte, error) {
+	partial := make(map[string]json.RawMessage)
+	var err error
+	var fieldKey []byte
+	fieldKey, err = r._marshalJSONstring(x.Key)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Record[A]._marshalJSONRecordLb_A_bL: field name Key; %w", err)
+	}
+	partial["Key"] = fieldKey
+	var fieldData []byte
+	fieldData, err = r._marshalJSONA(x.Data)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Record[A]._marshalJSONRecordLb_A_bL: field name Data; %w", err)
+	}
+	partial["Data"] = fieldData
+	var fieldEventTime []byte
+	fieldEventTime, err = r._marshalJSONEventTime(x.EventTime)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Record[A]._marshalJSONRecordLb_A_bL: field name EventTime; %w", err)
+	}
+	partial["EventTime"] = fieldEventTime
+	result, err := json.Marshal(partial)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Record[A]._marshalJSONRecordLb_A_bL: struct; %w", err)
+	}
+	return result, nil
+}
+func (r *Record[A]) _marshalJSONstring(x string) ([]byte, error) {
+	result, err := json.Marshal(x)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Record[A]._marshalJSONstring:; %w", err)
+	}
+	return result, nil
+}
+func (r *Record[A]) _marshalJSONA(x A) ([]byte, error) {
+	result, err := shared.JSONMarshal[A](x)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Record[A]._marshalJSONA:; %w", err)
+	}
+	return result, nil
+}
+func (r *Record[A]) _marshalJSONEventTime(x EventTime) ([]byte, error) {
+	result, err := shared.JSONMarshal[EventTime](x)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Record[A]._marshalJSONEventTime:; %w", err)
+	}
+	return result, nil
+}
+func (r *Record[A]) UnmarshalJSON(data []byte) error {
+	result, err := r._unmarshalJSONRecordLb_A_bL(data)
+	if err != nil {
+		return fmt.Errorf("projection: Record[A].UnmarshalJSON: %w", err)
+	}
+	*r = result
+	return nil
+}
+func (r *Record[A]) _unmarshalJSONRecordLb_A_bL(data []byte) (Record[A], error) {
+	result := Record[A]{}
+	var partial map[string]json.RawMessage
+	err := json.Unmarshal(data, &partial)
+	if err != nil {
+		return result, fmt.Errorf("projection: Record[A]._unmarshalJSONRecordLb_A_bL: native struct unwrap; %w", err)
+	}
+	if fieldKey, ok := partial["Key"]; ok {
+		result.Key, err = r._unmarshalJSONstring(fieldKey)
+		if err != nil {
+			return result, fmt.Errorf("projection: Record[A]._unmarshalJSONRecordLb_A_bL: field Key; %w", err)
+		}
+	}
+	if fieldData, ok := partial["Data"]; ok {
+		result.Data, err = r._unmarshalJSONA(fieldData)
+		if err != nil {
+			return result, fmt.Errorf("projection: Record[A]._unmarshalJSONRecordLb_A_bL: field Data; %w", err)
+		}
+	}
+	if fieldEventTime, ok := partial["EventTime"]; ok {
+		result.EventTime, err = r._unmarshalJSONEventTime(fieldEventTime)
+		if err != nil {
+			return result, fmt.Errorf("projection: Record[A]._unmarshalJSONRecordLb_A_bL: field EventTime; %w", err)
+		}
+	}
+	return result, nil
+}
+func (r *Record[A]) _unmarshalJSONstring(data []byte) (string, error) {
+	var result string
+	err := json.Unmarshal(data, &result)
+	if err != nil {
+		return result, fmt.Errorf("projection: Record[A]._unmarshalJSONstring: native primitive unwrap; %w", err)
+	}
+	return result, nil
+}
+func (r *Record[A]) _unmarshalJSONA(data []byte) (A, error) {
+	result, err := shared.JSONUnmarshal[A](data)
+	if err != nil {
+		return result, fmt.Errorf("projection: Record[A]._unmarshalJSONA: native ref unwrap; %w", err)
+	}
+	return result, nil
+}
+func (r *Record[A]) _unmarshalJSONEventTime(data []byte) (EventTime, error) {
+	result, err := shared.JSONUnmarshal[EventTime](data)
+	if err != nil {
+		return result, fmt.Errorf("projection: Record[A]._unmarshalJSONEventTime: native ref unwrap; %w", err)
+	}
+	return result, nil
+}
+
+func WatermarkFromJSON[A any](x []byte) (*Watermark[A], error) {
+	result := new(Watermark[A])
+	err := result.UnmarshalJSON(x)
+	if err != nil {
+		return nil, fmt.Errorf("projection.WatermarkFromJSON[A]: %w", err)
+	}
+	return result, nil
+}
+
+func WatermarkToJSON[A any](x *Watermark[A]) ([]byte, error) {
+	return x.MarshalJSON()
+}
+
+var (
+	_ json.Unmarshaler = (*Watermark[any])(nil)
+	_ json.Marshaler   = (*Watermark[any])(nil)
+)
+
+func (r *Watermark[A]) MarshalJSON() ([]byte, error) {
+	if r == nil {
+		return nil, nil
+	}
+	return r._marshalJSONWatermarkLb_A_bL(*r)
+}
+func (r *Watermark[A]) _marshalJSONWatermarkLb_A_bL(x Watermark[A]) ([]byte, error) {
+	partial := make(map[string]json.RawMessage)
+	var err error
+	var fieldEventTime []byte
+	fieldEventTime, err = r._marshalJSONEventTime(x.EventTime)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Watermark[A]._marshalJSONWatermarkLb_A_bL: field name EventTime; %w", err)
+	}
+	partial["EventTime"] = fieldEventTime
+	result, err := json.Marshal(partial)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Watermark[A]._marshalJSONWatermarkLb_A_bL: struct; %w", err)
+	}
+	return result, nil
+}
+func (r *Watermark[A]) _marshalJSONEventTime(x EventTime) ([]byte, error) {
+	result, err := shared.JSONMarshal[EventTime](x)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Watermark[A]._marshalJSONEventTime:; %w", err)
+	}
+	return result, nil
+}
+func (r *Watermark[A]) UnmarshalJSON(data []byte) error {
+	result, err := r._unmarshalJSONWatermarkLb_A_bL(data)
+	if err != nil {
+		return fmt.Errorf("projection: Watermark[A].UnmarshalJSON: %w", err)
+	}
+	*r = result
+	return nil
+}
+func (r *Watermark[A]) _unmarshalJSONWatermarkLb_A_bL(data []byte) (Watermark[A], error) {
+	result := Watermark[A]{}
+	var partial map[string]json.RawMessage
+	err := json.Unmarshal(data, &partial)
+	if err != nil {
+		return result, fmt.Errorf("projection: Watermark[A]._unmarshalJSONWatermarkLb_A_bL: native struct unwrap; %w", err)
+	}
+	if fieldEventTime, ok := partial["EventTime"]; ok {
+		result.EventTime, err = r._unmarshalJSONEventTime(fieldEventTime)
+		if err != nil {
+			return result, fmt.Errorf("projection: Watermark[A]._unmarshalJSONWatermarkLb_A_bL: field EventTime; %w", err)
+		}
+	}
+	return result, nil
+}
+func (r *Watermark[A]) _unmarshalJSONEventTime(data []byte) (EventTime, error) {
+	result, err := shared.JSONUnmarshal[EventTime](data)
+	if err != nil {
+		return result, fmt.Errorf("projection: Watermark[A]._unmarshalJSONEventTime: native ref unwrap; %w", err)
+	}
+	return result, nil
+}
 
 type EitherVisitor[A any, B any] interface {
 	VisitLeft(v *Left[A, B]) any
@@ -156,3 +437,225 @@ func MatchEitherR0[A any, B any](
 		f2(v)
 	}
 }
+func init() {
+	shared.JSONMarshallerRegister("github.com/widmogrod/mkunion/x/projection.Either[any,any]", EitherFromJSON[any, any], EitherToJSON[any, any])
+	shared.JSONMarshallerRegister("github.com/widmogrod/mkunion/x/projection.Left[any,any]", LeftFromJSON[any, any], LeftToJSON[any, any])
+	shared.JSONMarshallerRegister("github.com/widmogrod/mkunion/x/projection.Right[any,any]", RightFromJSON[any, any], RightToJSON[any, any])
+}
+
+type EitherUnionJSON[A any, B any] struct {
+	Type  string          `json:"$type,omitempty"`
+	Left  json.RawMessage `json:"projection.Left,omitempty"`
+	Right json.RawMessage `json:"projection.Right,omitempty"`
+}
+
+func EitherFromJSON[A any, B any](x []byte) (Either[A, B], error) {
+	if x == nil || len(x) == 0 {
+		return nil, nil
+	}
+	if string(x[:4]) == "null" {
+		return nil, nil
+	}
+	var data EitherUnionJSON[A, B]
+	err := json.Unmarshal(x, &data)
+	if err != nil {
+		return nil, fmt.Errorf("projection.EitherFromJSON[A,B]: %w", err)
+	}
+
+	switch data.Type {
+	case "projection.Left":
+		return LeftFromJSON[A, B](data.Left)
+	case "projection.Right":
+		return RightFromJSON[A, B](data.Right)
+	}
+
+	if data.Left != nil {
+		return LeftFromJSON[A, B](data.Left)
+	} else if data.Right != nil {
+		return RightFromJSON[A, B](data.Right)
+	}
+	return nil, fmt.Errorf("projection.EitherFromJSON[A,B]: unknown type: %s", data.Type)
+}
+
+func EitherToJSON[A any, B any](x Either[A, B]) ([]byte, error) {
+	if x == nil {
+		return []byte(`null`), nil
+	}
+	return MatchEitherR2(
+		x,
+		func(y *Left[A, B]) ([]byte, error) {
+			body, err := LeftToJSON[A, B](y)
+			if err != nil {
+				return nil, fmt.Errorf("projection.EitherToJSON[A,B]: %w", err)
+			}
+			return json.Marshal(EitherUnionJSON[A, B]{
+				Type: "projection.Left",
+				Left: body,
+			})
+		},
+		func(y *Right[A, B]) ([]byte, error) {
+			body, err := RightToJSON[A, B](y)
+			if err != nil {
+				return nil, fmt.Errorf("projection.EitherToJSON[A,B]: %w", err)
+			}
+			return json.Marshal(EitherUnionJSON[A, B]{
+				Type:  "projection.Right",
+				Right: body,
+			})
+		},
+	)
+}
+
+func LeftFromJSON[A any, B any](x []byte) (*Left[A, B], error) {
+	result := new(Left[A, B])
+	err := result.UnmarshalJSON(x)
+	if err != nil {
+		return nil, fmt.Errorf("projection.LeftFromJSON[A,B]: %w", err)
+	}
+	return result, nil
+}
+
+func LeftToJSON[A any, B any](x *Left[A, B]) ([]byte, error) {
+	return x.MarshalJSON()
+}
+
+var (
+	_ json.Unmarshaler = (*Left[any, any])(nil)
+	_ json.Marshaler   = (*Left[any, any])(nil)
+)
+
+func (r *Left[A, B]) MarshalJSON() ([]byte, error) {
+	if r == nil {
+		return nil, nil
+	}
+	return r._marshalJSONLeftLb_ACommaB_bL(*r)
+}
+func (r *Left[A, B]) _marshalJSONLeftLb_ACommaB_bL(x Left[A, B]) ([]byte, error) {
+	partial := make(map[string]json.RawMessage)
+	var err error
+	var fieldLeft []byte
+	fieldLeft, err = r._marshalJSONA(x.Left)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Left[A,B]._marshalJSONLeftLb_ACommaB_bL: field name Left; %w", err)
+	}
+	partial["Left"] = fieldLeft
+	result, err := json.Marshal(partial)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Left[A,B]._marshalJSONLeftLb_ACommaB_bL: struct; %w", err)
+	}
+	return result, nil
+}
+func (r *Left[A, B]) _marshalJSONA(x A) ([]byte, error) {
+	result, err := shared.JSONMarshal[A](x)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Left[A,B]._marshalJSONA:; %w", err)
+	}
+	return result, nil
+}
+func (r *Left[A, B]) UnmarshalJSON(data []byte) error {
+	result, err := r._unmarshalJSONLeftLb_ACommaB_bL(data)
+	if err != nil {
+		return fmt.Errorf("projection: Left[A,B].UnmarshalJSON: %w", err)
+	}
+	*r = result
+	return nil
+}
+func (r *Left[A, B]) _unmarshalJSONLeftLb_ACommaB_bL(data []byte) (Left[A, B], error) {
+	result := Left[A, B]{}
+	var partial map[string]json.RawMessage
+	err := json.Unmarshal(data, &partial)
+	if err != nil {
+		return result, fmt.Errorf("projection: Left[A,B]._unmarshalJSONLeftLb_ACommaB_bL: native struct unwrap; %w", err)
+	}
+	if fieldLeft, ok := partial["Left"]; ok {
+		result.Left, err = r._unmarshalJSONA(fieldLeft)
+		if err != nil {
+			return result, fmt.Errorf("projection: Left[A,B]._unmarshalJSONLeftLb_ACommaB_bL: field Left; %w", err)
+		}
+	}
+	return result, nil
+}
+func (r *Left[A, B]) _unmarshalJSONA(data []byte) (A, error) {
+	result, err := shared.JSONUnmarshal[A](data)
+	if err != nil {
+		return result, fmt.Errorf("projection: Left[A,B]._unmarshalJSONA: native ref unwrap; %w", err)
+	}
+	return result, nil
+}
+
+func RightFromJSON[A any, B any](x []byte) (*Right[A, B], error) {
+	result := new(Right[A, B])
+	err := result.UnmarshalJSON(x)
+	if err != nil {
+		return nil, fmt.Errorf("projection.RightFromJSON[A,B]: %w", err)
+	}
+	return result, nil
+}
+
+func RightToJSON[A any, B any](x *Right[A, B]) ([]byte, error) {
+	return x.MarshalJSON()
+}
+
+var (
+	_ json.Unmarshaler = (*Right[any, any])(nil)
+	_ json.Marshaler   = (*Right[any, any])(nil)
+)
+
+func (r *Right[A, B]) MarshalJSON() ([]byte, error) {
+	if r == nil {
+		return nil, nil
+	}
+	return r._marshalJSONRightLb_ACommaB_bL(*r)
+}
+func (r *Right[A, B]) _marshalJSONRightLb_ACommaB_bL(x Right[A, B]) ([]byte, error) {
+	partial := make(map[string]json.RawMessage)
+	var err error
+	var fieldRight []byte
+	fieldRight, err = r._marshalJSONB(x.Right)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Right[A,B]._marshalJSONRightLb_ACommaB_bL: field name Right; %w", err)
+	}
+	partial["Right"] = fieldRight
+	result, err := json.Marshal(partial)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Right[A,B]._marshalJSONRightLb_ACommaB_bL: struct; %w", err)
+	}
+	return result, nil
+}
+func (r *Right[A, B]) _marshalJSONB(x B) ([]byte, error) {
+	result, err := shared.JSONMarshal[B](x)
+	if err != nil {
+		return nil, fmt.Errorf("projection: Right[A,B]._marshalJSONB:; %w", err)
+	}
+	return result, nil
+}
+func (r *Right[A, B]) UnmarshalJSON(data []byte) error {
+	result, err := r._unmarshalJSONRightLb_ACommaB_bL(data)
+	if err != nil {
+		return fmt.Errorf("projection: Right[A,B].UnmarshalJSON: %w", err)
+	}
+	*r = result
+	return nil
+}
+func (r *Right[A, B]) _unmarshalJSONRightLb_ACommaB_bL(data []byte) (Right[A, B], error) {
+	result := Right[A, B]{}
+	var partial map[string]json.RawMessage
+	err := json.Unmarshal(data, &partial)
+	if err != nil {
+		return result, fmt.Errorf("projection: Right[A,B]._unmarshalJSONRightLb_ACommaB_bL: native struct unwrap; %w", err)
+	}
+	if fieldRight, ok := partial["Right"]; ok {
+		result.Right, err = r._unmarshalJSONB(fieldRight)
+		if err != nil {
+			return result, fmt.Errorf("projection: Right[A,B]._unmarshalJSONRightLb_ACommaB_bL: field Right; %w", err)
+		}
+	}
+	return result, nil
+}
+func (r *Right[A, B]) _unmarshalJSONB(data []byte) (B, error) {
+	result, err := shared.JSONUnmarshal[B](data)
+	if err != nil {
+		return result, fmt.Errorf("projection: Right[A,B]._unmarshalJSONB: native ref unwrap; %w", err)
+	}
+	return result, nil
+}
diff --git a/x/projection/types_reg_gen.go b/x/projection/types_reg_gen.go
index 16a9cb01..d7c36de0 100644
--- a/x/projection/types_reg_gen.go
+++ b/x/projection/types_reg_gen.go
@@ -14,14 +14,22 @@ func init() {
 	shared.TypeRegistryStore[predicate.WherePredicates]("github.com/widmogrod/mkunion/x/storage/predicate.WherePredicates")
 	shared.TypeRegistryStore[AtWatermark]("github.com/widmogrod/mkunion/x/projection.AtWatermark")
 	shared.TypeRegistryStore[Data[Either[int, float64]]]("github.com/widmogrod/mkunion/x/projection.Data[Either[int,float64]]")
+	shared.JSONMarshallerRegister("github.com/widmogrod/mkunion/x/projection.Data[Either[int,float64]]", DataFromJSON[Either[int, float64]], DataToJSON[Either[int, float64]])
 	shared.TypeRegistryStore[Data[any]]("github.com/widmogrod/mkunion/x/projection.Data[any]")
+	shared.JSONMarshallerRegister("github.com/widmogrod/mkunion/x/projection.Data[any]", DataFromJSON[any], DataToJSON[any])
 	shared.TypeRegistryStore[Data[float64]]("github.com/widmogrod/mkunion/x/projection.Data[float64]")
+	shared.JSONMarshallerRegister("github.com/widmogrod/mkunion/x/projection.Data[float64]", DataFromJSON[float64], DataToJSON[float64])
 	shared.TypeRegistryStore[Data[int]]("github.com/widmogrod/mkunion/x/projection.Data[int]")
+	shared.JSONMarshallerRegister("github.com/widmogrod/mkunion/x/projection.Data[int]", DataFromJSON[int], DataToJSON[int])
 	shared.TypeRegistryStore[Data[string]]("github.com/widmogrod/mkunion/x/projection.Data[string]")
+	shared.JSONMarshallerRegister("github.com/widmogrod/mkunion/x/projection.Data[string]", DataFromJSON[string], DataToJSON[string])
 	shared.TypeRegistryStore[Discard]("github.com/widmogrod/mkunion/x/projection.Discard")
 	shared.TypeRegistryStore[Either[*Record[int], *Record[float64]]]("github.com/widmogrod/mkunion/x/projection.Either[*Record[int],*Record[float64]]")
+	shared.JSONMarshallerRegister("github.com/widmogrod/mkunion/x/projection.Either[*Record[int],*Record[float64]]", EitherFromJSON[*Record[int], *Record[float64]], EitherToJSON[*Record[int], *Record[float64]])
 	shared.TypeRegistryStore[Either[any, any]]("github.com/widmogrod/mkunion/x/projection.Either[any,any]")
+	shared.JSONMarshallerRegister("github.com/widmogrod/mkunion/x/projection.Either[any,any]", EitherFromJSON[any, any], EitherToJSON[any, any])
 	shared.TypeRegistryStore[Either[int, float64]]("github.com/widmogrod/mkunion/x/projection.Either[int,float64]")
+	shared.JSONMarshallerRegister("github.com/widmogrod/mkunion/x/projection.Either[int,float64]", EitherFromJSON[int, float64], EitherToJSON[int, float64])
 	shared.TypeRegistryStore[FixedWindow]("github.com/widmogrod/mkunion/x/projection.FixedWindow")
 	shared.TypeRegistryStore[JoinContextState]("github.com/widmogrod/mkunion/x/projection.JoinContextState")
 	shared.TypeRegistryStore[Left[*Record[int], *Record[float64]]]("github.com/widmogrod/mkunion/x/projection.Left[*Record[int],*Record[float64]]")
diff --git a/x/shape/testasset/type_example.go b/x/shape/testasset/type_example.go
index 5dcb5a73..37ba0dd7 100644
--- a/x/shape/testasset/type_example.go
+++ b/x/shape/testasset/type_example.go
@@ -4,7 +4,7 @@ import (
 	"time"
 )
 
-//go:generate go run ../../../cmd/mkunion/main.go
+//go:generate go run ../../../cmd/mkunion/main.go --type-registry=false
 
 //go:tag mkunion:"Example"
 type (
diff --git a/x/shape/testasset/type_other.go b/x/shape/testasset/type_other.go
index a9552b11..979fce08 100644
--- a/x/shape/testasset/type_other.go
+++ b/x/shape/testasset/type_other.go
@@ -1,6 +1,6 @@
 package testasset
 
-//go:generate go run ../../../cmd/mkunion/main.go
+//go:generate go run ../../../cmd/mkunion/main.go --type-registry=false
 
 //go:tag mkunion:"SomeDSL"
 type (

From 32b5f10d1596da76cb1ea4438fd3592624bca426 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 00:29:40 +0200
Subject: [PATCH 46/72] fix: fix moq problem with order of running of
 go:generate

---
 example/state/machine.go | 2 +-
 example/state/model.go   | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/example/state/machine.go b/example/state/machine.go
index 5de2f381..5e3e01a2 100644
--- a/example/state/machine.go
+++ b/example/state/machine.go
@@ -28,7 +28,7 @@ var (
 	ErrWorkerIDRequired = fmt.Errorf("worker ID required; %w", ErrValidationFailed)
 )
 
-//go:generate moq -with-resets -stub -out machine_mock_test.go . Dependency
+// go:generate moq -with-resets -stub -out machine_mock_test.go . Dependency
 type Dependency interface {
 	TimeNow() *time.Time
 	WarehouseRemoveStock(quantity Quantity) error
diff --git a/example/state/model.go b/example/state/model.go
index 0655d1d5..06e232b1 100644
--- a/example/state/model.go
+++ b/example/state/model.go
@@ -95,3 +95,6 @@ const (
 	ProblemWarehouseAPIUnreachable ProblemCode = iota
 	ProblemPaymentAPIUnreachable
 )
+
+// moq must be run after union type is generated
+//go:generate moq -with-resets -stub -out machine_mock_test.go . Dependency

From a58c46017d4491aa6245a637c9b6057d260515a7 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 21:14:33 +0200
Subject: [PATCH 47/72] maintenance: go mod tidy

---
 example/my-app/go.mod |  8 ++++----
 go.mod                |  5 ++---
 go.sum                | 14 ++++----------
 3 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/example/my-app/go.mod b/example/my-app/go.mod
index a6c79405..d0d2ea81 100644
--- a/example/my-app/go.mod
+++ b/example/my-app/go.mod
@@ -38,12 +38,12 @@ require (
 	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/crypto v0.18.0 // indirect
 	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
diff --git a/go.mod b/go.mod
index 7b8a1eeb..077106e7 100644
--- a/go.mod
+++ b/go.mod
@@ -42,13 +42,12 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/matryer/moq v0.3.4 // indirect
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
-	golang.org/x/sys v0.11.0 // indirect
-	golang.org/x/tools v0.17.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
diff --git a/go.sum b/go.sum
index a601a2dc..a96ccaa6 100644
--- a/go.sum
+++ b/go.sum
@@ -106,8 +106,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
 github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
-github.com/matryer/moq v0.3.4 h1:czCFIos9rI2tyOehN9ktc/6bQ76N9J4xQ2n3dk063ac=
-github.com/matryer/moq v0.3.4/go.mod h1:wqm9QObyoMuUtH81zFfs3EK6mXEcByy+TjvSROOXJ2U=
 github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs=
 github.com/moby/sys/mount v0.3.3/go.mod h1:PBaEorSNTLG5t/+4EgukEQVlAvVEc6ZjTySwKdqp5K0=
 github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
@@ -161,8 +159,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ=
 golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
 golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -170,8 +166,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -182,8 +178,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
-golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -196,8 +192,6 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
-golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 h1:0BOZf6qNozI3pkN3fJLwNubheHJYHhMh91GRFOWWK08=
 google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=

From cf0ecb4091b6ff3332d3b9f3d105157dbfe28f35 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 21:14:50 +0200
Subject: [PATCH 48/72] debug: ci.yaml find empty imports

---
 .github/workflows/ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 47d61744..f02daea6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -23,6 +23,8 @@ jobs:
       - run: pip install awscli-local
       - run: dev/bootstrap.sh -nologs
 
+      - run: |
+          find / -type f -name '*.go' -exec grep -l 'import ""' {} +
       # run tests
       - run: |
           export RUN_EXPERIMENTAL_TEST="false"

From 00150c0466a1ae2ccf5591432463517bc026db48 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 21:20:27 +0200
Subject: [PATCH 49/72] debug: supress find errors

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f02daea6..bd24ee23 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,7 +24,7 @@ jobs:
       - run: dev/bootstrap.sh -nologs
 
       - run: |
-          find / -type f -name '*.go' -exec grep -l 'import ""' {} +
+          find / -type f -name '*.go' -exec grep -l 'import ""' {} + 2>/dev/null
       # run tests
       - run: |
           export RUN_EXPERIMENTAL_TEST="false"

From 8c77002de38888c506ca4912fb0b99ff23f9c3a1 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 21:36:14 +0200
Subject: [PATCH 50/72] debug: supress find errors

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bd24ee23..44e32af8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,7 +24,7 @@ jobs:
       - run: dev/bootstrap.sh -nologs
 
       - run: |
-          find / -type f -name '*.go' -exec grep -l 'import ""' {} + 2>/dev/null
+          find $GOROOT -type f -name '*.go' -exec grep -l 'import ""' {} + 2>/dev/null
       # run tests
       - run: |
           export RUN_EXPERIMENTAL_TEST="false"

From 5d5752d074fa752e70c81982cb17ae2eee49aa4c Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 21:40:03 +0200
Subject: [PATCH 51/72] debug: make find return 0

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 44e32af8..5729cca1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,7 +24,7 @@ jobs:
       - run: dev/bootstrap.sh -nologs
 
       - run: |
-          find $GOROOT -type f -name '*.go' -exec grep -l 'import ""' {} + 2>/dev/null
+          find $GOROOT -type f -name '*.go' -exec grep -l 'import ""' {} + 2>/dev/null || true
       # run tests
       - run: |
           export RUN_EXPERIMENTAL_TEST="false"

From c02fa010433e538517fc24e6fa3bc69318d16790 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 21:46:17 +0200
Subject: [PATCH 52/72] debug: make find return 0

---
 .github/workflows/ci.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5729cca1..2879818d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,7 +24,10 @@ jobs:
       - run: dev/bootstrap.sh -nologs
 
       - run: |
-          find $GOROOT -type f -name '*.go' -exec grep -l 'import ""' {} + 2>/dev/null || true
+          find / -type f -name '*.go' -exec grep -l 'import ""' {} + &2>/dev/null || true
+
+      - run: |
+          go fmt ./... || true
       # run tests
       - run: |
           export RUN_EXPERIMENTAL_TEST="false"

From 9f0f49f91d8c495314c4cc9c4e5cfc7277913c90 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 21:50:12 +0200
Subject: [PATCH 53/72] debug: where is empty import

---
 .github/workflows/ci.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2879818d..53affd44 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,10 +24,13 @@ jobs:
       - run: dev/bootstrap.sh -nologs
 
       - run: |
-          find / -type f -name '*.go' -exec grep -l 'import ""' {} + &2>/dev/null || true
+          find . -type f -name '*.go' -exec grep -l 'import ()' {} + &2>/dev/null || true
+          find . -type f -name '*.go' -exec grep -l 'import ("")' {} + &2>/dev/null || true
+          find . -type f -name '*.go' -exec grep -l 'import ""' {} + &2>/dev/null || true
+          find . -type f -name '*.go' -exec grep -l 'import (' {} + &2>/dev/null || true
 
       - run: |
-          go fmt ./... || true
+          go vet ./... || true
       # run tests
       - run: |
           export RUN_EXPERIMENTAL_TEST="false"

From d13b7b9238618fa41b4489b936be3d27e6862267 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 21:57:18 +0200
Subject: [PATCH 54/72] fix: attempt to filter out empty imports

---
 x/shape/togo.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/x/shape/togo.go b/x/shape/togo.go
index 68e0c842..f2d7a827 100644
--- a/x/shape/togo.go
+++ b/x/shape/togo.go
@@ -467,7 +467,7 @@ func ExtractPkgImportNames(x Shape) map[string]string {
 		},
 		func(y *RefName) map[string]string {
 			result := make(map[string]string)
-			if y.PkgName != "" {
+			if y.PkgName != "" && y.PkgImportName != "" {
 				result[y.PkgName] = y.PkgImportName
 			}
 
@@ -484,7 +484,7 @@ func ExtractPkgImportNames(x Shape) map[string]string {
 		},
 		func(x *AliasLike) map[string]string {
 			result := make(map[string]string)
-			if x.PkgName != "" {
+			if x.PkgName != "" && x.PkgImportName != "" {
 				result[x.PkgName] = x.PkgImportName
 			}
 
@@ -511,7 +511,7 @@ func ExtractPkgImportNames(x Shape) map[string]string {
 		},
 		func(x *StructLike) map[string]string {
 			result := make(map[string]string)
-			if x.PkgName != "" {
+			if x.PkgName != "" && x.PkgImportName != "" {
 				result[x.PkgName] = x.PkgImportName
 			}
 
@@ -528,7 +528,7 @@ func ExtractPkgImportNames(x Shape) map[string]string {
 		},
 		func(x *UnionLike) map[string]string {
 			result := make(map[string]string)
-			if x.PkgName != "" {
+			if x.PkgName != "" && x.PkgImportName != "" {
 				result[x.PkgName] = x.PkgImportName
 			}
 

From 8aa7ec2971873087bd356053850922679d02e788 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 21:58:24 +0200
Subject: [PATCH 55/72] debug: print lines

---
 .github/workflows/ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 53affd44..3d91d9e4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,10 +24,10 @@ jobs:
       - run: dev/bootstrap.sh -nologs
 
       - run: |
-          find . -type f -name '*.go' -exec grep -l 'import ()' {} + &2>/dev/null || true
-          find . -type f -name '*.go' -exec grep -l 'import ("")' {} + &2>/dev/null || true
-          find . -type f -name '*.go' -exec grep -l 'import ""' {} + &2>/dev/null || true
-          find . -type f -name '*.go' -exec grep -l 'import (' {} + &2>/dev/null || true
+          find . -type f -name '*.go' -exec grep -H 'import ()' {} + &2>/dev/null || true
+          find . -type f -name '*.go' -exec grep -H 'import ("")' {} + &2>/dev/null || true
+          find . -type f -name '*.go' -exec grep -H 'import ""' {} + &2>/dev/null || true
+          find . -type f -name '*.go' -exec grep -H 'import (' {} + &2>/dev/null || true
 
       - run: |
           go vet ./... || true

From 05c9b9e10bdd0449ac6a18cf47da7bb6fc4d9164 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 22:01:54 +0200
Subject: [PATCH 56/72] debug: print lines

---
 .github/workflows/ci.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3d91d9e4..def4f2cd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,10 +24,7 @@ jobs:
       - run: dev/bootstrap.sh -nologs
 
       - run: |
-          find . -type f -name '*.go' -exec grep -H 'import ()' {} + &2>/dev/null || true
-          find . -type f -name '*.go' -exec grep -H 'import ("")' {} + &2>/dev/null || true
-          find . -type f -name '*.go' -exec grep -H 'import ""' {} + &2>/dev/null || true
-          find . -type f -name '*.go' -exec grep -H 'import (' {} + &2>/dev/null || true
+          find . -type f -name '*.go' -exec grep -H '""' {} + &2>/dev/null || true
 
       - run: |
           go vet ./... || true

From e85c80be31182373cf3615c29006afaa1816ffa0 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 22:06:55 +0200
Subject: [PATCH 57/72] debug: more debugging

---
 .github/workflows/ci.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index def4f2cd..7ad0a82e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,10 +24,12 @@ jobs:
       - run: dev/bootstrap.sh -nologs
 
       - run: |
-          find . -type f -name '*.go' -exec grep -H '""' {} + &2>/dev/null || true
+          find . -type f -name '*_gen.go' -exec grep -C 2 -H '""' {} + &2>/dev/null || true
 
       - run: |
-          go vet ./... || true
+          cat ./x/storage/schemaless/types_reg_gen.go
+          
+          go vet ./x/storage/schemaless/types_reg_gen.go || true
       # run tests
       - run: |
           export RUN_EXPERIMENTAL_TEST="false"

From 4aff6f2c658ceae6fc24a6bdaf6af2bba5989b4b Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 22:13:42 +0200
Subject: [PATCH 58/72] fix: attempt to filter out empty imports

---
 x/shape/togo.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/x/shape/togo.go b/x/shape/togo.go
index f2d7a827..0d1553a6 100644
--- a/x/shape/togo.go
+++ b/x/shape/togo.go
@@ -552,7 +552,7 @@ func ExtractPkgImportNamesForTypeInitialisation(x Shape) map[string]string {
 		},
 		func(y *RefName) map[string]string {
 			result := make(map[string]string)
-			if y.PkgName != "" {
+			if y.PkgName != "" && y.PkgImportName != "" {
 				result[y.PkgName] = y.PkgImportName
 			}
 
@@ -569,7 +569,7 @@ func ExtractPkgImportNamesForTypeInitialisation(x Shape) map[string]string {
 		},
 		func(x *AliasLike) map[string]string {
 			result := make(map[string]string)
-			if x.PkgName != "" {
+			if x.PkgName != "" && x.PkgImportName != "" {
 				result[x.PkgName] = x.PkgImportName
 			}
 
@@ -596,7 +596,7 @@ func ExtractPkgImportNamesForTypeInitialisation(x Shape) map[string]string {
 		},
 		func(x *StructLike) map[string]string {
 			result := make(map[string]string)
-			if x.PkgName != "" {
+			if x.PkgName != "" && x.PkgImportName != "" {
 				result[x.PkgName] = x.PkgImportName
 			}
 
@@ -604,7 +604,7 @@ func ExtractPkgImportNamesForTypeInitialisation(x Shape) map[string]string {
 		},
 		func(x *UnionLike) map[string]string {
 			result := make(map[string]string)
-			if x.PkgName != "" {
+			if x.PkgName != "" && x.PkgImportName != "" {
 				result[x.PkgName] = x.PkgImportName
 			}
 

From 55fa8f73f6615cd5dc7f99cc812cb5160b7dc69b Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 22:28:49 +0200
Subject: [PATCH 59/72] debug: more debugging

---
 .github/workflows/ci.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7ad0a82e..d382dfcf 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,12 +24,11 @@ jobs:
       - run: dev/bootstrap.sh -nologs
 
       - run: |
-          find . -type f -name '*_gen.go' -exec grep -C 2 -H '""' {} + &2>/dev/null || true
+          find . -type f -name '*.go' -exec grep -C 2 -H 'github.com/opensearch-project/opensearch-go/v2' {} + &2>/dev/null || true
 
       - run: |
-          cat ./x/storage/schemaless/types_reg_gen.go
+          cat x/storage/schemaless/types_reg_gen.go
           
-          go vet ./x/storage/schemaless/types_reg_gen.go || true
       # run tests
       - run: |
           export RUN_EXPERIMENTAL_TEST="false"

From 979c3e8b50fc4a296667556c132296048d1cc85c Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 22:29:35 +0200
Subject: [PATCH 60/72] ci.yml: go mod download

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d382dfcf..106cd806 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -17,6 +17,7 @@ jobs:
 
       - run: go install github.com/matryer/moq@latest
 
+      - run: go mod download
       - run: go get -v -t -d ./...
       - run: go generate ./...
 

From 2176e1148218f14ccc943af0079ae02f755fff96 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 22:44:17 +0200
Subject: [PATCH 61/72] debug: more debugging

---
 .github/workflows/ci.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 106cd806..714a2b94 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -17,7 +17,6 @@ jobs:
 
       - run: go install github.com/matryer/moq@latest
 
-      - run: go mod download
       - run: go get -v -t -d ./...
       - run: go generate ./...
 
@@ -27,6 +26,9 @@ jobs:
       - run: |
           find . -type f -name '*.go' -exec grep -C 2 -H 'github.com/opensearch-project/opensearch-go/v2' {} + &2>/dev/null || true
 
+      - run: |
+          find / -type d -name 'opensearch-go' &2>/dev/null || true
+
       - run: |
           cat x/storage/schemaless/types_reg_gen.go
           

From 3395c6db9366866489607d6649a995ae4a4c53da Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 22:51:47 +0200
Subject: [PATCH 62/72] debug: more debugging

---
 .github/workflows/ci.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 714a2b94..c2233a0e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -27,7 +27,11 @@ jobs:
           find . -type f -name '*.go' -exec grep -C 2 -H 'github.com/opensearch-project/opensearch-go/v2' {} + &2>/dev/null || true
 
       - run: |
-          find / -type d -name 'opensearch-go' &2>/dev/null || true
+          la -la /home/runner/go/pkg/mod/github.com/opensearch-project/opensearch-go
+          la -la /home/runner/go/pkg/mod/cache/download/github.com/opensearch-project/opensearch-go
+          
+          tree /home/runner/go/pkg/mod/github.com/opensearch-project/opensearch-go
+          tree /home/runner/go/pkg/mod/cache/download/github.com/opensearch-project/opensearch-go
 
       - run: |
           cat x/storage/schemaless/types_reg_gen.go

From b488ee03aa8b4a9aed9a2aa131a8ec2fb7a9cf9f Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 22:55:36 +0200
Subject: [PATCH 63/72] debug: more debugging

---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c2233a0e..bc53c791 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -27,8 +27,8 @@ jobs:
           find . -type f -name '*.go' -exec grep -C 2 -H 'github.com/opensearch-project/opensearch-go/v2' {} + &2>/dev/null || true
 
       - run: |
-          la -la /home/runner/go/pkg/mod/github.com/opensearch-project/opensearch-go
-          la -la /home/runner/go/pkg/mod/cache/download/github.com/opensearch-project/opensearch-go
+          ls -la /home/runner/go/pkg/mod/github.com/opensearch-project/opensearch-go
+          ls -la /home/runner/go/pkg/mod/cache/download/github.com/opensearch-project/opensearch-go
           
           tree /home/runner/go/pkg/mod/github.com/opensearch-project/opensearch-go
           tree /home/runner/go/pkg/mod/cache/download/github.com/opensearch-project/opensearch-go

From b2959dc9a0c2411d9f1e967f453b33ce597ed8e8 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 23:04:56 +0200
Subject: [PATCH 64/72] debug: more debugging

---
 x/shape/lookup_refs.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/x/shape/lookup_refs.go b/x/shape/lookup_refs.go
index 8523fecc..6265d1da 100644
--- a/x/shape/lookup_refs.go
+++ b/x/shape/lookup_refs.go
@@ -300,6 +300,8 @@ func checkPkgExistsInPaths(pkgImportName string) (string, error) {
 		filepath.Join(cwd),
 	}
 
+	log.Debugf("shape.checkPkgExistsInPaths: %s checking paths: %s\n", pkgImportName, strings.Join(paths, ";\n"))
+
 	for _, p := range paths {
 		packPath := filepath.Join(p, pkgImportName)
 		if _, err := os.Stat(packPath); err == nil {

From a19f97cf3c0c6c991f11a215203d95da46ffcf05 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 23:20:09 +0200
Subject: [PATCH 65/72] fix: moq problem and type-registry

---
 example/state/machine.go | 2 +-
 example/state/model.go   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/example/state/machine.go b/example/state/machine.go
index 5e3e01a2..899e6fca 100644
--- a/example/state/machine.go
+++ b/example/state/machine.go
@@ -28,7 +28,7 @@ var (
 	ErrWorkerIDRequired = fmt.Errorf("worker ID required; %w", ErrValidationFailed)
 )
 
-// go:generate moq -with-resets -stub -out machine_mock_test.go . Dependency
+// go:generate moq -with-resets -stub -out machine_mock.go . Dependency
 type Dependency interface {
 	TimeNow() *time.Time
 	WarehouseRemoveStock(quantity Quantity) error
diff --git a/example/state/model.go b/example/state/model.go
index 06e232b1..ae0a498c 100644
--- a/example/state/model.go
+++ b/example/state/model.go
@@ -97,4 +97,4 @@ const (
 )
 
 // moq must be run after union type is generated
-//go:generate moq -with-resets -stub -out machine_mock_test.go . Dependency
+//go:generate moq -with-resets -stub -out machine_mock.go . Dependency

From 25322b4a80979f764f7cf5c9aa376a73e60c4507 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Sun, 12 May 2024 23:48:48 +0200
Subject: [PATCH 66/72] x/shape: improve package lookup

---
 x/shape/lookup_refs.go | 43 ++++++++++++++++++++++--------------------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/x/shape/lookup_refs.go b/x/shape/lookup_refs.go
index 6265d1da..017140bb 100644
--- a/x/shape/lookup_refs.go
+++ b/x/shape/lookup_refs.go
@@ -198,37 +198,38 @@ func findPackagePath(pkgImportName string) (string, error) {
 	if cwd == "" {
 		cwd, _ = os.Getwd()
 	}
+	if cwd != "" {
+		// hack: to make sure code is simple, we start with the current directory
+		// add append nonsense to the path,
+		// because it will be stripped by path.Dir
+		cwd = path.Join(cwd, "nonsense")
+	}
 
 	// if path has "go.mod" and package name is the same as x.PkgName
 	// then we can assume that it's root of the package
 
-	// hack: to make sure code is simple, we start with the current directory
-	// add append nonsense to the path,
-	// because it will be stripped by path.Dir
-	cwd = path.Join(cwd, "nonsense")
 	for {
 		cwd = path.Dir(cwd)
-		if cwd == "." || cwd == "/" {
+		if cwd == "." || cwd == "/" || cwd == "" {
 			log.Debugf("shape.findPackagePath: %s could not find go.mod file in CWD or parent directories %s, continue with other paths", pkgImportName, cwd)
 			break
 		}
 
 		modpath := path.Join(cwd, "go.mod")
 		_, err := os.Stat(modpath)
-		log.Debugf("shape.findPackagePath: %s checking modpath %s; err=%s", pkgImportName, modpath, err)
+		//log.Debugf("shape.findPackagePath: %s checking modpath %s; err=%s", pkgImportName, modpath, err)
 		if err == nil {
 			f, err := os.Open(modpath)
 			if err != nil {
-				log.Debugf("shape.findPackagePath: %s could not open %s", pkgImportName, cwd)
+				//log.Debugf("shape.findPackagePath: %s could not open %s", pkgImportName, cwd)
 				continue
 			}
 			defer f.Close()
 
 			data, err := io.ReadAll(f)
 			if err != nil {
-				log.Debugf("shape.findPackagePath: %s could not read %s", pkgImportName, cwd)
+				log.Errorf("shape.findPackagePath: %s could not read go.mod in %s", pkgImportName, cwd)
 				continue
-				//return "", fmt.Errorf("shape.findPackagePath: could not read %s; %w", cwd, err)
 			} else {
 				parsed, err := modfile.Parse(modpath, data, nil)
 				if err != nil {
@@ -288,27 +289,29 @@ func findPackagePath(pkgImportName string) (string, error) {
 }
 
 func checkPkgExistsInPaths(pkgImportName string) (string, error) {
-	cwd := os.Getenv("PWD")
-	if cwd == "" {
-		cwd, _ = os.Getwd()
+	gocache := os.Getenv("GOMODCACHE")
+	if gocache == "" {
+		gocache = os.Getenv("GOPATH")
+		if gocache != "" {
+			gocache = filepath.Join(os.Getenv("GOPATH"), "pkg/mod")
+		}
 	}
 
-	paths := []string{
-		filepath.Join(os.Getenv("GOPATH"), "pkg/mod"),
-		filepath.Join(cwd, "vendor"),
-		filepath.Join(os.Getenv("GOROOT"), "src"),
-		filepath.Join(cwd),
+	paths := []string{}
+
+	if gocache != "" {
+		paths = append(paths, gocache)
 	}
 
-	log.Debugf("shape.checkPkgExistsInPaths: %s checking paths: %s\n", pkgImportName, strings.Join(paths, ";\n"))
+	paths = append(paths, filepath.Join(os.Getenv("GOROOT"), "src"))
 
 	for _, p := range paths {
 		packPath := filepath.Join(p, pkgImportName)
 		if _, err := os.Stat(packPath); err == nil {
-			log.Infof("shape.checkPkgExistsInPaths: %s found package in fallback %s", pkgImportName, packPath)
+			log.Infof("shape.checkPkgExistsInPaths: '%s' found package in fallback %s", pkgImportName, packPath)
 			return packPath, nil
 		} else {
-			log.Debugf("shape.checkPkgExistsInPaths: %s could not find package in fallback path %s", pkgImportName, packPath)
+			log.Debugf("shape.checkPkgExistsInPaths: '%s' could not find package in fallback path %s", pkgImportName, packPath)
 		}
 	}
 

From b88c95192e2a79bc6e024147a2efb884517c4028 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Mon, 13 May 2024 00:08:14 +0200
Subject: [PATCH 67/72] x/shape: more debugging

---
 x/shape/lookup_refs.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/x/shape/lookup_refs.go b/x/shape/lookup_refs.go
index 017140bb..3ed0d5bb 100644
--- a/x/shape/lookup_refs.go
+++ b/x/shape/lookup_refs.go
@@ -297,6 +297,11 @@ func checkPkgExistsInPaths(pkgImportName string) (string, error) {
 		}
 	}
 
+	log.Debugf("GOMODCACHE=%s", os.Getenv("GOMODCACHE"))
+	log.Debugf("GOPATH=%s", os.Getenv("GOPATH"))
+	log.Debugf("GOROOT=%s", os.Getenv("GOROOT"))
+	log.Debugf("GOROOT=%s", os.Getenv("PWD"))
+
 	paths := []string{}
 
 	if gocache != "" {

From 65f57e04eeb3d77efa754d98c7c17a59c68a60a5 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Mon, 13 May 2024 00:15:59 +0200
Subject: [PATCH 68/72] x/shape: more debugging

---
 x/shape/lookup_refs.go | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/x/shape/lookup_refs.go b/x/shape/lookup_refs.go
index 3ed0d5bb..98aae6fe 100644
--- a/x/shape/lookup_refs.go
+++ b/x/shape/lookup_refs.go
@@ -292,7 +292,14 @@ func checkPkgExistsInPaths(pkgImportName string) (string, error) {
 	gocache := os.Getenv("GOMODCACHE")
 	if gocache == "" {
 		gocache = os.Getenv("GOPATH")
-		if gocache != "" {
+		if gocache == "" {
+			gocache = os.Getenv("HOME")
+			if gocache != "" {
+				gocache = filepath.Join(gocache, "go")
+			}
+		}
+
+		if gocache == "" {
 			gocache = filepath.Join(os.Getenv("GOPATH"), "pkg/mod")
 		}
 	}
@@ -300,7 +307,8 @@ func checkPkgExistsInPaths(pkgImportName string) (string, error) {
 	log.Debugf("GOMODCACHE=%s", os.Getenv("GOMODCACHE"))
 	log.Debugf("GOPATH=%s", os.Getenv("GOPATH"))
 	log.Debugf("GOROOT=%s", os.Getenv("GOROOT"))
-	log.Debugf("GOROOT=%s", os.Getenv("PWD"))
+	log.Debugf("HOME=%s", os.Getenv("HOME"))
+	log.Debugf("gocache=%s", os.Getenv("gocache"))
 
 	paths := []string{}
 

From 8ebe0780c3d45fec372e57cb591f5e4202432764 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Mon, 13 May 2024 00:19:59 +0200
Subject: [PATCH 69/72] x/shape: more debugging

---
 x/shape/lookup_refs.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x/shape/lookup_refs.go b/x/shape/lookup_refs.go
index 98aae6fe..bb40df01 100644
--- a/x/shape/lookup_refs.go
+++ b/x/shape/lookup_refs.go
@@ -308,7 +308,7 @@ func checkPkgExistsInPaths(pkgImportName string) (string, error) {
 	log.Debugf("GOPATH=%s", os.Getenv("GOPATH"))
 	log.Debugf("GOROOT=%s", os.Getenv("GOROOT"))
 	log.Debugf("HOME=%s", os.Getenv("HOME"))
-	log.Debugf("gocache=%s", os.Getenv("gocache"))
+	log.Debugf("gocache=%s", gocache)
 
 	paths := []string{}
 

From 55b2496571de5648bcbfd27c38de24048d37f194 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Mon, 13 May 2024 00:21:38 +0200
Subject: [PATCH 70/72] x/shape: more debugging

---
 x/shape/lookup_refs.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x/shape/lookup_refs.go b/x/shape/lookup_refs.go
index bb40df01..14eb7a3b 100644
--- a/x/shape/lookup_refs.go
+++ b/x/shape/lookup_refs.go
@@ -299,7 +299,7 @@ func checkPkgExistsInPaths(pkgImportName string) (string, error) {
 			}
 		}
 
-		if gocache == "" {
+		if gocache != "" {
 			gocache = filepath.Join(os.Getenv("GOPATH"), "pkg/mod")
 		}
 	}

From a1747f0bb435b622233351a225da587899987325 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Mon, 13 May 2024 00:24:29 +0200
Subject: [PATCH 71/72] x/shape: more debugging

---
 x/shape/lookup_refs.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x/shape/lookup_refs.go b/x/shape/lookup_refs.go
index 14eb7a3b..7c732aad 100644
--- a/x/shape/lookup_refs.go
+++ b/x/shape/lookup_refs.go
@@ -300,7 +300,7 @@ func checkPkgExistsInPaths(pkgImportName string) (string, error) {
 		}
 
 		if gocache != "" {
-			gocache = filepath.Join(os.Getenv("GOPATH"), "pkg/mod")
+			gocache = filepath.Join(gocache, "pkg/mod")
 		}
 	}
 

From 73b719d00ee998d85e028a7d501f681aff37d0b5 Mon Sep 17 00:00:00 2001
From: widmogrod <widmogrod@gmail.com>
Date: Mon, 13 May 2024 21:53:47 +0200
Subject: [PATCH 72/72] x/shape: reduce debugging

---
 x/shape/lookup_refs.go | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/x/shape/lookup_refs.go b/x/shape/lookup_refs.go
index 7c732aad..d52dd9fd 100644
--- a/x/shape/lookup_refs.go
+++ b/x/shape/lookup_refs.go
@@ -304,12 +304,6 @@ func checkPkgExistsInPaths(pkgImportName string) (string, error) {
 		}
 	}
 
-	log.Debugf("GOMODCACHE=%s", os.Getenv("GOMODCACHE"))
-	log.Debugf("GOPATH=%s", os.Getenv("GOPATH"))
-	log.Debugf("GOROOT=%s", os.Getenv("GOROOT"))
-	log.Debugf("HOME=%s", os.Getenv("HOME"))
-	log.Debugf("gocache=%s", gocache)
-
 	paths := []string{}
 
 	if gocache != "" {