diff --git a/integration_test/metrics_test.go b/integration_test/metrics_test.go new file mode 100644 index 00000000..26c3ac99 --- /dev/null +++ b/integration_test/metrics_test.go @@ -0,0 +1,111 @@ +package integrationtest + +import ( + "bytes" + "io" + "net/http" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type Metrics struct { + SigningOpsTotal int + Sum float64 + Count int + Error int +} + +func AssertMetricsSuccessIncremented(t *testing.T, before Metrics, after Metrics, op string) { + assert.Greater(t, after.Count, before.Count) + assert.Greater(t, after.Sum, before.Sum) + //because Issue #376 + if op == "generic" { + assert.Greater(t, after.SigningOpsTotal, before.SigningOpsTotal) + } + assert.Equal(t, after.Error, before.Error) +} + +func AssertMetricsFailure(t *testing.T, before Metrics, after Metrics) { + AssertMetricsSuccessUnchanged(t, before, after) + assert.Greater(t, after.Error, before.Error) +} + +func AssertMetricsSuccessUnchanged(t *testing.T, before Metrics, after Metrics) { + assert.Equal(t, after.Count, before.Count) + assert.Equal(t, after.Sum, before.Sum) + assert.Equal(t, after.SigningOpsTotal, before.SigningOpsTotal) +} + +// this is the most simple test of metrics. metrics validation is also added to vault and operations/kinds tests +func TestMetrics(t *testing.T) { + metrics0 := GetMetrics("tz1VSUr8wwNhLAzempoch5d6hLRiTh8Cjcjb", "transaction", "generic", "File") + _, err := OctezClient("transfer", "1", "from", "alice", "to", "bob") + require.Nil(t, err) + metrics1 := GetMetrics("tz1VSUr8wwNhLAzempoch5d6hLRiTh8Cjcjb", "transaction", "generic", "File") + AssertMetricsSuccessIncremented(t, metrics0, metrics1, "generic") +} + +func GetMetrics(address string, kind string, operation string, vault string) Metrics { + metrics := Metrics{SigningOpsTotal: 0, Sum: 0, Count: 0, Error: 0} + _, b := getBytes() + lines := bytes.Split(b, []byte("\n")) + for _, line := range lines { + s := string(line) + + if strings.HasPrefix(s, "vault_sign_request_error_total") { + if strings.Contains(s, "vault=\""+vault) { + v := parseValue(s) + metrics.Error, _ = strconv.Atoi(v) + } + } + if strings.HasPrefix(s, "signing_ops_total") { + if strings.Contains(s, "vault=\""+vault) && + strings.Contains(s, "address=\""+address) && + strings.Contains(s, "kind=\""+kind) { + metrics.SigningOpsTotal, _ = strconv.Atoi(parseValue(s)) + } + } + if strings.HasPrefix(s, "vault_sign_request_duration_milliseconds_count") { + if strings.Contains(s, "vault=\""+vault) && + strings.Contains(s, "address=\""+address) && + strings.Contains(s, "op=\""+operation) { + metrics.Count, _ = strconv.Atoi(parseValue(s)) + } + } + if strings.HasPrefix(s, "vault_sign_request_duration_milliseconds_sum") { + if strings.Contains(s, "vault=\""+vault) && + strings.Contains(s, "address=\""+address) && + strings.Contains(s, "op=\""+operation) { + metrics.Sum, _ = strconv.ParseFloat(parseValue(s), 64) + } + } + } + return metrics +} + +func parseValue(line string) string { + return strings.Split(line, "} ")[1] +} + +func getBytes() (int, []byte) { + url := "http://localhost:9583/metrics" + client := &http.Client{} + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + panic(err) + } + resp, err := client.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + bytes, err := io.ReadAll(resp.Body) + if err != nil { + panic(err) + } + return resp.StatusCode, bytes +} diff --git a/integration_test/operationkinds_test.go b/integration_test/operationkinds_test.go index 3014308c..f0304108 100644 --- a/integration_test/operationkinds_test.go +++ b/integration_test/operationkinds_test.go @@ -17,10 +17,12 @@ const ( contract = "contract.event.tz" contractAlias = "emit_event" flextesanob = "http://flextesanobaking:20000" + vault = "File" ) type testCase struct { - opName string + op string + kind string testSetupOps [][]string testOp []string account string @@ -33,97 +35,98 @@ type testCase struct { // these test cases are not atomic -- some tests depend on previous tests (order matters) var testcases = []testCase{ { - opName: "preendorsement", - testSetupOps: nil, - testOp: []string{"--endpoint", flextesanob, "preendorse", "for", alias, "--force"}, - account: account, - allowPolicy: map[string][]string{"generic": {"preendorsement"}, "preendorsement": {}}, - notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"preendorsement"}), "endorsement": {}, "block": {}}, - successMessage: "injected preendorsement", - validateOctezReturn: false, + kind: "preendorsement", + op: "preendorsement", + testSetupOps: nil, + testOp: []string{"--endpoint", flextesanob, "preendorse", "for", alias, "--force"}, + account: account, + allowPolicy: map[string][]string{"generic": {"preendorsement"}, "preendorsement": {}}, + notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"preendorsement"}), "endorsement": {}, "block": {}}, + successMessage: "injected preendorsement", }, { - opName: "endorsement", - testSetupOps: nil, - testOp: []string{"--endpoint", flextesanob, "endorse", "for", alias, "--force"}, - account: account, - allowPolicy: map[string][]string{"generic": {"endorsement"}, "endorsement": {}}, - notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"endorsement"}), "preendorsement": {}, "block": {}}, - successMessage: "injected endorsement", - validateOctezReturn: false, + kind: "endorsement", + op: "endorsement", + testSetupOps: nil, + testOp: []string{"--endpoint", flextesanob, "endorse", "for", alias, "--force"}, + account: account, + allowPolicy: map[string][]string{"generic": {"endorsement"}, "endorsement": {}}, + notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"endorsement"}), "preendorsement": {}, "block": {}}, + successMessage: "injected endorsement", }, { - opName: "block", - testSetupOps: nil, - testOp: []string{"--endpoint", flextesanob, "bake", "for", alias, "--force"}, - account: account, - allowPolicy: map[string][]string{"generic": {}, "block": {}}, - notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"block"}), "preendorsement": {}, "endorsement": {}}, - successMessage: "injected for " + alias + " (" + account + ")", - validateOctezReturn: false, + kind: "block", + op: "block", + testSetupOps: nil, + testOp: []string{"--endpoint", flextesanob, "bake", "for", alias, "--force"}, + account: account, + allowPolicy: map[string][]string{"generic": {}, "block": {}}, + notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"block"}), "preendorsement": {}, "endorsement": {}}, + successMessage: "injected for " + alias + " (" + account + ")", }, { - opName: "reveal", - testSetupOps: [][]string{{"-w", "1", "transfer", "100", "from", "alice", "to", alias, "--burn-cap", "0.06425"}}, - testOp: []string{"reveal", "key", "for", alias}, - account: account, - allowPolicy: map[string][]string{"generic": {"reveal"}}, - notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"reveal"})}, - successMessage: "Operation successfully injected in the node", - validateOctezReturn: true, + kind: "reveal", + op: "generic", + testSetupOps: [][]string{{"-w", "1", "transfer", "100", "from", "alice", "to", alias, "--burn-cap", "0.06425"}}, + testOp: []string{"reveal", "key", "for", alias}, + account: account, + allowPolicy: map[string][]string{"generic": {"reveal"}}, + notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"reveal"})}, + successMessage: "Operation successfully injected in the node", }, { - opName: "register_global_constant", - testSetupOps: nil, - testOp: []string{"register", "global", "constant", "999", "from", alias, "--burn-cap", "0.017"}, - account: account, - allowPolicy: map[string][]string{"generic": {"register_global_constant"}}, - notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"register_global_constant"})}, - successMessage: "Operation successfully injected in the node", - validateOctezReturn: true, + kind: "register_global_constant", + op: "generic", + testSetupOps: nil, + testOp: []string{"register", "global", "constant", "999", "from", alias, "--burn-cap", "0.017"}, + account: account, + allowPolicy: map[string][]string{"generic": {"register_global_constant"}}, + notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"register_global_constant"})}, + successMessage: "Operation successfully injected in the node", }, { - opName: "transaction", - testSetupOps: nil, - account: account, - testOp: []string{"transfer", "1", "from", alias, "to", "alice", "--burn-cap", "0.06425"}, - allowPolicy: map[string][]string{"generic": {"transaction"}}, - notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"transaction"})}, - successMessage: "Operation successfully injected in the node", - validateOctezReturn: true, + kind: "transaction", + op: "generic", + testSetupOps: nil, + account: account, + testOp: []string{"transfer", "1", "from", alias, "to", "alice", "--burn-cap", "0.06425"}, + allowPolicy: map[string][]string{"generic": {"transaction"}}, + notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"transaction"})}, + successMessage: "Operation successfully injected in the node", }, { - opName: "delegation", - testSetupOps: nil, - account: account, - testOp: []string{"register", "key", alias, "as", "delegate"}, - allowPolicy: map[string][]string{"generic": {"delegation"}}, - notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"delegation"})}, - successMessage: "Operation successfully injected in the node", - validateOctezReturn: true, + kind: "delegation", + op: "generic", + testSetupOps: nil, + account: account, + testOp: []string{"register", "key", alias, "as", "delegate"}, + allowPolicy: map[string][]string{"generic": {"delegation"}}, + notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"delegation"})}, + successMessage: "Operation successfully injected in the node", }, { - opName: "set_deposits_limit", - testSetupOps: nil, - account: account, - testOp: []string{"set", "deposits", "limit", "for", alias, "to", "10000"}, - allowPolicy: map[string][]string{"generic": {"set_deposits_limit"}}, - notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"set_deposits_limit"})}, - successMessage: "Operation successfully injected in the node", - validateOctezReturn: true, + kind: "set_deposits_limit", + op: "generic", + testSetupOps: nil, + account: account, + testOp: []string{"set", "deposits", "limit", "for", alias, "to", "10000"}, + allowPolicy: map[string][]string{"generic": {"set_deposits_limit"}}, + notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"set_deposits_limit"})}, + successMessage: "Operation successfully injected in the node", }, { - opName: "update_consensus_key", - testSetupOps: nil, - account: account, - testOp: []string{"set", "consensus", "key", "for", alias, "to", alias1}, - allowPolicy: map[string][]string{"generic": {"update_consensus_key"}}, - notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"update_consensus_key"})}, - successMessage: "Operation successfully injected in the node", - validateOctezReturn: true, + kind: "update_consensus_key", + op: "generic", + testSetupOps: nil, + account: account, + testOp: []string{"set", "consensus", "key", "for", alias, "to", alias1}, + allowPolicy: map[string][]string{"generic": {"update_consensus_key"}}, + notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"update_consensus_key"})}, + successMessage: "Operation successfully injected in the node", }, { - opName: "origination", + kind: "origination", + op: "generic", testSetupOps: nil, account: account, testOp: []string{"originate", "contract", contractAlias, "transferring", "1", "from", alias, "running", contract, "--burn-cap", "0.4"}, @@ -133,21 +136,21 @@ var testcases = []testCase{ validateOctezReturn: true, }, { - opName: "increase_paid_storage", - testSetupOps: nil, - account: account, - testOp: []string{"increase", "the", "paid", "storage", "of", contractAlias, "by", "0x5c", "bytes", "from", alias}, - allowPolicy: map[string][]string{"generic": {"increase_paid_storage"}}, - notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"increase_paid_storage"})}, - successMessage: "Operation successfully injected in the node", - validateOctezReturn: true, + kind: "increase_paid_storage", + op: "generic", + testSetupOps: nil, + account: account, + testOp: []string{"increase", "the", "paid", "storage", "of", contractAlias, "by", "0x5c", "bytes", "from", alias}, + allowPolicy: map[string][]string{"generic": {"increase_paid_storage"}}, + notAllowPolicy: map[string][]string{"generic": getAllOpsExcluding([]string{"increase_paid_storage"})}, + successMessage: "Operation successfully injected in the node", }, } func TestOperationAllowPolicy(t *testing.T) { defer clean_tezos_folder() for _, test := range testcases { - t.Run(test.opName, func(t *testing.T) { + t.Run(test.kind, func(t *testing.T) { //first, do any setup steps that have to happen before the operation to be tested for _, setupOp := range test.testSetupOps { out, err := OctezClient(setupOp...) @@ -155,6 +158,8 @@ func TestOperationAllowPolicy(t *testing.T) { require.Contains(t, string(out), "Operation successfully injected in the node") } + metrics0 := GetMetrics(test.account, test.kind, test.op, vault) + //next, configure every operation allowed except for the one tested, to test it will be denied var c Config c.Read() @@ -163,12 +168,16 @@ func TestOperationAllowPolicy(t *testing.T) { defer restore_config() restart_signatory() out, err := OctezClient(test.testOp...) - if test.validateOctezReturn { + if test.op == "generic" { //the baking operations in octez-client do not return an error when they fail //so, we do this assert when we can assert.Error(t, err) } - assert.Contains(t, string(out), "`"+test.opName+"' is not allowed") + assert.Contains(t, string(out), "`"+test.kind+"' is not allowed") + + metrics1 := GetMetrics(test.account, test.kind, test.op, vault) + //this should be changed to AssertMetricsFailure + AssertMetricsSuccessUnchanged(t, metrics0, metrics1) //finally, configure the operation being tested as the only one allowed and test it is successful c.Read() @@ -181,6 +190,8 @@ func TestOperationAllowPolicy(t *testing.T) { } assert.NoError(t, err) assert.Contains(t, string(out), test.successMessage) + metrics2 := GetMetrics(test.account, test.kind, test.op, vault) + AssertMetricsSuccessIncremented(t, metrics1, metrics2, test.op) }) } } diff --git a/integration_test/watermark_test.go b/integration_test/watermark_test.go index e2e4417e..01cbb1fb 100644 --- a/integration_test/watermark_test.go +++ b/integration_test/watermark_test.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "io" - "log" "net/http" "os/exec" "strings" @@ -321,33 +320,33 @@ func request_sign_concurrent(request string) { func mkdir() { _, err := exec.Command("docker", "exec", container, "mkdir", "-p", dir).CombinedOutput() if err != nil { - log.Fatal("failed to make watermark directory") + panic("failed to make watermark directory") } } func remove_watermark_files() { out, err := exec.Command("docker", "exec", container, "rm", "-rf", dir).CombinedOutput() if err != nil { - log.Fatal("failed to remove watermark files: " + err.Error() + " " + string(out)) + panic("failed to remove watermark files: " + err.Error() + " " + string(out)) } } func write_watermark_file(ktw keyToWatermark, filename string) { json, err := json.Marshal(ktw) if err != nil { - log.Fatal("json marshal failed") + panic("json marshal failed") } shell := "echo '" + string(json) + "' >" + dir + filename _, err = exec.Command("docker", "exec", container, "bash", "-c", shell).CombinedOutput() if err != nil { - log.Fatal("failed to write watermark file") + panic("failed to write watermark file") } } func read_watermark_file(chainId string) (out []byte) { out, err := exec.Command("docker", "exec", container, "cat", dir+chainId+".json").CombinedOutput() if err != nil { - log.Fatal("failed to read watermark file") + panic("failed to read watermark file") } return } @@ -357,16 +356,16 @@ func request_sign(body string) (int, []byte) { client := &http.Client{} req, err := http.NewRequest(http.MethodPost, url, reqbody) if err != nil { - log.Fatal(err) + panic(err) } resp, err := client.Do(req) if err != nil { - log.Fatal(err) + panic(err) } defer resp.Body.Close() bytes, err := io.ReadAll(resp.Body) if err != nil { - log.Fatal(err) + panic(err) } //fmt.Println(string(bytes)) return resp.StatusCode, bytes