diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 17d135dbf6..23c57dbbb0 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -106,6 +106,7 @@ jobs: runs-on: ubuntu-large timeout-minutes: 30 env: + JOB_TIMEOUT: 28m AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} DAPP_TESTS_MNEMONIC: ${{ secrets.DAPP_TESTS_MNEMONIC }} @@ -128,9 +129,7 @@ jobs: { name: "Mint & Staking & Bank Module", scripts: [ - "python3 integration_test/scripts/runner.py integration_test/staking_module/staking_test.yaml", - "python3 integration_test/scripts/runner.py integration_test/bank_module/send_funds_test.yaml", - "python3 integration_test/scripts/runner.py integration_test/mint_module/mint_test.yaml" + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run \"TestMintModule|TestStakingModule|TestBankModule\"" ] }, { diff --git a/go.mod b/go.mod index 009b009979..35808976e0 100644 --- a/go.mod +++ b/go.mod @@ -107,6 +107,7 @@ require ( google.golang.org/grpc v1.75.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 k8s.io/component-base v0.35.0 pgregory.net/rapid v1.2.0 ) @@ -129,7 +130,6 @@ require ( github.com/skeema/knownhosts v1.3.1 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( diff --git a/integration_test/bank_module/multi_sig_send_test.yaml b/integration_test/bank_module/multi_sig_send_test.yaml index c8341ac070..cdc025c27f 100644 --- a/integration_test/bank_module/multi_sig_send_test.yaml +++ b/integration_test/bank_module/multi_sig_send_test.yaml @@ -21,6 +21,8 @@ - cmd: seid tx sign unsigned-tx.json --multisig=$MULTI_SIG_ACC --keyring-backend test --from=wallet2 --output-document=wallet2.json --chain-id sei -b block --fees 1sei - cmd: seid tx multisign unsigned-tx.json multisig wallet1.json wallet2.json --chain-id sei --keyring-backend test > signed-tx.json - cmd: seid tx broadcast signed-tx.json --chain-id sei -b block -y + # cleanup + - cmd: rm wallet1.json wallet2.json signed-tx.json unsigned-tx.json # Check multi-sig balance - cmd: seid q bank balances $MULTI_SIG_ACC --output json | jq -r .balances[0].amount diff --git a/integration_test/bank_module/simulation_tx.yaml b/integration_test/bank_module/simulation_tx.yaml index d0b138b33f..b4bfb762f4 100644 --- a/integration_test/bank_module/simulation_tx.yaml +++ b/integration_test/bank_module/simulation_tx.yaml @@ -10,7 +10,7 @@ # Send funds - cmd: printf "12345678\n" | seid tx bank send $ADMIN_ACC $SIMULATION_TEST_ACC 1sei -b block --fees 2000usei --chain-id sei -y - - cmd: seid tx bank send $ADMIN_ACC $SIMULATION_TEST_ACC 1000sei --from $ADMIN_ACC --chain-id sei -b block -y --dry-run --keyring-backend test + - cmd: seid tx bank send $ADMIN_ACC $SIMULATION_TEST_ACC 1000sei --from $ADMIN_ACC --chain-id sei -b block -y --dry-run --keyring-backend test 2>&1 env: GAS_ESIMATE # Validate that only the 1sei is sent diff --git a/integration_test/runner/runner.go b/integration_test/runner/runner.go new file mode 100644 index 0000000000..94717d2811 --- /dev/null +++ b/integration_test/runner/runner.go @@ -0,0 +1,359 @@ +// Package runner is a Go-native driver for the YAML integration test suite, +// +// It integrates with standard `go test`, so each test case becomes a subtest +// with proper failure attribution, -run filtering, and -v output. +// +// Requires a running Docker cluster. Start one with `make docker-cluster-start`. +// +// Run a single module: +// +// go test -tags yaml_integration -v ./integration_test/runner/... -run TestBankModule +// +// Run everything: +// +// go test -tags yaml_integration -v ./integration_test/runner/... +package runner + +import ( + "errors" + "fmt" + "math/big" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +// TestCase mirrors the YAML schema used by the existing test files. +type TestCase struct { + Name string `yaml:"name"` + Inputs []Input `yaml:"inputs"` + Verifiers []Verifier `yaml:"verifiers"` +} + +// Input is one command step within a test case. +type Input struct { + Cmd string `yaml:"cmd"` + Env string `yaml:"env,omitempty"` // capture trimmed stdout into this name + Node string `yaml:"node,omitempty"` // docker container; defaults to Options.DefaultContainer +} + +// Verifier checks the accumulated env map after all inputs have run. +type Verifier struct { + Type string `yaml:"type"` // "eval" or "regex" + Expr string `yaml:"expr"` + Result string `yaml:"result,omitempty"` // env var to match (regex only) +} + +// Options controls how RunFile executes commands. +type Options struct { + // DefaultContainer is the docker container used when an Input has no Node set. + DefaultContainer string + // ExtraPath is appended to PATH inside docker containers. + ExtraPath string + // Shell is the shell used to execute commands (e.g. "sh", "bash"). + // Resolved via PATH at runtime. Defaults to "sh". + Shell string +} + +// Option is a functional option for Options. +type Option func(*Options) + +// WithContainer sets the default docker container for inputs that don't specify one. +func WithContainer(container string) Option { + return func(o *Options) { o.DefaultContainer = container } +} + +// WithExtraPath overrides the PATH suffix injected into docker containers. +func WithExtraPath(path string) Option { + return func(o *Options) { o.ExtraPath = path } +} + +// WithShell overrides the shell used to execute commands. Resolved via PATH at runtime. +func WithShell(shell string) Option { + return func(o *Options) { o.Shell = shell } +} + +func newOptions(opts []Option) Options { + var o Options + //applying default options + for _, f := range []Option{WithContainer("sei-node-0"), WithExtraPath("/root/go/bin:/root/.foundry/bin"), WithShell("bash")} { + f(&o) + } + for _, opt := range opts { + opt(&o) + } + return o +} + +// RunFile reads path as a YAML list of TestCases and runs each as a subtest of t. +func RunFile(t *testing.T, path string, opts ...Option) { + t.Helper() + o := newOptions(opts) + data, err := os.ReadFile(path) //nolint:gosec + require.NoError(t, err, "read %s: %v", path, err) + var cases []TestCase + require.NoError(t, yaml.Unmarshal(data, &cases), "unmarshal %s: %v", path, err) + for _, tc := range cases { + t.Run(tc.Name, func(t *testing.T) { + runCase(t, tc, o) + }) + } +} + +func runCase(t *testing.T, tc TestCase, opts Options) { + t.Helper() + envMap := make(map[string]string) + + for i, inp := range tc.Inputs { + container := inp.Node + if container == "" { + container = opts.DefaultContainer + } + out, err := execCmd(t, inp.Cmd, container, envMap, opts) + t.Logf("[%d] $ %s\n => %s", i, inp.Cmd, out) + require.NoError(t, err, "input[%d] failed: %v", i, err) + if inp.Env != "" { + envMap[inp.Env] = out + } + } + + for _, v := range tc.Verifiers { + if err := verify(v, envMap); err != nil { + t.Errorf("verifier %s %q: %v", v.Type, v.Expr, err) + } + } +} + +// execCmd runs cmd in the given docker container (or locally if container is empty), +// injecting the accumulated envMap. Non-zero exit is logged but not fatal — this +// matches runner.py behaviour where commands that echo error codes exit 0 from +// bash but the captured output is the code. +func execCmd(t *testing.T, cmd, container string, envMap map[string]string, opts Options) (string, error) { + t.Helper() + var c *exec.Cmd + + if container != "" { + // capacity: 1 ("exec") + 2*len(envMap) ("-e" + "k=v" per entry) + 4 (container, "/bin/bash", "-c", cmd) + args := make([]string, 1, 1+2*len(envMap)+4) + args[0] = "exec" + for k, v := range envMap { + args = append(args, "-e", k+"="+v) + } + args = append(args, container, opts.Shell, "-c", + "export PATH=$PATH:"+opts.ExtraPath+" && "+cmd) + c = exec.Command("docker", args...) //nolint:gosec + } else { + c = exec.Command(opts.Shell, "-c", cmd) //nolint:gosec + c.Env = append(os.Environ(), envMapSlice(envMap)...) + } + + out, err := c.Output() + stdout := strings.TrimSpace(string(out)) + if err != nil { + var exit *exec.ExitError + if errors.As(err, &exit) { + t.Logf(" (exit %d) stderr: %s", exit.ExitCode(), strings.TrimSpace(string(exit.Stderr))) + return stdout, nil + } + return stdout, err + } + return stdout, nil +} + +// envMapSlice converts envMap to a slice of "K=V" strings suitable for exec.Cmd.Env. +func envMapSlice(envMap map[string]string) []string { + s := make([]string, 0, len(envMap)) + for k, v := range envMap { + s = append(s, k+"="+v) + } + return s +} + +// verify evaluates a single Verifier against the accumulated env map. +func verify(v Verifier, envMap map[string]string) error { + switch v.Type { + case "eval": + ok, err := evalExpr(v.Expr, envMap) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("false: %s%s", v.Expr, fmtRelevant(v.Expr, envMap)) + } + return nil + case "regex": + val := envMap[v.Result] + ok, err := regexp.MatchString(v.Expr, val) + if err != nil { + return fmt.Errorf("bad pattern %q: %w", v.Expr, err) + } + if !ok { + return fmt.Errorf("pattern %q did not match %q", v.Expr, val) + } + return nil + default: + return fmt.Errorf("unknown verifier type %q", v.Type) + } +} + +// evalExpr evaluates a Python-runner-compatible eval expression. +// Handles: VAR op literal, VAR op VAR, expr and expr, expr or expr. +// Operators: ==, !=, >, <, >=, <=. +func evalExpr(expr string, envMap map[string]string) (bool, error) { + if parts := strings.Split(expr, " and "); len(parts) > 1 { + for _, p := range parts { + ok, err := evalExpr(strings.TrimSpace(p), envMap) + if err != nil { + return false, err + } + if !ok { + return false, nil + } + } + return true, nil + } + if parts := strings.Split(expr, " or "); len(parts) > 1 { + for _, p := range parts { + ok, err := evalExpr(strings.TrimSpace(p), envMap) + if err == nil && ok { + return true, nil + } + } + return false, nil + } + return cmpExpr(strings.TrimSpace(expr), envMap) +} + +// cmpExpr evaluates a single "LHS op RHS" comparison. +// It tokenizes by whitespace first and substitutes env var names before +// locating the operator, so arithmetic sub-expressions like EXPECTED_COUNTS + 1 +// are evaluated rather than compared as literal strings. +func cmpExpr(expr string, envMap map[string]string) (bool, error) { + tokens := strings.Fields(expr) + for i, t := range tokens { + if len(t) >= 2 && t[0] == '"' && t[len(t)-1] == '"' { + tokens[i] = t[1 : len(t)-1] + } else if v, ok := envMap[t]; ok { + tokens[i] = v + } + } + for i, t := range tokens { + for _, op := range []string{"!=", ">=", "<=", "==", ">", "<"} { + if t == op { + lhs, err := evalArith(tokens[:i]) + if err != nil { + return false, err + } + rhs, err := evalArith(tokens[i+1:]) + if err != nil { + return false, err + } + return cmpValues(lhs, rhs, op) + } + } + } + return false, fmt.Errorf("no operator in %q", expr) +} + +// evalArith evaluates a sequence of tokens as big.Int arithmetic (e.g. ["4", "+", "1"] → "5"). +// Falls back to the raw joined string when the first token is not a valid integer, +// allowing float and string comparisons to fall through to cmpValues. +func evalArith(tokens []string) (string, error) { + if len(tokens) == 0 { + return "", fmt.Errorf("empty side of comparison") + } + if len(tokens) == 1 { + return tokens[0], nil + } + result := new(big.Int) + if _, ok := result.SetString(tokens[0], 10); !ok { + return strings.Join(tokens, " "), nil + } + for i := 1; i+1 < len(tokens); i += 2 { + operand := new(big.Int) + if _, ok := operand.SetString(tokens[i+1], 10); !ok { + return strings.Join(tokens, " "), nil + } + switch tokens[i] { + case "+": + result.Add(result, operand) + case "-": + result.Sub(result, operand) + case "*": + result.Mul(result, operand) + case "/": + if operand.Sign() == 0 { + return "", fmt.Errorf("division by zero in %v", tokens) + } + result.Div(result, operand) + default: + return strings.Join(tokens, " "), nil + } + } + return result.String(), nil +} + +// cmpValues compares a and b with op. Tries big.Int (for large integers without +// float64 precision loss), then float64, then string. +func cmpValues(a, b, op string) (bool, error) { + a, b = strings.TrimSpace(a), strings.TrimSpace(b) + + ai, bi := new(big.Int), new(big.Int) + if _, ok1 := ai.SetString(a, 10); ok1 { + if _, ok2 := bi.SetString(b, 10); ok2 { + return applyOp(ai.Cmp(bi), op) + } + } + + fa, errA := strconv.ParseFloat(a, 64) + fb, errB := strconv.ParseFloat(b, 64) + if errA == nil && errB == nil { + c := 0 + if fa < fb { + c = -1 + } else if fa > fb { + c = 1 + } + return applyOp(c, op) + } + + return applyOp(strings.Compare(a, b), op) +} + +func applyOp(cmp int, op string) (bool, error) { + switch op { + case "==": + return cmp == 0, nil + case "!=": + return cmp != 0, nil + case ">": + return cmp > 0, nil + case "<": + return cmp < 0, nil + case ">=": + return cmp >= 0, nil + case "<=": + return cmp <= 0, nil + } + return false, fmt.Errorf("unknown operator %q", op) +} + +// fmtRelevant returns a diagnostic string listing env vars referenced in expr. +func fmtRelevant(expr string, envMap map[string]string) string { + var refs []string + for k, v := range envMap { + if strings.Contains(expr, k) { + refs = append(refs, k+"="+v) + } + } + if len(refs) == 0 { + return "" + } + return " [" + strings.Join(refs, ", ") + "]" +} diff --git a/integration_test/runner/runner_test.go b/integration_test/runner/runner_test.go new file mode 100644 index 0000000000..a2bbb363c8 --- /dev/null +++ b/integration_test/runner/runner_test.go @@ -0,0 +1,92 @@ +//go:build yaml_integration + +// Package runner_test wires the existing YAML test files into standard `go test`. +// +// Requires a running Docker cluster (`make docker-cluster-start`). +// +// Run a single module: +// +// go test -tags yaml_integration -v ./integration_test/runner/... -run TestBankModule +// +// Run all modules: +// +// go test -tags yaml_integration -v ./integration_test/runner/... +// +// Each YAML test case becomes a named subtest, so -run accepts the full path: +// +// -run TestBankModule/Test_sending_funds +package runner_test + +import ( + "testing" + + "github.com/sei-protocol/sei-chain/integration_test/runner" +) + +func TestStartup(t *testing.T) { + runner.RunFile(t, "../startup/startup_test.yaml") +} + +// Tests are declared in the order the CI matrix ran them as Python scripts +// (go test executes tests in declaration order): staking, then bank, then mint. + +func TestStakingModule(t *testing.T) { + runner.RunFile(t, "../staking_module/staking_test.yaml") +} + +func TestBankModule(t *testing.T) { + runner.RunFile(t, "../bank_module/send_funds_test.yaml") + runner.RunFile(t, "../bank_module/multi_sig_send_test.yaml") + runner.RunFile(t, "../bank_module/simulation_tx.yaml") +} + +func TestMintModule(t *testing.T) { + runner.RunFile(t, "../mint_module/mint_test.yaml") +} + +func TestGovModule(t *testing.T) { + runner.RunFile(t, "../gov_module/gov_proposal_test.yaml") + runner.RunFile(t, "../gov_module/staking_proposal_test.yaml") +} + +func TestOracleModule(t *testing.T) { + runner.RunFile(t, "../oracle_module/verify_penalty_counts.yaml") + runner.RunFile(t, "../oracle_module/set_feeder_test.yaml") +} + +func TestDistributionModule(t *testing.T) { + runner.RunFile(t, "../distribution_module/community_pool.yaml") + runner.RunFile(t, "../distribution_module/rewards.yaml") +} + +func TestTokenFactoryModule(t *testing.T) { + runner.RunFile(t, "../tokenfactory_module/create_tokenfactory_test.yaml") +} + +func TestAuthzModule(t *testing.T) { + runner.RunFile(t, "../authz_module/send_authorization_test.yaml") + runner.RunFile(t, "../authz_module/staking_authorization_test.yaml") + runner.RunFile(t, "../authz_module/generic_authorization_test.yaml") +} + +func TestWasmModule(t *testing.T) { + runner.RunFile(t, "../wasm_module/timelocked_token_delegation_test.yaml") + runner.RunFile(t, "../wasm_module/timelocked_token_admin_test.yaml") + runner.RunFile(t, "../wasm_module/timelocked_token_withdraw_test.yaml") + runner.RunFile(t, "../wasm_module/timelocked_token_emergency_withdraw_test.yaml") +} + +func TestSeiDB(t *testing.T) { + runner.RunFile(t, "../seidb/flatkv_evm_test.yaml") + runner.RunFile(t, "../seidb/state_store_test.yaml") +} + +func TestChainOperation(t *testing.T) { + runner.RunFile(t, "../chain_operation/snapshot_operation.yaml") +} + +// TestStateSync requires the sei-rpc-node container, which is only present in +// the CI RPC-node cluster (make run-rpc-node-integration-ci). +func TestStateSync(t *testing.T) { + runner.RunFile(t, "../chain_operation/statesync_operation.yaml") +}