Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions builder/builder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -598,8 +598,9 @@ func TestSaveAndSetMeta(t *testing.T) {
`with "quotes"`,
builder.ExprString("value"),
)
stmt5 := builder.StmtDeleteAccountMetadata(builder.UnsafeAccount("acc2"), "k")

_, _, script := builder.BuildProgram(stmt1, stmt2, stmt3, stmt4)
_, _, script := builder.BuildProgram(stmt1, stmt2, stmt3, stmt4, stmt5)
snaps.MatchInlineSnapshot(t, script, snaps.Inline(`vars {
string $string_0
asset $asset_0
Expand All @@ -611,7 +612,9 @@ save [$asset_0 *] from @acc1

set_tx_meta("my key", 42)

set_account_meta(@acc2, "with \"quotes\"", $string_0)`))
set_account_meta(@acc2, "with \"quotes\"", $string_0)

delete_account_metadata(@acc2, "k")`))
}

func TestOriginVars(t *testing.T) {
Expand Down
11 changes: 11 additions & 0 deletions builder/statement_extra.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,17 @@ func StmtSetTxMeta[T ExprType](key string, value Expression[T]) Statement {
}
}

// StmtDeleteAccountMetadata: `delete_account_metadata(<account>, "<key>")`.
func StmtDeleteAccountMetadata(account Expression[ExprTypeAccount], key string) Statement {
return func(env *env, w int) {
env.builder.WriteString("delete_account_metadata(")
account(env, w)
env.builder.WriteString(", ")
writeStringLiteral(env, key)
env.builder.WriteString(")")
}
}

// StmtSetAccountMeta: `set_account_meta(<account>, "<key>", <value>)`.
func StmtSetAccountMeta[T ExprType](
account Expression[ExprTypeAccount],
Expand Down
107 changes: 92 additions & 15 deletions bytecode_corpus_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package numscript_test
import (
"bytes"
"context"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
Expand Down Expand Up @@ -90,6 +91,13 @@ func TestFreezeBytecodeCorpus(t *testing.T) {
require.NoError(t, err)
require.NotEmpty(t, rawSpecs)

var earlier []string
for _, freeze := range readCorpusFreezes(t) {
if runsOnCurrentVm(freezeVersion(t, freeze)) {
earlier = append(earlier, freeze)
}
}

frozen := 0
for _, rawSpec := range rawSpecs {
rel, err := filepath.Rel(bytecodeFixturesDir, rawSpec.NumscriptPath)
Expand All @@ -115,8 +123,7 @@ func TestFreezeBytecodeCorpus(t *testing.T) {
}
require.NoError(t, cErr, rel)

require.NoError(t, os.MkdirAll(filepath.Join(dir, programName), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, programName+corpusProgramExt), program.Encode(), 0o644))
files := map[string][]byte{programName + corpusProgramExt: program.Encode()}

used := map[string]bool{}
for _, tc := range specs.TestCases {
Expand All @@ -131,22 +138,29 @@ func TestFreezeBytecodeCorpus(t *testing.T) {

res, execErr := numscript.ExecVm(context.Background(), numscript.NewVm(program), &vars, newCorpusStore(balances, metadata))

caseDir := filepath.Join(dir, programName, caseFolderName(tc.It, used))
require.NoError(t, os.MkdirAll(caseDir, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(caseDir, corpusVarsFile), vars.Encode(), 0o644))
writeCorpusJSON(t, filepath.Join(caseDir, corpusBalancesFile), balances)
caseDir := filepath.Join(programName, caseFolderName(tc.It, used))
files[filepath.Join(caseDir, corpusVarsFile)] = vars.Encode()
files[filepath.Join(caseDir, corpusBalancesFile)] = corpusJSON(t, balances)
if len(metadata) != 0 {
writeCorpusJSON(t, filepath.Join(caseDir, corpusMetadataFile), metadata)
files[filepath.Join(caseDir, corpusMetadataFile)] = corpusJSON(t, metadata)
}
if execErr != nil {
writeCorpusJSON(t, filepath.Join(caseDir, corpusErrorFile), corpusErrorOf(t, execErr))
files[filepath.Join(caseDir, corpusErrorFile)] = corpusJSON(t, corpusErrorOf(t, execErr))
} else {
writeCorpusJSON(t, filepath.Join(caseDir, corpusPostingsFile), corpusResult(res))
files[filepath.Join(caseDir, corpusPostingsFile)] = corpusJSON(t, corpusResult(res))
}
frozen++
}

if slices.ContainsFunc(earlier, func(freeze string) bool { return freezeHasFiles(t, freeze, programName, files) }) {
continue
}
for path, data := range files {
require.NoError(t, os.MkdirAll(filepath.Join(dir, filepath.Dir(path)), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, path), data, 0o644))
}
frozen++
}
t.Logf("froze %d cases into %s", frozen, dir)
t.Logf("froze %d programs into %s", frozen, dir)
}

func TestBytecodeCorpusHasCurrentVersion(t *testing.T) {
Expand Down Expand Up @@ -205,6 +219,7 @@ func checkFrozenProgram(t *testing.T, dir, name string, version numscript.Byteco
require.NotEmpty(t, cases, "no cases")

readable := numscript.CurrentBytecodeVersion.CanRead(programVersion)
restamp := !readable && runsOnCurrentVm(programVersion)
for _, c := range cases {
require.True(t, c.IsDir(), "%s: not a case folder", c.Name())
t.Run(c.Name(), func(t *testing.T) {
Expand All @@ -223,14 +238,21 @@ func checkFrozenProgram(t *testing.T, dir, name string, version numscript.Byteco
_, err = numscript.DecodeVars(varsBytes)
require.ErrorAs(t, err, &unsupported)
require.Equal(t, varsVersion, unsupported.Encoded)
return
if !restamp {
return
}
varsBytes = withCurrentVersion(varsBytes)
}
programBytes := programBytes
if restamp {
programBytes = withCurrentVersion(programBytes)
}

program, err := numscript.DecodeCompiledProgram(programBytes)
require.NoError(t, err)
vars, err := numscript.DecodeVars(varsBytes)
require.NoError(t, err)
if programVersion == numscript.CurrentBytecodeVersion {
if programVersion == numscript.CurrentBytecodeVersion || restamp {
require.Equal(t, programBytes, program.Encode(), "re-encoding changed the program bytes")
require.Equal(t, varsBytes, vars.Encode(), "re-encoding changed the vars bytes")
}
Expand Down Expand Up @@ -349,10 +371,61 @@ func sortedKeys(m map[string]bool) []string {
return keys
}

func writeCorpusJSON(t *testing.T, path string, v any) {
func corpusJSON(t *testing.T, v any) []byte {
data, err := json.MarshalIndent(v, "", " ")
require.NoError(t, err)
require.NoError(t, os.WriteFile(path, append(data, '\n'), 0o644))
return append(data, '\n')
}

// Under 0.x a reader rejects every other version, but an older 0.x freeze still
// runs, re-stamped, so unchanged programs are not frozen again on a bump.
func runsOnCurrentVm(v numscript.BytecodeVersion) bool {
current := numscript.CurrentBytecodeVersion
return current.CanRead(v) || (current.Major == 0 && v.Major == 0)
}

// bytes 4..8 of a program or vars blob are the little-endian major and minor
func withCurrentVersion(blob []byte) []byte {
out := slices.Clone(blob)
binary.LittleEndian.PutUint16(out[4:], numscript.CurrentBytecodeVersion.Major)
binary.LittleEndian.PutUint16(out[6:], numscript.CurrentBytecodeVersion.Minor)
return out
}

// freezeHasFiles reports whether freeze holds exactly files for programName,
// comparing .numb and .nvar blobs without their version.
func freezeHasFiles(t *testing.T, freeze, programName string, files map[string][]byte) bool {
root := filepath.Join(bytecodeCorpusDir, freeze)
onDisk := []string{programName + corpusProgramExt}
if _, err := os.Stat(filepath.Join(root, onDisk[0])); err != nil {
return false
}
require.NoError(t, filepath.WalkDir(filepath.Join(root, programName), func(p string, d fs.DirEntry, err error) error {
if err != nil || d.IsDir() {
return err
}
rel, err := filepath.Rel(root, p)
onDisk = append(onDisk, rel)
return err
}))
if len(onDisk) != len(files) {
return false
}
for _, path := range onDisk {
want, ok := files[path]
if !ok {
return false
}
got, err := os.ReadFile(filepath.Join(root, path))
require.NoError(t, err)
if strings.HasSuffix(path, corpusProgramExt) || strings.HasSuffix(path, corpusVarsFile) {
got, want = withCurrentVersion(got), withCurrentVersion(want)
}
if !bytes.Equal(got, want) {
return false
}
}
return true
}

func readCorpusJSON(t *testing.T, path string, v any) {
Expand All @@ -372,3 +445,7 @@ func decodeJSONExact(t *testing.T, data []byte) any {
require.NoError(t, dec.Decode(&v))
return v
}

func (s corpusStore) DeleteAccountMetadata(context.Context, string, string, string) error {
return nil
}
6 changes: 3 additions & 3 deletions compiler-architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -188,9 +188,9 @@ Major 0 is **unstable**, and the format is at `0.1` today. Any 0.x change may ch

#### Frozen bytecode

`testdata/bytecode/` holds, per freeze, the bytecode the compiler produced from every script fixture at one point in time, with each test case's vars blob, inputs and the output it produced then. A freeze directory is named after the bytecode version (`0.1`, or `0.1-<suffix>` for a later freeze at the same version) and is never edited once committed. Inside it, `<script>.numb` is the program and `<script>/<case>/` holds `vars.nvar`, `balances.json`, `metadata.json` (when the case has any) and either `postings.json` or `error.json`.
`testdata/bytecode/` holds, per freeze, the bytecode the compiler produced from every script fixture at one point in time, with each test case's vars blob, inputs and the output it produced then. A freeze directory is named after the bytecode version (`0.1`, or `0.1-<suffix>` for a later freeze at the same version) and is never edited once committed. A freeze only holds the programs that no older freeze run by this build already holds with the same files, ignoring the version header. Inside it, `<script>.numb` is the program and `<script>/<case>/` holds `vars.nvar`, `balances.json`, `metadata.json` (when the case has any) and either `postings.json` or `error.json`.

`TestBytecodeCorpus` (module root) runs every frozen case through the public API the ledger uses. A blob this build can read is decoded, verified and executed, and must produce the frozen output. A blob it cannot read must be rejected with `UnsupportedBytecodeVersionError`, which is what makes a host recompile from source. `TestBytecodeCorpusHasCurrentVersion` fails when `CurrentBytecodeVersion` has no directory.
`TestBytecodeCorpus` (module root) runs every frozen case through the public API the ledger uses. A blob this build can read is decoded, verified and executed, and must produce the frozen output. A blob it cannot read must be rejected with `UnsupportedBytecodeVersionError`, which is what makes a host recompile from source. Under 0.x an older 0.x blob is then also run with its header changed to the current version, so it keeps covering this VM. `TestBytecodeCorpusHasCurrentVersion` fails when `CurrentBytecodeVersion` has no directory.

When to freeze (`just freeze-bytecode [suffix]`):

Expand All @@ -199,7 +199,7 @@ When to freeze (`just freeze-bytecode [suffix]`):

Compiler changes never invalidate a freeze: the frozen files test the decoder, the verifier and the VM, not the compiler. A red `TestBytecodeCorpus` on a readable directory means this build runs existing bytecode differently. Either the change is a bug, or the bytecode version has to be bumped so that older blobs are rejected.

Under 0.x every bump rejects all older bytecode, so until 1.0 the corpus only catches a forgotten bump. From 1.0 a 1.1 VM must run frozen 1.0 bytecode identically, and a VM fix that changes a frozen output is a major bump.
Under 0.x every bump rejects all older bytecode, but the corpus still runs it re-stamped: a 0.x bump that deliberately changes what an old program does fails that case. That is the one edit an older freeze gets: delete the program from it, then freeze again. From 1.0 a 1.1 VM must run frozen 1.0 bytecode identically, and a VM fix that changes a frozen output is a major bump.

### Constant pools

Expand Down
7 changes: 6 additions & 1 deletion instruction-encoding.md
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,12 @@ Instructions are **4 bytes** wide: `[Opcode: 8] [A: 8] [B: 8] [C: 8]`.
<td><code>int_regs[A] =</code> amount; scope reg in <code>B</code> (<code>0xFF</code> = unscoped)</td>
</tr>
<tr>
<td colspan="7" align="center"><em>0x26..0x2F reserved</em></td>
<td>38</td><td><code>0x26</code></td><td><strong>DELETE_ACCOUNT_META</strong></td>
<td>acc</td><td>key</td><td>scope</td>
<td>Calls <code>Store.DeleteAccountMetadata</code> for account <code>A</code>, key <code>B</code>, scope <code>C</code> (<code>0xFF</code> = unscoped). Not allowed while a mark is open</td>
</tr>
<tr>
<td colspan="7" align="center"><em>0x27..0x2F reserved</em></td>
</tr>
</tbody>
</table>
Expand Down
21 changes: 17 additions & 4 deletions internal/analysis/check.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,9 @@ type VarOriginFnCallResolution struct {
VersionConstraints []VersionClause
}
type StatementFnCallResolution struct {
Params []string
Docs string
Params []string
Docs string
VersionConstraints []VersionClause
}

func (VarOriginFnCallResolution) ContextName() string { return "variable origin" }
Expand All @@ -57,8 +58,9 @@ func (r StatementFnCallResolution) GetParams() []string { return r.Params }

const (
// Statement fns
FnSetTxMeta = builtins.SetTxMeta
FnSetAccountMeta = builtins.SetAccountMeta
FnSetTxMeta = builtins.SetTxMeta
FnSetAccountMeta = builtins.SetAccountMeta
FnDeleteAccountMetadata = builtins.DeleteAccountMetadata

// Expr fns
FnVarOriginMeta = builtins.Meta
Expand All @@ -78,6 +80,16 @@ var Builtins = map[string]FnCallResolution{
Params: []string{TypeAccount, TypeString, TypeAny},
Docs: "set account metadata",
},
FnDeleteAccountMetadata: StatementFnCallResolution{
Params: []string{TypeAccount, TypeString},
Docs: "delete account metadata",
VersionConstraints: []VersionClause{
{
Version: parser.NewVersionInterpreter(0, 1, 4),
FeatureFlag: flags.ExperimentalDeleteAccountMetadata,
},
},
},
FnVarOriginMeta: VarOriginFnCallResolution{
Params: []string{TypeAccount, TypeString},
Return: TypeAny,
Expand Down Expand Up @@ -363,6 +375,7 @@ func (res *CheckResult) checkStatement(statement parser.Statement) {
if ok {
if varOrigin, ok := resolution.(StatementFnCallResolution); ok {
res.fnCallResolution[statement.Caller] = varOrigin
res.requireVersion(statement.Range, varOrigin.VersionConstraints...)
}
}

Expand Down
79 changes: 79 additions & 0 deletions internal/analysis/check_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1216,3 +1216,82 @@ func TestIncompleteVarOriginDoesNotPanic(t *testing.T) {
})
}
}

func TestRequireFlagForDeleteAccountMetadata(t *testing.T) {
t.Parallel()

input := `
// @version interpreter 0.1.4
delete_account_metadata(@acc, "k")
`

require.Equal(t,
[]analysis.Diagnostic{
{
Range: parser.RangeOfIndexed(input, `delete_account_metadata(@acc, "k")`, 0),
Kind: analysis.ExperimentalFeature{
Name: flags.ExperimentalDeleteAccountMetadata,
},
},
},
checkSource(input),
)
}

func TestRequireVersionForDeleteAccountMetadata(t *testing.T) {
t.Parallel()

input := `
// @version interpreter 0.1.3
delete_account_metadata(@acc, "k")
`

require.Equal(t,
[]analysis.Diagnostic{
{
Range: parser.RangeOfIndexed(input, `delete_account_metadata(@acc, "k")`, 0),
Kind: analysis.VersionMismatch{
GotVersion: parser.NewVersionInterpreter(0, 1, 3),
RequiredVersion: parser.NewVersionInterpreter(0, 1, 4),
},
},
},
checkSource(input),
)
}

func TestDeleteAccountMetadataWithFlag(t *testing.T) {
t.Parallel()

input := `
// @version interpreter 0.1.4
// @feature_flag experimental-delete-account-metadata
delete_account_metadata(@acc, "k")
`

require.Empty(t, checkSource(input))
}

func TestDeleteAccountMetadataSignature(t *testing.T) {
t.Parallel()

input := `delete_account_metadata("acc", @k, @extra)`

require.Equal(t,
[]analysis.Diagnostic{
{
Range: parser.RangeOfIndexed(input, "@extra", 0),
Kind: analysis.BadArity{Expected: 2, Actual: 3},
},
{
Range: parser.RangeOfIndexed(input, `"acc"`, 0),
Kind: analysis.TypeMismatch{Expected: analysis.TypeAccount, Got: analysis.TypeString},
},
{
Range: parser.RangeOfIndexed(input, "@k", 0),
Kind: analysis.TypeMismatch{Expected: analysis.TypeString, Got: analysis.TypeAccount},
},
},
checkSource(input),
)
}
Loading
Loading