From 90bd10f7422844e722a222f59572d058b370efb9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 13 Aug 2026 06:57:49 +0000 Subject: [PATCH 1/7] Add language features for self-hosting: import/export defs, int<->pointer casts, heap builtins - 'import def name(params) type;' declares a host function imported from WASM module 'env' - 'export def' exports a function from the compiled module - int<->pointer casts allow user code to implement heap allocators - pointer-to-struct cast syntax: 'Token~(expr)' - __heap_end__/__grow_heap__ builtins wrap memory.size/memory.grow - address-of now supports struct members and dereferences (&p~.member) - CLI tools: puffc (compile), wat2wasm (assemble), run (execute with env imports) - turn off WAT debug comments to keep output manageable for large programs Co-authored-by: Andrew Chan --- src/backend.ts | 49 +++++++-- src/nodes.ts | 54 +++++++++- src/parser.ts | 126 ++++++++++++++++++----- src/resolver.ts | 7 +- src/tokens.ts | 4 + test.ts | 251 +++++++++++++++++++++++++++++++++++++++++++++- tools/puffc.ts | 37 +++++++ tools/run.ts | 135 +++++++++++++++++++++++++ tools/wat2wasm.ts | 33 ++++++ 9 files changed, 655 insertions(+), 41 deletions(-) create mode 100644 tools/puffc.ts create mode 100644 tools/run.ts create mode 100644 tools/wat2wasm.ts diff --git a/src/backend.ts b/src/backend.ts index 55d2e9f..0da875a 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -91,7 +91,7 @@ function isVariableInRegister(symbol: ast.VariableSymbol | ast.ParamSymbol): boo return type !== null && ast.isScalar(type) } -const DEBUG_COMMENTS = true +const DEBUG_COMMENTS = false interface LoopLabel { outerLabel: string @@ -709,7 +709,8 @@ export function emit(context: ast.Context): string { break } case ast.TypeCategory.BOOL: - case ast.TypeCategory.INT: { + case ast.TypeCategory.INT: + case ast.TypeCategory.POINTER: { // no conversion needed break } @@ -743,7 +744,8 @@ export function emit(context: ast.Context): string { break } case ast.TypeCategory.POINTER: { - if (op.value.resolvedType?.category === ast.TypeCategory.POINTER) { + if (op.value.resolvedType?.category === ast.TypeCategory.POINTER || + op.value.resolvedType?.category === ast.TypeCategory.INT) { // no conversions needed } else { throw new Error(`Unexpected type ${ast.typeToString(op.type)} for cast source`) @@ -991,8 +993,11 @@ export function emit(context: ast.Context): string { } break } - case ast.NodeKind.INDEX_EXPR: { + case ast.NodeKind.INDEX_EXPR: + case ast.NodeKind.DOT_EXPR: + case ast.NodeKind.DEREF_EXPR: { // `&arr[i]` should return address of the ith element in `arr`. + // `&s.member` and `&p~` similarly return addresses of their operands. visit(op.value, ExprMode.LVALUE) break } @@ -1063,8 +1068,8 @@ export function emit(context: ast.Context): string { break } localLocs = new Map() - if (op.name.lexeme === "main") { - line(`(func ${wasmId("main")} (export "main")`) + if (op.name.lexeme === "main" || op.isExported) { + line(`(func ${wasmId(op.name.lexeme)} (export "${op.name.lexeme}")`) } else { line(`(func ${wasmId(op.name.lexeme)}`) } @@ -1290,6 +1295,23 @@ export function emit(context: ast.Context): string { line(`(import "io" "puti" (func ${wasmId("__puti__")} (param i32)))`) line(`(import "io" "flush" (func ${wasmId("__flush__")}))`) + // Imports must precede all non-import definitions in the module. + context.topLevelStatements.forEach((statement) => { + if (statement.kind === ast.NodeKind.FUNCTION_STMT) { + const fn = statement as ast.FunctionStmt + if (fn.hostModule !== null) { + let sig = "" + fn.params.forEach((param) => { + sig += ` (param ${registerType(param.type)})` + }) + if (!ast.isEqual(fn.returnType, ast.VoidType)) { + sig += ` (result ${registerType(fn.returnType)})` + } + line(`(import "${fn.hostModule}" "${fn.name.lexeme}" (func ${wasmId(fn.name.lexeme)}${sig}))`) + } + } + }) + line(`(memory $memory ${INITIAL_PAGES})`) line(`(global ${wasmId("__stack_ptr__")} (mut i32) i32.const ${STACK_TOP_BYTE_OFFSET})`) @@ -1402,6 +1424,21 @@ export function emit(context: ast.Context): string { } line(`)`) + line(`(func ${wasmId("__heap_end__")} (result i32)`) + { + line(`memory.size`) + line(`i32.const 65536`) + line(`i32.mul`) + } + line(`)`) + + line(`(func ${wasmId("__grow_heap__")} (param $numPages i32) (result i32)`) + { + line(`local.get $numPages`) + line(`memory.grow`) + } + line(`)`) + context.topLevelStatements.forEach((statement) => { visit(statement) }) diff --git a/src/nodes.ts b/src/nodes.ts index 6ae934f..619adeb 100644 --- a/src/nodes.ts +++ b/src/nodes.ts @@ -504,6 +504,14 @@ export function canCast(from: Type, to: Type): boolean { // Pointers are a type escape hatch and can always be casted to/from each other. return true } + if (from.category === TypeCategory.INT && to.category === TypeCategory.POINTER) { + // Allow int-to-pointer casts so user code can implement allocators. + return true + } + if (from.category === TypeCategory.POINTER && to.category === TypeCategory.INT) { + // Allow pointer-to-int casts for pointer bookkeeping (e.g. alignment). + return true + } return isEqual(from, to) } @@ -702,6 +710,12 @@ export interface FunctionStmt extends Node { block: Stmt[] scope: Scope } | null // "null" means the function is imported or built-in + // For functions imported from the host environment, the module + // to import from (e.g. `(import "env" "getchar" ...)`). + // null for normal functions and compiler built-ins. + hostModule: string | null + // Whether the function should be exported from the WASM module. + isExported: boolean symbol: FunctionSymbol | null // filled in by parser // After resolve pass, `hoistedLocals` should contain // all local variables declared in descendant scopes @@ -711,13 +725,14 @@ export interface FunctionStmt extends Node { } export function functionStmt( - { name, params, returnType, block, scope, symbol }: { + { name, params, returnType, block, scope, symbol, isExported }: { name: Token; params: Param[]; returnType: Type; block: Stmt[]; scope: Scope; - symbol: FunctionSymbol | null + symbol: FunctionSymbol | null; + isExported?: boolean }): FunctionStmt { return { kind: NodeKind.FUNCTION_STMT, @@ -728,17 +743,20 @@ export function functionStmt( block, scope }, + hostModule: null, + isExported: isExported ?? false, symbol, hoistedLocals: null } } export function importedFunctionStmt( - { name, params, returnType, symbol }: { + { name, params, returnType, symbol, hostModule }: { name: Token; params: Param[]; returnType: Type; - symbol: FunctionSymbol | null + symbol: FunctionSymbol | null; + hostModule?: string }): FunctionStmt { return { kind: NodeKind.FUNCTION_STMT, @@ -746,6 +764,8 @@ export function importedFunctionStmt( params, returnType, body: null, + hostModule: hostModule ?? null, + isExported: false, symbol, hoistedLocals: null } @@ -987,8 +1007,29 @@ export class Context { symbol: null }) sqrt.symbol = this.functionSymbol(sqrt) + const heapEnd = importedFunctionStmt({ + name: fakeToken(TokenType.IDENTIFIER, "__heap_end__"), + params: [], + returnType: IntType, + symbol: null + }) + heapEnd.symbol = this.functionSymbol(heapEnd) + const grow = importedFunctionStmt({ + name: fakeToken(TokenType.IDENTIFIER, "__grow_heap__"), + params: [ + { + name: fakeToken(TokenType.IDENTIFIER, "numPages"), + type: IntType + }, + ], + returnType: IntType, + symbol: null + }) + grow.symbol = this.functionSymbol(grow) this.global.define(memcpy.name.lexeme, memcpy.symbol) this.global.define(sqrt.name.lexeme, sqrt.symbol) + this.global.define(heapEnd.name.lexeme, heapEnd.symbol) + this.global.define(grow.name.lexeme, grow.symbol) } variableSymbol(node: VarStmt, isGlobal: boolean): VariableSymbol { @@ -1220,6 +1261,11 @@ export function astToSExpr(node: Node): string { case NodeKind.FUNCTION_STMT: { const op = node as FunctionStmt out += "(" + if (op.hostModule !== null) { + out += "import " + } else if (op.isExported) { + out += "export " + } out += `def ${op.name.lexeme} ` out += "(" op.params.forEach((param, i) => { diff --git a/src/parser.ts b/src/parser.ts index b4db65f..71a83f9 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -116,14 +116,65 @@ export function parse(tokens: Token[], reportError: ReportError): ast.Context { } function topDecl(): ast.TopStmt { - if (match(TokenType.DEF)) return funDecl() + if (match(TokenType.IMPORT)) { + consume(TokenType.DEF, "Expect 'def' after 'import'.") + return importDecl() + } + if (match(TokenType.EXPORT)) { + consume(TokenType.DEF, "Expect 'def' after 'export'.") + return funDecl(/* isExported */ true) + } + if (match(TokenType.DEF)) return funDecl(/* isExported */ false) if (match(TokenType.STRUCT)) return structDecl() if (match(TokenType.VAR)) return varDecl() throw parseError("Only variable declarations and function definitions allowed at the top-level.") } - function funDecl(): ast.FunctionStmt { + function importDecl(): ast.FunctionStmt { + const name = consume(TokenType.IDENTIFIER, "Expect identifier after 'def'.") + + consume(TokenType.LEFT_PAREN, "Expect '(' after function name.") + const params: ast.Param[] = [] + while (!check(TokenType.RIGHT_PAREN) && !isAtEnd()) { + if (params.length > 0) { + consume(TokenType.COMMA, "Missing comma after parameter.") + } + const paramName = consume(TokenType.IDENTIFIER, "Expect identifier.") + const paramType = type() + params.push({ + name: paramName, + type: paramType + }) + } + consume(TokenType.RIGHT_PAREN, "Expect ')' after parameters.") + + let returnType: ast.Type = ast.VoidType + if (!check(TokenType.SEMICOLON)) { + returnType = type() + } + consume(TokenType.SEMICOLON, "Expect ';' after import declaration.") + + const node = ast.importedFunctionStmt({ + name, + params, + returnType, + symbol: null, + hostModule: "env" + }) + const outerScope = peekScope() + if (outerScope.hasDirect(name.lexeme)) { + // Throw; we want to ignore this function and synchronize to next statement + throw parseErrorForToken(name, `'${name.lexeme}' is already declared in this scope.`) + } else { + const symbol = context.functionSymbol(node) + outerScope.define(name.lexeme, symbol) + node.symbol = symbol + } + return node + } + + function funDecl(isExported: boolean): ast.FunctionStmt { const name = consume(TokenType.IDENTIFIER, "Expect identifier after 'def'.") consume(TokenType.LEFT_PAREN, "Expect '(' after function name.") @@ -167,7 +218,8 @@ export function parse(tokens: Token[], reportError: ReportError): ast.Context { returnType, block: statements, scope, - symbol: null + symbol: null, + isExported }) const outerScope = peekScope() if (outerScope.hasDirect(name.lexeme)) { @@ -737,6 +789,46 @@ export function parse(tokens: Token[], reportError: ReportError): ast.Context { return expr } + // Returns true if the token stream looks like the start of a cast to + // a pointer-to-struct type, e.g. `Foo~(expr)` or `Foo~~(expr)`. + function checkStructPtrCast(): boolean { + if (!check(TokenType.IDENTIFIER)) { + return false + } + let i = current + 1 + while (i < tokens.length && tokens[i].type === TokenType.TILDE) { + i++ + } + return i > current + 1 && i < tokens.length && tokens[i].type === TokenType.LEFT_PAREN + } + + function castPrimary(): ast.Expr { + // cast expression + // TODO: allow pointers to arrays + const castType = type() + switch (castType.category) { + case ast.TypeCategory.INT: + case ast.TypeCategory.FLOAT: + case ast.TypeCategory.BYTE: + case ast.TypeCategory.BOOL: + case ast.TypeCategory.POINTER: { + break + } + default: { + throw parseError("Cannot cast to this type.") + } + } + consume(TokenType.LEFT_PAREN, "Expect '(' after type in cast expression.") + const paren = previous() + const value = expression() + consume(TokenType.RIGHT_PAREN, "Expect ')' after cast expression.") + return ast.castExpr({ + token: paren, + type: castType, + value + }) + } + function exprPrimary(): ast.Expr { if (match(TokenType.TRUE) || match(TokenType.FALSE)) { return ast.literalExpr({ @@ -786,6 +878,9 @@ export function parse(tokens: Token[], reportError: ReportError): ast.Context { type: ast.ByteType }) } + if (checkStructPtrCast()) { + return castPrimary() + } if (match(TokenType.IDENTIFIER)) { return ast.variableExpr({ name: previous() @@ -830,30 +925,7 @@ export function parse(tokens: Token[], reportError: ReportError): ast.Context { } if (check(TokenType.INT) || check(TokenType.FLOAT) || check(TokenType.BYTE) || check(TokenType.BOOL)) { - // cast expression - // TODO: allow pointers to arrays - const castType = type() - switch (castType.category) { - case ast.TypeCategory.INT: - case ast.TypeCategory.FLOAT: - case ast.TypeCategory.BYTE: - case ast.TypeCategory.BOOL: - case ast.TypeCategory.POINTER: { - break - } - default: { - throw parseError("Cannot cast to this type.") - } - } - consume(TokenType.LEFT_PAREN, "Expect '(' after type in cast expression.") - const paren = previous() - const value = expression() - consume(TokenType.RIGHT_PAREN, "Expect ')' after cast expression.") - return ast.castExpr({ - token: paren, - type: castType, - value - }) + return castPrimary() } if (match(TokenType.LEN)) { diff --git a/src/resolver.ts b/src/resolver.ts index 6679803..209bbd0 100644 --- a/src/resolver.ts +++ b/src/resolver.ts @@ -266,6 +266,7 @@ export function resolve(context: ast.Context, reportError: ReportError) { } case ast.NodeKind.CAST_EXPR: { const op = node as ast.CastExpr + op.type = resolveType(op.type) resolveNode(op.value, isLiveAtEnd) if (!ast.canCast(op.value.resolvedType!, op.type)) { resolveError(op.token, `Cannot cast from ${ast.typeToString(op.value.resolvedType!)} to ${ast.typeToString(op.type)}.`) @@ -446,7 +447,11 @@ export function resolve(context: ast.Context, reportError: ReportError) { category: ast.TypeCategory.POINTER, elementType: op.value.resolvedType! } - } else if (op.value.kind === ast.NodeKind.INDEX_EXPR) { + } else if ( + op.value.kind === ast.NodeKind.INDEX_EXPR || + op.value.kind === ast.NodeKind.DOT_EXPR || + op.value.kind === ast.NodeKind.DEREF_EXPR + ) { op.resolvedType = { category: ast.TypeCategory.POINTER, elementType: op.value.resolvedType! diff --git a/src/tokens.ts b/src/tokens.ts index 40a9797..f04f7e9 100644 --- a/src/tokens.ts +++ b/src/tokens.ts @@ -53,10 +53,12 @@ export enum TokenType { CONTINUE, DEF, ELSE, + EXPORT, FALSE, FOR, FLOAT, IF, + IMPORT, INT, LEN, PRINT, @@ -122,10 +124,12 @@ export const TokenPattern: Readonly> = { [TokenType.CONTINUE]: /continue/y, [TokenType.DEF]: /def/y, [TokenType.ELSE]: /else/y, + [TokenType.EXPORT]: /export/y, [TokenType.FALSE]: /false/y, [TokenType.FOR]: /for/y, [TokenType.FLOAT]: /float/y, [TokenType.IF]: /if/y, + [TokenType.IMPORT]: /import/y, [TokenType.INT]: /int/y, [TokenType.LEN]: /len/y, [TokenType.PRINT]: /print/y, diff --git a/test.ts b/test.ts index 263a2c1..6403f5e 100644 --- a/test.ts +++ b/test.ts @@ -54,7 +54,14 @@ function expectErrors(source: string, expectedErrors: string[], passes: Passes): expect(errors).toEqual(expectedErrors) return context } -async function expectOutput(source: string, expectedOutput: string) { +class ExitCalled extends Error { + code: number + constructor(code: number) { + super(`exit ${code}`) + this.code = code + } +} +async function expectOutput(source: string, expectedOutput: string, stdin: string = "", expectedExitCode: number = 0) { const context = expectErrors(source, [], Passes.THROUGH_RESOLVE) if (context) { const code = emit(context) @@ -62,6 +69,8 @@ async function expectOutput(source: string, expectedOutput: string) { child_process.execSync(`npx -p wabt wat2wasm test/tmp.wat -o test/tmp.wasm`) const codec = new UTF8Codec() + const stdinBuf = codec.encodeString(stdin) + let stdinPos = 0 let ioBuffer = "" let output = "" @@ -83,12 +92,36 @@ async function expectOutput(source: string, expectedOutput: string) { output += ioBuffer + "\n" ioBuffer = "" } + }, + env: { + getchar: (): number => { + return stdinPos < stdinBuf.length ? stdinBuf[stdinPos++] : -1 + }, + putchar: (c: number) => { + output += codec.decodeASCIIChar(c & 0xFF) + }, + puterr: (c: number) => { + output += codec.decodeASCIIChar(c & 0xFF) + }, + exit: (code: number) => { + throw new ExitCalled(code) + } } }); const exports = instance.instance.exports as any - exports.__init_globals__() - exports.main() + let exitCode = 0 + try { + exports.__init_globals__() + exports.main() + } catch (e) { + if (e instanceof ExitCalled) { + exitCode = e.code + } else { + throw e + } + } expect(output).toBe(expectedOutput) + expect(exitCode).toBe(expectedExitCode) } } @@ -425,6 +458,81 @@ describe("parser", () => { "18: Expect expression.", ]) }) + + test("Import and export declarations", () => { + expectAST(` + import def getchar() int; + import def putchar(c int); + export def main() {} + `, + "(" + + "(import def getchar () ()) " + + "(import def putchar ((param c int)) ()) " + + "(export def main () ())" + + ")") + + expectParseErrors(` + import getchar() int; + def main() {} + `, + [ + "1: Expect 'def' after 'import'." + ]) + + expectParseErrors(` + import def getchar() int + def main() {} + `, + [ + "2: Expect ';' after import declaration." + ]) + + expectParseErrors(` + export struct Point { x int, y int } + def main() {} + `, + [ + "1: Expect 'def' after 'export'." + ]) + + expectParseErrors(` + import def getchar() int; + def getchar() int { return -1; } + def main() {} + `, + [ + "2: 'getchar' is already declared in this scope." + ]) + + expectParseErrors(` + def main() { + import def getchar() int; + } + `, + [ + "2: Expect expression.", // at 'import' + "2: Expect expression." // after synchronizing to 'def' + ]) + }) + + test("Pointer cast syntax", () => { + expectAST(` + struct Token { pos int } + def main() { + var p = int~(1024); + var t = Token~(p); + var i = int(t); + } + `, + "(" + + "(struct Token ((pos int))) " + + "(def main () (" + + "(var p ((ptr int) 1024)) " + + "(var t ((ptr (struct (unresolved 'Token'))) p)) " + + "(var i (int t))" + + "))" + + ")") + }) }) describe("type checking", () => { @@ -1607,6 +1715,64 @@ describe("type checking", () => { "14: Cyclic member declaration for struct 'Bad1'.", ]) }) + + test("pointer casts", () => { + expectResolveErrors(` + struct Vec { x float, y float } + def main() { + var p = int~(1024); // ok: int-to-pointer + var v = Vec~(p); // ok: pointer-to-pointer + var i = int(v); // ok: pointer-to-int + var f = float~(p); // ok: pointer-to-pointer + var e1 = float~(1.5); // error! float-to-pointer + var e2 = int~(true); // error! bool-to-pointer + var e3 = float(p); // error! pointer-to-float + var e4 = Missing~(p); // error! unknown typename + } + `, + [ + "7: Cannot cast from float to float~.", + "8: Cannot cast from bool to int~.", + "9: Cannot cast from int~ to float.", + "10: Undefined typename 'Missing'.", + ]) + }) + + test("imports and builtins", () => { + expectResolveErrors(` + import def getchar() int; + import def putchar(c int); + def main() { + putchar(getchar()); + putchar(); // error! arity + var x int = getchar(3); // error! arity + var pages = __grow_heap__(1); + var end int = __heap_end__(); + } + `, + [ + "5: Expected 1 arguments but got 0 in call to putchar.", + "6: Expected 0 arguments but got 1 in call to getchar.", + ]) + }) + + test("address-of struct members and dereferences", () => { + expectResolveErrors(` + struct Point { x int, y int } + def main() { + var pt = Point{1, 2}; + var px = &pt.x; // ok + var pp = &pt; // ok + var py = &pp~.y; // ok + var pd = &pp~; // ok + px~ = 5; + var bad = &5; // error! + } + `, + [ + "9: Invalid operand for unary operator '&'.", + ]) + }) }) describe("end to end", () => { @@ -2868,6 +3034,85 @@ done [1, 0] `.trim() + "\n") }) + + test("imports, exports, and pointer casts", async () => { + await expectOutput(` + import def getchar() int; + import def putchar(c int); + + struct Point { x int, y int } + + export def heapStart() int { + return 1024*1024; + } + + def main() { + var p = Point~(int~(heapStart())); + p~.x = 3; + p~.y = 4; + var py = &p~.y; + py~ = 5; + print p~.x + p~.y; + print int(py) - int(p); + print int~(int(p)) == int~(heapStart()); + + // echo stdin to stdout, uppercasing lowercase ascii + var c = getchar(); + while (c >= 0) { + if (c >= int('a') && c <= int('z')) { + c = c - 32; + } + putchar(c); + c = getchar(); + } + } + `, + ` +8 +4 +1 +`.trim() + "\n" + "WASM!\n", + "wasm!\n") + }) + + test("heap builtins", async () => { + await expectOutput(` + def main() { + var initialPages = __heap_end__() / 65536; + print initialPages; + var res = __grow_heap__(2); + print res == initialPages; + print __heap_end__() / 65536 - initialPages; + + // write to and read from newly grown memory + var p = int~(__heap_end__() - 4); + p~ = 12345; + print p~; + } + `, + ` +128 +1 +2 +12345 +`.trim() + "\n") + }) + + test("exit builtin", async () => { + await expectOutput(` + import def exit(code int); + def main() { + print 1; + exit(42); + print 2; + } + `, + ` +1 +`.trim() + "\n", + "", + 42) + }) }) // TODO: string literals with non-ascii UTF-8 chars \ No newline at end of file diff --git a/tools/puffc.ts b/tools/puffc.ts new file mode 100644 index 0000000..016f4c3 --- /dev/null +++ b/tools/puffc.ts @@ -0,0 +1,37 @@ +// CLI driver for the TypeScript reference compiler. +// Usage: node dist/tools/puffc.js [-o out.wat] input1.puff [input2.puff ...] +// Multiple inputs are concatenated in order before compilation (poor-man's modules). +import fs from 'fs' +import { compile } from '../index' + +function main() { + const args = process.argv.slice(2) + let outFile: string | null = null + const inputs: string[] = [] + for (let i = 0; i < args.length; i++) { + if (args[i] === "-o") { + outFile = args[++i] + } else { + inputs.push(args[i]) + } + } + if (inputs.length === 0) { + console.error("usage: puffc [-o out.wat] input1.puff [input2.puff ...]") + process.exit(2) + } + const source = inputs.map((f) => fs.readFileSync(f, "utf8")).join("\n") + const result = compile(source) + if (result.errors.length > 0) { + for (const err of result.errors) { + console.error(err) + } + process.exit(1) + } + if (outFile !== null) { + fs.writeFileSync(outFile, result.program!) + } else { + process.stdout.write(result.program!) + } +} + +main() diff --git a/tools/run.ts b/tools/run.ts new file mode 100644 index 0000000..f92369a --- /dev/null +++ b/tools/run.ts @@ -0,0 +1,135 @@ +// Runs a compiled puffscript WASM module with the standard host environment: +// - io.*: the `print` statement machinery (writes lines to stdout) +// - env.getchar: reads bytes from stdin (or the file given by --stdin) +// - env.putchar: writes bytes to stdout +// - env.puterr: writes bytes to stderr +// - env.exit: terminates with the given exit code +// +// Usage: node dist/tools/run.js module.wasm [--stdin file] [--stdout file] +import fs from 'fs' +import { UTF8Codec } from '../src/util' + +class ExitError extends Error { + code: number + constructor(code: number) { + super(`exit ${code}`) + this.code = code + } +} + +async function main() { + const args = process.argv.slice(2) + let wasmFile: string | null = null + let stdinFile: string | null = null + let stdoutFile: string | null = null + for (let i = 0; i < args.length; i++) { + if (args[i] === "--stdin") { + stdinFile = args[++i] + } else if (args[i] === "--stdout") { + stdoutFile = args[++i] + } else { + wasmFile = args[i] + } + } + if (wasmFile === null) { + console.error("usage: run module.wasm [--stdin file] [--stdout file]") + process.exit(2) + } + + const input: Buffer = stdinFile !== null ? fs.readFileSync(stdinFile) : (() => { + try { + return fs.readFileSync(0) // stdin + } catch { + return Buffer.alloc(0) + } + })() + let inputPos = 0 + + const codec = new UTF8Codec() + let ioBuffer = "" + const stdoutChunks: number[] = [] + const stderrChunks: number[] = [] + + function flushAll() { + if (stdoutChunks.length > 0) { + const buf = Buffer.from(stdoutChunks) + if (stdoutFile !== null) { + fs.writeFileSync(stdoutFile, buf) + } else { + process.stdout.write(buf) + } + stdoutChunks.length = 0 + } + if (stderrChunks.length > 0) { + process.stderr.write(Buffer.from(stderrChunks)) + stderrChunks.length = 0 + } + } + + function pushString(target: number[], s: string) { + const bytes = codec.encodeString(s) + for (let i = 0; i < bytes.length; i++) { + target.push(bytes[i]) + } + } + + const imports = { + io: { + log: (x: any) => { + pushString(stdoutChunks, x + "\n") + }, + putchar: (x: number) => { + ioBuffer += codec.decodeASCIIChar(x) + }, + putf: (x: number) => { + ioBuffer += x + }, + puti: (x: number) => { + ioBuffer += x + }, + flush: () => { + pushString(stdoutChunks, ioBuffer + "\n") + ioBuffer = "" + } + }, + env: { + getchar: (): number => { + if (inputPos >= input.length) { + return -1 + } + return input[inputPos++] + }, + putchar: (c: number) => { + stdoutChunks.push(c & 0xFF) + }, + puterr: (c: number) => { + stderrChunks.push(c & 0xFF) + }, + exit: (code: number) => { + throw new ExitError(code) + } + } + } + + const instance = await WebAssembly.instantiate(fs.readFileSync(wasmFile), imports) + const exports = instance.instance.exports as any + let exitCode = 0 + try { + exports.__init_globals__() + exports.main() + } catch (e) { + if (e instanceof ExitError) { + exitCode = e.code + } else { + flushAll() + throw e + } + } + flushAll() + process.exit(exitCode) +} + +main().catch((e) => { + console.error(e.stack ?? e) + process.exit(1) +}) diff --git a/tools/wat2wasm.ts b/tools/wat2wasm.ts new file mode 100644 index 0000000..a2e60f5 --- /dev/null +++ b/tools/wat2wasm.ts @@ -0,0 +1,33 @@ +// Converts a WAT file to a WASM binary using the wabt JS API. +// Usage: node dist/tools/wat2wasm.js input.wat -o output.wasm +import fs from 'fs' +import wabtFactory from 'wabt' + +async function main() { + const args = process.argv.slice(2) + let outFile: string | null = null + let inFile: string | null = null + for (let i = 0; i < args.length; i++) { + if (args[i] === "-o") { + outFile = args[++i] + } else { + inFile = args[i] + } + } + if (inFile === null || outFile === null) { + console.error("usage: wat2wasm input.wat -o output.wasm") + process.exit(2) + } + const wabt = await wabtFactory() + const source = fs.readFileSync(inFile, "utf8") + const module = wabt.parseWat(inFile, source) + module.resolveNames() + module.validate() + const binary = module.toBinary({ log: false, write_debug_names: false }) + fs.writeFileSync(outFile, Buffer.from(binary.buffer)) +} + +main().catch((e) => { + console.error(e.message ?? e) + process.exit(1) +}) From a28259955747d0f268f23c458a902dea44c7a461 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 13 Aug 2026 07:03:08 +0000 Subject: [PATCH 2/7] Emit float literals from source lexemes; enlarge puff stack region to 4MB Both changes make WAT output reproducible by a self-hosted compiler: - float constants no longer depend on JS shortest-round-trip double formatting - the larger in-memory stack accommodates compiler workloads (string temporaries) Co-authored-by: Andrew Chan --- src/backend.ts | 33 ++++++++++++++++++++++++++++++--- src/nodes.ts | 6 +++++- src/parser.ts | 3 ++- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/src/backend.ts b/src/backend.ts index 0da875a..2a33b98 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -30,8 +30,8 @@ const codec = new UTF8Codec() // // Like in C and most other models, Puff's stack grows downwards. // The stack pointer is stored as $__stack_ptr__ WASM global. -const STACK_TOP_BYTE_OFFSET = 512*1024 -const DATA_TOP_BYTE_OFFSET = 1024*1024 +const STACK_TOP_BYTE_OFFSET = 4*1024*1024 +const DATA_TOP_BYTE_OFFSET = 8*1024*1024 const INITIAL_PAGES = (8*1024*1024) / (64*1024); @@ -46,6 +46,29 @@ function escapeString(str: string): string { return str.replace(/'/g, '\'').replace(/\\/g, '\\\\').replace(/"/g, '\"') } +// Formats a float literal for WAT output from its source lexeme, e.g. +// "5.50" => "5.5", "5." => "5", "3.14" => "3.14". +// Emitting from the lexeme (rather than the parsed numeric value) keeps +// the backend independent of host float-formatting behavior, which makes +// output reproducible by the self-hosted compiler. +function formatFloatLexeme(lexeme: string): string { + let out = lexeme + if (out.indexOf(".") >= 0) { + let end = out.length + while (end > 0 && out.charAt(end - 1) === "0") { + end-- + } + if (end > 0 && out.charAt(end - 1) === ".") { + end-- + } + out = out.substring(0, end) + } + if (out.length === 0) { + out = "0" + } + return out +} + // Returns the WASM type used to represent values of the given type in the WASM (host) stack. // This may be different than the WASM type representing the value in the Puff (in-memory) stack. // E.g. integer arrays are represented by sequences of i32 in the in-memory stack but only @@ -925,7 +948,11 @@ export function emit(context: ast.Context): string { break } case ast.TypeCategory.FLOAT: { - line(`f32.const ${op.value}`) + if (op.sourceLexeme !== null) { + line(`f32.const ${formatFloatLexeme(op.sourceLexeme)}`) + } else { + line(`f32.const ${op.value}`) + } break } default: { diff --git a/src/nodes.ts b/src/nodes.ts index 619adeb..3a63a55 100644 --- a/src/nodes.ts +++ b/src/nodes.ts @@ -244,14 +244,18 @@ export interface LiteralExpr extends Node { kind: NodeKind.LITERAL_EXPR value: any type: Type + // Original source lexeme for numeric literals, used to emit + // reproducible WAT float constants. null for synthesized literals. + sourceLexeme: string | null resolvedType: Type | null // filled in by resolver pass } -export function literalExpr({ value, type }: { value: any; type: Type }): LiteralExpr { +export function literalExpr({ value, type, sourceLexeme }: { value: any; type: Type; sourceLexeme?: string }): LiteralExpr { return { kind: NodeKind.LITERAL_EXPR, value, type, + sourceLexeme: sourceLexeme ?? null, resolvedType: null } } diff --git a/src/parser.ts b/src/parser.ts index 71a83f9..a6045d7 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -845,7 +845,8 @@ export function parse(tokens: Token[], reportError: ReportError): ast.Context { if (match(TokenType.NUMBER_DECIMAL)) { return ast.literalExpr({ value: previous().literal, - type: ast.FloatType + type: ast.FloatType, + sourceLexeme: previous().lexeme }) } if (match(TokenType.NUMBER_HEX)) { From ec4f1a8e808691f80fc9b5d6e3136b7a6651fb6d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 13 Aug 2026 07:12:43 +0000 Subject: [PATCH 3/7] Self-hosted compiler: runtime utilities and scanner - selfhost/util.puff: bump allocator over grown WASM memory, Vec/Buf/Str helpers - selfhost/scanner.puff: token definitions and scanner, differentially tested against the reference scanner (token streams and error messages match) - fix __memcpy__ builtin declaration to match its 3-arg WASM implementation - tools: dump.ts (reference-compiler debug dumps), build-selfhost.sh Co-authored-by: Andrew Chan --- selfhost/main_dev_tokens.puff | 34 ++ selfhost/scanner.puff | 626 ++++++++++++++++++++++++++++++++++ selfhost/util.puff | 328 ++++++++++++++++++ src/nodes.ts | 4 + tools/build-selfhost.sh | 23 ++ tools/dump.ts | 75 ++++ 6 files changed, 1090 insertions(+) create mode 100644 selfhost/main_dev_tokens.puff create mode 100644 selfhost/scanner.puff create mode 100644 selfhost/util.puff create mode 100755 tools/build-selfhost.sh create mode 100644 tools/dump.ts diff --git a/selfhost/main_dev_tokens.puff b/selfhost/main_dev_tokens.puff new file mode 100644 index 0000000..4413d73 --- /dev/null +++ b/selfhost/main_dev_tokens.puff @@ -0,0 +1,34 @@ +// Development driver: dumps the token stream in the same format as +// `tools/dump.ts --tokens` for differential testing. + +var gOut Buf~ = bufNewWithCap(65536); + +def main() { + gSource = readAllInput(); + scanTokens(); + if (numErrors() == 0) { + for (var i = 0; i < numTokens(); i = i + 1) { + var t = tokenAt(i); + bufPushInt(gOut, t~.type); + bufPushChar(gOut, 32); + bufPushInt(gOut, t~.offset); + bufPushChar(gOut, 32); + bufPushInt(gOut, t~.lexLen); + if (t~.type == T_NUMBER || t~.type == T_NUMBER_HEX) { + bufPushChar(gOut, 32); + bufPushBytes(gOut, t~.decStr~.data, t~.decStr~.count); + } + bufPushChar(gOut, 10); + } + } + writeBufTo(gOut, false); + for (var i = 0; i < gErrors~.count; i = i + 1) { + var e = Str~(vecGet(gErrors, i)); + writeStrTo(e, true); + puterr(10); + } + if (numErrors() > 0) { + exit(1); + } + exit(0); +} diff --git a/selfhost/scanner.puff b/selfhost/scanner.puff new file mode 100644 index 0000000..7bad815 --- /dev/null +++ b/selfhost/scanner.puff @@ -0,0 +1,626 @@ +// scanner.puff +// Token definitions and scanner, ported from src/tokens.ts + src/scanner.ts. + +// TokenType constants. NOTE: values must mirror the TokenType enum in +// src/tokens.ts, and are ordered so that a scanner matching tokens to input +// will choose longer tokens when input can match multiple tokens. +var T_IDENTIFIER = 0; +var T_STRING = 1; +var T_SINGLE_QUOTE_STRING = 2; +var T_NUMBER_DECIMAL = 3; +var T_NUMBER_HEX = 4; +var T_NUMBER = 5; +var T_COMMENT = 6; +var T_LEFT_PAREN = 7; +var T_RIGHT_PAREN = 8; +var T_LEFT_BRACE = 9; +var T_RIGHT_BRACE = 10; +var T_LEFT_BRACKET = 11; +var T_RIGHT_BRACKET = 12; +var T_COMMA = 13; +var T_DOT = 14; +var T_SEMICOLON = 15; +var T_TILDE = 16; +var T_BANG_EQUAL = 17; +var T_BANG = 18; +var T_EQUAL_EQUAL = 19; +var T_EQUAL = 20; +var T_GREATER_EQUAL = 21; +var T_GREATER = 22; +var T_LESS_EQUAL = 23; +var T_LESS = 24; +var T_AMP_AMP = 25; +var T_AMP = 26; +var T_BAR_BAR = 27; +var T_MINUS_EQUAL = 28; +var T_MINUS = 29; +var T_PLUS_EQUAL = 30; +var T_PLUS = 31; +var T_SLASH_EQUAL = 32; +var T_SLASH = 33; +var T_STAR_EQUAL = 34; +var T_STAR = 35; +var T_PERCENT_EQUAL = 36; +var T_PERCENT = 37; +var T_BYTE = 38; +var T_BOOL = 39; +var T_BREAK = 40; +var T_CONTINUE = 41; +var T_DEF = 42; +var T_ELSE = 43; +var T_EXPORT = 44; +var T_FALSE = 45; +var T_FOR = 46; +var T_FLOAT = 47; +var T_IF = 48; +var T_IMPORT = 49; +var T_INT = 50; +var T_LEN = 51; +var T_PRINT = 52; +var T_RETURN = 53; +var T_STRUCT = 54; +var T_TRUE = 55; +var T_VAR = 56; +var T_VOID = 57; +var T_WHILE = 58; +var T_EOF = 59; + +// The program source. Set once by main before scanning. +var gSource Buf~ = bufNew(); + +struct Token { + type int, + // Lexeme: for tokens scanned from source, [lexStart, lexStart+lexLen) into + // gSource. Fake tokens store a static lexeme pointer instead. + // NOTE: like in the reference compiler, a token's lexeme need not equal + // source[offset..offset+lexLen); lexemes may be used to determine operator + // function while `offset` should only be used for error reporting. + lexPtr byte~, + lexLen int, + litInt int, // int/hex/char literal value (i32, wrapping) + litFloat float, // float literal value + litStr Str~, // string literal value (contents between quotes) + decStr Str~, // canonical decimal string for int/hex literal emission + offset int, + hasSource bool // false for fake tokens with no location provider +} + +def tokenNew(type int, lexPtr byte~, lexLen int, offset int) Token~ { + var t = Token~(alloc(32)); + t~.type = type; + t~.lexPtr = lexPtr; + t~.lexLen = lexLen; + t~.litStr = Str~(0); + t~.decStr = Str~(0); + t~.offset = offset; + t~.hasSource = true; + return t; +} + +def fakeToken(type int, lexeme Str~, locationProvider Token~) Token~ { + var t = Token~(alloc(32)); + t~.type = type; + t~.lexPtr = lexeme~.data; + t~.lexLen = lexeme~.count; + t~.litStr = Str~(0); + t~.decStr = Str~(0); + if (int(locationProvider) != 0) { + t~.offset = locationProvider~.offset; + t~.hasSource = locationProvider~.hasSource; + } else { + t~.offset = 0; + t~.hasSource = false; + } + return t; +} + +def tokenLexemeStr(t Token~) Str~ { + return strNew(t~.lexPtr, t~.lexLen); +} + +// Returns 1-based line number of the token. +def tokenLine(t Token~) int { + if (!t~.hasSource) { + return 1; + } + var line = 1; + for (var i = 0; i < t~.offset; i = i + 1) { + if (int((gSource~.data + i)~) == 10) { + line = line + 1; + } + } + return line; +} + +// Appends the token's source line (with an optional caret pointer line) +// to the given buffer. +def tokenLineStrTo(out Buf~, t Token~, showPointer bool) { + var src = gSource~.data; + var srcLen = gSource~.count; + var start = 0; + for (var i = 0; i < t~.offset; i = i + 1) { + if (int((src + i)~) == 10) { + start = i + 1; + } + } + var end = t~.offset + t~.lexLen; + for (var j = end; j < srcLen; j = j + 1) { + if (int((src + j)~) == 10) { + end = j; + break; + } + } + bufPushBytes(out, src + start, end - start); + if (showPointer) { + bufPushChar(out, 10); + for (var k = 0; k < t~.offset - start; k = k + 1) { + bufPushChar(out, 32); + } + bufPushChar(out, 94); // '^' + } +} + +// --------------------------------------------------------------------------- +// Error reporting (shared by all compiler passes) +// --------------------------------------------------------------------------- + +// List of Str~ error messages formatted as ": ". +var gErrors Vec~ = vecNew(); +// Scratch buffer for building the current error message. +var gErrBuf Buf~ = bufNew(); + +def errBegin(line int) { + gErrBuf~.count = 0; + bufPushInt(gErrBuf, line); + bufPushChar(gErrBuf, 58); // ':' + bufPushChar(gErrBuf, 32); // ' ' +} + +def errStr(s Str~) { + bufPushBytes(gErrBuf, s~.data, s~.count); +} + +def errRaw(p byte~, n int) { + bufPushBytes(gErrBuf, p, n); +} + +def errChar(c int) { + bufPushChar(gErrBuf, c); +} + +def errInt(x int) { + bufPushInt(gErrBuf, x); +} + +def errEnd() { + vecPush(gErrors, int(strFromBuf(gErrBuf))); +} + +def numErrors() int { + return gErrors~.count; +} + +// --------------------------------------------------------------------------- +// Keyword table +// --------------------------------------------------------------------------- + +// Str~ for each keyword, indexed by (tokenType - T_BYTE). +var gKeywords Vec~ = makeKeywords(); + +def kwAdd(t Vec~, p byte~, n int) { + vecPush(t, int(strCopy(p, n))); +} + +def makeKeywords() Vec~ { + var t = vecNew(); + var s0 = "byte"; kwAdd(t, &s0[0], len(s0)); + var s1 = "bool"; kwAdd(t, &s1[0], len(s1)); + var s2 = "break"; kwAdd(t, &s2[0], len(s2)); + var s3 = "continue"; kwAdd(t, &s3[0], len(s3)); + var s4 = "def"; kwAdd(t, &s4[0], len(s4)); + var s5 = "else"; kwAdd(t, &s5[0], len(s5)); + var s6 = "export"; kwAdd(t, &s6[0], len(s6)); + var s7 = "false"; kwAdd(t, &s7[0], len(s7)); + var s8 = "for"; kwAdd(t, &s8[0], len(s8)); + var s9 = "float"; kwAdd(t, &s9[0], len(s9)); + var s10 = "if"; kwAdd(t, &s10[0], len(s10)); + var s11 = "import"; kwAdd(t, &s11[0], len(s11)); + var s12 = "int"; kwAdd(t, &s12[0], len(s12)); + var s13 = "len"; kwAdd(t, &s13[0], len(s13)); + var s14 = "print"; kwAdd(t, &s14[0], len(s14)); + var s15 = "return"; kwAdd(t, &s15[0], len(s15)); + var s16 = "struct"; kwAdd(t, &s16[0], len(s16)); + var s17 = "true"; kwAdd(t, &s17[0], len(s17)); + var s18 = "var"; kwAdd(t, &s18[0], len(s18)); + var s19 = "void"; kwAdd(t, &s19[0], len(s19)); + var s20 = "while"; kwAdd(t, &s20[0], len(s20)); + return t; +} + +def keywordStr(tokenType int) Str~ { + return Str~(vecGet(gKeywords, tokenType - T_BYTE)); +} + +// --------------------------------------------------------------------------- +// Scanner +// --------------------------------------------------------------------------- + +def isDigit(c int) bool { + return c >= 48 && c <= 57; +} + +def isHexDigit(c int) bool { + return isDigit(c) || (c >= 97 && c <= 102) || (c >= 65 && c <= 70); +} + +def isAlpha(c int) bool { + return (c >= 97 && c <= 122) || (c >= 65 && c <= 90) || c == 95; +} + +def isAlphaNumeric(c int) bool { + return isAlpha(c) || isDigit(c); +} + +def isWhitespace(c int) bool { + // Mirrors the ASCII subset of the /\s/ regex class: + // space, \t, \n, \v, \f, \r + return c == 32 || c == 9 || c == 10 || c == 11 || c == 12 || c == 13; +} + +def srcByte(i int) int { + return int((gSource~.data + i)~); +} + +// Match helpers below return the length of the match starting at `pos`, +// or 0 if there is no match. They mirror the TokenPattern regexes. + +def matchIdentifier(pos int) int { + var n = gSource~.count; + if (pos >= n || !isAlpha(srcByte(pos))) { + return 0; + } + var i = pos + 1; + while (i < n && isAlphaNumeric(srcByte(i))) { + i = i + 1; + } + return i - pos; +} + +// Matches a quoted string with the given quote char (no escapes). +def matchQuoted(pos int, quote int) int { + var n = gSource~.count; + if (pos >= n || srcByte(pos) != quote) { + return 0; + } + var i = pos + 1; + while (i < n && srcByte(i) != quote) { + i = i + 1; + } + if (i >= n) { + return 0; // unterminated + } + return i + 1 - pos; +} + +// Matches /\d+\.\d*/ +def matchNumberDecimal(pos int) int { + var n = gSource~.count; + var i = pos; + while (i < n && isDigit(srcByte(i))) { + i = i + 1; + } + if (i == pos || i >= n || srcByte(i) != 46) { + return 0; + } + i = i + 1; + while (i < n && isDigit(srcByte(i))) { + i = i + 1; + } + return i - pos; +} + +// Matches /0x[a-fA-F0-9]+/ +def matchNumberHex(pos int) int { + var n = gSource~.count; + if (pos >= n || srcByte(pos) != 48) { + return 0; + } + if (pos + 1 >= n || srcByte(pos + 1) != 120) { + return 0; + } + var i = pos + 2; + while (i < n && isHexDigit(srcByte(i))) { + i = i + 1; + } + if (i == pos + 2) { + return 0; + } + return i - pos; +} + +// Matches /\d+/ +def matchNumber(pos int) int { + var n = gSource~.count; + var i = pos; + while (i < n && isDigit(srcByte(i))) { + i = i + 1; + } + return i - pos; +} + +// Matches /\/\/.*/ +def matchComment(pos int) int { + var n = gSource~.count; + if (pos + 1 >= n || srcByte(pos) != 47 || srcByte(pos + 1) != 47) { + return 0; + } + var i = pos + 2; + while (i < n && srcByte(i) != 10) { + i = i + 1; + } + return i - pos; +} + +// Matches a 1-char token. +def match1(pos int, c int) int { + if (pos < gSource~.count && srcByte(pos) == c) { + return 1; + } + return 0; +} + +// Matches a 2-char token. +def match2(pos int, c1 int, c2 int) int { + var n = gSource~.count; + if (pos + 1 < n && srcByte(pos) == c1 && srcByte(pos + 1) == c2) { + return 2; + } + return 0; +} + +// Tries to match the pattern for token type `t` at `pos`. +// Mirrors TokenPattern in src/tokens.ts for non-keyword token types. +def matchPattern(t int, pos int) int { + if (t == T_IDENTIFIER) { return matchIdentifier(pos); } + if (t == T_STRING) { return matchQuoted(pos, 34); } + if (t == T_SINGLE_QUOTE_STRING) { return matchQuoted(pos, 39); } + if (t == T_NUMBER_DECIMAL) { return matchNumberDecimal(pos); } + if (t == T_NUMBER_HEX) { return matchNumberHex(pos); } + if (t == T_NUMBER) { return matchNumber(pos); } + if (t == T_COMMENT) { return matchComment(pos); } + if (t == T_LEFT_PAREN) { return match1(pos, 40); } + if (t == T_RIGHT_PAREN) { return match1(pos, 41); } + if (t == T_LEFT_BRACE) { return match1(pos, 123); } + if (t == T_RIGHT_BRACE) { return match1(pos, 125); } + if (t == T_LEFT_BRACKET) { return match1(pos, 91); } + if (t == T_RIGHT_BRACKET) { return match1(pos, 93); } + if (t == T_COMMA) { return match1(pos, 44); } + if (t == T_DOT) { return match1(pos, 46); } + if (t == T_SEMICOLON) { return match1(pos, 59); } + if (t == T_TILDE) { return match1(pos, 126); } + if (t == T_BANG_EQUAL) { return match2(pos, 33, 61); } + if (t == T_BANG) { return match1(pos, 33); } + if (t == T_EQUAL_EQUAL) { return match2(pos, 61, 61); } + if (t == T_EQUAL) { return match1(pos, 61); } + if (t == T_GREATER_EQUAL) { return match2(pos, 62, 61); } + if (t == T_GREATER) { return match1(pos, 62); } + if (t == T_LESS_EQUAL) { return match2(pos, 60, 61); } + if (t == T_LESS) { return match1(pos, 60); } + if (t == T_AMP_AMP) { return match2(pos, 38, 38); } + if (t == T_AMP) { return match1(pos, 38); } + if (t == T_BAR_BAR) { return match2(pos, 124, 124); } + if (t == T_MINUS_EQUAL) { return match2(pos, 45, 61); } + if (t == T_MINUS) { return match1(pos, 45); } + if (t == T_PLUS_EQUAL) { return match2(pos, 43, 61); } + if (t == T_PLUS) { return match1(pos, 43); } + if (t == T_SLASH_EQUAL) { return match2(pos, 47, 61); } + if (t == T_SLASH) { return match1(pos, 47); } + if (t == T_STAR_EQUAL) { return match2(pos, 42, 61); } + if (t == T_STAR) { return match1(pos, 42); } + if (t == T_PERCENT_EQUAL) { return match2(pos, 37, 61); } + if (t == T_PERCENT) { return match1(pos, 37); } + return 0; +} + +// Parses the decimal digits at [start, start+length) as an i32 (wrapping). +def parseIntLexeme(start int, length int) int { + var v = 0; + for (var i = 0; i < length; i = i + 1) { + v = v * 10 + (srcByte(start + i) - 48); + } + return v; +} + +def hexDigitValue(c int) int { + if (isDigit(c)) { + return c - 48; + } + if (c >= 97) { + return c - 97 + 10; + } + return c - 65 + 10; +} + +// Parses the hex literal (starting with "0x") as an i32 (wrapping). +def parseHexLexeme(start int, length int) int { + var v = 0; + for (var i = 2; i < length; i = i + 1) { + v = v * 16 + hexDigitValue(srcByte(start + i)); + } + return v; +} + +// Parses the float literal digits as f32. +def parseFloatLexeme(start int, length int) float { + var v = 0.0; + var i = 0; + while (i < length && srcByte(start + i) != 46) { + v = v * 10.0 + float(srcByte(start + i) - 48); + i = i + 1; + } + i = i + 1; // skip '.' + var scale = 0.1; + while (i < length) { + v = v + float(srcByte(start + i) - 48) * scale; + scale = scale / 10.0; + i = i + 1; + } + return v; +} + +// Builds the canonical decimal string for the lexeme's numeric value. +// For decimal literals this is the lexeme without leading zeros; for hex +// literals we convert to the unsigned decimal representation (this matches +// how the reference compiler prints `parseInt(lexeme)` values). +def decStrForDecimal(start int, length int) Str~ { + var i = 0; + while (i < length - 1 && srcByte(start + i) == 48) { + i = i + 1; + } + return strCopy(gSource~.data + start + i, length - i); +} + +def decStrForHex(start int, length int) Str~ { + // Little-endian decimal digit accumulator; u32 values need at most 10 + // digits. Hex literals longer than 8 digits are reported as errors by the + // scanner (so their exact value never matters), but we still keep enough + // headroom to avoid overflowing on them. + var digits = [0; 16]; + var numDigits = 1; + for (var i = 2; i < length; i = i + 1) { + if (numDigits >= 15) { + break; + } + var carry = hexDigitValue(srcByte(start + i)); + for (var j = 0; j < numDigits; j = j + 1) { + var x = digits[j] * 16 + carry; + digits[j] = x % 10; + carry = x / 10; + } + while (carry > 0 && numDigits < 15) { + digits[numDigits] = carry % 10; + carry = carry / 10; + numDigits = numDigits + 1; + } + } + var b = bufNew(); + while (numDigits > 0) { + numDigits = numDigits - 1; + bufPushChar(b, 48 + digits[numDigits]); + } + return strFromBuf(b); +} + +// The scanned token stream (Vec of Token~). +var gTokens Vec~ = vecNew(); + +def tokensPush(t Token~) { + vecPush(gTokens, int(t)); +} + +def tokenAt(i int) Token~ { + return Token~(vecGet(gTokens, i)); +} + +def numTokens() int { + return gTokens~.count; +} + +// Scans gSource into gTokens. Ported from scanTokens in src/scanner.ts. +def scanTokens() { + var srcLen = gSource~.count; + var src = gSource~.data; + var current = 0; + while (current < srcLen) { + // 1. check if current lexeme is whitespace + if (isWhitespace(srcByte(current))) { + while (current < srcLen && isWhitespace(srcByte(current))) { + current = current + 1; + } + continue; + } + // 2. check if current lexeme matches a valid token (not including keywords) + var matched = false; + for (var t = T_IDENTIFIER; t < T_BYTE; t = t + 1) { + var mLen = matchPattern(t, current); + if (mLen > 0) { + if (t == T_IDENTIFIER) { + // Check if it's also a keyword and set token type accordingly + var outType = t; + for (var k = T_BYTE; k <= T_WHILE; k = k + 1) { + var kw = keywordStr(k); + if (kw~.count == mLen && strEqRaw(kw, src + current, mLen)) { + outType = k; + break; + } + } + tokensPush(tokenNew(outType, src + current, mLen, current)); + } else if (t == T_STRING) { + var tok = tokenNew(t, src + current, mLen, current); + tok~.litStr = strNew(src + current + 1, mLen - 2); + tokensPush(tok); + } else if (t == T_SINGLE_QUOTE_STRING) { + var tok2 = tokenNew(t, src + current, mLen, current); + tok2~.litStr = strNew(src + current + 1, mLen - 2); + tokensPush(tok2); + } else if (t == T_NUMBER_DECIMAL) { + var tok3 = tokenNew(t, src + current, mLen, current); + tok3~.litFloat = parseFloatLexeme(current, mLen); + tokensPush(tok3); + } else if (t == T_NUMBER) { + var tok4 = tokenNew(t, src + current, mLen, current); + tok4~.litInt = parseIntLexeme(current, mLen); + tok4~.decStr = decStrForDecimal(current, mLen); + tokensPush(tok4); + } else if (t == T_NUMBER_HEX) { + var tok5 = tokenNew(t, src + current, mLen, current); + tok5~.litInt = parseHexLexeme(current, mLen); + tok5~.decStr = decStrForHex(current, mLen); + tokensPush(tok5); + if (mLen > 2 + 8) { + errBegin(tokenLine(tok5)); + var m0 = "Hex literal does not fit in any numeric type."; + errRaw(&m0[0], len(m0)); + errEnd(); + } + } else if (t != T_COMMENT) { + // comments are skipped and not added as tokens + tokensPush(tokenNew(t, src + current, mLen, current)); + } + current = current + mLen; + // Lexemes are scanned to exactly 1 valid token + matched = true; + break; + } + } + if (matched) { + continue; + } + // 3. current lexeme is an invalid character. report error and advance. + var line = 1; + for (var i = 0; i < current; i = i + 1) { + if (srcByte(i) == 10) { + line = line + 1; + } + } + errBegin(line); + var m1 = "Unexpected character '"; + errRaw(&m1[0], len(m1)); + errChar(srcByte(current)); + var m2 = "'."; + errRaw(&m2[0], len(m2)); + errEnd(); + current = current + 1; + } + + var eofLex = strEmpty(); + var eof = Token~(alloc(32)); + eof~.type = T_EOF; + eof~.lexPtr = eofLex~.data; + eof~.lexLen = 0; + eof~.litStr = Str~(0); + eof~.decStr = Str~(0); + eof~.offset = current; + eof~.hasSource = true; + tokensPush(eof); +} diff --git a/selfhost/util.puff b/selfhost/util.puff new file mode 100644 index 0000000..2330ffc --- /dev/null +++ b/selfhost/util.puff @@ -0,0 +1,328 @@ +// util.puff +// Runtime support for the self-hosted puffscript compiler: +// host imports, heap allocator, growable vectors and byte buffers, +// string helpers, and number parsing/printing. + +import def getchar() int; +import def putchar(c int); +import def puterr(c int); +import def exit(code int); + +// --------------------------------------------------------------------------- +// Heap +// +// The puff memory layout puts the in-memory stack at [0, 4MB) and static data +// at [4MB, 8MB). Memory beyond the initial 8MB is all ours; we implement a +// simple bump allocator over it, growing the WASM memory as needed. +// Fresh WASM memory is always zeroed, and we never free, so allocations are +// guaranteed zero-initialized. +// --------------------------------------------------------------------------- + +var heapPtr = __heap_end__(); + +def alloc(numBytes int) byte~ { + // align allocations to 8 bytes + var n = (numBytes + 7) / 8 * 8; + var p = heapPtr; + while (p + n > __heap_end__()) { + var deficit = p + n - __heap_end__(); + var pagesNeeded = (deficit + 65535) / 65536; + if (pagesNeeded < 16) { + pagesNeeded = 16; + } + if (__grow_heap__(pagesNeeded) < 0) { + die(70); // out of memory + } + } + heapPtr = p + n; + return byte~(p); +} + +// Aborts the program, writing "internal error " to stderr. +def die(code int) { + var msg = "internal error "; + var i = 0; + while (i < len(msg)) { + puterr(int(msg[i])); + i = i + 1; + } + // print code digits + if (code < 0) { + puterr(45); // '-' + code = -code; + } + var scale = 1; + while (code / scale >= 10) { + scale = scale * 10; + } + while (scale > 0) { + puterr(48 + code / scale % 10); + scale = scale / 10; + } + puterr(10); + exit(101); +} + +def memCopy(src byte~, dst byte~, numBytes int) { + __memcpy__(src, dst, numBytes); +} + +// --------------------------------------------------------------------------- +// Vec: growable array of i32 (also used for pointers via casts) +// --------------------------------------------------------------------------- + +struct Vec { + data int~, + count int, + cap int +} + +def vecNew() Vec~ { + var v = Vec~(alloc(12)); + v~.cap = 8; + v~.count = 0; + v~.data = int~(alloc(v~.cap * 4)); + return v; +} + +def vecGrow(v Vec~, newCap int) { + if (newCap <= v~.cap) { + return; + } + var nd = int~(alloc(newCap * 4)); + memCopy(byte~(v~.data), byte~(nd), v~.count * 4); + v~.data = nd; + v~.cap = newCap; +} + +def vecPush(v Vec~, x int) { + if (v~.count == v~.cap) { + vecGrow(v, v~.cap * 2); + } + (v~.data + v~.count)~ = x; + v~.count = v~.count + 1; +} + +def vecPop(v Vec~) int { + v~.count = v~.count - 1; + return (v~.data + v~.count)~; +} + +def vecGet(v Vec~, i int) int { + if (i < 0 || i >= v~.count) { + die(71); // vec index out of bounds + } + return (v~.data + i)~; +} + +def vecSet(v Vec~, i int, x int) { + if (i < 0 || i >= v~.count) { + die(72); // vec index out of bounds + } + (v~.data + i)~ = x; +} + +def vecPeek(v Vec~) int { + return vecGet(v, v~.count - 1); +} + +// --------------------------------------------------------------------------- +// Buf: growable byte buffer (used for strings and program output) +// --------------------------------------------------------------------------- + +struct Buf { + data byte~, + count int, + cap int +} + +def bufNew() Buf~ { + return bufNewWithCap(16); +} + +def bufNewWithCap(cap int) Buf~ { + var b = Buf~(alloc(12)); + b~.cap = cap; + b~.count = 0; + b~.data = alloc(cap); + return b; +} + +def bufGrow(b Buf~, newCap int) { + if (newCap <= b~.cap) { + return; + } + var nd = alloc(newCap); + memCopy(b~.data, nd, b~.count); + b~.data = nd; + b~.cap = newCap; +} + +def bufPushByte(b Buf~, c byte) { + if (b~.count == b~.cap) { + bufGrow(b, b~.cap * 2); + } + (b~.data + b~.count)~ = c; + b~.count = b~.count + 1; +} + +def bufPushChar(b Buf~, c int) { + bufPushByte(b, byte(c)); +} + +def bufPushBytes(b Buf~, p byte~, n int) { + if (b~.count + n > b~.cap) { + var newCap = b~.cap * 2; + while (b~.count + n > newCap) { + newCap = newCap * 2; + } + bufGrow(b, newCap); + } + memCopy(p, b~.data + b~.count, n); + b~.count = b~.count + n; +} + +def bufGetByte(b Buf~, i int) byte { + if (i < 0 || i >= b~.count) { + die(73); // buf index out of bounds + } + return (b~.data + i)~; +} + +// Appends the signed decimal representation of x. +def bufPushInt(b Buf~, x int) { + if (x == 0) { + bufPushChar(b, 48); // '0' + return; + } + // Handle INT32_MIN (cannot be negated): x / 10 and x % 10 are safe on it. + var digits = [byte(0); 12]; + var numDigits = 0; + if (x < 0) { + bufPushChar(b, 45); // '-' + // Collect digits from the negative value; i32 % is truncated so digits + // come out negative. + while (x != 0) { + var d = x % 10; + digits[numDigits] = byte(48 - d); + numDigits = numDigits + 1; + x = x / 10; + } + } else { + while (x != 0) { + digits[numDigits] = byte(48 + x % 10); + numDigits = numDigits + 1; + x = x / 10; + } + } + while (numDigits > 0) { + numDigits = numDigits - 1; + bufPushByte(b, digits[numDigits]); + } +} + +// --------------------------------------------------------------------------- +// Str: immutable (pointer, length) string on the heap +// --------------------------------------------------------------------------- + +struct Str { + data byte~, + count int +} + +def strNew(p byte~, n int) Str~ { + var s = Str~(alloc(8)); + s~.data = p; + s~.count = n; + return s; +} + +// Copies the given bytes to the heap and wraps them in a Str. +def strCopy(p byte~, n int) Str~ { + var d = alloc(n); + memCopy(p, d, n); + return strNew(d, n); +} + +def strEmpty() Str~ { + return strNew(byte~(0), 0); +} + +def strFromBuf(b Buf~) Str~ { + return strCopy(b~.data, b~.count); +} + +def strGet(s Str~, i int) byte { + if (i < 0 || i >= s~.count) { + die(74); // str index out of bounds + } + return (s~.data + i)~; +} + +def strEq(a Str~, b Str~) bool { + if (a~.count != b~.count) { + return false; + } + for (var i = 0; i < a~.count; i = i + 1) { + if ((a~.data + i)~ != (b~.data + i)~) { + return false; + } + } + return true; +} + +def strEqRaw(a Str~, p byte~, n int) bool { + if (a~.count != n) { + return false; + } + for (var i = 0; i < n; i = i + 1) { + if ((a~.data + i)~ != (p + i)~) { + return false; + } + } + return true; +} + +def strSub(s Str~, start int, end int) Str~ { + if (start < 0 || end > s~.count || start > end) { + die(75); // invalid substring range + } + return strNew(s~.data + start, end - start); +} + +// --------------------------------------------------------------------------- +// stdout/stderr helpers +// --------------------------------------------------------------------------- + +def writeBufTo(b Buf~, isErr bool) { + for (var i = 0; i < b~.count; i = i + 1) { + var c = int((b~.data + i)~); + if (isErr) { + puterr(c); + } else { + putchar(c); + } + } +} + +def writeStrTo(s Str~, isErr bool) { + for (var i = 0; i < s~.count; i = i + 1) { + var c = int((s~.data + i)~); + if (isErr) { + puterr(c); + } else { + putchar(c); + } + } +} + +// Reads all of stdin into a Buf. +def readAllInput() Buf~ { + var b = bufNewWithCap(65536); + var c = getchar(); + while (c >= 0) { + bufPushChar(b, c); + c = getchar(); + } + return b; +} diff --git a/src/nodes.ts b/src/nodes.ts index 3a63a55..0afd106 100644 --- a/src/nodes.ts +++ b/src/nodes.ts @@ -993,6 +993,10 @@ export class Context { { name: fakeToken(TokenType.IDENTIFIER, "dst"), type: ptrType(ByteType) + }, + { + name: fakeToken(TokenType.IDENTIFIER, "numBytes"), + type: IntType } ], returnType: VoidType, diff --git a/tools/build-selfhost.sh b/tools/build-selfhost.sh new file mode 100755 index 0000000..b38c48c --- /dev/null +++ b/tools/build-selfhost.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Compiles the self-hosted compiler sources (plus an optional dev main) +# with the TypeScript reference compiler into a runnable wasm module. +# Usage: tools/build-selfhost.sh [extra-main.puff] +set -euo pipefail +cd "$(dirname "$0")/.." + +OUT="${1:?usage: build-selfhost.sh [main.puff]}" +MAIN="${2:-selfhost/main.puff}" + +SOURCES=( + selfhost/util.puff + selfhost/scanner.puff +) +[ -f selfhost/ast.puff ] && SOURCES+=(selfhost/ast.puff) +[ -f selfhost/parser.puff ] && SOURCES+=(selfhost/parser.puff) +[ -f selfhost/resolver.puff ] && SOURCES+=(selfhost/resolver.puff) +[ -f selfhost/backend.puff ] && SOURCES+=(selfhost/backend.puff) +SOURCES+=("$MAIN") + +WAT="${OUT%.wasm}.wat" +node dist/tools/puffc.js "${SOURCES[@]}" -o "$WAT" +node dist/tools/wat2wasm.js "$WAT" -o "$OUT" diff --git a/tools/dump.ts b/tools/dump.ts new file mode 100644 index 0000000..bda6e70 --- /dev/null +++ b/tools/dump.ts @@ -0,0 +1,75 @@ +// Debug dumps from the TypeScript reference compiler, for differential +// testing against the self-hosted compiler. +// Usage: node dist/tools/dump.js --tokens|--sexpr|--errors file.puff [more.puff ...] +import fs from 'fs' +import { scanTokens } from '../src/scanner' +import { parse } from '../src/parser' +import { resolve } from '../src/resolver' +import { TokenType } from '../src/tokens' +import * as ast from '../src/nodes' +import { ReportError } from '../src/util' + +function main() { + const args = process.argv.slice(2) + let mode: string | null = null + const inputs: string[] = [] + for (const a of args) { + if (a.startsWith("--")) { + mode = a.substring(2) + } else { + inputs.push(a) + } + } + if (mode === null || inputs.length === 0) { + console.error("usage: dump --tokens|--sexpr|--errors file.puff [more.puff ...]") + process.exit(2) + } + const source = inputs.map((f) => fs.readFileSync(f, "utf8")).join("\n") + const errors: string[] = [] + const reportError: ReportError = (line, msg) => { + errors.push(`${line}: ${msg}`) + } + + const tokens = scanTokens(source, reportError) + let out = "" + if (mode === "tokens") { + if (errors.length === 0) { + for (const t of tokens) { + out += `${t.type} ${t.offset} ${t.lexeme.length}` + if (t.type === TokenType.NUMBER || t.type === TokenType.NUMBER_HEX) { + out += ` ${t.literal}` + } + out += "\n" + } + } + } else if (mode === "sexpr") { + if (errors.length === 0) { + const context = parse(tokens, reportError) + if (errors.length === 0) { + out += "(" + context.topLevelStatements.forEach((stmt, i) => { + if (i > 0) out += " " + out += ast.astToSExpr(stmt) + }) + out += ")\n" + } + } + } else if (mode === "errors") { + if (errors.length === 0) { + const context = parse(tokens, reportError) + if (errors.length === 0) { + resolve(context, reportError) + } + } + } else { + console.error(`unknown mode ${mode}`) + process.exit(2) + } + process.stdout.write(out) + for (const e of errors) { + process.stderr.write(e + "\n") + } + process.exit(errors.length > 0 ? 1 : 0) +} + +main() From 22e94071b2ac2f589f9773330d92cbf887780959 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 13 Aug 2026 07:27:41 +0000 Subject: [PATCH 4/7] Self-hosted compiler: AST definitions, parser, s-expression printer Differentially tested against the reference parser: token streams, s-expressions, parse errors, and exit codes match over the fixture corpus (extracted from test.ts) and the selfhost sources themselves. Also fixes a latent reference-compiler bug where emitDebugComments computed s-expressions eagerly (infinite recursion on self-referential struct pointer types) even with debug comments disabled. Co-authored-by: Andrew Chan --- selfhost/ast.puff | 645 +++++++++++++++++++ selfhost/main_dev_sexpr.puff | 33 + selfhost/parser.puff | 1179 ++++++++++++++++++++++++++++++++++ selfhost/scanner.puff | 28 +- selfhost/sexpr.puff | 342 ++++++++++ src/backend.ts | 33 +- tools/build-selfhost.sh | 1 + tools/compare-selfhost.sh | 29 + tools/extract-fixtures.ts | 27 + 9 files changed, 2300 insertions(+), 17 deletions(-) create mode 100644 selfhost/ast.puff create mode 100644 selfhost/main_dev_sexpr.puff create mode 100644 selfhost/parser.puff create mode 100644 selfhost/sexpr.puff create mode 100755 tools/compare-selfhost.sh create mode 100644 tools/extract-fixtures.ts diff --git a/selfhost/ast.puff b/selfhost/ast.puff new file mode 100644 index 0000000..d2c158b --- /dev/null +++ b/selfhost/ast.puff @@ -0,0 +1,645 @@ +// ast.puff +// AST node/type/symbol/scope definitions, ported from src/nodes.ts. +// Nodes are represented as one "fat" struct with a kind tag; field usage +// per kind is documented below. + +// NodeKind constants (must mirror the NodeKind enum in src/nodes.ts). +var NK_ASSIGN_EXPR = 0; +var NK_BINARY_EXPR = 1; +var NK_CALL_EXPR = 2; +var NK_CAST_EXPR = 3; +var NK_DEREF_EXPR = 4; +var NK_DOT_EXPR = 5; +var NK_GROUP_EXPR = 6; +var NK_INDEX_EXPR = 7; +var NK_LEN_EXPR = 8; +var NK_LIST_EXPR = 9; +var NK_LITERAL_EXPR = 10; +var NK_LOGICAL_EXPR = 11; +var NK_UNARY_EXPR = 12; +var NK_VARIABLE_EXPR = 13; +var NK_BLOCK_STMT = 14; +var NK_EXPRESSION_STMT = 15; +var NK_FUNCTION_STMT = 16; +var NK_IF_STMT = 17; +var NK_LOOP_CONTROL_STMT = 18; +var NK_PRINT_STMT = 19; +var NK_RETURN_STMT = 20; +var NK_STRUCT_STMT = 21; +var NK_VAR_STMT = 22; +var NK_WHILE_STMT = 23; + +// TypeCategory constants (must mirror TypeCategory in src/nodes.ts). +var TC_ARRAY = 0; +var TC_BOOL = 1; +var TC_BYTE = 2; +var TC_ERROR = 3; +var TC_FLOAT = 4; +var TC_INT = 5; +var TC_POINTER = 6; +var TC_STRUCT = 7; +var TC_VOID = 8; + +// SymbolKind constants (must mirror SymbolKind in src/nodes.ts). +var SK_VARIABLE = 0; +var SK_FUNCTION = 1; +var SK_PARAM = 2; +var SK_STRUCT = 3; + +// ListKind constants. +var LK_LIST = 0; +var LK_REPEAT = 1; + +var INT_MIN = -2147483647; +var INT_MAX = 2147483647; +var BYTE_MIN = 0; +var BYTE_MAX = 255; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +struct Type { + category int, + elementType Type~, // for ARRAY and POINTER + length int, // for ARRAY + name Token~, // for STRUCT + resolvedStruct Node~ // for STRUCT; the StructStmt node, filled by resolver +} + +def newType(category int) Type~ { + var t = Type~(alloc(20)); + t~.category = category; + t~.elementType = Type~(0); + t~.name = Token~(0); + t~.resolvedStruct = Node~(0); + return t; +} + +// Type singletons, mirroring IntType/FloatType/etc in src/nodes.ts. +var gTypeError Type~ = newType(TC_ERROR); +var gTypeVoid Type~ = newType(TC_VOID); +var gTypeInt Type~ = newType(TC_INT); +var gTypeFloat Type~ = newType(TC_FLOAT); +var gTypeByte Type~ = newType(TC_BYTE); +var gTypeBool Type~ = newType(TC_BOOL); + +def arrayType(elementType Type~, length int) Type~ { + var t = newType(TC_ARRAY); + t~.elementType = elementType; + t~.length = length; + return t; +} + +def ptrType(elementType Type~) Type~ { + var t = newType(TC_POINTER); + t~.elementType = elementType; + return t; +} + +def unresolvedStructType(name Token~) Type~ { + var t = newType(TC_STRUCT); + t~.name = name; + return t; +} + +def resolvedStructType(structNode Node~) Type~ { + var t = newType(TC_STRUCT); + t~.name = structNode~.tokA; + t~.resolvedStruct = structNode; + return t; +} + +def typeIsEqual(a Type~, b Type~) bool { + if (a~.category == TC_ARRAY && b~.category == TC_ARRAY) { + return typeIsEqual(a~.elementType, b~.elementType) && a~.length == b~.length; + } + if (a~.category == TC_STRUCT && b~.category == TC_STRUCT) { + if (int(a~.resolvedStruct) == 0 || int(b~.resolvedStruct) == 0) { + die(80); // comparing unresolved struct types + } + return int(a~.resolvedStruct) == int(b~.resolvedStruct); + } + if (a~.category == TC_POINTER && b~.category == TC_POINTER) { + return typeIsEqual(a~.elementType, b~.elementType); + } + return a~.category == b~.category; +} + +def isValidElementType(t Type~) bool { + if (t~.category == TC_ERROR) { + return true; + } + return !(t~.category == TC_VOID); +} + +def sizeOf(t Type~) int { + if (t~.category == TC_ARRAY) { + return sizeOf(t~.elementType) * t~.length; + } + if (t~.category == TC_STRUCT) { + if (int(t~.resolvedStruct) == 0) { + die(81); // sizeof unresolved struct type + } + var size = 0; + var members = t~.resolvedStruct~.params; + for (var i = 0; i < members~.count; i = i + 1) { + var member = Param~(vecGet(members, i)); + size = size + sizeOf(member~.type); + } + return size; + } + if (t~.category == TC_POINTER || t~.category == TC_INT || t~.category == TC_FLOAT) { + return 4; + } + if (t~.category == TC_BYTE || t~.category == TC_BOOL) { + return 1; + } + if (t~.category == TC_VOID) { + return 0; + } + die(82); // sizeof error type + return 0; +} + +def isScalar(t Type~) bool { + var c = t~.category; + return c == TC_BOOL || c == TC_BYTE || c == TC_FLOAT || c == TC_INT || c == TC_POINTER; +} + +def isNumeric(t Type~) bool { + var c = t~.category; + return c == TC_INT || c == TC_FLOAT || c == TC_BYTE; +} + +def canCast(from Type~, to Type~) bool { + if (from~.category == TC_ERROR || to~.category == TC_ERROR) { + // We already threw an error somewhere. + // Pretend we can cast the result so we don't cascade errors. + return true; + } + if ((isNumeric(from) || from~.category == TC_BOOL) && (isNumeric(to) || to~.category == TC_BOOL)) { + // Numerics and bools can always be casted to and from each other. + return true; + } + if (from~.category == TC_POINTER && to~.category == TC_POINTER) { + // Pointers are a type escape hatch and can always be casted to/from each other. + return true; + } + if (from~.category == TC_INT && to~.category == TC_POINTER) { + return true; + } + if (from~.category == TC_POINTER && to~.category == TC_INT) { + return true; + } + return typeIsEqual(from, to); +} + +def canCoerce(from Type~, to Type~) bool { + if (from~.category == TC_ERROR || to~.category == TC_ERROR) { + return true; + } + if (typeIsEqual(from, to)) { + return true; + } + var fc = from~.category; + var tc = to~.category; + if (fc == TC_ARRAY || fc == TC_VOID || fc == TC_POINTER || fc == TC_STRUCT) { + return false; + } + if (tc == TC_ARRAY || tc == TC_VOID || tc == TC_POINTER || tc == TC_STRUCT) { + return false; + } + if (fc == TC_INT) { + return tc == TC_INT || tc == TC_FLOAT || tc == TC_BOOL; + } + if (fc == TC_FLOAT) { + return tc == TC_FLOAT; + } + if (fc == TC_BYTE) { + return tc == TC_INT || tc == TC_FLOAT || tc == TC_BYTE; + } + if (fc == TC_BOOL) { + return tc == TC_BOOL; + } + return false; +} + +// Checks whether a literal node's numeric value can be coerced to the target +// type. Mirrors canCoerceNumberLiteral in src/nodes.ts; takes the literal +// node so it can handle both int-flavored and float-flavored values. +def canCoerceNumberLiteralNode(node Node~, to Type~) bool { + if (!isNumeric(to)) { + return false; + } + var isFloatLit = node~.typeA~.category == TC_FLOAT; + if (to~.category == TC_FLOAT) { + return true; + } + // Target is byte or int: value must be an integer in range. + if (isFloatLit) { + var v = node~.litFloat; + // Out-of-range values cannot be represented (guard i32.trunc traps; + // 2147483520 is the largest f32 below 2^31). + if (!(v >= -2147483520.0 && v <= 2147483520.0)) { + return false; + } + var iv = int(v); + if (float(iv) != v) { + return false; // not an integer + } + if (to~.category == TC_BYTE) { + return iv >= BYTE_MIN && iv <= BYTE_MAX; + } + return iv >= INT_MIN && iv <= INT_MAX; + } + var value = node~.litInt; + if (to~.category == TC_BYTE) { + return value >= BYTE_MIN && value <= BYTE_MAX; + } + return value >= INT_MIN && value <= INT_MAX; +} + +def isNumberLiteral(node Node~) bool { + if (node~.kind != NK_LITERAL_EXPR) { + return false; + } + return isNumeric(node~.typeA); +} + +// Get lowest common numeric type to which we can coerce both `a` and `b`. +// If one of the args is not a numeric, returns null. +def getLowestCommonNumeric(a Type~, b Type~) Type~ { + if (!isNumeric(a) || !isNumeric(b)) { + return Type~(0); + } + if (typeIsEqual(a, b)) { + return a; + } + if (canCoerce(a, gTypeByte) && canCoerce(b, gTypeByte)) { + return gTypeByte; + } + if (canCoerce(a, gTypeInt) && canCoerce(b, gTypeInt)) { + return gTypeInt; + } + if (canCoerce(a, gTypeFloat) && canCoerce(b, gTypeFloat)) { + return gTypeFloat; + } + return Type~(0); +} + +// Appends the human-readable type name to `out`. +// Mirrors typeToString in src/nodes.ts. +def typeToString(out Buf~, t Type~) { + if (t~.category == TC_ARRAY) { + bufPushChar(out, 91); // '[' + typeToString(out, t~.elementType); + bufPushChar(out, 59); // ';' + bufPushChar(out, 32); + bufPushInt(out, t~.length); + bufPushChar(out, 93); // ']' + return; + } + if (t~.category == TC_POINTER) { + typeToString(out, t~.elementType); + bufPushChar(out, 126); // '~' + return; + } + if (t~.category == TC_STRUCT) { + bufPushBytes(out, t~.name~.lexPtr, t~.name~.lexLen); + return; + } + if (t~.category == TC_ERROR) { + var s0 = ""; + bufPushBytes(out, &s0[0], len(s0)); + return; + } + if (t~.category == TC_INT) { + var s1 = "int"; + bufPushBytes(out, &s1[0], len(s1)); + return; + } + if (t~.category == TC_FLOAT) { + var s2 = "float"; + bufPushBytes(out, &s2[0], len(s2)); + return; + } + if (t~.category == TC_BYTE) { + var s3 = "byte"; + bufPushBytes(out, &s3[0], len(s3)); + return; + } + if (t~.category == TC_BOOL) { + var s4 = "bool"; + bufPushBytes(out, &s4[0], len(s4)); + return; + } + if (t~.category == TC_VOID) { + var s5 = "void"; + bufPushBytes(out, &s5[0], len(s5)); + return; + } + die(83); // unhandled type in typeToString +} + +// --------------------------------------------------------------------------- +// Params +// --------------------------------------------------------------------------- + +struct Param { + name Token~, + type Type~ +} + +def newParam(name Token~, type Type~) Param~ { + var p = Param~(alloc(8)); + p~.name = name; + p~.type = type; + return p; +} + +// --------------------------------------------------------------------------- +// Symbols +// --------------------------------------------------------------------------- + +struct Symbol { + kind int, + id int, + node Node~, // VAR_STMT / FUNCTION_STMT / STRUCT_STMT node + param Param~, // for PARAM symbols + // backend bookkeeping: + globalLoc int, // data-segment address of a global variable + localLoc int, // distance of a local from the function base pointer + isGlobal bool, + isAddressTaken bool, + hasGlobalLoc bool, + hasLocalLoc bool, + inHoisted bool +} + +var gNextSymbolId = 0; + +def newSymbol(kind int) Symbol~ { + var s = Symbol~(alloc(32)); + s~.kind = kind; + s~.id = gNextSymbolId; + gNextSymbolId = gNextSymbolId + 1; + s~.node = Node~(0); + s~.param = Param~(0); + return s; +} + +def variableSymbol(node Node~, isGlobal bool) Symbol~ { + var s = newSymbol(SK_VARIABLE); + s~.node = node; + s~.isGlobal = isGlobal; + return s; +} + +def functionSymbol(node Node~) Symbol~ { + var s = newSymbol(SK_FUNCTION); + s~.node = node; + return s; +} + +def paramSymbol(param Param~) Symbol~ { + var s = newSymbol(SK_PARAM); + s~.param = param; + return s; +} + +def structSymbol(node Node~) Symbol~ { + var s = newSymbol(SK_STRUCT); + s~.node = node; + return s; +} + +// --------------------------------------------------------------------------- +// Scopes +// --------------------------------------------------------------------------- + +struct Scope { + parent Scope~, + names Vec~, // Vec of Str~ + syms Vec~ // Vec of Symbol~ +} + +def scopeNew(parent Scope~) Scope~ { + var s = Scope~(alloc(12)); + s~.parent = parent; + s~.names = vecNew(); + s~.syms = vecNew(); + return s; +} + +def scopeDefine(s Scope~, name Str~, sym Symbol~) { + vecPush(s~.names, int(name)); + vecPush(s~.syms, int(sym)); +} + +def scopeFindDirect(s Scope~, name Str~) Symbol~ { + for (var i = 0; i < s~.names~.count; i = i + 1) { + if (strEq(Str~(vecGet(s~.names, i)), name)) { + return Symbol~(vecGet(s~.syms, i)); + } + } + return Symbol~(0); +} + +def scopeHasDirect(s Scope~, name Str~) bool { + return int(scopeFindDirect(s, name)) != 0; +} + +// Filters for scopeLookup, mirroring the lookup filters used in the +// reference resolver. +var LOOKUP_ANY = 0; +// Variables must be visited already or global; params/functions/structs pass. +var LOOKUP_VISIBLE_SYMBOLS = 1; + +def lookupFilterAccepts(filter int, sym Symbol~) bool { + if (filter == LOOKUP_ANY) { + return true; + } + // LOOKUP_VISIBLE_SYMBOLS + if (sym~.kind == SK_PARAM || sym~.kind == SK_FUNCTION || sym~.kind == SK_STRUCT) { + return true; + } + // SK_VARIABLE + return sym~.node~.visited || sym~.isGlobal; +} + +def scopeLookup(s Scope~, name Str~, filter int) Symbol~ { + var sym = scopeFindDirect(s, name); + if (int(sym) != 0 && lookupFilterAccepts(filter, sym)) { + return sym; + } + if (int(s~.parent) != 0) { + return scopeLookup(s~.parent, name, filter); + } + return Symbol~(0); +} + +// --------------------------------------------------------------------------- +// Nodes +// +// Field usage by kind: +// ASSIGN_EXPR: tokA=operator, a=left, b=right +// BINARY_EXPR: tokA=operator, a=left, b=right +// CALL_EXPR: tokA=paren, a=callee, list=args +// CAST_EXPR: tokA=token, typeA=target type, a=value +// DEREF_EXPR: tokA=operator, a=value +// DOT_EXPR: tokA=dot, tokB=identifier, a=callee +// GROUP_EXPR: a=expression +// INDEX_EXPR: tokA=bracket, a=callee, b=index +// LEN_EXPR: a=value, resolvedLength +// LIST_EXPR: tokA=bracket, listKind, list=values (LIST), +// a=value + repeatLen (REPEAT) +// LITERAL_EXPR: typeA=literal type, litInt/litFloat/litBool/litStr, +// decStr (canonical int emission), lexeme (float lexeme) +// LOGICAL_EXPR: tokA=operator, a=left, b=right +// UNARY_EXPR: tokA=operator, a=value +// VARIABLE_EXPR: tokA=name, symbol=resolvedSymbol +// BLOCK_STMT: list=statements, scope +// EXPRESSION_STMT: a=expression +// FUNCTION_STMT: tokA=name, params, typeA=returnType, list=body block, +// scope, hasBody, hostModule, isExported, symbol, hoisted +// IF_STMT: a=expression, b=thenBranch, c=elseBranch +// LOOP_CONTROL_STMT:tokA=keyword +// PRINT_STMT: tokA=keyword, a=expression +// RETURN_STMT: tokA=keyword, a=value (may be null) +// STRUCT_STMT: tokA=name, params=members, symbol +// VAR_STMT: tokA=name, a=initializer, typeA=declared type (may be +// null = infer), symbol +// WHILE_STMT: a=expression, b=body, c=increment (may be null) +// --------------------------------------------------------------------------- + +struct Node { + kind int, + resolvedType Type~, + tokA Token~, + tokB Token~, + a Node~, + b Node~, + c Node~, + list Vec~, + typeA Type~, + litInt int, + litFloat float, + litStr Str~, + lexeme Str~, + decStr Str~, + listKind int, + repeatLen int, + params Vec~, + scope Scope~, + symbol Symbol~, + hoisted Vec~, + hostModule Str~, + isLiveAtEnd int, // -1 = not filled in yet, 0 = false, 1 = true + resolvedLength int, + strLoc int, // backend: data segment location of string literal + litBool bool, + isExported bool, + hasBody bool, + hasResolvedLength bool, + visited bool, // resolver bookkeeping + walked bool, // resolver bookkeeping + skipEmit bool // backend bookkeeping +} + +def newNode(kind int) Node~ { + // 24 i32/ptr/float fields * 4 bytes + 7 bool bytes = 103 bytes + var n = Node~(alloc(103)); + n~.kind = kind; + n~.resolvedType = Type~(0); + n~.tokA = Token~(0); + n~.tokB = Token~(0); + n~.a = Node~(0); + n~.b = Node~(0); + n~.c = Node~(0); + n~.list = Vec~(0); + n~.typeA = Type~(0); + n~.litStr = Str~(0); + n~.lexeme = Str~(0); + n~.decStr = Str~(0); + n~.params = Vec~(0); + n~.scope = Scope~(0); + n~.symbol = Symbol~(0); + n~.hoisted = Vec~(0); + n~.hostModule = Str~(0); + n~.isLiveAtEnd = -1; + return n; +} + +// --------------------------------------------------------------------------- +// Context: global compilation state (mirrors ast.Context) +// --------------------------------------------------------------------------- + +var gGlobalScope Scope~ = scopeNew(Scope~(0)); +// Interned string literals: parallel key/node vectors. +var gStrLitKeys Vec~ = vecNew(); +var gStrLitNodes Vec~ = vecNew(); +// Top-level statements in parse order. +var gTopLevel Vec~ = vecNew(); +// Global variable statements in initialization (dependency) order; +// filled in by the resolver. +var gGlobalInitOrder Vec~ = vecNew(); + +def strLitLookup(s Str~) Node~ { + for (var i = 0; i < gStrLitKeys~.count; i = i + 1) { + if (strEq(Str~(vecGet(gStrLitKeys, i)), s)) { + return Node~(vecGet(gStrLitNodes, i)); + } + } + return Node~(0); +} + +def strLitIntern(s Str~, node Node~) { + vecPush(gStrLitKeys, int(s)); + vecPush(gStrLitNodes, int(node)); +} + +// Registers a built-in function with no puff-level body. +def defineBuiltin(name Str~, params Vec~, returnType Type~) { + var node = newNode(NK_FUNCTION_STMT); + node~.tokA = fakeToken(T_IDENTIFIER, name, Token~(0)); + node~.params = params; + node~.typeA = returnType; + node~.hasBody = false; + node~.symbol = functionSymbol(node); + scopeDefine(gGlobalScope, name, node~.symbol); +} + +def staticStr(p byte~, n int) Str~ { + return strCopy(p, n); +} + +// Defines the built-in functions __memcpy__, __sqrt__, __heap_end__, and +// __grow_heap__ with the same symbol ids as the reference compiler. +// Must be called exactly once, before parsing. +def initContext() { + var sMemcpy = "__memcpy__"; + var sSrc = "src"; + var sDst = "dst"; + var sNumBytes = "numBytes"; + var memcpyParams = vecNew(); + vecPush(memcpyParams, int(newParam(fakeToken(T_IDENTIFIER, staticStr(&sSrc[0], len(sSrc)), Token~(0)), ptrType(gTypeByte)))); + vecPush(memcpyParams, int(newParam(fakeToken(T_IDENTIFIER, staticStr(&sDst[0], len(sDst)), Token~(0)), ptrType(gTypeByte)))); + vecPush(memcpyParams, int(newParam(fakeToken(T_IDENTIFIER, staticStr(&sNumBytes[0], len(sNumBytes)), Token~(0)), gTypeInt))); + defineBuiltin(staticStr(&sMemcpy[0], len(sMemcpy)), memcpyParams, gTypeVoid); + + var sSqrt = "__sqrt__"; + var sX = "x"; + var sqrtParams = vecNew(); + vecPush(sqrtParams, int(newParam(fakeToken(T_IDENTIFIER, staticStr(&sX[0], len(sX)), Token~(0)), gTypeFloat))); + defineBuiltin(staticStr(&sSqrt[0], len(sSqrt)), sqrtParams, gTypeFloat); + + var sHeapEnd = "__heap_end__"; + defineBuiltin(staticStr(&sHeapEnd[0], len(sHeapEnd)), vecNew(), gTypeInt); + + var sGrow = "__grow_heap__"; + var sNumPages = "numPages"; + var growParams = vecNew(); + vecPush(growParams, int(newParam(fakeToken(T_IDENTIFIER, staticStr(&sNumPages[0], len(sNumPages)), Token~(0)), gTypeInt))); + defineBuiltin(staticStr(&sGrow[0], len(sGrow)), growParams, gTypeInt); +} diff --git a/selfhost/main_dev_sexpr.puff b/selfhost/main_dev_sexpr.puff new file mode 100644 index 0000000..b67d7e1 --- /dev/null +++ b/selfhost/main_dev_sexpr.puff @@ -0,0 +1,33 @@ +// Development driver: dumps the parsed AST as s-expressions in the same +// format as `tools/dump.ts --sexpr` for differential testing. + +var gOut Buf~ = bufNewWithCap(65536); + +def main() { + gSource = readAllInput(); + initContext(); + scanTokens(); + if (numErrors() == 0) { + parseProgram(); + if (numErrors() == 0) { + bufPushChar(gOut, 40); + for (var i = 0; i < gTopLevel~.count; i = i + 1) { + if (i > 0) { + bufPushChar(gOut, 32); + } + astToSExpr(gOut, Node~(vecGet(gTopLevel, i))); + } + bufPushChar(gOut, 41); + bufPushChar(gOut, 10); + } + } + writeBufTo(gOut, false); + for (var i = 0; i < gErrors~.count; i = i + 1) { + writeStrTo(Str~(vecGet(gErrors, i)), true); + puterr(10); + } + if (numErrors() > 0) { + exit(1); + } + exit(0); +} diff --git a/selfhost/parser.puff b/selfhost/parser.puff new file mode 100644 index 0000000..7e5660b --- /dev/null +++ b/selfhost/parser.puff @@ -0,0 +1,1179 @@ +// parser.puff +// Recursive descent parser, ported from src/parser.ts. +// +// The reference parser uses exceptions (ParseError) for panic-mode error +// recovery; here we use an explicit panic flag (pPanic). Functions that can +// "throw" set the flag; callers must check it after each call and bail out, +// which mirrors exception unwinding (including the reference compiler's +// behavior of not popping scopes while unwinding). +// +// 1. Construct AST from stream of tokens +// 2. Create scopes for blocks and functions +// 3. Emplace symbol declarations into scopes +// 4. Ensure no duplicate symbols in same scope + +var pCurrent = 0; +var pPanic = false; +// Stack of Scope~. +var pScopes Vec~ = vecNew(); + +def pPeekScope() Scope~ { + return Scope~(vecPeek(pScopes)); +} + +def pPushScope() { + vecPush(pScopes, int(scopeNew(pPeekScope()))); +} + +def pPopScope() Scope~ { + return Scope~(vecPop(pScopes)); +} + +def pPeek() Token~ { + return tokenAt(pCurrent); +} + +def pPrevious() Token~ { + return tokenAt(pCurrent - 1); +} + +def pIsAtEnd() bool { + return pPeek()~.type == T_EOF; +} + +def pAdvance() { + if (!pIsAtEnd()) { + pCurrent = pCurrent + 1; + } +} + +// same as `pMatch`, but does not advance +def pCheck(t int) bool { + if (pIsAtEnd()) { + return false; + } + return pPeek()~.type == t; +} + +// advances and returns true if next token matches input token type, +// else returns false +def pMatch(t int) bool { + if (pCheck(t)) { + pAdvance(); + return true; + } + return false; +} + +// Reports an error at the given token without panicking. +def pReportAt(tok Token~, msg byte~, msgLen int) { + errBegin(tokenLine(tok)); + errRaw(msg, msgLen); + errEnd(); +} + +// Reports an error at the given token and enters panic mode +// (the equivalent of `throw parseErrorForToken(...)`). +def pPanicAt(tok Token~, msg byte~, msgLen int) { + pReportAt(tok, msg, msgLen); + pPanic = true; +} + +// The equivalent of `throw parseError(...)`: reports at peek(). +def pPanicHere(msg byte~, msgLen int) { + pPanicAt(pPeek(), msg, msgLen); +} + +// advances and returns the next token if it matches the input token type, +// else reports an error with the given message and panics +def pConsume(t int, msg byte~, msgLen int) Token~ { + if (pCheck(t)) { + var token = pPeek(); + pAdvance(); + return token; + } + pPanicHere(msg, msgLen); + return pPeek(); +} + +// on catching a parse error, discard tokens until we're at the beginning +// of the next statement/declaration so we can continue parsing w/o cascading +// errors +def pSynchronize() { + pPanic = false; + pAdvance(); + while (!pIsAtEnd()) { + if (pPrevious()~.type == T_SEMICOLON) { + return; + } + var t = pPeek()~.type; + if (t == T_DEF || t == T_STRUCT || t == T_VAR || t == T_IF || t == T_PRINT || t == T_RETURN || t == T_WHILE) { + return; + } + pAdvance(); + } +} + +def pTopDecl() Node~ { + if (pMatch(T_IMPORT)) { + var m0 = "Expect 'def' after 'import'."; + pConsume(T_DEF, &m0[0], len(m0)); + if (pPanic) { return Node~(0); } + return pImportDecl(); + } + if (pMatch(T_EXPORT)) { + var m1 = "Expect 'def' after 'export'."; + pConsume(T_DEF, &m1[0], len(m1)); + if (pPanic) { return Node~(0); } + return pFunDecl(true); + } + if (pMatch(T_DEF)) { + return pFunDecl(false); + } + if (pMatch(T_STRUCT)) { + return pStructDecl(); + } + if (pMatch(T_VAR)) { + return pVarDecl(); + } + var m2 = "Only variable declarations and function definitions allowed at the top-level."; + pPanicHere(&m2[0], len(m2)); + return Node~(0); +} + +// Reports "'' is already declared in this scope." at the given token. +def pReportAlreadyDeclared(name Token~, panic bool) { + errBegin(tokenLine(name)); + errChar(39); // '\'' + errRaw(name~.lexPtr, name~.lexLen); + var m0 = "' is already declared in this scope."; + errRaw(&m0[0], len(m0)); + errEnd(); + if (panic) { + pPanic = true; + } +} + +// Parses the parameter list of a function declaration (after the name). +// Returns a Vec of Param~, or panics. +def pParams() Vec~ { + var m0 = "Expect '(' after function name."; + pConsume(T_LEFT_PAREN, &m0[0], len(m0)); + if (pPanic) { return Vec~(0); } + var params = vecNew(); + while (!pCheck(T_RIGHT_PAREN) && !pIsAtEnd()) { + if (params~.count > 0) { + var m1 = "Missing comma after parameter."; + pConsume(T_COMMA, &m1[0], len(m1)); + if (pPanic) { return Vec~(0); } + } + var m2 = "Expect identifier."; + var paramName = pConsume(T_IDENTIFIER, &m2[0], len(m2)); + if (pPanic) { return Vec~(0); } + var paramType = Type~(0); + paramType = pType(); + if (pPanic) { return Vec~(0); } + vecPush(params, int(newParam(paramName, paramType))); + } + var m3 = "Expect ')' after parameters."; + pConsume(T_RIGHT_PAREN, &m3[0], len(m3)); + if (pPanic) { return Vec~(0); } + return params; +} + +def pImportDecl() Node~ { + var m0 = "Expect identifier after 'def'."; + var name = pConsume(T_IDENTIFIER, &m0[0], len(m0)); + if (pPanic) { return Node~(0); } + + var params = Vec~(0); + params = pParams(); + if (pPanic) { return Node~(0); } + + var returnType = gTypeVoid; + if (!pCheck(T_SEMICOLON)) { + returnType = pType(); + if (pPanic) { return Node~(0); } + } + var m1 = "Expect ';' after import declaration."; + pConsume(T_SEMICOLON, &m1[0], len(m1)); + if (pPanic) { return Node~(0); } + + var node = newNode(NK_FUNCTION_STMT); + node~.tokA = name; + node~.params = params; + node~.typeA = returnType; + node~.hasBody = false; + var sEnv = "env"; + node~.hostModule = strCopy(&sEnv[0], len(sEnv)); + + var outerScope = pPeekScope(); + if (scopeHasDirect(outerScope, tokenLexemeStr(name))) { + // Panic; we want to ignore this function and synchronize to next statement + pReportAlreadyDeclared(name, true); + return Node~(0); + } + var symbol = functionSymbol(node); + scopeDefine(outerScope, tokenLexemeStr(name), symbol); + node~.symbol = symbol; + return node; +} + +def pFunDecl(isExported bool) Node~ { + var m0 = "Expect identifier after 'def'."; + var name = pConsume(T_IDENTIFIER, &m0[0], len(m0)); + if (pPanic) { return Node~(0); } + + var params = Vec~(0); + params = pParams(); + if (pPanic) { return Node~(0); } + + var returnType = gTypeVoid; + if (!pCheck(T_LEFT_BRACE)) { + returnType = pType(); + if (pPanic) { return Node~(0); } + } + + var m1 = "Expect '{' before function body."; + pConsume(T_LEFT_BRACE, &m1[0], len(m1)); + if (pPanic) { return Node~(0); } + pPushScope(); + for (var i = 0; i < params~.count; i = i + 1) { + var param = Param~(vecGet(params, i)); + var scope = pPeekScope(); + if (scopeHasDirect(scope, tokenLexemeStr(param~.name))) { + // Don't panic; Function body will be parsed + resolved as if duplicate doesn't exist. + // Calls will still be parsed + resolved with arity including duplicate. + pReportAlreadyDeclared(param~.name, false); + } else { + scopeDefine(scope, tokenLexemeStr(param~.name), paramSymbol(param)); + } + } + var statements = Vec~(0); + statements = pBlock(); + if (pPanic) { return Node~(0); } + var scope2 = pPopScope(); + + var node = newNode(NK_FUNCTION_STMT); + node~.tokA = name; + node~.params = params; + node~.typeA = returnType; + node~.list = statements; + node~.scope = scope2; + node~.hasBody = true; + node~.isExported = isExported; + + var outerScope = pPeekScope(); + if (scopeHasDirect(outerScope, tokenLexemeStr(name))) { + // Panic; we want to ignore this function and synchronize to next statement + pReportAlreadyDeclared(name, true); + return Node~(0); + } + var symbol = functionSymbol(node); + scopeDefine(outerScope, tokenLexemeStr(name), symbol); + node~.symbol = symbol; + return node; +} + +def pStructDecl() Node~ { + var m0 = "Expect identifier after 'struct'."; + var name = pConsume(T_IDENTIFIER, &m0[0], len(m0)); + if (pPanic) { return Node~(0); } + + var m1 = "Expect '{' after struct name."; + pConsume(T_LEFT_BRACE, &m1[0], len(m1)); + if (pPanic) { return Node~(0); } + var members = vecNew(); + while (!pCheck(T_RIGHT_BRACE) && !pIsAtEnd()) { + if (members~.count > 0) { + var m2 = "Missing comma after member."; + pConsume(T_COMMA, &m2[0], len(m2)); + if (pPanic) { return Node~(0); } + } + var m3 = "Expect identifier."; + var paramName = pConsume(T_IDENTIFIER, &m3[0], len(m3)); + if (pPanic) { return Node~(0); } + var paramType = Type~(0); + paramType = pType(); + if (pPanic) { return Node~(0); } + // check for duplicate member names + var isDuplicate = false; + for (var i = 0; i < members~.count; i = i + 1) { + var existing = Param~(vecGet(members, i)); + if (strEq(tokenLexemeStr(existing~.name), tokenLexemeStr(paramName))) { + isDuplicate = true; + break; + } + } + if (isDuplicate) { + // Don't panic; Remaining members will be parsed + resolved as if duplicate doesn't exist. + errBegin(tokenLine(paramName)); + errChar(39); // '\'' + errRaw(paramName~.lexPtr, paramName~.lexLen); + var m4 = "' is already declared in member list."; + errRaw(&m4[0], len(m4)); + errEnd(); + } else { + vecPush(members, int(newParam(paramName, paramType))); + } + } + var m5 = "Expect '}' after member list."; + pConsume(T_RIGHT_BRACE, &m5[0], len(m5)); + if (pPanic) { return Node~(0); } + + var node = newNode(NK_STRUCT_STMT); + node~.tokA = name; + node~.params = members; + + var scope = pPeekScope(); + if (scopeHasDirect(scope, tokenLexemeStr(name))) { + // Panic; ignore this struct and synchronize to next statement + pReportAlreadyDeclared(name, true); + return Node~(0); + } + var symbol = structSymbol(node); + scopeDefine(scope, tokenLexemeStr(name), symbol); + node~.symbol = symbol; + return node; +} + +def pVarDecl() Node~ { + var m0 = "Expect identifier after 'var'."; + var name = pConsume(T_IDENTIFIER, &m0[0], len(m0)); + if (pPanic) { return Node~(0); } + + // null means `infer from initializer`. + var varType = Type~(0); + if (!pCheck(T_EQUAL)) { + varType = pType(); + if (pPanic) { return Node~(0); } + } + var m1 = "Expect '=' after variable declaration."; + pConsume(T_EQUAL, &m1[0], len(m1)); + if (pPanic) { return Node~(0); } + var expr = Node~(0); + expr = pExpression(); + if (pPanic) { return Node~(0); } + var m2 = "Expect ';' after statement."; + pConsume(T_SEMICOLON, &m2[0], len(m2)); + if (pPanic) { return Node~(0); } + + var node = newNode(NK_VAR_STMT); + node~.tokA = name; + node~.a = expr; + node~.typeA = varType; + + var scope = pPeekScope(); + if (scopeHasDirect(scope, tokenLexemeStr(name))) { + pReportAlreadyDeclared(name, false); + } else { + var symbol = variableSymbol(node, int(scope) == int(gGlobalScope)); + scopeDefine(scope, tokenLexemeStr(name), symbol); + node~.symbol = symbol; + } + return node; +} + +def pStatement() Node~ { + if (pMatch(T_IF)) { + return pIfStmt(); + } + if (pMatch(T_PRINT)) { + return pPrintStmt(); + } + if (pMatch(T_WHILE)) { + return pWhileStmt(); + } + if (pMatch(T_FOR)) { + return pForStmt(); + } + if (pMatch(T_RETURN)) { + return pReturnStmt(); + } + if (pMatch(T_LEFT_BRACE)) { + pPushScope(); + var statements = Vec~(0); + statements = pBlock(); + if (pPanic) { return Node~(0); } + var scope = pPopScope(); + var node = newNode(NK_BLOCK_STMT); + node~.list = statements; + node~.scope = scope; + return node; + } + if (pMatch(T_BREAK) || pMatch(T_CONTINUE)) { + var keyword = pPrevious(); + var m0 = "expect ';' after statement."; + pConsume(T_SEMICOLON, &m0[0], len(m0)); + if (pPanic) { return Node~(0); } + var node2 = newNode(NK_LOOP_CONTROL_STMT); + node2~.tokA = keyword; + return node2; + } + return pExpressionStmt(); +} + +def pExpressionStmt() Node~ { + var expr = Node~(0); + expr = pExpression(); + if (pPanic) { return Node~(0); } + var m0 = "Expect ';' after expression statement."; + pConsume(T_SEMICOLON, &m0[0], len(m0)); + if (pPanic) { return Node~(0); } + var node = newNode(NK_EXPRESSION_STMT); + node~.a = expr; + return node; +} + +def pIfStmt() Node~ { + var m0 = "Expect '(' after 'if'."; + pConsume(T_LEFT_PAREN, &m0[0], len(m0)); + if (pPanic) { return Node~(0); } + var expr = Node~(0); + expr = pExpression(); + if (pPanic) { return Node~(0); } + var m1 = "Expect ')' after if condition."; + pConsume(T_RIGHT_PAREN, &m1[0], len(m1)); + if (pPanic) { return Node~(0); } + + var thenBranch = Node~(0); + thenBranch = pStatement(); + if (pPanic) { return Node~(0); } + var elseBranch = Node~(0); + if (pMatch(T_ELSE)) { + elseBranch = pStatement(); + if (pPanic) { return Node~(0); } + } + var node = newNode(NK_IF_STMT); + node~.a = expr; + node~.b = thenBranch; + node~.c = elseBranch; + return node; +} + +def pPrintStmt() Node~ { + var keyword = pPrevious(); + var expr = Node~(0); + expr = pExpression(); + if (pPanic) { return Node~(0); } + var m0 = "expect ';' after print statement."; + pConsume(T_SEMICOLON, &m0[0], len(m0)); + if (pPanic) { return Node~(0); } + var node = newNode(NK_PRINT_STMT); + node~.tokA = keyword; + node~.a = expr; + return node; +} + +def pReturnStmt() Node~ { + var keyword = pPrevious(); + if (pMatch(T_SEMICOLON)) { + var node = newNode(NK_RETURN_STMT); + node~.tokA = keyword; + node~.a = Node~(0); + return node; + } + var value = Node~(0); + value = pExpression(); + if (pPanic) { return Node~(0); } + var m0 = "Expect ';' after return statement."; + pConsume(T_SEMICOLON, &m0[0], len(m0)); + if (pPanic) { return Node~(0); } + var node2 = newNode(NK_RETURN_STMT); + node2~.tokA = keyword; + node2~.a = value; + return node2; +} + +def pWhileStmt() Node~ { + var m0 = "Expect '(' after 'while'."; + pConsume(T_LEFT_PAREN, &m0[0], len(m0)); + if (pPanic) { return Node~(0); } + var condition = Node~(0); + condition = pExpression(); + if (pPanic) { return Node~(0); } + var m1 = "Expect ')' after loop condition."; + pConsume(T_RIGHT_PAREN, &m1[0], len(m1)); + if (pPanic) { return Node~(0); } + + var body = Node~(0); + body = pStatement(); + if (pPanic) { return Node~(0); } + var node = newNode(NK_WHILE_STMT); + node~.a = condition; + node~.b = body; + node~.c = Node~(0); + return node; +} + +def pForStmt() Node~ { + // This is desugared into initializer, while loop, and increment statements + var m0 = "Expect '(' after 'for'."; + pConsume(T_LEFT_PAREN, &m0[0], len(m0)); + if (pPanic) { return Node~(0); } + pPushScope(); + + var initializer = Node~(0); + if (pMatch(T_SEMICOLON)) { + initializer = Node~(0); + } else if (pMatch(T_VAR)) { + initializer = pVarDecl(); + if (pPanic) { return Node~(0); } + } else { + initializer = pExpressionStmt(); + if (pPanic) { return Node~(0); } + } + var condition = Node~(0); + if (!pCheck(T_SEMICOLON)) { + condition = pExpression(); + if (pPanic) { return Node~(0); } + } else { + condition = newNode(NK_LITERAL_EXPR); + condition~.typeA = gTypeBool; + condition~.litBool = true; + } + var m1 = "Expect ';' after loop condition."; + pConsume(T_SEMICOLON, &m1[0], len(m1)); + if (pPanic) { return Node~(0); } + var increment = Node~(0); + if (!pCheck(T_RIGHT_PAREN)) { + increment = pExpression(); + if (pPanic) { return Node~(0); } + } + var m2 = "Expect ')' after for clauses."; + pConsume(T_RIGHT_PAREN, &m2[0], len(m2)); + if (pPanic) { return Node~(0); } + pPushScope(); + var body = Node~(0); + body = pStatement(); + if (pPanic) { return Node~(0); } + var innerScope = pPopScope(); + var outerScope = pPopScope(); + + var innerBlock = newNode(NK_BLOCK_STMT); + innerBlock~.list = vecNew(); + vecPush(innerBlock~.list, int(body)); + innerBlock~.scope = innerScope; + + var loop = newNode(NK_WHILE_STMT); + loop~.a = condition; + loop~.b = innerBlock; + if (int(increment) != 0) { + var incStmt = newNode(NK_EXPRESSION_STMT); + incStmt~.a = increment; + loop~.c = incStmt; + } else { + loop~.c = Node~(0); + } + + var outerBlock = newNode(NK_BLOCK_STMT); + outerBlock~.list = vecNew(); + if (int(initializer) != 0) { + vecPush(outerBlock~.list, int(initializer)); + } + vecPush(outerBlock~.list, int(loop)); + outerBlock~.scope = outerScope; + return outerBlock; +} + +// Parses statements until '}'. Returns a Vec of statement nodes. +def pBlock() Vec~ { + var statements = vecNew(); + while (!pCheck(T_RIGHT_BRACE) && !pIsAtEnd()) { + if (pMatch(T_VAR)) { + var varStmt = Node~(0); + varStmt = pVarDecl(); + if (pPanic) { + pSynchronize(); + continue; + } + vecPush(statements, int(varStmt)); + } else { + var stmt = Node~(0); + stmt = pStatement(); + if (pPanic) { + pSynchronize(); + continue; + } + vecPush(statements, int(stmt)); + } + } + var m0 = "Expect '}' after block."; + pConsume(T_RIGHT_BRACE, &m0[0], len(m0)); + return statements; +} + +def pInvalidTypePanic() { + errBegin(tokenLine(pPeek())); + var m0 = "Invalid type specifier starting at '"; + errRaw(&m0[0], len(m0)); + errRaw(pPeek()~.lexPtr, pPeek()~.lexLen); + var m1 = "'."; + errRaw(&m1[0], len(m1)); + errEnd(); + pPanic = true; +} + +def pType() Type~ { + var baseType = Type~(0); + if (pMatch(T_LEFT_BRACKET)) { + var elementType = Type~(0); + elementType = pType(); + if (pPanic) { return gTypeError; } + var m0 = "Expect ';' in array type."; + pConsume(T_SEMICOLON, &m0[0], len(m0)); + if (pPanic) { return gTypeError; } + var m1 = "Expect array length specifier."; + var lengthTok = pConsume(T_NUMBER, &m1[0], len(m1)); + if (pPanic) { return gTypeError; } + var m2 = "Expect ']' after array type."; + pConsume(T_RIGHT_BRACKET, &m2[0], len(m2)); + if (pPanic) { return gTypeError; } + baseType = arrayType(elementType, lengthTok~.litInt); + } else if (pMatch(T_INT)) { + baseType = gTypeInt; + } else if (pMatch(T_FLOAT)) { + baseType = gTypeFloat; + } else if (pMatch(T_BYTE)) { + baseType = gTypeByte; + } else if (pMatch(T_BOOL)) { + baseType = gTypeBool; + } else if (pMatch(T_IDENTIFIER)) { + baseType = unresolvedStructType(pPrevious()); + } + + if (int(baseType) != 0) { + var outType = baseType; + while (pMatch(T_TILDE)) { + if (isValidElementType(outType)) { + outType = ptrType(outType); + } else { + pInvalidTypePanic(); + return gTypeError; + } + } + return outType; + } + pInvalidTypePanic(); + return gTypeError; +} + +def pExpression() Node~ { + return pExprAssignment(); +} + +// Builds the assign node for compound assignment operators (+=, -=, ...), +// desugaring into `left = left right`. +def pCompoundAssign(left Node~, combinedOperator Token~, opType int, opChar int) Node~ { + var right = Node~(0); + right = pExprAssignment(); + if (pPanic) { return Node~(0); } + var opStr = bufNew(); + bufPushChar(opStr, opChar); + var binary = newNode(NK_BINARY_EXPR); + binary~.a = left; + binary~.tokA = fakeToken(opType, strFromBuf(opStr), combinedOperator); + binary~.b = right; + var node = newNode(NK_ASSIGN_EXPR); + node~.tokA = combinedOperator; + node~.a = left; + node~.b = binary; + return node; +} + +def pInvalidAssignmentTarget() { + // No need to panic and synchronize. + // Report error; it's still valuable to continue parsing + // the rest of whatever statement we're in to indicate + // remaining syntax errors to the user. + errBegin(tokenLine(pPeek())); + var m0 = "Invalid assignment target."; + errRaw(&m0[0], len(m0)); + errEnd(); +} + +def pExprAssignment() Node~ { + var expr = Node~(0); + expr = pExprOr(); + if (pPanic) { return Node~(0); } + var isValidAssignmentTarget = + expr~.kind == NK_VARIABLE_EXPR || + expr~.kind == NK_INDEX_EXPR || + expr~.kind == NK_DEREF_EXPR || + expr~.kind == NK_DOT_EXPR; + if (pMatch(T_EQUAL)) { + var operator = pPrevious(); + var right = Node~(0); + right = pExprAssignment(); + if (pPanic) { return Node~(0); } + if (isValidAssignmentTarget) { + var node = newNode(NK_ASSIGN_EXPR); + node~.tokA = operator; + node~.a = expr; + node~.b = right; + return node; + } + pInvalidAssignmentTarget(); + } else if (pMatch(T_PLUS_EQUAL)) { + if (isValidAssignmentTarget) { + return pCompoundAssign(expr, pPrevious(), T_PLUS, 43); + } + var r1 = Node~(0); + r1 = pExprAssignment(); + if (pPanic) { return Node~(0); } + pInvalidAssignmentTarget(); + } else if (pMatch(T_MINUS_EQUAL)) { + if (isValidAssignmentTarget) { + return pCompoundAssign(expr, pPrevious(), T_MINUS, 45); + } + var r2 = Node~(0); + r2 = pExprAssignment(); + if (pPanic) { return Node~(0); } + pInvalidAssignmentTarget(); + } else if (pMatch(T_STAR_EQUAL)) { + if (isValidAssignmentTarget) { + return pCompoundAssign(expr, pPrevious(), T_STAR, 42); + } + var r3 = Node~(0); + r3 = pExprAssignment(); + if (pPanic) { return Node~(0); } + pInvalidAssignmentTarget(); + } else if (pMatch(T_SLASH_EQUAL)) { + if (isValidAssignmentTarget) { + return pCompoundAssign(expr, pPrevious(), T_SLASH, 47); + } + var r4 = Node~(0); + r4 = pExprAssignment(); + if (pPanic) { return Node~(0); } + pInvalidAssignmentTarget(); + } else if (pMatch(T_PERCENT_EQUAL)) { + if (isValidAssignmentTarget) { + return pCompoundAssign(expr, pPrevious(), T_PERCENT, 37); + } + var r5 = Node~(0); + r5 = pExprAssignment(); + if (pPanic) { return Node~(0); } + pInvalidAssignmentTarget(); + } + return expr; +} + +def pExprOr() Node~ { + var expr = Node~(0); + expr = pExprAnd(); + if (pPanic) { return Node~(0); } + while (pMatch(T_BAR_BAR)) { + var operator = pPrevious(); + var right = Node~(0); + right = pExprAnd(); + if (pPanic) { return Node~(0); } + var node = newNode(NK_LOGICAL_EXPR); + node~.a = expr; + node~.tokA = operator; + node~.b = right; + expr = node; + } + return expr; +} + +def pExprAnd() Node~ { + var expr = Node~(0); + expr = pExprEquality(); + if (pPanic) { return Node~(0); } + while (pMatch(T_AMP_AMP)) { + var operator = pPrevious(); + var right = Node~(0); + right = pExprEquality(); + if (pPanic) { return Node~(0); } + var node = newNode(NK_LOGICAL_EXPR); + node~.a = expr; + node~.tokA = operator; + node~.b = right; + expr = node; + } + return expr; +} + +def pBinaryStep(left Node~, operator Token~, right Node~) Node~ { + var node = newNode(NK_BINARY_EXPR); + node~.a = left; + node~.tokA = operator; + node~.b = right; + return node; +} + +def pExprEquality() Node~ { + var expr = Node~(0); + expr = pExprComparison(); + if (pPanic) { return Node~(0); } + while (pMatch(T_EQUAL_EQUAL) || pMatch(T_BANG_EQUAL)) { + var operator = pPrevious(); + var right = Node~(0); + right = pExprComparison(); + if (pPanic) { return Node~(0); } + expr = pBinaryStep(expr, operator, right); + } + return expr; +} + +def pExprComparison() Node~ { + var expr = Node~(0); + expr = pExprTerm(); + if (pPanic) { return Node~(0); } + while (pMatch(T_GREATER) || pMatch(T_GREATER_EQUAL) || pMatch(T_LESS) || pMatch(T_LESS_EQUAL)) { + var operator = pPrevious(); + var right = Node~(0); + right = pExprTerm(); + if (pPanic) { return Node~(0); } + expr = pBinaryStep(expr, operator, right); + } + return expr; +} + +def pExprTerm() Node~ { + var expr = Node~(0); + expr = pExprFactor(); + if (pPanic) { return Node~(0); } + while (pMatch(T_MINUS) || pMatch(T_PLUS)) { + var operator = pPrevious(); + var right = Node~(0); + right = pExprFactor(); + if (pPanic) { return Node~(0); } + expr = pBinaryStep(expr, operator, right); + } + return expr; +} + +def pExprFactor() Node~ { + var expr = Node~(0); + expr = pExprUnary(); + if (pPanic) { return Node~(0); } + while (pMatch(T_SLASH) || pMatch(T_STAR) || pMatch(T_PERCENT)) { + var operator = pPrevious(); + var right = Node~(0); + right = pExprUnary(); + if (pPanic) { return Node~(0); } + expr = pBinaryStep(expr, operator, right); + } + return expr; +} + +def pExprUnary() Node~ { + if (pMatch(T_BANG) || pMatch(T_MINUS) || pMatch(T_AMP)) { + var operator = pPrevious(); + var value = Node~(0); + value = pExprUnary(); + if (pPanic) { return Node~(0); } + var node = newNode(NK_UNARY_EXPR); + node~.tokA = operator; + node~.a = value; + return node; + } + return pExprCall(); +} + +def pExprCall() Node~ { + var expr = Node~(0); + expr = pExprPrimary(); + if (pPanic) { return Node~(0); } + // can have at most one call or construct in series of calls/indexes (no first class functions) + if (pMatch(T_LEFT_PAREN)) { + var paren = pPrevious(); + var args = vecNew(); + while (!pCheck(T_RIGHT_PAREN) && !pIsAtEnd()) { + if (args~.count > 0) { + var m0 = "Expect ',' between arguments."; + pConsume(T_COMMA, &m0[0], len(m0)); + if (pPanic) { return Node~(0); } + } + var arg = Node~(0); + arg = pExpression(); + if (pPanic) { return Node~(0); } + vecPush(args, int(arg)); + } + var m1 = "Expect ')' after arguments."; + pConsume(T_RIGHT_PAREN, &m1[0], len(m1)); + if (pPanic) { return Node~(0); } + var node = newNode(NK_CALL_EXPR); + node~.a = expr; + node~.tokA = paren; + node~.list = args; + expr = node; + } else if (pMatch(T_LEFT_BRACE)) { + var brace = pPrevious(); + var args2 = vecNew(); + while (!pCheck(T_RIGHT_BRACE) && !pIsAtEnd()) { + if (args2~.count > 0) { + var m2 = "Expect ',' between member initializers."; + pConsume(T_COMMA, &m2[0], len(m2)); + if (pPanic) { return Node~(0); } + } + var arg2 = Node~(0); + arg2 = pExpression(); + if (pPanic) { return Node~(0); } + vecPush(args2, int(arg2)); + } + var m3 = "Expect '}' after member initializers."; + pConsume(T_RIGHT_BRACE, &m3[0], len(m3)); + if (pPanic) { return Node~(0); } + var node2 = newNode(NK_CALL_EXPR); + node2~.a = expr; + node2~.tokA = brace; + node2~.list = args2; + expr = node2; + } + // can have any number of indexes or pointer dereferences + while (pMatch(T_LEFT_BRACKET) || pMatch(T_TILDE) || pMatch(T_DOT)) { + var operator = pPrevious(); + if (operator~.type == T_LEFT_BRACKET) { + var index = Node~(0); + index = pExpression(); + if (pPanic) { return Node~(0); } + var m4 = "Expect ']' after index."; + pConsume(T_RIGHT_BRACKET, &m4[0], len(m4)); + if (pPanic) { return Node~(0); } + var idxNode = newNode(NK_INDEX_EXPR); + idxNode~.a = expr; + idxNode~.tokA = operator; + idxNode~.b = index; + expr = idxNode; + } else if (operator~.type == T_TILDE) { + var derefNode = newNode(NK_DEREF_EXPR); + derefNode~.tokA = operator; + derefNode~.a = expr; + expr = derefNode; + } else { + var m5 = "Expect identifier after '.'."; + var identifier = pConsume(T_IDENTIFIER, &m5[0], len(m5)); + if (pPanic) { return Node~(0); } + var dotNode = newNode(NK_DOT_EXPR); + dotNode~.a = expr; + dotNode~.tokA = operator; + dotNode~.tokB = identifier; + expr = dotNode; + } + } + return expr; +} + +// Returns true if the token stream looks like the start of a cast to +// a pointer-to-struct type, e.g. `Foo~(expr)` or `Foo~~(expr)`. +def pCheckStructPtrCast() bool { + if (!pCheck(T_IDENTIFIER)) { + return false; + } + var i = pCurrent + 1; + while (i < numTokens() && tokenAt(i)~.type == T_TILDE) { + i = i + 1; + } + return i > pCurrent + 1 && i < numTokens() && tokenAt(i)~.type == T_LEFT_PAREN; +} + +def pCastPrimary() Node~ { + // cast expression + var castType = Type~(0); + castType = pType(); + if (pPanic) { return Node~(0); } + var c = castType~.category; + if (!(c == TC_INT || c == TC_FLOAT || c == TC_BYTE || c == TC_BOOL || c == TC_POINTER)) { + var m0 = "Cannot cast to this type."; + pPanicHere(&m0[0], len(m0)); + return Node~(0); + } + var m1 = "Expect '(' after type in cast expression."; + pConsume(T_LEFT_PAREN, &m1[0], len(m1)); + if (pPanic) { return Node~(0); } + var paren = pPrevious(); + var value = Node~(0); + value = pExpression(); + if (pPanic) { return Node~(0); } + var m2 = "Expect ')' after cast expression."; + pConsume(T_RIGHT_PAREN, &m2[0], len(m2)); + if (pPanic) { return Node~(0); } + var node = newNode(NK_CAST_EXPR); + node~.tokA = paren; + node~.typeA = castType; + node~.a = value; + return node; +} + +def pExprPrimary() Node~ { + if (pMatch(T_TRUE) || pMatch(T_FALSE)) { + var node = newNode(NK_LITERAL_EXPR); + node~.typeA = gTypeBool; + node~.litBool = pPrevious()~.type == T_TRUE; + return node; + } + if (pMatch(T_NUMBER)) { + var node2 = newNode(NK_LITERAL_EXPR); + node2~.typeA = gTypeInt; + node2~.litInt = pPrevious()~.litInt; + node2~.decStr = pPrevious()~.decStr; + return node2; + } + if (pMatch(T_NUMBER_DECIMAL)) { + var node3 = newNode(NK_LITERAL_EXPR); + node3~.typeA = gTypeFloat; + node3~.litFloat = pPrevious()~.litFloat; + node3~.lexeme = tokenLexemeStr(pPrevious()); + return node3; + } + if (pMatch(T_NUMBER_HEX)) { + var node4 = newNode(NK_LITERAL_EXPR); + node4~.typeA = gTypeInt; + node4~.litInt = pPrevious()~.litInt; + node4~.decStr = pPrevious()~.decStr; + return node4; + } + if (pMatch(T_STRING)) { + var s = pPrevious()~.litStr; + var existing = strLitLookup(s); + if (int(existing) != 0) { + return existing; + } + var litNode = newNode(NK_LITERAL_EXPR); + litNode~.typeA = arrayType(gTypeByte, s~.count); + litNode~.litStr = s; + strLitIntern(s, litNode); + return litNode; + } + if (pMatch(T_SINGLE_QUOTE_STRING)) { + var cs = pPrevious()~.litStr; + // The reference compiler checks the UTF-16 length of the literal; + // compute it from the UTF-8 bytes. + var utf16Len = 0; + for (var ci = 0; ci < cs~.count; ci = ci + 1) { + var cb = int(strGet(cs, ci)); + if (cb < 128) { + utf16Len = utf16Len + 1; // ASCII + } else if (cb >= 240) { + utf16Len = utf16Len + 2; // 4-byte sequence: surrogate pair + } else if (cb >= 192) { + utf16Len = utf16Len + 1; // 2- or 3-byte sequence lead + } // else: continuation byte + } + if (utf16Len != 1) { + var m0 = "Invalid character literal (use double quotes for strings)."; + pPanicHere(&m0[0], len(m0)); + return Node~(0); + } + if (int(strGet(cs, 0)) >= 128) { + var m1 = "Invalid character literal (only ASCII characters allowed)."; + pPanicHere(&m1[0], len(m1)); + return Node~(0); + } + var chNode = newNode(NK_LITERAL_EXPR); + chNode~.typeA = gTypeByte; + chNode~.litInt = int(strGet(cs, 0)); + return chNode; + } + if (pCheckStructPtrCast()) { + return pCastPrimary(); + } + if (pMatch(T_IDENTIFIER)) { + var varNode = newNode(NK_VARIABLE_EXPR); + varNode~.tokA = pPrevious(); + return varNode; + } + if (pMatch(T_LEFT_PAREN)) { + var expr = Node~(0); + expr = pExpression(); + if (pPanic) { return Node~(0); } + var m2 = "Expect ')' matching '('."; + pConsume(T_RIGHT_PAREN, &m2[0], len(m2)); + if (pPanic) { return Node~(0); } + var groupNode = newNode(NK_GROUP_EXPR); + groupNode~.a = expr; + return groupNode; + } + if (pMatch(T_LEFT_BRACKET)) { + var bracket = pPrevious(); + var values = vecNew(); + if (!pCheck(T_RIGHT_BRACKET)) { + // disambiguate between [x; N] and [x, y, z] literals + var value = Node~(0); + value = pExpression(); + if (pPanic) { return Node~(0); } + if (pMatch(T_SEMICOLON)) { + var m3 = "Expect length specifier in array repeat literal."; + var lengthTok = pConsume(T_NUMBER, &m3[0], len(m3)); + if (pPanic) { return Node~(0); } + if (lengthTok~.litInt < 0) { + var m4 = "Array length specifier must be >=0."; + pPanicHere(&m4[0], len(m4)); + return Node~(0); + } + var m5 = "Expect ']' after list literal."; + pConsume(T_RIGHT_BRACKET, &m5[0], len(m5)); + if (pPanic) { return Node~(0); } + var repNode = newNode(NK_LIST_EXPR); + repNode~.tokA = bracket; + repNode~.listKind = LK_REPEAT; + repNode~.a = value; + repNode~.repeatLen = lengthTok~.litInt; + return repNode; + } + vecPush(values, int(value)); + while (!pCheck(T_RIGHT_BRACKET) && !pIsAtEnd()) { + if (values~.count > 0) { + var m6 = "Expect ',' between items in list literal."; + pConsume(T_COMMA, &m6[0], len(m6)); + if (pPanic) { return Node~(0); } + } + var val = Node~(0); + val = pExpression(); + if (pPanic) { return Node~(0); } + vecPush(values, int(val)); + } + } + var m7 = "Expect ']' after list literal."; + pConsume(T_RIGHT_BRACKET, &m7[0], len(m7)); + if (pPanic) { return Node~(0); } + var listNode = newNode(NK_LIST_EXPR); + listNode~.tokA = bracket; + listNode~.listKind = LK_LIST; + listNode~.list = values; + return listNode; + } + + if (pCheck(T_INT) || pCheck(T_FLOAT) || pCheck(T_BYTE) || pCheck(T_BOOL)) { + return pCastPrimary(); + } + + if (pMatch(T_LEN)) { + var m8 = "Expect '(' before len expression."; + pConsume(T_LEFT_PAREN, &m8[0], len(m8)); + if (pPanic) { return Node~(0); } + var lenValue = Node~(0); + lenValue = pExpression(); + if (pPanic) { return Node~(0); } + var m9 = "Expect ')' after len expression."; + pConsume(T_RIGHT_PAREN, &m9[0], len(m9)); + if (pPanic) { return Node~(0); } + var lenNode = newNode(NK_LEN_EXPR); + lenNode~.a = lenValue; + lenNode~.resolvedType = gTypeInt; + return lenNode; + } + + var m10 = "Expect expression."; + pPanicHere(&m10[0], len(m10)); + return Node~(0); +} + +// Parses gTokens into gTopLevel. Mirrors `parse` in src/parser.ts. +// initContext() must have been called already. +def parseProgram() { + pCurrent = 0; + pPanic = false; + vecPush(pScopes, int(gGlobalScope)); + while (!pIsAtEnd()) { + var stmt = Node~(0); + stmt = pTopDecl(); + if (pPanic) { + pSynchronize(); + continue; + } + vecPush(gTopLevel, int(stmt)); + } +} diff --git a/selfhost/scanner.puff b/selfhost/scanner.puff index 7bad815..685cc4c 100644 --- a/selfhost/scanner.puff +++ b/selfhost/scanner.puff @@ -86,7 +86,7 @@ struct Token { } def tokenNew(type int, lexPtr byte~, lexLen int, offset int) Token~ { - var t = Token~(alloc(32)); + var t = Token~(alloc(40)); t~.type = type; t~.lexPtr = lexPtr; t~.lexLen = lexLen; @@ -98,7 +98,7 @@ def tokenNew(type int, lexPtr byte~, lexLen int, offset int) Token~ { } def fakeToken(type int, lexeme Str~, locationProvider Token~) Token~ { - var t = Token~(alloc(32)); + var t = Token~(alloc(40)); t~.type = type; t~.lexPtr = lexeme~.data; t~.lexLen = lexeme~.count; @@ -450,14 +450,32 @@ def parseHexLexeme(start int, length int) int { return v; } -// Parses the float literal digits as f32. +// Parses the float literal digits as f32. The integer part is accumulated +// exactly in i32 while it fits (then converted, which rounds correctly); +// we only fall back to lossy f32 accumulation for huge values. def parseFloatLexeme(start int, length int) float { var v = 0.0; + var intPart = 0; + var overflowed = false; var i = 0; while (i < length && srcByte(start + i) != 46) { - v = v * 10.0 + float(srcByte(start + i) - 48); + var d = srcByte(start + i) - 48; + if (!overflowed) { + if (intPart > (2147483647 - d) / 10) { + overflowed = true; + v = float(intPart); + } else { + intPart = intPart * 10 + d; + } + } + if (overflowed) { + v = v * 10.0 + float(d); + } i = i + 1; } + if (!overflowed) { + v = float(intPart); + } i = i + 1; // skip '.' var scale = 0.1; while (i < length) { @@ -614,7 +632,7 @@ def scanTokens() { } var eofLex = strEmpty(); - var eof = Token~(alloc(32)); + var eof = Token~(alloc(40)); eof~.type = T_EOF; eof~.lexPtr = eofLex~.data; eof~.lexLen = 0; diff --git a/selfhost/sexpr.puff b/selfhost/sexpr.puff new file mode 100644 index 0000000..9d68658 --- /dev/null +++ b/selfhost/sexpr.puff @@ -0,0 +1,342 @@ +// sexpr.puff +// AST s-expression printer, ported from astToSExpr/typeToSExpr in +// src/nodes.ts. Used for differential testing of the parser against the +// reference compiler. + +def sxRaw(out Buf~, p byte~, n int) { + bufPushBytes(out, p, n); +} + +def typeToSExpr(out Buf~, t Type~) { + if (t~.category == TC_ARRAY) { + var s0 = "(arraytype "; + sxRaw(out, &s0[0], len(s0)); + bufPushInt(out, t~.length); + bufPushChar(out, 32); + typeToSExpr(out, t~.elementType); + bufPushChar(out, 41); + return; + } + if (t~.category == TC_POINTER) { + var s1 = "(ptr "; + sxRaw(out, &s1[0], len(s1)); + typeToSExpr(out, t~.elementType); + bufPushChar(out, 41); + return; + } + if (t~.category == TC_STRUCT) { + var s2 = "(struct ("; + sxRaw(out, &s2[0], len(s2)); + if (int(t~.resolvedStruct) != 0) { + var members = t~.resolvedStruct~.params; + for (var i = 0; i < members~.count; i = i + 1) { + var member = Param~(vecGet(members, i)); + bufPushChar(out, 40); + bufPushBytes(out, member~.name~.lexPtr, member~.name~.lexLen); + bufPushChar(out, 32); + typeToSExpr(out, member~.type); + bufPushChar(out, 41); + } + } else { + var s3 = "unresolved '"; + sxRaw(out, &s3[0], len(s3)); + bufPushBytes(out, t~.name~.lexPtr, t~.name~.lexLen); + bufPushChar(out, 39); + } + bufPushChar(out, 41); + bufPushChar(out, 41); + return; + } + if (t~.category == TC_ERROR) { + var s4 = ""; + sxRaw(out, &s4[0], len(s4)); + return; + } + // int/float/byte/bool/void: reuse typeToString's lowercase names + typeToString(out, t); +} + +// Appends the literal string value escaped like JSON.stringify. +def sxQuotedString(out Buf~, s Str~) { + bufPushChar(out, 34); + for (var i = 0; i < s~.count; i = i + 1) { + var c = int(strGet(s, i)); + if (c == 34) { + bufPushChar(out, 92); + bufPushChar(out, 34); + } else if (c == 92) { + bufPushChar(out, 92); + bufPushChar(out, 92); + } else if (c == 10) { + bufPushChar(out, 92); + bufPushChar(out, 110); // 'n' + } else if (c == 9) { + bufPushChar(out, 92); + bufPushChar(out, 116); // 't' + } else if (c == 13) { + bufPushChar(out, 92); + bufPushChar(out, 114); // 'r' + } else { + bufPushChar(out, c); + } + } + bufPushChar(out, 34); +} + +def astToSExpr(out Buf~, node Node~) { + var kind = node~.kind; + if (kind == NK_ASSIGN_EXPR) { + var s0 = "(assign "; + sxRaw(out, &s0[0], len(s0)); + astToSExpr(out, node~.a); + bufPushChar(out, 32); + astToSExpr(out, node~.b); + bufPushChar(out, 41); + } else if (kind == NK_BINARY_EXPR || kind == NK_LOGICAL_EXPR) { + bufPushChar(out, 40); + bufPushBytes(out, node~.tokA~.lexPtr, node~.tokA~.lexLen); + bufPushChar(out, 32); + astToSExpr(out, node~.a); + bufPushChar(out, 32); + astToSExpr(out, node~.b); + bufPushChar(out, 41); + } else if (kind == NK_CALL_EXPR) { + var s1 = "(call "; + sxRaw(out, &s1[0], len(s1)); + astToSExpr(out, node~.a); + bufPushChar(out, 32); + bufPushChar(out, 40); + for (var i = 0; i < node~.list~.count; i = i + 1) { + if (i > 0) { + bufPushChar(out, 32); + } + astToSExpr(out, Node~(vecGet(node~.list, i))); + } + bufPushChar(out, 41); + bufPushChar(out, 41); + } else if (kind == NK_CAST_EXPR) { + bufPushChar(out, 40); + typeToSExpr(out, node~.typeA); + bufPushChar(out, 32); + astToSExpr(out, node~.a); + bufPushChar(out, 41); + } else if (kind == NK_DEREF_EXPR) { + var s2 = "(deref "; + sxRaw(out, &s2[0], len(s2)); + astToSExpr(out, node~.a); + bufPushChar(out, 41); + } else if (kind == NK_DOT_EXPR) { + var s3 = "(. "; + sxRaw(out, &s3[0], len(s3)); + astToSExpr(out, node~.a); + bufPushChar(out, 32); + bufPushBytes(out, node~.tokB~.lexPtr, node~.tokB~.lexLen); + bufPushChar(out, 41); + } else if (kind == NK_GROUP_EXPR) { + bufPushChar(out, 40); + astToSExpr(out, node~.a); + bufPushChar(out, 41); + } else if (kind == NK_INDEX_EXPR) { + var s4 = "(index "; + sxRaw(out, &s4[0], len(s4)); + astToSExpr(out, node~.a); + bufPushChar(out, 32); + astToSExpr(out, node~.b); + bufPushChar(out, 41); + } else if (kind == NK_LEN_EXPR) { + var s5 = "(len "; + sxRaw(out, &s5[0], len(s5)); + astToSExpr(out, node~.a); + bufPushChar(out, 41); + } else if (kind == NK_LIST_EXPR) { + bufPushChar(out, 40); + if (node~.listKind == LK_LIST) { + var s6 = "list-initializer"; + sxRaw(out, &s6[0], len(s6)); + for (var j = 0; j < node~.list~.count; j = j + 1) { + bufPushChar(out, 32); + astToSExpr(out, Node~(vecGet(node~.list, j))); + } + } else { + var s7 = "repeat-initializer "; + sxRaw(out, &s7[0], len(s7)); + bufPushInt(out, node~.repeatLen); + bufPushChar(out, 32); + astToSExpr(out, node~.a); + } + bufPushChar(out, 41); + } else if (kind == NK_LITERAL_EXPR) { + var cat = node~.typeA~.category; + if (cat == TC_FLOAT) { + // Mirrors: integral values print via toFixed(1), others via + // JSON.stringify (which we approximate with the normalized lexeme). + var v = node~.litFloat; + var isIntegral = false; + if (v >= -2147483520.0 && v <= 2147483520.0) { + if (float(int(v)) == v) { + isIntegral = true; + } + } + if (isIntegral) { + bufPushInt(out, int(v)); + var s8 = ".0"; + sxRaw(out, &s8[0], len(s8)); + } else { + // strip trailing zeros (and a trailing '.') from the lexeme + var end = node~.lexeme~.count; + while (end > 0 && int(strGet(node~.lexeme, end - 1)) == 48) { + end = end - 1; + } + if (end > 0 && int(strGet(node~.lexeme, end - 1)) == 46) { + end = end - 1; + } + bufPushBytes(out, node~.lexeme~.data, end); + } + } else if (cat == TC_BOOL) { + if (node~.litBool) { + var s9 = "true"; + sxRaw(out, &s9[0], len(s9)); + } else { + var s10 = "false"; + sxRaw(out, &s10[0], len(s10)); + } + } else if (cat == TC_ARRAY) { + // string literal + sxQuotedString(out, node~.litStr); + } else if (int(node~.decStr) != 0) { + bufPushBytes(out, node~.decStr~.data, node~.decStr~.count); + } else { + bufPushInt(out, node~.litInt); + } + } else if (kind == NK_UNARY_EXPR) { + bufPushChar(out, 40); + bufPushBytes(out, node~.tokA~.lexPtr, node~.tokA~.lexLen); + bufPushChar(out, 32); + astToSExpr(out, node~.a); + bufPushChar(out, 41); + } else if (kind == NK_VARIABLE_EXPR) { + bufPushBytes(out, node~.tokA~.lexPtr, node~.tokA~.lexLen); + } else if (kind == NK_BLOCK_STMT) { + var s11 = "(block "; + sxRaw(out, &s11[0], len(s11)); + for (var k = 0; k < node~.list~.count; k = k + 1) { + if (k > 0) { + bufPushChar(out, 32); + } + astToSExpr(out, Node~(vecGet(node~.list, k))); + } + bufPushChar(out, 41); + } else if (kind == NK_EXPRESSION_STMT) { + astToSExpr(out, node~.a); + } else if (kind == NK_FUNCTION_STMT) { + bufPushChar(out, 40); + if (int(node~.hostModule) != 0) { + var s12 = "import "; + sxRaw(out, &s12[0], len(s12)); + } else if (node~.isExported) { + var s13 = "export "; + sxRaw(out, &s13[0], len(s13)); + } + var s14 = "def "; + sxRaw(out, &s14[0], len(s14)); + bufPushBytes(out, node~.tokA~.lexPtr, node~.tokA~.lexLen); + bufPushChar(out, 32); + bufPushChar(out, 40); + for (var m = 0; m < node~.params~.count; m = m + 1) { + if (m > 0) { + bufPushChar(out, 32); + } + var param = Param~(vecGet(node~.params, m)); + var s15 = "(param "; + sxRaw(out, &s15[0], len(s15)); + bufPushBytes(out, param~.name~.lexPtr, param~.name~.lexLen); + bufPushChar(out, 32); + typeToSExpr(out, param~.type); + bufPushChar(out, 41); + } + bufPushChar(out, 41); + bufPushChar(out, 32); + bufPushChar(out, 40); + if (node~.hasBody) { + for (var n2 = 0; n2 < node~.list~.count; n2 = n2 + 1) { + if (n2 > 0) { + bufPushChar(out, 32); + } + astToSExpr(out, Node~(vecGet(node~.list, n2))); + } + } + bufPushChar(out, 41); + bufPushChar(out, 41); + } else if (kind == NK_IF_STMT) { + var s16 = "(if "; + sxRaw(out, &s16[0], len(s16)); + astToSExpr(out, node~.a); + bufPushChar(out, 32); + astToSExpr(out, node~.b); + bufPushChar(out, 32); + if (int(node~.c) != 0) { + astToSExpr(out, node~.c); + bufPushChar(out, 32); + } + bufPushChar(out, 41); + } else if (kind == NK_LOOP_CONTROL_STMT) { + bufPushChar(out, 40); + bufPushBytes(out, node~.tokA~.lexPtr, node~.tokA~.lexLen); + bufPushChar(out, 41); + } else if (kind == NK_PRINT_STMT) { + var s17 = "(print "; + sxRaw(out, &s17[0], len(s17)); + astToSExpr(out, node~.a); + bufPushChar(out, 41); + } else if (kind == NK_RETURN_STMT) { + var s18 = "(return "; + sxRaw(out, &s18[0], len(s18)); + if (int(node~.a) != 0) { + astToSExpr(out, node~.a); + } else { + var s19 = "void"; + sxRaw(out, &s19[0], len(s19)); + } + bufPushChar(out, 41); + } else if (kind == NK_STRUCT_STMT) { + var s20 = "(struct "; + sxRaw(out, &s20[0], len(s20)); + bufPushBytes(out, node~.tokA~.lexPtr, node~.tokA~.lexLen); + bufPushChar(out, 32); + bufPushChar(out, 40); + for (var q = 0; q < node~.params~.count; q = q + 1) { + var member = Param~(vecGet(node~.params, q)); + bufPushChar(out, 40); + bufPushBytes(out, member~.name~.lexPtr, member~.name~.lexLen); + bufPushChar(out, 32); + typeToSExpr(out, member~.type); + bufPushChar(out, 41); + } + bufPushChar(out, 41); + bufPushChar(out, 41); + } else if (kind == NK_VAR_STMT) { + var s21 = "(var "; + sxRaw(out, &s21[0], len(s21)); + bufPushBytes(out, node~.tokA~.lexPtr, node~.tokA~.lexLen); + bufPushChar(out, 32); + if (int(node~.typeA) != 0) { + typeToSExpr(out, node~.typeA); + bufPushChar(out, 32); + } + astToSExpr(out, node~.a); + bufPushChar(out, 41); + } else if (kind == NK_WHILE_STMT) { + var s22 = "(while "; + sxRaw(out, &s22[0], len(s22)); + astToSExpr(out, node~.a); + bufPushChar(out, 32); + astToSExpr(out, node~.b); + if (int(node~.c) != 0) { + bufPushChar(out, 32); + astToSExpr(out, node~.c); + } + bufPushChar(out, 41); + } else { + die(90); // unhandled node kind in astToSExpr + } +} diff --git a/src/backend.ts b/src/backend.ts index 2a33b98..cb6a61c 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -439,6 +439,9 @@ export function emit(context: ast.Context): string { } function emitDebugComments(node: ast.Node) { + if (!DEBUG_COMMENTS) { + return + } switch (node.kind) { case ast.NodeKind.BINARY_EXPR: case ast.NodeKind.CAST_EXPR: @@ -469,10 +472,12 @@ export function emit(context: ast.Context): string { // expressions case ast.NodeKind.ASSIGN_EXPR: { const op = node as ast.AssignExpr - op.operator.lineStr(true).split("\n").forEach((l) => { - debugLine(`;; ${l}`) - }) - debugLine(``) + if (DEBUG_COMMENTS) { + op.operator.lineStr(true).split("\n").forEach((l) => { + debugLine(`;; ${l}`) + }) + debugLine(``) + } if (op.left.kind === ast.NodeKind.VARIABLE_EXPR) { const symbol = op.left.resolvedSymbol! @@ -803,10 +808,12 @@ export function emit(context: ast.Context): string { } case ast.NodeKind.DOT_EXPR: { const op = node as ast.DotExpr - op.dot.lineStr(true).split("\n").forEach((l) => { - debugLine(`;; ${l}`) - }) - debugLine(``) + if (DEBUG_COMMENTS) { + op.dot.lineStr(true).split("\n").forEach((l) => { + debugLine(`;; ${l}`) + }) + debugLine(``) + } const memberType = op.resolvedType const structType = op.callee.resolvedType @@ -845,10 +852,12 @@ export function emit(context: ast.Context): string { case ast.NodeKind.INDEX_EXPR: { // TODO: trap on out-of-bounds access const op = node as ast.IndexExpr - op.bracket.lineStr(true).split("\n").forEach((l) => { - debugLine(`;; ${l}`) - }) - debugLine(``) + if (DEBUG_COMMENTS) { + op.bracket.lineStr(true).split("\n").forEach((l) => { + debugLine(`;; ${l}`) + }) + debugLine(``) + } const elementType = op.resolvedType! visit(op.callee, ExprMode.LVALUE) // get address of array start diff --git a/tools/build-selfhost.sh b/tools/build-selfhost.sh index b38c48c..243bdab 100755 --- a/tools/build-selfhost.sh +++ b/tools/build-selfhost.sh @@ -13,6 +13,7 @@ SOURCES=( selfhost/scanner.puff ) [ -f selfhost/ast.puff ] && SOURCES+=(selfhost/ast.puff) +[ -f selfhost/sexpr.puff ] && SOURCES+=(selfhost/sexpr.puff) [ -f selfhost/parser.puff ] && SOURCES+=(selfhost/parser.puff) [ -f selfhost/resolver.puff ] && SOURCES+=(selfhost/resolver.puff) [ -f selfhost/backend.puff ] && SOURCES+=(selfhost/backend.puff) diff --git a/tools/compare-selfhost.sh b/tools/compare-selfhost.sh new file mode 100755 index 0000000..21e9f36 --- /dev/null +++ b/tools/compare-selfhost.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Differentially tests the self-hosted compiler against the TS reference +# compiler over the fixture corpus. +# Usage: tools/compare-selfhost.sh +# mode: sexpr | errors | wat +set -uo pipefail +cd "$(dirname "$0")/.." + +MODE="$1"; shift +WASM="$1"; shift + +PASS=0 +FAIL=0 +for f in "$@"; do + if [ "$MODE" = "wat" ]; then + node dist/tools/puffc.js "$f" > /tmp/cmp-a.out 2>/tmp/cmp-a.err; AE=$? + else + node dist/tools/dump.js "--$MODE" "$f" > /tmp/cmp-a.out 2>/tmp/cmp-a.err; AE=$? + fi + node dist/tools/run.js "$WASM" --stdin "$f" > /tmp/cmp-b.out 2>/tmp/cmp-b.err; BE=$? + if diff -q /tmp/cmp-a.out /tmp/cmp-b.out >/dev/null && diff -q /tmp/cmp-a.err /tmp/cmp-b.err >/dev/null && [ "$AE" -eq "$BE" ]; then + PASS=$((PASS+1)) + else + FAIL=$((FAIL+1)) + echo "DIFF $f (exit $AE vs $BE)" + fi +done +echo "$MODE: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] diff --git a/tools/extract-fixtures.ts b/tools/extract-fixtures.ts new file mode 100644 index 0000000..75d7fbc --- /dev/null +++ b/tools/extract-fixtures.ts @@ -0,0 +1,27 @@ +// Extracts embedded puffscript sources from test.ts into fixture files for +// differential testing of the self-hosted compiler. +// Usage: node dist/tools/extract-fixtures.js +import fs from 'fs' +import path from 'path' + +function main() { + const outDir = process.argv[2] ?? "test/fixtures" + fs.mkdirSync(outDir, { recursive: true }) + const testSource = fs.readFileSync(path.join(__dirname, "../../test.ts"), "utf8") + + // Matches the first template-literal argument of test helper calls. + const re = /(expectOutput|expectAST|expectParseErrors|expectResolveErrors)\(\s*`([^`]*)`/g + let m: RegExpExecArray | null + let counts: Record = {} + while ((m = re.exec(testSource)) !== null) { + const kind = m[1] === "expectOutput" ? "e2e" : + m[1] === "expectAST" ? "ast" : + m[1] === "expectParseErrors" ? "parseerr" : "resolveerr" + counts[kind] = (counts[kind] ?? 0) + 1 + const name = `${kind}-${String(counts[kind]).padStart(3, "0")}.puff` + fs.writeFileSync(path.join(outDir, name), m[2].trim() + "\n") + } + console.log(JSON.stringify(counts)) +} + +main() From db203f70ee134e8e125700f95ae88337b1437210 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 13 Aug 2026 07:34:21 +0000 Subject: [PATCH 5/7] Self-hosted compiler: resolver Full port of type resolution/checking including out-of-order global resolution, symbol cycle detection, liveness analysis, and coercions. Error output matches the reference resolver over all 116 fixtures (parser/resolver error fixtures, e2e programs, and selfhost sources). Co-authored-by: Andrew Chan --- selfhost/main_dev_errors.puff | 22 + selfhost/resolver.puff | 897 ++++++++++++++++++++++++++++++++++ 2 files changed, 919 insertions(+) create mode 100644 selfhost/main_dev_errors.puff create mode 100644 selfhost/resolver.puff diff --git a/selfhost/main_dev_errors.puff b/selfhost/main_dev_errors.puff new file mode 100644 index 0000000..48165b1 --- /dev/null +++ b/selfhost/main_dev_errors.puff @@ -0,0 +1,22 @@ +// Development driver: runs scan/parse/resolve and reports errors in the same +// format as `tools/dump.ts --errors` for differential testing. + +def main() { + gSource = readAllInput(); + initContext(); + scanTokens(); + if (numErrors() == 0) { + parseProgram(); + if (numErrors() == 0) { + resolveProgram(); + } + } + for (var i = 0; i < gErrors~.count; i = i + 1) { + writeStrTo(Str~(vecGet(gErrors, i)), true); + puterr(10); + } + if (numErrors() > 0) { + exit(1); + } + exit(0); +} diff --git a/selfhost/resolver.puff b/selfhost/resolver.puff new file mode 100644 index 0000000..b8cbe3e --- /dev/null +++ b/selfhost/resolver.puff @@ -0,0 +1,897 @@ +// resolver.puff +// Type resolution and checking, ported from src/resolver.ts. +// +// 1. Resolve AST types and type check all expressions + initializers +// 2. Determine dependencies of global symbols +// 3. Match `return` statements to enclosing functions +// 4. Check arity of function calls +// 5. Ensure no cyclic variable or type declarations +// +// NOTE: like elsewhere in the self-hosted compiler, recursive calls must not +// appear in `var` initializers (the resolver's own cycle detection forbids +// it), so declare-then-assign is used instead. + +// Stack of Scope~. Reassigned (swapped) during out-of-order global resolution. +var rScopes Vec~ = vecNew(); +// Stack of FunctionStmt Node~. Also swapped during out-of-order resolution. +var rFunctionStack Vec~ = vecNew(); +// Stack of WhileStmt Node~. +var rLoopStack Vec~ = vecNew(); +// Stack of all AST nodes walked from a top-level statement (incl. non-tree +// dependency edges). Used for symbol cycle detection. +var rWalked Vec~ = vecNew(); + +def rPeekScope() Scope~ { + return Scope~(vecPeek(rScopes)); +} + +def rPushScope(scope Scope~) { + vecPush(rScopes, int(scope)); +} + +def rPopScope() { + vecPop(rScopes); +} + +def rPeekFunction() Node~ { + if (rFunctionStack~.count > 0) { + return Node~(vecPeek(rFunctionStack)); + } + return Node~(0); +} + +def rPeekLoop() Node~ { + if (rLoopStack~.count > 0) { + return Node~(vecPeek(rLoopStack)); + } + return Node~(0); +} + +def rPreVisit(node Node~) { + node~.walked = true; + vecPush(rWalked, int(node)); +} + +def rPostVisit(node Node~) { + if (node~.kind == NK_VAR_STMT) { + var isGlobal = rScopes~.count == 1; + if (isGlobal) { + vecPush(gGlobalInitOrder, int(node)); + } + } + node~.walked = false; + vecPop(rWalked); +} + +// --------------------------------------------------------------------------- +// Error helpers +// --------------------------------------------------------------------------- + +def rErrType(t Type~) { + typeToString(gErrBuf, t); +} + +def rErrLexeme(tok Token~) { + errRaw(tok~.lexPtr, tok~.lexLen); +} + +// Reports "Cannot implicitly convert operand to ''." +def rErrCannotConvert(token Token~, type Type~) { + errBegin(tokenLine(token)); + var m0 = "Cannot implicitly convert operand to '"; + errRaw(&m0[0], len(m0)); + rErrType(type); + var m1 = "'."; + errRaw(&m1[0], len(m1)); + errEnd(); +} + +// Reports "Invalid operand types for binary operator ''." +def rErrInvalidBinaryOperands(operator Token~) { + errBegin(tokenLine(operator)); + var m0 = "Invalid operand types for binary operator '"; + errRaw(&m0[0], len(m0)); + rErrLexeme(operator); + var m1 = "'."; + errRaw(&m1[0], len(m1)); + errEnd(); +} + +// Reports "Cannot compare to ." +def rErrCannotCompare(operator Token~, a Type~, b Type~) { + errBegin(tokenLine(operator)); + var m0 = "Cannot compare "; + errRaw(&m0[0], len(m0)); + rErrType(a); + var m1 = " to "; + errRaw(&m1[0], len(m1)); + rErrType(b); + var m2 = "."; + errRaw(&m2[0], len(m2)); + errEnd(); +} + +// --------------------------------------------------------------------------- +// Coercion +// --------------------------------------------------------------------------- + +// Resolve `node` while attempting to coerce it to `type`. +// If coercion is possible, returns `node` wrapped in a cast expression if needed. +// If coercion is not possible, reports an error at `token` and returns `node`. +def resolveNodeWithCoercion(node Node~, isLiveAtEnd bool, type Type~, token Token~) Node~ { + var out = node; + resolveNode(node, isLiveAtEnd); + if (!typeIsEqual(node~.resolvedType, type)) { + var coercible = canCoerce(node~.resolvedType, type) || + (isNumberLiteral(node) && canCoerceNumberLiteralNode(node, type)); + if (coercible) { + var castNode = newNode(NK_CAST_EXPR); + castNode~.tokA = fakeToken(T_EOF, strEmpty(), Token~(0)); + castNode~.typeA = type; + castNode~.a = node; + out = castNode; + resolveNode(out, isLiveAtEnd); + } else { + rErrCannotConvert(token, type); + } + } + return out; +} + +// --------------------------------------------------------------------------- +// resolveType +// --------------------------------------------------------------------------- + +// Is called when visiting AST nodes with explicit type names, e.g. variable +// declarations, struct member declarations, function param/return type +// declarations. +// - Resolves any struct types s.t. they are tagged with associated struct +// declaration. +// - Detects cyclic type declarations. +// Returns a resolved type (can be the same object but mutated). +def resolveType(type Type~, isForPointerElement bool) Type~ { + if (type~.category == TC_ARRAY) { + type~.elementType = resolveType(type~.elementType, false); + return type; + } + if (type~.category == TC_STRUCT) { + var symbol = scopeLookup(rPeekScope(), tokenLexemeStr(type~.name), LOOKUP_ANY); + if (int(symbol) == 0 || symbol~.kind != SK_STRUCT) { + errBegin(tokenLine(type~.name)); + var m0 = "Undefined typename '"; + errRaw(&m0[0], len(m0)); + rErrLexeme(type~.name); + var m1 = "'."; + errRaw(&m1[0], len(m1)); + errEnd(); + return gTypeError; + } + if (!symbol~.node~.visited) { + // Structs can be defined and used out-of-order. + // Resolve the out-of-order struct definition. This is needed to + // detect cyclic declarations in case this variable expression + // is part of a global's initializer. + resolveNode(symbol~.node, true); + } else if (symbol~.node~.walked && !isForPointerElement) { + // We're resolving a struct definition with a member that + // (directly/indirectly) depends on itself. + // Note pointer element types can be 'cyclic'. + errBegin(tokenLine(type~.name)); + var m2 = "Cyclic member declaration for struct '"; + errRaw(&m2[0], len(m2)); + rErrLexeme(symbol~.node~.tokA); + var m3 = "'."; + errRaw(&m3[0], len(m3)); + errEnd(); + } + type~.resolvedStruct = symbol~.node; + return type; + } + if (type~.category == TC_POINTER) { + type~.elementType = resolveType(type~.elementType, true); + return type; + } + return type; +} + +// --------------------------------------------------------------------------- +// resolveNode: expressions +// --------------------------------------------------------------------------- + +def rBinaryExpr(op Node~, isLiveAtEnd bool) { + resolveNode(op~.a, isLiveAtEnd); + resolveNode(op~.b, isLiveAtEnd); + var opType = op~.tokA~.type; + if (opType == T_LESS || opType == T_LESS_EQUAL || opType == T_GREATER || opType == T_GREATER_EQUAL) { + var lct = getLowestCommonNumeric(op~.a~.resolvedType, op~.b~.resolvedType); + if (int(lct) != 0) { + op~.a = resolveNodeWithCoercion(op~.a, isLiveAtEnd, lct, op~.tokA); + op~.b = resolveNodeWithCoercion(op~.b, isLiveAtEnd, lct, op~.tokA); + } else { + rErrInvalidBinaryOperands(op~.tokA); + } + op~.resolvedType = gTypeBool; + return; + } + if (opType == T_BANG_EQUAL || opType == T_EQUAL_EQUAL) { + var leftType = op~.a~.resolvedType; + var rightType = op~.b~.resolvedType; + if (isScalar(leftType) && isScalar(rightType)) { + var lct2 = getLowestCommonNumeric(leftType, rightType); + if (int(lct2) != 0) { + op~.a = resolveNodeWithCoercion(op~.a, isLiveAtEnd, lct2, op~.tokA); + op~.b = resolveNodeWithCoercion(op~.b, isLiveAtEnd, lct2, op~.tokA); + } else if (!typeIsEqual(leftType, rightType)) { + rErrCannotCompare(op~.tokA, leftType, rightType); + } + } else { + // TODO: We should auto-generate equals operator (member-wise equals) for structs. + // memcmp is a bad idea because of float equality (NaNs, zero). + rErrCannotCompare(op~.tokA, leftType, rightType); + } + op~.resolvedType = gTypeBool; + return; + } + if (opType == T_PERCENT) { + var lct3 = getLowestCommonNumeric(op~.a~.resolvedType, op~.b~.resolvedType); + if (int(lct3) != 0 && (lct3~.category == TC_INT || lct3~.category == TC_BYTE)) { + op~.a = resolveNodeWithCoercion(op~.a, isLiveAtEnd, lct3, op~.tokA); + op~.b = resolveNodeWithCoercion(op~.b, isLiveAtEnd, lct3, op~.tokA); + } else { + rErrInvalidBinaryOperands(op~.tokA); + } + op~.resolvedType = op~.a~.resolvedType; + return; + } + // + - * / + var lt = op~.a~.resolvedType; + var rt = op~.b~.resolvedType; + var lct4 = getLowestCommonNumeric(lt, rt); + if (int(lct4) != 0) { + op~.a = resolveNodeWithCoercion(op~.a, isLiveAtEnd, lct4, op~.tokA); + op~.b = resolveNodeWithCoercion(op~.b, isLiveAtEnd, lct4, op~.tokA); + op~.resolvedType = lct4; + return; + } + if (opType == T_PLUS || opType == T_MINUS) { + // Rules for pointer arithmetic: + // 1. Only allowed between a pointer and a numeric. Operating on 2 pointers is not allowed. + // 2. (numeric + pointer) and (pointer + numeric) are both allowed. + // 3. (pointer - numeric) is allowed, (numeric - pointer) is not allowed. + if (lt~.category == TC_POINTER && isNumeric(rt)) { + var sizeLit = newNode(NK_LITERAL_EXPR); + sizeLit~.typeA = gTypeInt; + sizeLit~.litInt = sizeOf(lt~.elementType); + var mul = newNode(NK_BINARY_EXPR); + mul~.a = sizeLit; + var sStar = "*"; + mul~.tokA = fakeToken(T_STAR, strCopy(&sStar[0], len(sStar)), Token~(0)); + mul~.b = resolveNodeWithCoercion(op~.b, isLiveAtEnd, gTypeInt, op~.tokA); + op~.b = mul; + resolveNode(op~.b, isLiveAtEnd); + op~.resolvedType = lt; + return; + } + if (rt~.category == TC_POINTER && isNumeric(lt) && opType == T_PLUS) { + var sizeLit2 = newNode(NK_LITERAL_EXPR); + sizeLit2~.typeA = gTypeInt; + sizeLit2~.litInt = sizeOf(rt~.elementType); + var mul2 = newNode(NK_BINARY_EXPR); + mul2~.a = sizeLit2; + var sStar2 = "*"; + mul2~.tokA = fakeToken(T_STAR, strCopy(&sStar2[0], len(sStar2)), Token~(0)); + mul2~.b = resolveNodeWithCoercion(op~.a, isLiveAtEnd, gTypeInt, op~.tokA); + op~.a = mul2; + resolveNode(op~.a, isLiveAtEnd); + op~.resolvedType = rt; + return; + } + rErrInvalidBinaryOperands(op~.tokA); + op~.resolvedType = gTypeError; + return; + } + if (!(lt~.category == TC_ERROR) && !(rt~.category == TC_ERROR)) { + rErrInvalidBinaryOperands(op~.tokA); + } + op~.resolvedType = gTypeError; +} + +def rCallExpr(op Node~, isLiveAtEnd bool) { + resolveNode(op~.a, isLiveAtEnd); + // If we ever add first-class functions, this should instead + // check resolvedType of callee is "Function" type + if (op~.a~.kind != NK_VARIABLE_EXPR) { + errBegin(tokenLine(op~.tokA)); + var m0 = "Cannot call this type."; + errRaw(&m0[0], len(m0)); + errEnd(); + } else { + var callee = op~.a; + var symbol = callee~.symbol; + if (int(symbol) == 0) { + // We already reported 'Undefined symbol' error earlier when resolving the callee VariableExpr + } else if ((symbol~.kind == SK_FUNCTION && op~.tokA~.type == T_LEFT_PAREN) || + (symbol~.kind == SK_STRUCT && op~.tokA~.type == T_LEFT_BRACE)) { + var fnNode = symbol~.node; + var params = fnNode~.params; + // Check arity and types of arguments + if (op~.list~.count != params~.count) { + errBegin(tokenLine(op~.tokA)); + var m1 = "Expected "; + errRaw(&m1[0], len(m1)); + errInt(params~.count); + var m2 = " arguments but got "; + errRaw(&m2[0], len(m2)); + errInt(op~.list~.count); + var m3 = " in call to "; + errRaw(&m3[0], len(m3)); + rErrLexeme(fnNode~.tokA); + var m4 = "."; + errRaw(&m4[0], len(m4)); + errEnd(); + } else { + for (var i = 0; i < op~.list~.count; i = i + 1) { + var param = Param~(vecGet(params, i)); + var argNode = Node~(vecGet(op~.list, i)); + argNode = resolveNodeWithCoercion(argNode, isLiveAtEnd, param~.type, op~.tokA); + vecSet(op~.list, i, int(argNode)); + } + } + if (symbol~.kind == SK_FUNCTION) { + op~.resolvedType = fnNode~.typeA; + } else { + op~.resolvedType = resolvedStructType(fnNode); + } + } else { + errBegin(tokenLine(callee~.tokA)); + var m5 = "Cannot "; + errRaw(&m5[0], len(m5)); + if (op~.tokA~.type == T_LEFT_BRACE) { + var m6 = "construct"; + errRaw(&m6[0], len(m6)); + } else { + var m7 = "call"; + errRaw(&m7[0], len(m7)); + } + var m8 = " this type."; + errRaw(&m8[0], len(m8)); + errEnd(); + } + } + if (int(op~.resolvedType) == 0) { + op~.resolvedType = gTypeError; + } +} + +def rVariableExpr(op Node~) { + var symbol = scopeLookup(rPeekScope(), tokenLexemeStr(op~.tokA), LOOKUP_VISIBLE_SYMBOLS); + if (int(symbol) == 0) { + errBegin(tokenLine(op~.tokA)); + var m0 = "Undefined symbol '"; + errRaw(&m0[0], len(m0)); + rErrLexeme(op~.tokA); + var m1 = "'."; + errRaw(&m1[0], len(m1)); + errEnd(); + op~.resolvedType = gTypeError; + return; + } + op~.symbol = symbol; + var resolveTypeFromSymbol = true; + + // The symbol's declaration node (null for params). + var symbolDecl = Node~(0); + if (symbol~.kind != SK_PARAM) { + symbolDecl = symbol~.node; + } + + if (int(symbolDecl) != 0 && !symbolDecl~.visited) { + // Globals can be declared and used out-of-order. + // Resolve the out-of-order global declaration. This is needed to: + // 1. Resolve type of this variable expression. + // 2. Detect cyclic declarations in case this variable expression + // is part of a global's initializer. + var oldScopes = rScopes; + var oldFunctionStack = rFunctionStack; + // Reset stacks since we're following non-tree edge back to top-level + rScopes = vecNew(); + vecPush(rScopes, int(gGlobalScope)); + rFunctionStack = vecNew(); + resolveNode(symbolDecl, true); + rScopes = oldScopes; + rFunctionStack = oldFunctionStack; + } else if (int(symbolDecl) != 0 && symbolDecl~.walked) { + // Detect cyclic variable declarations (declarations using this variable + // expr in initializer). Note `symbolDecl` may not be the cyclic variable; + // backtrack in the walked stack and return any variable symbol between + // the top and `symbolDecl`. + var cyclicVar = Node~(0); + if (symbolDecl~.kind == NK_VAR_STMT) { + cyclicVar = symbolDecl; + } else { + for (var i = rWalked~.count - 1; int(vecGet(rWalked, i)) != int(symbolDecl); i = i - 1) { + var walkedNode = Node~(vecGet(rWalked, i)); + if (walkedNode~.kind == NK_VAR_STMT) { + cyclicVar = walkedNode; + break; + } + } + } + if (int(cyclicVar) != 0) { + errBegin(tokenLine(cyclicVar~.tokA)); + var m2 = "Declaration of '"; + errRaw(&m2[0], len(m2)); + rErrLexeme(cyclicVar~.tokA); + var m3 = "' is cyclic. Defined here:"; + errRaw(&m3[0], len(m3)); + errChar(10); + tokenLineStrTo(gErrBuf, cyclicVar~.tokA, true); + errEnd(); + op~.resolvedType = gTypeError; + resolveTypeFromSymbol = false; + } + } + if (resolveTypeFromSymbol) { + if (symbol~.kind == SK_FUNCTION || symbol~.kind == SK_STRUCT) { + // TODO: Either make `CallExpr` only use names and not sub-expressions, + // or add a function type + op~.resolvedType = gTypeVoid; + } else if (symbol~.kind == SK_PARAM) { + op~.resolvedType = symbol~.param~.type; + } else { + // SK_VARIABLE: we should've filled this in after resolving the declaration + op~.resolvedType = symbol~.node~.typeA; + } + } +} + +def rUnaryExpr(op Node~, isLiveAtEnd bool) { + var opType = op~.tokA~.type; + if (opType == T_BANG) { + op~.a = resolveNodeWithCoercion(op~.a, isLiveAtEnd, gTypeBool, op~.tokA); + op~.resolvedType = gTypeBool; + return; + } + if (opType == T_MINUS) { + resolveNode(op~.a, isLiveAtEnd); + if (!isNumeric(op~.a~.resolvedType)) { + errBegin(tokenLine(op~.tokA)); + var m0 = "Invalid operand type for unary operator '-'."; + errRaw(&m0[0], len(m0)); + errEnd(); + op~.resolvedType = gTypeInt; + } else { + if (op~.a~.resolvedType~.category == TC_BYTE) { + op~.a = resolveNodeWithCoercion(op~.a, isLiveAtEnd, gTypeInt, op~.tokA); + } + op~.resolvedType = op~.a~.resolvedType; + } + return; + } + // '&' + resolveNode(op~.a, isLiveAtEnd); + if (isValidElementType(op~.a~.resolvedType)) { + if (op~.a~.kind == NK_VARIABLE_EXPR) { + var symbol = op~.a~.symbol; + if (int(symbol) != 0 && (symbol~.kind == SK_PARAM || symbol~.kind == SK_VARIABLE)) { + symbol~.isAddressTaken = true; + } + op~.resolvedType = ptrType(op~.a~.resolvedType); + } else if (op~.a~.kind == NK_INDEX_EXPR || op~.a~.kind == NK_DOT_EXPR || op~.a~.kind == NK_DEREF_EXPR) { + op~.resolvedType = ptrType(op~.a~.resolvedType); + } + } + if (int(op~.resolvedType) == 0) { + errBegin(tokenLine(op~.tokA)); + var m1 = "Invalid operand for unary operator '&'."; + errRaw(&m1[0], len(m1)); + errEnd(); + op~.resolvedType = gTypeError; + } +} + +def rListExpr(op Node~, isLiveAtEnd bool) { + if (op~.listKind == LK_LIST) { + var elementType = Type~(0); + if (op~.list~.count > 0) { + var first = Node~(vecGet(op~.list, 0)); + resolveNode(first, isLiveAtEnd); + elementType = first~.resolvedType; + for (var i = 1; i < op~.list~.count; i = i + 1) { + var item = Node~(vecGet(op~.list, i)); + resolveNode(item, isLiveAtEnd); + if (!typeIsEqual(item~.resolvedType, elementType)) { + elementType = Type~(0); + break; + } + } + } + // Note resolving an empty array will always throw a resolve error + // even if a type specifier for e.g. declaration or return value + // is given. Callers should not resolve the literal in that case. + if (int(elementType) != 0 && isValidElementType(elementType)) { + op~.resolvedType = arrayType(elementType, op~.list~.count); + } else { + if (op~.list~.count == 0) { + errBegin(tokenLine(op~.tokA)); + var m0 = "Zero-length arrays are not allowed."; + errRaw(&m0[0], len(m0)); + errEnd(); + } else { + errBegin(tokenLine(op~.tokA)); + var m1 = "Cannot infer type for literal."; + errRaw(&m1[0], len(m1)); + errEnd(); + } + op~.resolvedType = gTypeError; + } + return; + } + // repeat initializer + resolveNode(op~.a, isLiveAtEnd); + if (op~.repeatLen == 0) { + errBegin(tokenLine(op~.tokA)); + var m2 = "Zero-length arrays are not allowed."; + errRaw(&m2[0], len(m2)); + errEnd(); + op~.resolvedType = gTypeError; + } else if (!isValidElementType(op~.a~.resolvedType)) { + errBegin(tokenLine(op~.tokA)); + var m3 = "Cannot infer type for literal."; + errRaw(&m3[0], len(m3)); + errEnd(); + op~.resolvedType = gTypeError; + } else { + op~.resolvedType = arrayType(op~.a~.resolvedType, op~.repeatLen); + } +} + +// --------------------------------------------------------------------------- +// resolveNode: statements +// --------------------------------------------------------------------------- + +def rFunctionStmt(op Node~) { + // 1. Resolve parameter and return types + for (var i = 0; i < op~.params~.count; i = i + 1) { + var param = Param~(vecGet(op~.params, i)); + param~.type = resolveType(param~.type, false); + } + op~.typeA = resolveType(op~.typeA, false); + if (op~.hasBody) { + rPushScope(op~.scope); + vecPush(rFunctionStack, int(op)); + // 2. Ensure all return statements match the return type of the function + // 3. If the function has a return type, ensure all control paths return a value + var missingReturn = false; + if (op~.list~.count > 0) { + var prevIsLiveAtEnd = true; // functions start as live + for (var j = 0; j < op~.list~.count; j = j + 1) { + var stmt = Node~(vecGet(op~.list, j)); + resolveNode(stmt, prevIsLiveAtEnd); + prevIsLiveAtEnd = stmt~.isLiveAtEnd == 1; + } + var lastStmt = Node~(vecGet(op~.list, op~.list~.count - 1)); + if (lastStmt~.isLiveAtEnd == 1) { + missingReturn = true; + } + } else { + missingReturn = true; + } + if (missingReturn && !(op~.typeA~.category == TC_VOID) && !(op~.typeA~.category == TC_ERROR)) { + errBegin(tokenLine(op~.tokA)); + var m0 = "All control paths for "; + errRaw(&m0[0], len(m0)); + rErrLexeme(op~.tokA); + var m1 = " must return a value of type '"; + errRaw(&m1[0], len(m1)); + rErrType(op~.typeA); + var m2 = "'."; + errRaw(&m2[0], len(m2)); + errEnd(); + } + vecPop(rFunctionStack); + rPopScope(); + } +} + +def rVarStmt(op Node~, isLiveAtEnd bool) { + resolveNode(op~.a, isLiveAtEnd); + if (int(op~.a~.resolvedType) == 0) { + die(91); // variable initializer failed to resolve + } + if (int(op~.typeA) == 0) { + op~.typeA = op~.a~.resolvedType; + } else { + op~.typeA = resolveType(op~.typeA, false); + if (!typeIsEqual(op~.typeA, op~.a~.resolvedType)) { + if (!(op~.a~.resolvedType~.category == TC_ERROR) && !(op~.typeA~.category == TC_ERROR)) { + errBegin(tokenLine(op~.tokA)); + var m0 = "Cannot assign value of type '"; + errRaw(&m0[0], len(m0)); + rErrType(op~.a~.resolvedType); + var m1 = "' to variable of type '"; + errRaw(&m1[0], len(m1)); + rErrType(op~.typeA); + var m2 = "'."; + errRaw(&m2[0], len(m2)); + errEnd(); + } + } + } + var inFunction = rPeekFunction(); + if (int(inFunction) != 0 && inFunction~.hasBody && int(inFunction~.scope) != int(rPeekScope()) && int(op~.symbol) != 0) { + if (int(inFunction~.hoisted) == 0) { + inFunction~.hoisted = vecNew(); + } + // insertion-ordered set + if (!op~.symbol~.inHoisted) { + op~.symbol~.inHoisted = true; + vecPush(inFunction~.hoisted, int(op~.symbol)); + } + } + op~.isLiveAtEnd = boolToLive(isLiveAtEnd); +} + +def boolToLive(b bool) int { + if (b) { + return 1; + } + return 0; +} + +// --------------------------------------------------------------------------- +// resolveNode dispatcher +// --------------------------------------------------------------------------- + +def resolveNode(node Node~, isLiveAtEnd bool) { + if (node~.visited) { + return; + } + node~.visited = true; + rPreVisit(node); + var kind = node~.kind; + + if (kind == NK_ASSIGN_EXPR) { + resolveNode(node~.a, isLiveAtEnd); + node~.b = resolveNodeWithCoercion(node~.b, isLiveAtEnd, node~.a~.resolvedType, node~.tokA); + node~.resolvedType = node~.a~.resolvedType; + } else if (kind == NK_BINARY_EXPR) { + rBinaryExpr(node, isLiveAtEnd); + } else if (kind == NK_CALL_EXPR) { + rCallExpr(node, isLiveAtEnd); + } else if (kind == NK_CAST_EXPR) { + node~.typeA = resolveType(node~.typeA, false); + resolveNode(node~.a, isLiveAtEnd); + if (!canCast(node~.a~.resolvedType, node~.typeA)) { + errBegin(tokenLine(node~.tokA)); + var m0 = "Cannot cast from "; + errRaw(&m0[0], len(m0)); + rErrType(node~.a~.resolvedType); + var m1 = " to "; + errRaw(&m1[0], len(m1)); + rErrType(node~.typeA); + var m2 = "."; + errRaw(&m2[0], len(m2)); + errEnd(); + } + node~.resolvedType = node~.typeA; + } else if (kind == NK_DEREF_EXPR) { + resolveNode(node~.a, isLiveAtEnd); + var valueType = node~.a~.resolvedType; + if (int(valueType) != 0 && valueType~.category == TC_POINTER) { + node~.resolvedType = valueType~.elementType; + } else { + if (int(valueType) == 0 || valueType~.category != TC_ERROR) { + errBegin(tokenLine(node~.tokA)); + var m3 = "Invalid operand for dereferencing operator '~'."; + errRaw(&m3[0], len(m3)); + errEnd(); + } + node~.resolvedType = gTypeError; + } + } else if (kind == NK_DOT_EXPR) { + resolveNode(node~.a, isLiveAtEnd); + var calleeType = node~.a~.resolvedType; + if (int(calleeType) == 0 || calleeType~.category != TC_STRUCT) { + if (!(calleeType~.category == TC_ERROR)) { + errBegin(tokenLine(node~.tokA)); + var m4 = "Invalid operand for member access operator '.'."; + errRaw(&m4[0], len(m4)); + errEnd(); + } + node~.resolvedType = gTypeError; + } else { + var structNode = calleeType~.resolvedStruct; + if (int(structNode) != 0) { + for (var i = 0; i < structNode~.params~.count; i = i + 1) { + var member = Param~(vecGet(structNode~.params, i)); + if (strEq(tokenLexemeStr(member~.name), tokenLexemeStr(node~.tokB))) { + node~.resolvedType = member~.type; + break; + } + } + } + if (int(node~.resolvedType) == 0) { + errBegin(tokenLine(node~.tokB)); + var m5 = "Struct "; + errRaw(&m5[0], len(m5)); + if (int(structNode) != 0) { + rErrLexeme(structNode~.tokA); + } else { + var m6 = "undefined"; + errRaw(&m6[0], len(m6)); + } + var m7 = " has no member '"; + errRaw(&m7[0], len(m7)); + rErrLexeme(node~.tokB); + var m8 = "'."; + errRaw(&m8[0], len(m8)); + errEnd(); + node~.resolvedType = gTypeError; + } + } + } else if (kind == NK_GROUP_EXPR) { + resolveNode(node~.a, isLiveAtEnd); + node~.resolvedType = node~.a~.resolvedType; + } else if (kind == NK_INDEX_EXPR) { + resolveNode(node~.a, isLiveAtEnd); + resolveNode(node~.b, isLiveAtEnd); + if (node~.a~.resolvedType~.category != TC_ARRAY) { + errBegin(tokenLine(node~.tokA)); + var m9 = "Index operator requires array type."; + errRaw(&m9[0], len(m9)); + errEnd(); + node~.resolvedType = gTypeError; + } else { + var arrType = node~.a~.resolvedType; + var idxCat = node~.b~.resolvedType~.category; + if (idxCat == TC_INT || idxCat == TC_BYTE) { + node~.resolvedType = arrType~.elementType; + } else { + errBegin(tokenLine(node~.tokA)); + var m10 = "Index operator requires int or byte type."; + errRaw(&m10[0], len(m10)); + errEnd(); + node~.resolvedType = arrType~.elementType; + } + } + } else if (kind == NK_LEN_EXPR) { + resolveNode(node~.a, isLiveAtEnd); + var lenValType = node~.a~.resolvedType; + if (int(lenValType) != 0 && lenValType~.category == TC_ARRAY) { + node~.resolvedLength = lenValType~.length; + } else { + node~.resolvedLength = 0; + } + node~.hasResolvedLength = true; + } else if (kind == NK_LIST_EXPR) { + rListExpr(node, isLiveAtEnd); + } else if (kind == NK_LITERAL_EXPR) { + node~.resolvedType = node~.typeA; + } else if (kind == NK_LOGICAL_EXPR) { + node~.a = resolveNodeWithCoercion(node~.a, isLiveAtEnd, gTypeBool, node~.tokA); + node~.b = resolveNodeWithCoercion(node~.b, isLiveAtEnd, gTypeBool, node~.tokA); + node~.resolvedType = gTypeBool; + } else if (kind == NK_UNARY_EXPR) { + rUnaryExpr(node, isLiveAtEnd); + } else if (kind == NK_VARIABLE_EXPR) { + rVariableExpr(node); + } else if (kind == NK_BLOCK_STMT) { + rPushScope(node~.scope); + if (node~.list~.count > 0) { + var prevIsLiveAtEnd = isLiveAtEnd; + for (var i = 0; i < node~.list~.count; i = i + 1) { + var stmt = Node~(vecGet(node~.list, i)); + resolveNode(stmt, prevIsLiveAtEnd); + prevIsLiveAtEnd = stmt~.isLiveAtEnd == 1; + } + var lastStmt = Node~(vecGet(node~.list, node~.list~.count - 1)); + node~.isLiveAtEnd = lastStmt~.isLiveAtEnd; + } else { + node~.isLiveAtEnd = boolToLive(isLiveAtEnd); + } + rPopScope(); + } else if (kind == NK_EXPRESSION_STMT) { + resolveNode(node~.a, isLiveAtEnd); + node~.isLiveAtEnd = boolToLive(isLiveAtEnd); + } else if (kind == NK_FUNCTION_STMT) { + rFunctionStmt(node); + } else if (kind == NK_IF_STMT) { + resolveNode(node~.a, isLiveAtEnd); + resolveNode(node~.b, isLiveAtEnd); + var isLiveAfterThen = node~.b~.isLiveAtEnd == 1; + var isLiveAfterElse = isLiveAtEnd; + if (int(node~.c) != 0) { + resolveNode(node~.c, isLiveAtEnd); + isLiveAfterElse = node~.c~.isLiveAtEnd == 1; + } + node~.isLiveAtEnd = boolToLive(isLiveAfterThen || isLiveAfterElse); + } else if (kind == NK_LOOP_CONTROL_STMT) { + if (int(rPeekLoop()) == 0) { + errBegin(tokenLine(node~.tokA)); + var m11 = "Cannot "; + errRaw(&m11[0], len(m11)); + rErrLexeme(node~.tokA); + var m12 = " outside a loop."; + errRaw(&m12[0], len(m12)); + errEnd(); + } + // Control flow is considered live as long as we don't hit a "return". + // This is not affected by breaks/continues. + node~.isLiveAtEnd = boolToLive(isLiveAtEnd); + } else if (kind == NK_PRINT_STMT) { + resolveNode(node~.a, isLiveAtEnd); + var valueType = node~.a~.resolvedType; + if (valueType~.category == TC_VOID || valueType~.category == TC_POINTER || valueType~.category == TC_STRUCT) { + // TODO: allow hex address printing for pointers + errBegin(tokenLine(node~.tokA)); + var m13 = "Cannot print value of type '"; + errRaw(&m13[0], len(m13)); + rErrType(valueType); + var m14 = "'."; + errRaw(&m14[0], len(m14)); + errEnd(); + } + node~.isLiveAtEnd = boolToLive(isLiveAtEnd); + } else if (kind == NK_RETURN_STMT) { + var inFunction = rPeekFunction(); + if (int(inFunction) == 0) { + errBegin(tokenLine(node~.tokA)); + var m15 = "Cannot return from top-level code."; + errRaw(&m15[0], len(m15)); + errEnd(); + } else if (int(node~.a) != 0) { + resolveNode(node~.a, isLiveAtEnd); + if (!typeIsEqual(inFunction~.typeA, node~.a~.resolvedType)) { + errBegin(tokenLine(node~.tokA)); + var m16 = "Expected a value of type '"; + errRaw(&m16[0], len(m16)); + rErrType(inFunction~.typeA); + var m17 = "'."; + errRaw(&m17[0], len(m17)); + errEnd(); + } + } else { + if (!(inFunction~.typeA~.category == TC_VOID)) { + errBegin(tokenLine(node~.tokA)); + var m18 = "Expected a value of type '"; + errRaw(&m18[0], len(m18)); + rErrType(inFunction~.typeA); + var m19 = "'."; + errRaw(&m19[0], len(m19)); + errEnd(); + } + } + node~.isLiveAtEnd = 0; + } else if (kind == NK_STRUCT_STMT) { + for (var i = 0; i < node~.params~.count; i = i + 1) { + var member = Param~(vecGet(node~.params, i)); + member~.type = resolveType(member~.type, false); + } + node~.isLiveAtEnd = boolToLive(isLiveAtEnd); + } else if (kind == NK_VAR_STMT) { + rVarStmt(node, isLiveAtEnd); + } else if (kind == NK_WHILE_STMT) { + resolveNode(node~.a, isLiveAtEnd); + vecPush(rLoopStack, int(node)); + resolveNode(node~.b, isLiveAtEnd); + vecPop(rLoopStack); + if (int(node~.c) != 0) { + // This is an expression statement and cannot affect node.isLiveAtEnd + resolveNode(node~.c, isLiveAtEnd); + } + node~.isLiveAtEnd = node~.b~.isLiveAtEnd; + } else { + die(92); // unhandled node kind in resolveNode + } + rPostVisit(node); +} + +// Resolves all top-level statements. Mirrors `resolve` in src/resolver.ts. +def resolveProgram() { + vecPush(rScopes, int(gGlobalScope)); + for (var i = 0; i < gTopLevel~.count; i = i + 1) { + var stmt = Node~(vecGet(gTopLevel, i)); + resolveNode(stmt, true); + } +} From fd322b8422d918a95f09d19b57591b5a84de9f6c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 13 Aug 2026 07:40:29 +0000 Subject: [PATCH 6/7] =?UTF-8?q?Self-hosted=20compiler:=20backend=20and=20d?= =?UTF-8?q?river=20=E2=80=94=20bootstrap=20fixpoint=20achieved?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The puffscript compiler written in puffscript (selfhost/) now: - emits WAT byte-identical to the reference compiler on all fixtures - compiles its own source byte-identically to the reference compiler - reaches a bootstrap fixpoint: stage2 (self-compiled) output == stage1 output Co-authored-by: Andrew Chan --- selfhost/backend.puff | 1681 +++++++++++++++++++++++++++++++++++++++++ selfhost/main.puff | 28 + 2 files changed, 1709 insertions(+) create mode 100644 selfhost/backend.puff create mode 100644 selfhost/main.puff diff --git a/selfhost/backend.puff b/selfhost/backend.puff new file mode 100644 index 0000000..6fb9fbc --- /dev/null +++ b/selfhost/backend.puff @@ -0,0 +1,1681 @@ +// backend.puff +// WAT code generator, ported from src/backend.ts (with DEBUG_COMMENTS off). +// Emits byte-identical output to the reference compiler. +// +//////////////////////////////////////////////////// +// Puff memory layout +//////////////////////////////////////////////////// +// ----------------------- 0 +// +// ----------------------- +// ^ grows toward zero +// stack (locals) +// ----------------------- STACK_TOP_BYTE_OFFSET (4MB) +// data (globals) +// ----------------------- DATA_TOP_BYTE_OFFSET (8MB) +// +// ----------------------- max byte offset +//////////////////////////////////////////////////// + +var STACK_TOP_BYTE_OFFSET = 4194304; +var DATA_TOP_BYTE_OFFSET = 8388608; +var INITIAL_PAGES = 128; + +var EM_LVALUE = 0; +var EM_RVALUE = 1; + +// The generated WAT output. +var gOut Buf~ = bufNewWithCap(1048576); + +var eIndent = 0; +var eNextLabelID = 0; +// Loop label stack: 3 ints per entry (outer, inner, increment; -1 = none). +var eLoopStack Vec~ = vecNew(); +// Locations of locals are stored on Symbol via localLoc/hasLocalLoc. +// Locations of globals are stored on Symbol via globalLoc/hasGlobalLoc. +// Locations of string literals are stored on the literal node via strLoc. + +def wInd() { + for (var i = 0; i < eIndent; i = i + 1) { + bufPushChar(gOut, 32); + bufPushChar(gOut, 32); + } +} + +def wRaw(p byte~, n int) { + bufPushBytes(gOut, p, n); +} + +def wStr(s Str~) { + bufPushBytes(gOut, s~.data, s~.count); +} + +def wInt(x int) { + bufPushInt(gOut, x); +} + +def wEnd() { + bufPushChar(gOut, 10); +} + +def wLine(p byte~, n int) { + wInd(); + wRaw(p, n); + wEnd(); +} + +// Emits a token's lexeme. +def wLexeme(tok Token~) { + bufPushBytes(gOut, tok~.lexPtr, tok~.lexLen); +} + +// Formats a float literal for WAT output from its source lexeme, e.g. +// "5.50" => "5.5", "5." => "5", "3.14" => "3.14". +// Mirrors formatFloatLexeme in src/backend.ts. +def wFloatLexeme(lexeme Str~) { + var hasDot = false; + for (var i = 0; i < lexeme~.count; i = i + 1) { + if (int(strGet(lexeme, i)) == 46) { + hasDot = true; + } + } + var end = lexeme~.count; + if (hasDot) { + while (end > 0 && int(strGet(lexeme, end - 1)) == 48) { + end = end - 1; + } + if (end > 0 && int(strGet(lexeme, end - 1)) == 46) { + end = end - 1; + } + } + if (end == 0) { + bufPushChar(gOut, 48); // "0" + } else { + bufPushBytes(gOut, lexeme~.data, end); + } +} + +// Returns true if values of the given type are represented as f32 on the +// WASM stack ("registers"), false for i32. Mirrors registerType. +def regIsF32(t Type~) bool { + if (t~.category == TC_FLOAT) { + return true; + } + if (t~.category == TC_ERROR || t~.category == TC_VOID) { + die(93); // unhandled type for WASM backend + } + return false; +} + +def wRegType(isF32 bool) { + if (isF32) { + var s0 = "f32"; + wRaw(&s0[0], len(s0)); + } else { + var s1 = "i32"; + wRaw(&s1[0], len(s1)); + } +} + +def isVariableInRegister(symbol Symbol~) bool { + if (symbol~.isAddressTaken) { + return false; + } + var type = symbolValueType(symbol); + return int(type) != 0 && isScalar(type); +} + +// The type of a variable or param symbol. +def symbolValueType(symbol Symbol~) Type~ { + if (symbol~.kind == SK_PARAM) { + return symbol~.param~.type; + } + return symbol~.node~.typeA; +} + +def symbolName(symbol Symbol~) Token~ { + if (symbol~.kind == SK_PARAM) { + return symbol~.param~.name; + } + return symbol~.node~.tokA; +} + +// --------------------------------------------------------------------------- +// Common line emitters +// --------------------------------------------------------------------------- + +def oStackGet() { + var s = "global.get $__stack_ptr__"; + wLine(&s[0], len(s)); +} + +def oStackSet() { + var s = "global.set $__stack_ptr__"; + wLine(&s[0], len(s)); +} + +def oI32Const(x int) { + wInd(); + var s = "i32.const "; + wRaw(&s[0], len(s)); + wInt(x); + wEnd(); +} + +def oI32Add() { + var s = "i32.add"; + wLine(&s[0], len(s)); +} + +def oI32Sub() { + var s = "i32.sub"; + wLine(&s[0], len(s)); +} + +def oI32Mul() { + var s = "i32.mul"; + wLine(&s[0], len(s)); +} + +def oCallMemcpy() { + var s = "call $__memcpy__"; + wLine(&s[0], len(s)); +} + +def oDrop() { + var s = "drop"; + wLine(&s[0], len(s)); +} + +def oBasePtrGet() { + var s = "local.get $__base_ptr__"; + wLine(&s[0], len(s)); +} + +def oTee(isF32 bool) { + var op = "local.tee $__tee_"; + wInd(); + wRaw(&op[0], len(op)); + wRegType(isF32); + var s = "__"; + wRaw(&s[0], len(s)); + wEnd(); +} + +def oGetTee(isF32 bool) { + var op = "local.get $__tee_"; + wInd(); + wRaw(&op[0], len(op)); + wRegType(isF32); + var s = "__"; + wRaw(&s[0], len(s)); + wEnd(); +} + +// Emit code to duplicate the value at the top of the stack. +// PRECOND: inside a function. +def emitDupTop(isF32 bool) { + oTee(isF32); + oGetTee(isF32); +} + +// Emit code to swap stack[n] and stack[n-1]. +// PRECOND: inside a function. +def emitSwapTop(topIsF32 bool, secondIsF32 bool) { + var s0 = "local.set $__swapa_"; + wInd(); + wRaw(&s0[0], len(s0)); + wRegType(topIsF32); + var u = "__"; + wRaw(&u[0], len(u)); + wEnd(); + var s1 = "local.set $__swapb_"; + wInd(); + wRaw(&s1[0], len(s1)); + wRegType(secondIsF32); + wRaw(&u[0], len(u)); + wEnd(); + var s2 = "local.get $__swapa_"; + wInd(); + wRaw(&s2[0], len(s2)); + wRegType(topIsF32); + wRaw(&u[0], len(u)); + wEnd(); + var s3 = "local.get $__swapb_"; + wInd(); + wRaw(&s3[0], len(s3)); + wRegType(secondIsF32); + wRaw(&u[0], len(u)); + wEnd(); +} + +// Emit code to store a scalar value to given address. +// PRECOND: stack contains address then scalar value to store. +def emitStoreScalar(t Type~) { + if (t~.category == TC_BOOL || t~.category == TC_BYTE) { + var s0 = "i32.store8"; + wLine(&s0[0], len(s0)); + return; + } + if (t~.category == TC_FLOAT) { + var s1 = "f32.store"; + wLine(&s1[0], len(s1)); + return; + } + if (t~.category == TC_INT || t~.category == TC_POINTER) { + var s2 = "i32.store"; + wLine(&s2[0], len(s2)); + return; + } + die(94); // unhandled element type for emitStoreScalar +} + +// Emit code to load and return a scalar value from given address. +def emitLoadScalar(t Type~) { + if (t~.category == TC_BOOL || t~.category == TC_BYTE) { + var s0 = "i32.load8_u"; + wLine(&s0[0], len(s0)); + return; + } + if (t~.category == TC_FLOAT) { + var s1 = "f32.load"; + wLine(&s1[0], len(s1)); + return; + } + if (t~.category == TC_INT || t~.category == TC_POINTER) { + var s2 = "i32.load"; + wLine(&s2[0], len(s2)); + return; + } + die(95); // unhandled type for emitLoadScalar +} + +// Emit code to grow stack by sizeof(type) and adjust __stack_ptr__. +def emitAllocStackVal(t Type~) { + oStackGet(); + oI32Const(sizeOf(t)); + oI32Sub(); + oStackSet(); +} + +// Emit code to: +// 1. push a value of the given type to the in-memory stack +// 2. return the new stack ptr, which points to the pushed value +// PRECOND: Address of value to push is top item of WASM stack. +def emitPushMem(t Type~) { + emitAllocStackVal(t); + oStackGet(); + oI32Const(sizeOf(t)); + oCallMemcpy(); + oStackGet(); +} + +// Emit code to: +// 1. push a scalar value of the given type to the in-memory stack +// 2. return the new stack ptr, which points to the pushed value +// PRECOND: Value is stored in the tee register for the type. +def emitPushScalarFromTee(t Type~) { + emitAllocStackVal(t); + oStackGet(); + oGetTee(regIsF32(t)); + emitStoreScalar(t); + oStackGet(); +} + +// Emit code to compute the address of a local or global variable. +def emitLoc(symbol Symbol~) { + if (symbol~.kind == SK_VARIABLE && symbol~.isGlobal) { + if (symbol~.hasGlobalLoc && symbol~.globalLoc != 0) { + oI32Const(symbol~.globalLoc); + } else { + die(96); // cannot find global in emitLoc + } + } else { + if (symbol~.hasLocalLoc && symbol~.localLoc != 0) { + oBasePtrGet(); + oI32Const(symbol~.localLoc); + oI32Sub(); + } else { + die(97); // cannot find local in emitLoc + } + } +} + +// Emits "global.get $" / "global.set $" for register globals. +def oGlobalOp(isSet bool, name Token~) { + wInd(); + if (isSet) { + var s0 = "global.set $"; + wRaw(&s0[0], len(s0)); + } else { + var s1 = "global.get $"; + wRaw(&s1[0], len(s1)); + } + wLexeme(name); + wEnd(); +} + +// Emits "local.get $_" / "local.tee $_" for register locals. +def oLocalOp(isTee bool, name Token~, id int) { + wInd(); + if (isTee) { + var s0 = "local.tee $"; + wRaw(&s0[0], len(s0)); + } else { + var s1 = "local.get $"; + wRaw(&s1[0], len(s1)); + } + wLexeme(name); + bufPushChar(gOut, 95); // '_' + wInt(id); + wEnd(); +} + +// Emit code to set the given symbol to the value currently at top of the +// host WASM stack. Returns the set value or address of non-scalar symbol. +def emitSetSymbol(symbol Symbol~) { + var type = symbolValueType(symbol); + if (isVariableInRegister(symbol)) { + if (symbol~.kind == SK_VARIABLE && symbol~.isGlobal) { + emitDupTop(regIsF32(type)); + oGlobalOp(true, symbolName(symbol)); + } else { + oLocalOp(true, symbolName(symbol), symbol~.id); + } + } else { + if (isScalar(type)) { + emitDupTop(regIsF32(type)); // for return value + emitLoc(symbol); + emitSwapTop(false, regIsF32(type)); + emitStoreScalar(type); + } else { + emitLoc(symbol); + oI32Const(sizeOf(type)); + oCallMemcpy(); + emitLoc(symbol); // return dest address + } + } +} + +// Emits code to get the given symbol. +// - If symbol is scalar, returns value. +// - If symbol is non-scalar, returns address. +def emitGetSymbol(symbol Symbol~) { + var type = symbolValueType(symbol); + if (isVariableInRegister(symbol)) { + if (symbol~.kind == SK_VARIABLE && symbol~.isGlobal) { + oGlobalOp(false, symbolName(symbol)); + } else { + oLocalOp(false, symbolName(symbol), symbol~.id); + } + } else { + emitLoc(symbol); + if (isScalar(type)) { + emitLoadScalar(type); + } + // else: nothing to do, return address above + } +} + +// Emit code to print the given ASCII characters. +def emitPrintASCIIChars(p byte~, n int) { + for (var i = 0; i < n; i = i + 1) { + oI32Const(int((p + i)~)); + var s = "call $__putc__"; + wLine(&s[0], len(s)); + } +} + +// Emit code to print a value of the given type. +// PRECOND: Stack contains scalar value or address of non-scalar value. +def emitPrintVal(t Type~) { + if (t~.category == TC_ARRAY) { + var elementType = t~.elementType; + if (elementType~.category == TC_BYTE) { + // Byte arrays are printed as string literals + for (var i = 0; i < t~.length; i = i + 1) { + var isLast = i == t~.length - 1; + if (!isLast) { + emitDupTop(false); + } + oI32Const(i * sizeOf(elementType)); + oI32Add(); + emitLoadScalar(elementType); + var s0 = "call $__putc__"; + wLine(&s0[0], len(s0)); + } + } else { + var sOpen = "["; + emitPrintASCIIChars(&sOpen[0], len(sOpen)); + for (var j = 0; j < t~.length; j = j + 1) { + var isLast2 = j == t~.length - 1; + if (!isLast2) { + emitDupTop(false); + } + oI32Const(j * sizeOf(elementType)); + oI32Add(); + if (isScalar(elementType)) { + emitLoadScalar(elementType); + emitPrintVal(elementType); + } else { + emitPrintVal(elementType); + } + if (!isLast2) { + var sSep = ", "; + emitPrintASCIIChars(&sSep[0], len(sSep)); + } + } + var sClose = "]"; + emitPrintASCIIChars(&sClose[0], len(sClose)); + } + return; + } + if (t~.category == TC_BYTE) { + // mask 24 MSB before logging + var s1 = "i32.const 0x000000FF"; + wLine(&s1[0], len(s1)); + var s2 = "i32.and"; + wLine(&s2[0], len(s2)); + var s3 = "call $__puti__"; + wLine(&s3[0], len(s3)); + return; + } + if (t~.category == TC_BOOL || t~.category == TC_INT) { + var s4 = "call $__puti__"; + wLine(&s4[0], len(s4)); + return; + } + if (t~.category == TC_FLOAT) { + var s5 = "call $__putf__"; + wLine(&s5[0], len(s5)); + return; + } + die(98); // unexpected type for print +} + +// Emits "$" label reference. +def wLabel(n int) { + bufPushChar(gOut, 36); // '$' + wInt(n); +} + +// --------------------------------------------------------------------------- +// visit: expressions +// --------------------------------------------------------------------------- + +def vAssignExpr(op Node~) { + if (op~.a~.kind == NK_VARIABLE_EXPR) { + var symbol = op~.a~.symbol; + visit(op~.b, EM_RVALUE); + if (int(symbol) != 0 && (symbol~.kind == SK_VARIABLE || symbol~.kind == SK_PARAM)) { + emitSetSymbol(symbol); + } else { + die(99); // cannot assign to function symbol + } + } else { + // Assigning to an index, dot, or dereference expression + var elementType = op~.resolvedType; + if (isScalar(elementType)) { + visit(op~.a, EM_LVALUE); // gets address of indexed element + visit(op~.b, EM_RVALUE); // gets value + oTee(regIsF32(op~.resolvedType)); // for chained assignment + emitStoreScalar(elementType); + oGetTee(regIsF32(op~.resolvedType)); + } else { + visit(op~.b, EM_RVALUE); // gets address of value + visit(op~.a, EM_LVALUE); // gets address of indexed element + oTee(regIsF32(op~.resolvedType)); // for chained assignment + oI32Const(sizeOf(elementType)); + oCallMemcpy(); + oGetTee(regIsF32(op~.resolvedType)); + } + } +} + +def vBinaryExpr(op Node~) { + visit(op~.a, EM_RVALUE); + visit(op~.b, EM_RVALUE); + var leftIsFloat = op~.a~.resolvedType~.category == TC_FLOAT; + var leftIsByte = op~.a~.resolvedType~.category == TC_BYTE; + var opType = op~.tokA~.type; + if (opType == T_LESS) { + if (leftIsFloat) { + var s0 = "f32.lt"; + wLine(&s0[0], len(s0)); + } else if (leftIsByte) { + var s1 = "i32.lt_u"; + wLine(&s1[0], len(s1)); + } else { + var s2 = "i32.lt_s"; + wLine(&s2[0], len(s2)); + } + } else if (opType == T_LESS_EQUAL) { + if (leftIsFloat) { + var s3 = "f32.le"; + wLine(&s3[0], len(s3)); + } else if (leftIsByte) { + var s4 = "i32.le_u"; + wLine(&s4[0], len(s4)); + } else { + var s5 = "i32.le_s"; + wLine(&s5[0], len(s5)); + } + } else if (opType == T_GREATER) { + if (leftIsFloat) { + var s6 = "f32.gt"; + wLine(&s6[0], len(s6)); + } else if (leftIsByte) { + var s7 = "i32.gt_u"; + wLine(&s7[0], len(s7)); + } else { + var s8 = "i32.gt_s"; + wLine(&s8[0], len(s8)); + } + } else if (opType == T_GREATER_EQUAL) { + if (leftIsFloat) { + var s9 = "f32.ge"; + wLine(&s9[0], len(s9)); + } else if (leftIsByte) { + var s10 = "i32.ge_u"; + wLine(&s10[0], len(s10)); + } else { + var s11 = "i32.ge_s"; + wLine(&s11[0], len(s11)); + } + } else if (opType == T_BANG_EQUAL) { + if (leftIsFloat) { + var s12 = "f32.ne"; + wLine(&s12[0], len(s12)); + } else { + var s13 = "i32.ne"; + wLine(&s13[0], len(s13)); + } + } else if (opType == T_EQUAL_EQUAL) { + if (leftIsFloat) { + var s14 = "f32.eq"; + wLine(&s14[0], len(s14)); + } else { + var s15 = "i32.eq"; + wLine(&s15[0], len(s15)); + } + } else if (opType == T_PLUS) { + if (leftIsFloat) { + var s16 = "f32.add"; + wLine(&s16[0], len(s16)); + } else { + oI32Add(); + } + } else if (opType == T_MINUS) { + if (leftIsFloat) { + var s17 = "f32.sub"; + wLine(&s17[0], len(s17)); + } else { + oI32Sub(); + } + } else if (opType == T_STAR) { + if (leftIsFloat) { + var s18 = "f32.mul"; + wLine(&s18[0], len(s18)); + } else { + oI32Mul(); + } + } else if (opType == T_SLASH) { + if (leftIsFloat) { + var s19 = "f32.div"; + wLine(&s19[0], len(s19)); + } else { + var s20 = "i32.div_s"; + wLine(&s20[0], len(s20)); + } + } else if (opType == T_PERCENT) { + if (leftIsByte) { + var s21 = "i32.rem_u"; + wLine(&s21[0], len(s21)); + } else { + var s22 = "i32.rem_s"; + wLine(&s22[0], len(s22)); + } + } else { + die(100); // unhandled binary operator + } +} + +def vCallExpr(op Node~) { + if (op~.a~.kind != NK_VARIABLE_EXPR) { + die(101); // unexpected callee + } + var symbol = op~.a~.symbol; + if (int(symbol) == 0) { + die(102); // unexpected callee + } + if (symbol~.kind == SK_FUNCTION) { + for (var i = 0; i < op~.list~.count; i = i + 1) { + visit(Node~(vecGet(op~.list, i)), EM_RVALUE); + } + var returnType = op~.resolvedType; + var pushReturnValToStack = !isScalar(returnType) && !(returnType~.category == TC_VOID); + if (pushReturnValToStack) { + emitAllocStackVal(returnType); + } + wInd(); + var s0 = "call $"; + wRaw(&s0[0], len(s0)); + wLexeme(symbol~.node~.tokA); + wEnd(); + if (pushReturnValToStack) { + // Call above returned address of return value. + // Memcpy it to the earlier reservation + oStackGet(); + oI32Const(sizeOf(returnType)); + oCallMemcpy(); + oStackGet(); + } + } else if (symbol~.kind == SK_STRUCT) { + // Constructors are inlined + emitAllocStackVal(op~.resolvedType); + oStackGet(); + var structNode = op~.resolvedType~.resolvedStruct; + var offset = 0; + for (var j = 0; j < structNode~.params~.count; j = j + 1) { + emitDupTop(false); + oI32Const(offset); + oI32Add(); + visit(Node~(vecGet(op~.list, j)), EM_RVALUE); // returns scalar or address of non-scalar temporary + var member = Param~(vecGet(structNode~.params, j)); + if (isScalar(member~.type)) { + emitStoreScalar(member~.type); + } else { + emitSwapTop(false, false); + oI32Const(sizeOf(member~.type)); + oCallMemcpy(); + } + offset = offset + sizeOf(member~.type); + } + } else { + die(103); // unexpected callee + } +} + +def vCastExpr(op Node~) { + visit(op~.a, EM_RVALUE); + var target = op~.typeA~.category; + var source = op~.a~.resolvedType~.category; + if (target == TC_BOOL) { + if (source == TC_BOOL || source == TC_BYTE || source == TC_INT) { + var s0 = "i32.eqz"; + wLine(&s0[0], len(s0)); + wLine(&s0[0], len(s0)); + } else if (source == TC_FLOAT) { + var s1 = "f32.const 0"; + wLine(&s1[0], len(s1)); + var s2 = "f32.ne"; + wLine(&s2[0], len(s2)); + } else { + die(104); // unexpected type for cast source + } + return; + } + if (target == TC_BYTE) { + if (source == TC_BOOL || source == TC_BYTE || source == TC_INT) { + // no conversions needed + } else if (source == TC_FLOAT) { + // convert from f32 to signed i32 rounding towards zero (.5 will be lost) + // byte will interpret the 8 LSB as unsigned + var s3 = "i32.trunc_f32_s"; + wLine(&s3[0], len(s3)); + } else { + die(105); // unexpected type for cast source + } + return; + } + if (target == TC_INT) { + if (source == TC_BYTE) { + // mask 24 MSB + var s4 = "i32.const 0x000000FF"; + wLine(&s4[0], len(s4)); + var s5 = "i32.and"; + wLine(&s5[0], len(s5)); + } else if (source == TC_BOOL || source == TC_INT || source == TC_POINTER) { + // no conversion needed + } else if (source == TC_FLOAT) { + // convert from f32 to signed i32 rounding towards zero (.5 will be lost) + var s6 = "i32.trunc_f32_s"; + wLine(&s6[0], len(s6)); + } else { + die(106); // unexpected type for cast source + } + return; + } + if (target == TC_FLOAT) { + if (source == TC_BOOL || source == TC_BYTE || source == TC_INT) { + var s7 = "f32.convert_i32_s"; + wLine(&s7[0], len(s7)); + } else if (source == TC_FLOAT) { + // no conversions needed + } else { + die(107); // unexpected type for cast source + } + return; + } + if (target == TC_POINTER) { + if (source == TC_POINTER || source == TC_INT) { + // no conversions needed + } else { + die(108); // unexpected type for cast source + } + return; + } + die(109); // unexpected type for cast target +} + +def vLiteralExpr(op Node~) { + var cat = op~.typeA~.category; + if (cat == TC_ARRAY) { + var elementType = op~.typeA~.elementType; + if (elementType~.category != TC_BYTE) { + // If we're here, we probably meant to use a LIST_EXPR + die(110); // literal node contains non-string array + } + oI32Const(op~.strLoc); + emitPushMem(op~.typeA); + return; + } + if (cat == TC_BOOL) { + if (op~.litBool) { + oI32Const(1); + } else { + oI32Const(0); + } + return; + } + if (cat == TC_BYTE || cat == TC_INT) { + if (int(op~.decStr) != 0) { + wInd(); + var s0 = "i32.const "; + wRaw(&s0[0], len(s0)); + wStr(op~.decStr); + wEnd(); + } else { + oI32Const(op~.litInt); + } + return; + } + if (cat == TC_FLOAT) { + wInd(); + var s1 = "f32.const "; + wRaw(&s1[0], len(s1)); + if (int(op~.lexeme) != 0) { + wFloatLexeme(op~.lexeme); + } else { + die(111); // synthesized float literal without lexeme + } + wEnd(); + return; + } + die(112); // unhandled literal type +} + +def vLogicalExpr(op Node~) { + // Can use bitwise equivalents assuming operands are bools (0 or 1) + var label = eNextLabelID; + eNextLabelID = eNextLabelID + 1; + wInd(); + var s0 = "(block "; + wRaw(&s0[0], len(s0)); + wLabel(label); + var s1 = " (result i32)"; + wRaw(&s1[0], len(s1)); + wEnd(); + eIndent = eIndent + 1; + if (op~.tokA~.type == T_AMP_AMP) { + visit(op~.a, EM_RVALUE); + emitDupTop(false); + var s2 = "i32.eqz"; + wLine(&s2[0], len(s2)); + wInd(); + var s3 = "br_if "; + wRaw(&s3[0], len(s3)); + wLabel(label); + wEnd(); + visit(op~.b, EM_RVALUE); + var s4 = "i32.eq"; + wLine(&s4[0], len(s4)); + } else { + visit(op~.a, EM_RVALUE); + emitDupTop(false); + wInd(); + var s5 = "br_if "; + wRaw(&s5[0], len(s5)); + wLabel(label); + wEnd(); + oDrop(); + visit(op~.b, EM_RVALUE); + } + eIndent = eIndent - 1; + var s6 = ")"; + wLine(&s6[0], len(s6)); +} + +def vUnaryExpr(op Node~) { + var opType = op~.tokA~.type; + if (opType == T_BANG) { + visit(op~.a, EM_RVALUE); + var s0 = "i32.eqz"; + wLine(&s0[0], len(s0)); + return; + } + if (opType == T_MINUS) { + var isF32 = regIsF32(op~.a~.resolvedType); + wInd(); + wRegType(isF32); + var s1 = ".const 0"; + wRaw(&s1[0], len(s1)); + wEnd(); + visit(op~.a, EM_RVALUE); + wInd(); + wRegType(isF32); + var s2 = ".sub"; + wRaw(&s2[0], len(s2)); + wEnd(); + return; + } + // '&' + if (op~.a~.kind == NK_VARIABLE_EXPR) { + var symbol = op~.a~.symbol; + if (int(symbol) != 0 && (symbol~.kind == SK_PARAM || symbol~.kind == SK_VARIABLE)) { + emitLoc(symbol); + } else { + die(113); // unhandled operand for operator '&' + } + return; + } + if (op~.a~.kind == NK_INDEX_EXPR || op~.a~.kind == NK_DOT_EXPR || op~.a~.kind == NK_DEREF_EXPR) { + // `&arr[i]` should return address of the ith element in `arr`. + // `&s.member` and `&p~` similarly return addresses of their operands. + visit(op~.a, EM_LVALUE); + return; + } + die(114); // unhandled operand for operator '&' +} + +def vListExpr(op Node~) { + var elementType = op~.resolvedType~.elementType; + if (op~.listKind == LK_LIST) { + emitAllocStackVal(op~.resolvedType); + oStackGet(); + if (isScalar(elementType)) { + for (var i = 0; i < op~.list~.count; i = i + 1) { + emitDupTop(false); + oI32Const(i * sizeOf(elementType)); + oI32Add(); + visit(Node~(vecGet(op~.list, i)), EM_RVALUE); // value to store + emitStoreScalar(elementType); + } + } else { + for (var j = 0; j < op~.list~.count; j = j + 1) { + emitDupTop(false); + oI32Const(j * sizeOf(elementType)); + oI32Add(); + visit(Node~(vecGet(op~.list, j)), EM_RVALUE); // address of value to store + emitSwapTop(false, false); + oI32Const(sizeOf(elementType)); + oCallMemcpy(); + } + } + } else { + if (isScalar(elementType)) { + visit(op~.a, EM_RVALUE); // returns value to store + wInd(); + var s0 = "local.set $__tee_"; + wRaw(&s0[0], len(s0)); + wRegType(regIsF32(elementType)); + var s1 = "__"; + wRaw(&s1[0], len(s1)); + wEnd(); + for (var k = 0; k < op~.repeatLen; k = k + 1) { + emitPushScalarFromTee(elementType); // returns mutated __stack_ptr__ + oDrop(); + } + } else if (op~.repeatLen > 0) { + // Call repeat expression to create first array item + // INVARIANT: Repeat expression pushes only the evaluation result to stack + visit(op~.a, EM_RVALUE); // pushes to stack and returns mutated __stack_ptr__ + for (var q = 1; q < op~.repeatLen; q = q + 1) { + emitPushMem(elementType); // returns mutated __stack_ptr__ + } + oDrop(); + } + oStackGet(); + } +} + +// --------------------------------------------------------------------------- +// visit: statements +// --------------------------------------------------------------------------- + +// Emits the (local ...) declarations shared by all function bodies. +def emitStandardLocals() { + var s0 = "(local $__base_ptr__ i32)"; + wLine(&s0[0], len(s0)); + var s1 = "(local $__tee_i32__ i32)"; + wLine(&s1[0], len(s1)); + var s2 = "(local $__tee_f32__ f32)"; + wLine(&s2[0], len(s2)); + var s3 = "(local $__swapa_i32__ i32)"; + wLine(&s3[0], len(s3)); + var s4 = "(local $__swapb_i32__ i32)"; + wLine(&s4[0], len(s4)); + var s5 = "(local $__swapb_f32__ f32)"; + wLine(&s5[0], len(s5)); + var s6 = "(local $__swapa_f32__ f32)"; + wLine(&s6[0], len(s6)); +} + +def oBasePtrSave() { + oStackGet(); + var s0 = "local.set $__base_ptr__"; + wLine(&s0[0], len(s0)); +} + +def oBasePtrRestore() { + oBasePtrGet(); + oStackSet(); +} + +// Allocates a WASM register or a stack location for the given local symbol. +// Returns the new local offset. +def allocateRegisterOrStackLoc(local Symbol~, localOffset int) int { + if (local~.kind == SK_FUNCTION || local~.kind == SK_STRUCT) { + return localOffset; + } + if (isVariableInRegister(local) && local~.kind == SK_VARIABLE) { + wInd(); + var s0 = "(local $"; + wRaw(&s0[0], len(s0)); + wLexeme(local~.node~.tokA); + bufPushChar(gOut, 95); // '_' + wInt(local~.id); + bufPushChar(gOut, 32); + wRegType(regIsF32(local~.node~.typeA)); + bufPushChar(gOut, 41); // ')' + wEnd(); + return localOffset; + } + var type = symbolValueType(local); + var newOffset = localOffset + sizeOf(type); + local~.localLoc = newOffset; + local~.hasLocalLoc = true; + return newOffset; +} + +def vFunctionStmt(op Node~) { + if (!op~.hasBody) { + // Imported/built-in functions generate code separately + return; + } + var sMain = "main"; + var isMain = strEqRaw(tokenLexemeStr(op~.tokA), &sMain[0], len(sMain)); + wInd(); + var s0 = "(func $"; + wRaw(&s0[0], len(s0)); + wLexeme(op~.tokA); + if (isMain || op~.isExported) { + // NOTE: puff string literals cannot contain double quotes; build manually. + var s1 = " (export "; + wRaw(&s1[0], len(s1)); + bufPushChar(gOut, 34); + wLexeme(op~.tokA); + bufPushChar(gOut, 34); + bufPushChar(gOut, 41); // ')' + } + wEnd(); + eIndent = eIndent + 1; + // params + for (var i = 0; i < op~.scope~.syms~.count; i = i + 1) { + var sym = Symbol~(vecGet(op~.scope~.syms, i)); + if (sym~.kind == SK_PARAM) { + wInd(); + var s2 = "(param $"; + wRaw(&s2[0], len(s2)); + wLexeme(sym~.param~.name); + bufPushChar(gOut, 95); // '_' + wInt(sym~.id); + bufPushChar(gOut, 32); + wRegType(regIsF32(sym~.param~.type)); + bufPushChar(gOut, 41); // ')' + wEnd(); + } + } + if (!(op~.typeA~.category == TC_VOID)) { + wInd(); + var s3 = "(result "; + wRaw(&s3[0], len(s3)); + wRegType(regIsF32(op~.typeA)); + bufPushChar(gOut, 41); // ')' + wEnd(); + } + // WASM doesn't have block scope, and `func` definitions require all local + // registers to be declared ahead-of-time, so we hoist all locals in + // descendant scopes to the top. + emitStandardLocals(); + var localOffset = 0; + for (var j = 0; j < op~.scope~.syms~.count; j = j + 1) { + localOffset = allocateRegisterOrStackLoc(Symbol~(vecGet(op~.scope~.syms, j)), localOffset); + } + if (int(op~.hoisted) != 0) { + for (var k = 0; k < op~.hoisted~.count; k = k + 1) { + localOffset = allocateRegisterOrStackLoc(Symbol~(vecGet(op~.hoisted, k)), localOffset); + } + } + oBasePtrSave(); + oStackGet(); + oI32Const(localOffset); + oI32Sub(); + oStackSet(); + + // Copy + push nonregister args to stack + for (var m = 0; m < op~.params~.count; m = m + 1) { + var param = Param~(vecGet(op~.params, m)); + var symbol = scopeLookup(op~.scope, tokenLexemeStr(param~.name), LOOKUP_ANY); + if (int(symbol) != 0 && !isVariableInRegister(symbol)) { + oLocalOp(false, param~.name, symbol~.id); + emitSetSymbol(symbol); + // After this, we should only ever be using `emitGetSymbol` to access + // the symbol. The local is meaningless. + } + } + + for (var n = 0; n < op~.list~.count; n = n + 1) { + visit(Node~(vecGet(op~.list, n)), EM_RVALUE); + } + + oBasePtrRestore(); + eIndent = eIndent - 1; + var s4 = ")"; + wLine(&s4[0], len(s4)); +} + +def vIfStmt(op Node~) { + visit(op~.a, EM_RVALUE); + var s0 = "(if"; + wLine(&s0[0], len(s0)); + eIndent = eIndent + 1; + var s1 = "(then"; + wLine(&s1[0], len(s1)); + eIndent = eIndent + 1; + visit(op~.b, EM_RVALUE); + eIndent = eIndent - 1; + var s2 = ")"; + wLine(&s2[0], len(s2)); + if (int(op~.c) != 0) { + var s3 = "(else"; + wLine(&s3[0], len(s3)); + eIndent = eIndent + 1; + visit(op~.c, EM_RVALUE); + eIndent = eIndent - 1; + var s4 = ")"; + wLine(&s4[0], len(s4)); + } + eIndent = eIndent - 1; + var s5 = ")"; + wLine(&s5[0], len(s5)); +} + +def vWhileStmt(op Node~) { + var outerLabel = eNextLabelID; + eNextLabelID = eNextLabelID + 1; + var innerLabel = eNextLabelID; + eNextLabelID = eNextLabelID + 1; + var incrementLabel = -1; + if (int(op~.c) != 0) { + incrementLabel = eNextLabelID; + eNextLabelID = eNextLabelID + 1; + } + wInd(); + var s0 = "(block "; + wRaw(&s0[0], len(s0)); + wLabel(outerLabel); + wEnd(); + eIndent = eIndent + 1; + wInd(); + var s1 = "(loop "; + wRaw(&s1[0], len(s1)); + wLabel(innerLabel); + wEnd(); + eIndent = eIndent + 1; + visit(op~.a, EM_RVALUE); + var s2 = "i32.eqz"; + wLine(&s2[0], len(s2)); + wInd(); + var s3 = "br_if "; + wRaw(&s3[0], len(s3)); + wLabel(outerLabel); + wEnd(); + if (int(op~.c) != 0) { + wInd(); + var s4 = "(block "; + wRaw(&s4[0], len(s4)); + wLabel(incrementLabel); + wEnd(); + eIndent = eIndent + 1; + } + + vecPush(eLoopStack, outerLabel); + vecPush(eLoopStack, innerLabel); + vecPush(eLoopStack, incrementLabel); + visit(op~.b, EM_RVALUE); + vecPop(eLoopStack); + vecPop(eLoopStack); + vecPop(eLoopStack); + + if (int(op~.c) != 0) { + eIndent = eIndent - 1; + var s5 = ")"; + wLine(&s5[0], len(s5)); + visit(op~.c, EM_RVALUE); + } + + wInd(); + var s6 = "br "; + wRaw(&s6[0], len(s6)); + wLabel(innerLabel); + wEnd(); + eIndent = eIndent - 1; + var s7 = ")"; + wLine(&s7[0], len(s7)); + eIndent = eIndent - 1; + wLine(&s7[0], len(s7)); +} + +def vLoopControlStmt(op Node~) { + if (eLoopStack~.count == 0) { + die(115); // unexpected loop control statement outside of loop + } + var outerLabel = vecGet(eLoopStack, eLoopStack~.count - 3); + var innerLabel = vecGet(eLoopStack, eLoopStack~.count - 2); + var incrementLabel = vecGet(eLoopStack, eLoopStack~.count - 1); + wInd(); + var s0 = "br "; + wRaw(&s0[0], len(s0)); + if (op~.tokA~.type == T_BREAK) { + wLabel(outerLabel); + } else { + if (incrementLabel >= 0) { + wLabel(incrementLabel); + } else { + wLabel(innerLabel); + } + } + wEnd(); +} + +// --------------------------------------------------------------------------- +// visit dispatcher +// --------------------------------------------------------------------------- + +def visit(node Node~, exprMode int) { + if (node~.skipEmit) { + return; + } + var kind = node~.kind; + if (kind == NK_ASSIGN_EXPR) { + vAssignExpr(node); + } else if (kind == NK_BINARY_EXPR) { + vBinaryExpr(node); + } else if (kind == NK_CALL_EXPR) { + vCallExpr(node); + } else if (kind == NK_CAST_EXPR) { + vCastExpr(node); + } else if (kind == NK_DEREF_EXPR) { + var elementType = node~.resolvedType; + visit(node~.a, EM_RVALUE); // returns an address pointing to value of type `elementType` + if (exprMode == EM_LVALUE) { + // Done; return address of value + } else { + if (isScalar(elementType)) { + emitLoadScalar(elementType); + } else { + emitPushMem(elementType); + } + } + } else if (kind == NK_DOT_EXPR) { + var memberType = node~.resolvedType; + var structType = node~.a~.resolvedType; + if (structType~.category != TC_STRUCT) { + die(116); // unexpected callee type for dot expr + } + visit(node~.a, EM_LVALUE); // get address of struct start + var offset = 0; + var structNode = structType~.resolvedStruct; + for (var i = 0; i < structNode~.params~.count; i = i + 1) { + var member = Param~(vecGet(structNode~.params, i)); + if (strEq(tokenLexemeStr(member~.name), tokenLexemeStr(node~.tokB))) { + break; + } + offset = offset + sizeOf(member~.type); + } + oI32Const(offset); + oI32Add(); + if (exprMode == EM_LVALUE) { + // done; address of member returned + } else { + // get value of member + if (isScalar(memberType)) { + emitLoadScalar(memberType); + } else { + emitPushMem(memberType); + } + } + } else if (kind == NK_GROUP_EXPR) { + visit(node~.a, EM_RVALUE); + } else if (kind == NK_INDEX_EXPR) { + // TODO: trap on out-of-bounds access + var idxElementType = node~.resolvedType; + visit(node~.a, EM_LVALUE); // get address of array start + visit(node~.b, EM_RVALUE); // get index + // eat 2, get address of indexed element + oI32Const(sizeOf(idxElementType)); + oI32Mul(); + oI32Add(); + if (exprMode == EM_LVALUE) { + // done; address of indexed element returned + } else { + // get value of indexed element + if (isScalar(idxElementType)) { + emitLoadScalar(idxElementType); + } else { + emitPushMem(idxElementType); + } + } + } else if (kind == NK_LEN_EXPR) { + oI32Const(node~.resolvedLength); + } else if (kind == NK_LIST_EXPR) { + vListExpr(node); + } else if (kind == NK_LITERAL_EXPR) { + vLiteralExpr(node); + } else if (kind == NK_LOGICAL_EXPR) { + vLogicalExpr(node); + } else if (kind == NK_UNARY_EXPR) { + vUnaryExpr(node); + } else if (kind == NK_VARIABLE_EXPR) { + var symbol = node~.symbol; + if (int(symbol) == 0) { + die(117); // unresolved symbol in variable expression + } + if (symbol~.kind == SK_VARIABLE || symbol~.kind == SK_PARAM) { + emitGetSymbol(symbol); + if (!isScalar(node~.resolvedType) && exprMode == EM_RVALUE) { + // When evaluating non-scalar variables as rvals, copy the value + // as a temporary to the stack + emitPushMem(node~.resolvedType); + } + } else { + // We shouldn't be visiting variable expressions of function type. + die(118); + } + } else if (kind == NK_BLOCK_STMT) { + for (var i = 0; i < node~.list~.count; i = i + 1) { + visit(Node~(vecGet(node~.list, i)), EM_RVALUE); + } + } else if (kind == NK_EXPRESSION_STMT) { + visit(node~.a, EM_RVALUE); + if (!(node~.a~.resolvedType~.category == TC_VOID)) { + // discard result + oDrop(); + } + } else if (kind == NK_FUNCTION_STMT) { + vFunctionStmt(node); + } else if (kind == NK_IF_STMT) { + vIfStmt(node); + } else if (kind == NK_LOOP_CONTROL_STMT) { + vLoopControlStmt(node); + } else if (kind == NK_PRINT_STMT) { + visit(node~.a, EM_RVALUE); + emitPrintVal(node~.a~.resolvedType); + var s0 = "call $__flush__"; + wLine(&s0[0], len(s0)); + } else if (kind == NK_RETURN_STMT) { + if (int(node~.a) != 0) { + visit(node~.a, EM_RVALUE); + } + oBasePtrRestore(); + var s1 = "return"; + wLine(&s1[0], len(s1)); + } else if (kind == NK_STRUCT_STMT) { + // These don't generate any code. Constructors, assignment, comparisons, + // and member accesses are all inlined. + } else if (kind == NK_VAR_STMT) { + visit(node~.a, EM_RVALUE); + emitSetSymbol(node~.symbol); // returns address/value that was set + oDrop(); + } else if (kind == NK_WHILE_STMT) { + vWhileStmt(node); + } else { + die(119); // unhandled node kind in visit + } +} + +// --------------------------------------------------------------------------- +// Module emission +// --------------------------------------------------------------------------- + +def emitBuiltins() { + // __memcpy__ + var s0 = "(func $__memcpy__ (param $src i32) (param $dst i32) (param $numBytes i32)"; + wLine(&s0[0], len(s0)); + eIndent = eIndent + 1; + var outerLabel = eNextLabelID; + eNextLabelID = eNextLabelID + 1; + var innerLabel = eNextLabelID; + eNextLabelID = eNextLabelID + 1; + wInd(); + var s1 = "(block "; + wRaw(&s1[0], len(s1)); + wLabel(outerLabel); + wEnd(); + eIndent = eIndent + 1; + wInd(); + var s2 = "(loop "; + wRaw(&s2[0], len(s2)); + wLabel(innerLabel); + wEnd(); + eIndent = eIndent + 1; + var s3 = "local.get $numBytes"; + wLine(&s3[0], len(s3)); + oI32Const(0); + var s4 = "i32.gt_s"; + wLine(&s4[0], len(s4)); + var s5 = "i32.eqz"; + wLine(&s5[0], len(s5)); + wInd(); + var s6 = "br_if "; + wRaw(&s6[0], len(s6)); + wLabel(outerLabel); + wEnd(); + // *dst = *src; + var s7 = "local.get $dst"; + wLine(&s7[0], len(s7)); + var s8 = "local.get $src"; + wLine(&s8[0], len(s8)); + var s9 = "i32.load8_u"; + wLine(&s9[0], len(s9)); + var s10 = "i32.store8"; + wLine(&s10[0], len(s10)); + // src++, dst++; + wLine(&s8[0], len(s8)); + oI32Const(1); + oI32Add(); + var s11 = "local.set $src"; + wLine(&s11[0], len(s11)); + wLine(&s7[0], len(s7)); + oI32Const(1); + oI32Add(); + var s12 = "local.set $dst"; + wLine(&s12[0], len(s12)); + // numBytes--; + wLine(&s3[0], len(s3)); + oI32Const(1); + oI32Sub(); + var s13 = "local.set $numBytes"; + wLine(&s13[0], len(s13)); + wInd(); + wEnd(); + wInd(); + var s14 = "br "; + wRaw(&s14[0], len(s14)); + wLabel(innerLabel); + wEnd(); + eIndent = eIndent - 1; + var s15 = ")"; + wLine(&s15[0], len(s15)); + eIndent = eIndent - 1; + wLine(&s15[0], len(s15)); + eIndent = eIndent - 1; + wLine(&s15[0], len(s15)); + + // __sqrt__ (note: body is emitted at the same indent level as the `(func`, + // mirroring the reference backend) + var s16 = "(func $__sqrt__ (param $x f32) (result f32)"; + wLine(&s16[0], len(s16)); + var s17 = "local.get $x"; + wLine(&s17[0], len(s17)); + var s18 = "f32.sqrt"; + wLine(&s18[0], len(s18)); + wLine(&s15[0], len(s15)); + + // __heap_end__ + var s19 = "(func $__heap_end__ (result i32)"; + wLine(&s19[0], len(s19)); + var s20 = "memory.size"; + wLine(&s20[0], len(s20)); + oI32Const(65536); + oI32Mul(); + wLine(&s15[0], len(s15)); + + // __grow_heap__ + var s21 = "(func $__grow_heap__ (param $numPages i32) (result i32)"; + wLine(&s21[0], len(s21)); + var s22 = "local.get $numPages"; + wLine(&s22[0], len(s22)); + var s23 = "memory.grow"; + wLine(&s23[0], len(s23)); + wLine(&s15[0], len(s15)); +} + +// Emits WAT code for the resolved program into gOut. +// Mirrors `emit` in src/backend.ts. +def emitProgram() { + var s0 = "(module"; + wLine(&s0[0], len(s0)); + eIndent = eIndent + 1; + + emitIoImport(0); + emitIoImport(1); + emitIoImport(2); + emitIoImport(3); + emitIoImport(4); + emitIoImport(5); + + // Imports must precede all non-import definitions in the module. + for (var i = 0; i < gTopLevel~.count; i = i + 1) { + var stmt = Node~(vecGet(gTopLevel, i)); + if (stmt~.kind == NK_FUNCTION_STMT && int(stmt~.hostModule) != 0) { + emitEnvImport(stmt); + } + } + + wInd(); + var s2 = "(memory $memory "; + wRaw(&s2[0], len(s2)); + wInt(INITIAL_PAGES); + bufPushChar(gOut, 41); + wEnd(); + + wInd(); + var s3 = "(global $__stack_ptr__ (mut i32) i32.const "; + wRaw(&s3[0], len(s3)); + wInt(STACK_TOP_BYTE_OFFSET); + bufPushChar(gOut, 41); + wEnd(); + + var globalByteOffset = DATA_TOP_BYTE_OFFSET; + for (var j = 0; j < gStrLitNodes~.count; j = j + 1) { + var litNode = Node~(vecGet(gStrLitNodes, j)); + globalByteOffset = globalByteOffset - sizeOf(litNode~.typeA); + litNode~.strLoc = globalByteOffset; + wInd(); + var s4 = "(data (i32.const "; + wRaw(&s4[0], len(s4)); + wInt(globalByteOffset); + var s5 = ") "; + wRaw(&s5[0], len(s5)); + bufPushChar(gOut, 34); // '"' + // WAT strings cannot contain backslashes, ASCII control sequences, + // or quotes; escape backslashes (quotes cannot appear in literals). + var lit = litNode~.litStr; + for (var k = 0; k < lit~.count; k = k + 1) { + var c = int(strGet(lit, k)); + if (c == 92) { + bufPushChar(gOut, 92); + bufPushChar(gOut, 92); + } else { + bufPushChar(gOut, c); + } + } + bufPushChar(gOut, 34); // '"' + bufPushChar(gOut, 41); // ')' + wEnd(); + } + + for (var m = 0; m < gGlobalInitOrder~.count; m = m + 1) { + var varDecl = Node~(vecGet(gGlobalInitOrder, m)); + if (int(varDecl~.symbol) != 0 && int(varDecl~.typeA) != 0) { + if (isVariableInRegister(varDecl~.symbol)) { + wInd(); + var s6 = "(global $"; + wRaw(&s6[0], len(s6)); + wLexeme(varDecl~.tokA); + var s7 = " (mut "; + wRaw(&s7[0], len(s7)); + var isF32 = regIsF32(varDecl~.typeA); + wRegType(isF32); + var s8 = ") "; + wRaw(&s8[0], len(s8)); + wRegType(isF32); + var s9 = ".const 0)"; + wRaw(&s9[0], len(s9)); + wEnd(); + } else { + globalByteOffset = globalByteOffset - sizeOf(varDecl~.typeA); + varDecl~.symbol~.globalLoc = globalByteOffset; + varDecl~.symbol~.hasGlobalLoc = true; + } + } + } + + emitInitGlobalsHeader(); + eIndent = eIndent + 1; + emitStandardLocals(); + oBasePtrSave(); + for (var n = 0; n < gGlobalInitOrder~.count; n = n + 1) { + var varDecl2 = Node~(vecGet(gGlobalInitOrder, n)); + if (int(varDecl2~.typeA) != 0) { + visit(varDecl2, EM_RVALUE); + varDecl2~.skipEmit = true; + } + } + oBasePtrRestore(); + eIndent = eIndent - 1; + var s11 = ")"; + wLine(&s11[0], len(s11)); + + emitBuiltins(); + + for (var q = 0; q < gTopLevel~.count; q = q + 1) { + visit(Node~(vecGet(gTopLevel, q)), EM_RVALUE); + } + + eIndent = eIndent - 1; + wLine(&s11[0], len(s11)); +} + +// Emits `(func (export "__init_globals__")` -- built with explicit quote +// chars since puff string literals cannot contain double quotes. +def emitInitGlobalsHeader() { + wInd(); + var s0 = "(func (export "; + wRaw(&s0[0], len(s0)); + bufPushChar(gOut, 34); + var s1 = "__init_globals__"; + wRaw(&s1[0], len(s1)); + bufPushChar(gOut, 34); + bufPushChar(gOut, 41); + wEnd(); +} + +// Emits one of the six fixed io imports; `which` selects the import. +def emitIoImport(which int) { + wInd(); + var s0 = "(import "; + wRaw(&s0[0], len(s0)); + bufPushChar(gOut, 34); + var s1 = "io"; + wRaw(&s1[0], len(s1)); + bufPushChar(gOut, 34); + bufPushChar(gOut, 32); + bufPushChar(gOut, 34); + if (which == 0 || which == 1) { + var s2 = "log"; + wRaw(&s2[0], len(s2)); + } else if (which == 2) { + var s3 = "putchar"; + wRaw(&s3[0], len(s3)); + } else if (which == 3) { + var s4 = "putf"; + wRaw(&s4[0], len(s4)); + } else if (which == 4) { + var s5 = "puti"; + wRaw(&s5[0], len(s5)); + } else { + var s6 = "flush"; + wRaw(&s6[0], len(s6)); + } + bufPushChar(gOut, 34); + var s7 = " (func $"; + wRaw(&s7[0], len(s7)); + if (which == 0) { + var s8 = "__log_i32__ (param i32)"; + wRaw(&s8[0], len(s8)); + } else if (which == 1) { + var s9 = "__log_f32__ (param f32)"; + wRaw(&s9[0], len(s9)); + } else if (which == 2) { + var s10 = "__putc__ (param i32)"; + wRaw(&s10[0], len(s10)); + } else if (which == 3) { + var s11 = "__putf__ (param f32)"; + wRaw(&s11[0], len(s11)); + } else if (which == 4) { + var s12 = "__puti__ (param i32)"; + wRaw(&s12[0], len(s12)); + } else { + var s13 = "__flush__"; + wRaw(&s13[0], len(s13)); + } + var s14 = "))"; + wRaw(&s14[0], len(s14)); + wEnd(); +} + +// Emits `(import "env" "" (func $ (param i32)... (result i32)))`. +def emitEnvImport(fn Node~) { + wInd(); + var s0 = "(import "; + wRaw(&s0[0], len(s0)); + bufPushChar(gOut, 34); + wStr(fn~.hostModule); + bufPushChar(gOut, 34); + bufPushChar(gOut, 32); + bufPushChar(gOut, 34); + wLexeme(fn~.tokA); + bufPushChar(gOut, 34); + var s1 = " (func $"; + wRaw(&s1[0], len(s1)); + wLexeme(fn~.tokA); + for (var i = 0; i < fn~.params~.count; i = i + 1) { + var param = Param~(vecGet(fn~.params, i)); + var s2 = " (param "; + wRaw(&s2[0], len(s2)); + wRegType(regIsF32(param~.type)); + bufPushChar(gOut, 41); + } + if (!(fn~.typeA~.category == TC_VOID)) { + var s3 = " (result "; + wRaw(&s3[0], len(s3)); + wRegType(regIsF32(fn~.typeA)); + bufPushChar(gOut, 41); + } + var s4 = "))"; + wRaw(&s4[0], len(s4)); + wEnd(); +} diff --git a/selfhost/main.puff b/selfhost/main.puff new file mode 100644 index 0000000..7364994 --- /dev/null +++ b/selfhost/main.puff @@ -0,0 +1,28 @@ +// main.puff +// Driver for the self-hosted puffscript compiler. +// Reads puffscript source from stdin; writes WAT to stdout. +// Compilation errors are written to stderr and exit code is 1. + +def main() { + gSource = readAllInput(); + initContext(); + scanTokens(); + if (numErrors() == 0) { + parseProgram(); + if (numErrors() == 0) { + resolveProgram(); + if (numErrors() == 0) { + emitProgram(); + } + } + } + if (numErrors() > 0) { + for (var i = 0; i < gErrors~.count; i = i + 1) { + writeStrTo(Str~(vecGet(gErrors, i)), true); + puterr(10); + } + exit(1); + } + writeBufTo(gOut, false); + exit(0); +} From 1d05971b8716b3ccea8e7c1c8e5acd0d70cc5aa1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 13 Aug 2026 07:43:43 +0000 Subject: [PATCH 7/7] Add self-hosting bootstrap tests, bootstrap script, and docs - test.ts: bootstrap fixpoint test (stage1==stage2==stage3), output-equivalence tests vs the reference compiler, and an end-to-end run of a program compiled by the self-hosted compiler - tools/bootstrap.sh + 'npm run bootstrap' - README: self-hosting instructions, import/export, pointer casts, heap builtins - regenerate demo.js bundle Co-authored-by: Andrew Chan --- README.md | 66 +++++++- demo.js | 403 ++++++++++++++++++++++++++++++++------------- package.json | 3 +- test.ts | 199 ++++++++++++++++++++++ tools/bootstrap.sh | 46 ++++++ 5 files changed, 599 insertions(+), 118 deletions(-) create mode 100755 tools/bootstrap.sh diff --git a/README.md b/README.md index 3416eb4..74b5e91 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,34 @@ Puffscript is a toy imperative programming language that compiles to WebAssembly - Fixed-length, contiguous multi-dimensional arrays - String literals are syntactic sugar for UTF-8 encoded byte arrays - Pointers, value semantics, and pointer arithmetic +- Imported and exported functions for host interop (I/O etc.) +- **Self-hosting**: the compiler is implemented both in TypeScript (`src/`) and in puffscript itself (`selfhost/`), and the self-hosted compiler compiles itself + +# Self-hosting + +`selfhost/` contains a complete puffscript compiler written in puffscript. It reads +puffscript source from stdin (via the imported `getchar` host function) and writes +WebAssembly text format to stdout (via `putchar`), reporting errors on stderr (via +`puterr`). Its output is byte-identical to the TypeScript reference compiler's. + +To bootstrap it and verify the fixpoint (stage2 = compiler compiled by itself, +stage3 = compiler compiled by stage2, stage2 == stage3): + +``` +npm install +npm run bootstrap +``` + +Compile a program with the bootstrapped compiler: + +``` +node dist/tools/run.js test/stage2.wasm --stdin program.puff --stdout program.wat +node dist/tools/wat2wasm.js program.wat -o program.wasm +node dist/tools/run.js program.wasm +``` + +The self-hosted compiler is also exercised by `npm test`, which checks the +bootstrap fixpoint and output equivalence with the reference compiler. # Language reference @@ -106,7 +134,15 @@ def main() { } ``` -Numerics cannot be casted directly to pointer types, so pointers can only be initialized with other pointers or by taking the address of variables. This discourages the use of null pointers. +`&` can also take the address of struct members and dereference expressions, e.g. `&point.x` or `&node~.next`. + +`int` values can be casted to and from pointer types via call syntax, which allows user code to implement its own allocators (see `selfhost/util.puff`). Pointer-to-struct casts use the type name, e.g. `Node~(p)`: + +``` +var p = int~(1048576); // an int~ pointing at the 1MB byte offset +var q = Node~(p); // reinterpreted as a Node~ +var r = int(q); // back to an int +``` **Structs** @@ -136,10 +172,36 @@ def fib(n int) int { Functions can only be defined at the top-level. Puffscript does not support first-class functions nor function pointers. +**Imported and exported functions** + +Host functions can be imported from the WASM `env` module with `import def`, and puffscript functions can be exported from the compiled module with `export def` (`main` is always exported): + +``` +import def getchar() int; // read a byte from the host (-1 on EOF) +import def putchar(c int); // write a byte to the host + +export def echo() { + var c = getchar(); + while (c >= 0) { + putchar(c); + c = getchar(); + } +} + +def main() { + echo(); +} +``` + +The CLI runner (`tools/run.ts`) provides `getchar`/`putchar`/`puterr`/`exit` wired to stdin/stdout/stderr, which is enough for programs — like the self-hosted compiler — to do file I/O. + +**Heap and builtins** + +Compiled programs place their in-memory stack at `[0, 4MB)` and static data at `[4MB, 8MB)`; memory beyond the initial 8MB is free for user-managed heaps. The builtins `__heap_end__() int` and `__grow_heap__(numPages int) int` wrap the WASM `memory.size`/`memory.grow` instructions, and `__memcpy__(src byte~, dst byte~, numBytes int)` and `__sqrt__(x float) float` are also available. + TODOs: - Function overloading (defining multiple functions with the same name but different parameters) -- Exported and imported functions - Default arguments **Control flow** diff --git a/demo.js b/demo.js index c20942f..f130a2a 100644 --- a/demo.js +++ b/demo.js @@ -101,20 +101,22 @@ TokenType2[TokenType2["CONTINUE"] = 41] = "CONTINUE"; TokenType2[TokenType2["DEF"] = 42] = "DEF"; TokenType2[TokenType2["ELSE"] = 43] = "ELSE"; - TokenType2[TokenType2["FALSE"] = 44] = "FALSE"; - TokenType2[TokenType2["FOR"] = 45] = "FOR"; - TokenType2[TokenType2["FLOAT"] = 46] = "FLOAT"; - TokenType2[TokenType2["IF"] = 47] = "IF"; - TokenType2[TokenType2["INT"] = 48] = "INT"; - TokenType2[TokenType2["LEN"] = 49] = "LEN"; - TokenType2[TokenType2["PRINT"] = 50] = "PRINT"; - TokenType2[TokenType2["RETURN"] = 51] = "RETURN"; - TokenType2[TokenType2["STRUCT"] = 52] = "STRUCT"; - TokenType2[TokenType2["TRUE"] = 53] = "TRUE"; - TokenType2[TokenType2["VAR"] = 54] = "VAR"; - TokenType2[TokenType2["VOID"] = 55] = "VOID"; - TokenType2[TokenType2["WHILE"] = 56] = "WHILE"; - TokenType2[TokenType2["EOF"] = 57] = "EOF"; + TokenType2[TokenType2["EXPORT"] = 44] = "EXPORT"; + TokenType2[TokenType2["FALSE"] = 45] = "FALSE"; + TokenType2[TokenType2["FOR"] = 46] = "FOR"; + TokenType2[TokenType2["FLOAT"] = 47] = "FLOAT"; + TokenType2[TokenType2["IF"] = 48] = "IF"; + TokenType2[TokenType2["IMPORT"] = 49] = "IMPORT"; + TokenType2[TokenType2["INT"] = 50] = "INT"; + TokenType2[TokenType2["LEN"] = 51] = "LEN"; + TokenType2[TokenType2["PRINT"] = 52] = "PRINT"; + TokenType2[TokenType2["RETURN"] = 53] = "RETURN"; + TokenType2[TokenType2["STRUCT"] = 54] = "STRUCT"; + TokenType2[TokenType2["TRUE"] = 55] = "TRUE"; + TokenType2[TokenType2["VAR"] = 56] = "VAR"; + TokenType2[TokenType2["VOID"] = 57] = "VOID"; + TokenType2[TokenType2["WHILE"] = 58] = "WHILE"; + TokenType2[TokenType2["EOF"] = 59] = "EOF"; return TokenType2; })(TokenType || {}); var TokenPattern = { @@ -162,20 +164,22 @@ [41 /* CONTINUE */]: /continue/y, [42 /* DEF */]: /def/y, [43 /* ELSE */]: /else/y, - [44 /* FALSE */]: /false/y, - [45 /* FOR */]: /for/y, - [46 /* FLOAT */]: /float/y, - [47 /* IF */]: /if/y, - [48 /* INT */]: /int/y, - [49 /* LEN */]: /len/y, - [50 /* PRINT */]: /print/y, - [51 /* RETURN */]: /return/y, - [52 /* STRUCT */]: /struct/y, - [53 /* TRUE */]: /true/y, - [54 /* VAR */]: /var/y, - [55 /* VOID */]: /void/y, - [56 /* WHILE */]: /while/y, - [57 /* EOF */]: /$/y + [44 /* EXPORT */]: /export/y, + [45 /* FALSE */]: /false/y, + [46 /* FOR */]: /for/y, + [47 /* FLOAT */]: /float/y, + [48 /* IF */]: /if/y, + [49 /* IMPORT */]: /import/y, + [50 /* INT */]: /int/y, + [51 /* LEN */]: /len/y, + [52 /* PRINT */]: /print/y, + [53 /* RETURN */]: /return/y, + [54 /* STRUCT */]: /struct/y, + [55 /* TRUE */]: /true/y, + [56 /* VAR */]: /var/y, + [57 /* VOID */]: /void/y, + [58 /* WHILE */]: /while/y, + [59 /* EOF */]: /$/y }; // src/scanner.ts @@ -259,7 +263,7 @@ ${ptr}` : snippet; const lexeme = m[0]; if (t === 0 /* IDENTIFIER */) { keyword: - for (let k = 38 /* BYTE */; k <= 56 /* WHILE */; k++) { + for (let k = 38 /* BYTE */; k <= 58 /* WHILE */; k++) { const keywordMatch = match(source, current, TokenPattern[k]); if (keywordMatch !== null && keywordMatch[0] === lexeme) { t = k; @@ -302,7 +306,7 @@ ${ptr}` : snippet; reportError(line, `Unexpected character '${source.charAt(current)}'.`); current++; } - tokens.push(new Token(57 /* EOF */, "", null, current, source)); + tokens.push(new Token(59 /* EOF */, "", null, current, source)); return tokens; } @@ -434,11 +438,12 @@ ${ptr}` : snippet; resolvedType: null }; } - function literalExpr({ value, type }) { + function literalExpr({ value, type, sourceLexeme }) { return { kind: 10 /* LITERAL_EXPR */, value, type, + sourceLexeme: sourceLexeme != null ? sourceLexeme : null, resolvedType: null }; } @@ -625,6 +630,12 @@ ${ptr}` : snippet; if (from.category === 6 /* POINTER */ && to.category === 6 /* POINTER */) { return true; } + if (from.category === 5 /* INT */ && to.category === 6 /* POINTER */) { + return true; + } + if (from.category === 6 /* POINTER */ && to.category === 5 /* INT */) { + return true; + } return isEqual(from, to); } function canCoerce(from, to) { @@ -780,7 +791,7 @@ ${ptr}` : snippet; isLiveAtEnd: null }; } - function functionStmt({ name, params, returnType, block, scope, symbol }) { + function functionStmt({ name, params, returnType, block, scope, symbol, isExported }) { return { kind: 16 /* FUNCTION_STMT */, name, @@ -790,17 +801,21 @@ ${ptr}` : snippet; block, scope }, + hostModule: null, + isExported: isExported != null ? isExported : false, symbol, hoistedLocals: null }; } - function importedFunctionStmt({ name, params, returnType, symbol }) { + function importedFunctionStmt({ name, params, returnType, symbol, hostModule }) { return { kind: 16 /* FUNCTION_STMT */, name, params, returnType, body: null, + hostModule: hostModule != null ? hostModule : null, + isExported: false, symbol, hoistedLocals: null }; @@ -911,6 +926,10 @@ ${ptr}` : snippet; { name: fakeToken(0 /* IDENTIFIER */, "dst"), type: ptrType(ByteType) + }, + { + name: fakeToken(0 /* IDENTIFIER */, "numBytes"), + type: IntType } ], returnType: VoidType, @@ -929,8 +948,29 @@ ${ptr}` : snippet; symbol: null }); sqrt.symbol = this.functionSymbol(sqrt); + const heapEnd = importedFunctionStmt({ + name: fakeToken(0 /* IDENTIFIER */, "__heap_end__"), + params: [], + returnType: IntType, + symbol: null + }); + heapEnd.symbol = this.functionSymbol(heapEnd); + const grow = importedFunctionStmt({ + name: fakeToken(0 /* IDENTIFIER */, "__grow_heap__"), + params: [ + { + name: fakeToken(0 /* IDENTIFIER */, "numPages"), + type: IntType + } + ], + returnType: IntType, + symbol: null + }); + grow.symbol = this.functionSymbol(grow); this.global.define(memcpy.name.lexeme, memcpy.symbol); this.global.define(sqrt.name.lexeme, sqrt.symbol); + this.global.define(heapEnd.name.lexeme, heapEnd.symbol); + this.global.define(grow.name.lexeme, grow.symbol); } variableSymbol(node, isGlobal) { return { @@ -1155,6 +1195,11 @@ ${ptr}` : snippet; case 16 /* FUNCTION_STMT */: { const op = node; out += "("; + if (op.hostModule !== null) { + out += "import "; + } else if (op.isExported) { + out += "export "; + } out += `def ${op.name.lexeme} `; out += "("; op.params.forEach((param, i) => { @@ -1306,12 +1351,12 @@ ${ptr}` : snippet; } switch (peek().type) { case 42 /* DEF */: - case 52 /* STRUCT */: - case 54 /* VAR */: - case 47 /* IF */: - case 50 /* PRINT */: - case 51 /* RETURN */: - case 56 /* WHILE */: { + case 54 /* STRUCT */: + case 56 /* VAR */: + case 48 /* IF */: + case 52 /* PRINT */: + case 53 /* RETURN */: + case 58 /* WHILE */: { return; } default: { @@ -1322,18 +1367,64 @@ ${ptr}` : snippet; } } function isAtEnd() { - return peek().type === 57 /* EOF */; + return peek().type === 59 /* EOF */; } function topDecl() { + if (match2(49 /* IMPORT */)) { + consume(42 /* DEF */, "Expect 'def' after 'import'."); + return importDecl(); + } + if (match2(44 /* EXPORT */)) { + consume(42 /* DEF */, "Expect 'def' after 'export'."); + return funDecl(true); + } if (match2(42 /* DEF */)) - return funDecl(); - if (match2(52 /* STRUCT */)) + return funDecl(false); + if (match2(54 /* STRUCT */)) return structDecl(); - if (match2(54 /* VAR */)) + if (match2(56 /* VAR */)) return varDecl(); throw parseError("Only variable declarations and function definitions allowed at the top-level."); } - function funDecl() { + function importDecl() { + const name = consume(0 /* IDENTIFIER */, "Expect identifier after 'def'."); + consume(7 /* LEFT_PAREN */, "Expect '(' after function name."); + const params = []; + while (!check(8 /* RIGHT_PAREN */) && !isAtEnd()) { + if (params.length > 0) { + consume(13 /* COMMA */, "Missing comma after parameter."); + } + const paramName = consume(0 /* IDENTIFIER */, "Expect identifier."); + const paramType = type(); + params.push({ + name: paramName, + type: paramType + }); + } + consume(8 /* RIGHT_PAREN */, "Expect ')' after parameters."); + let returnType = VoidType; + if (!check(15 /* SEMICOLON */)) { + returnType = type(); + } + consume(15 /* SEMICOLON */, "Expect ';' after import declaration."); + const node = importedFunctionStmt({ + name, + params, + returnType, + symbol: null, + hostModule: "env" + }); + const outerScope = peekScope(); + if (outerScope.hasDirect(name.lexeme)) { + throw parseErrorForToken(name, `'${name.lexeme}' is already declared in this scope.`); + } else { + const symbol = context.functionSymbol(node); + outerScope.define(name.lexeme, symbol); + node.symbol = symbol; + } + return node; + } + function funDecl(isExported) { const name = consume(0 /* IDENTIFIER */, "Expect identifier after 'def'."); consume(7 /* LEFT_PAREN */, "Expect '(' after function name."); const params = []; @@ -1371,7 +1462,8 @@ ${ptr}` : snippet; returnType, block: statements, scope, - symbol: null + symbol: null, + isExported }); const outerScope = peekScope(); if (outerScope.hasDirect(name.lexeme)) { @@ -1445,19 +1537,19 @@ ${ptr}` : snippet; return node; } function statement() { - if (match2(47 /* IF */)) { + if (match2(48 /* IF */)) { return ifStmt2(); } - if (match2(50 /* PRINT */)) { + if (match2(52 /* PRINT */)) { return printStmt2(); } - if (match2(56 /* WHILE */)) { + if (match2(58 /* WHILE */)) { return whileStmt2(); } - if (match2(45 /* FOR */)) { + if (match2(46 /* FOR */)) { return forStmt(); } - if (match2(51 /* RETURN */)) { + if (match2(53 /* RETURN */)) { return returnStmt2(); } if (match2(9 /* LEFT_BRACE */)) { @@ -1538,7 +1630,7 @@ ${ptr}` : snippet; let initializer = null; if (match2(15 /* SEMICOLON */)) { initializer = null; - } else if (match2(54 /* VAR */)) { + } else if (match2(56 /* VAR */)) { initializer = varDecl(); } else { initializer = expressionStmt2(); @@ -1579,7 +1671,7 @@ ${ptr}` : snippet; const statements = []; while (!check(10 /* RIGHT_BRACE */) && !isAtEnd()) { try { - if (match2(54 /* VAR */)) { + if (match2(56 /* VAR */)) { const varStmt2 = varDecl(); statements.push(varStmt2); } else { @@ -1609,9 +1701,9 @@ ${ptr}` : snippet; elementType, length }; - } else if (match2(48 /* INT */)) { + } else if (match2(50 /* INT */)) { baseType = IntType; - } else if (match2(46 /* FLOAT */)) { + } else if (match2(47 /* FLOAT */)) { baseType = FloatType; } else if (match2(38 /* BYTE */)) { baseType = ByteType; @@ -1893,10 +1985,44 @@ ${ptr}` : snippet; } return expr; } + function checkStructPtrCast() { + if (!check(0 /* IDENTIFIER */)) { + return false; + } + let i = current + 1; + while (i < tokens.length && tokens[i].type === 16 /* TILDE */) { + i++; + } + return i > current + 1 && i < tokens.length && tokens[i].type === 7 /* LEFT_PAREN */; + } + function castPrimary() { + const castType = type(); + switch (castType.category) { + case 5 /* INT */: + case 4 /* FLOAT */: + case 2 /* BYTE */: + case 1 /* BOOL */: + case 6 /* POINTER */: { + break; + } + default: { + throw parseError("Cannot cast to this type."); + } + } + consume(7 /* LEFT_PAREN */, "Expect '(' after type in cast expression."); + const paren = previous(); + const value = expression(); + consume(8 /* RIGHT_PAREN */, "Expect ')' after cast expression."); + return castExpr({ + token: paren, + type: castType, + value + }); + } function exprPrimary() { - if (match2(53 /* TRUE */) || match2(44 /* FALSE */)) { + if (match2(55 /* TRUE */) || match2(45 /* FALSE */)) { return literalExpr({ - value: previous().type === 53 /* TRUE */ ? true : false, + value: previous().type === 55 /* TRUE */ ? true : false, type: BoolType }); } @@ -1909,7 +2035,8 @@ ${ptr}` : snippet; if (match2(3 /* NUMBER_DECIMAL */)) { return literalExpr({ value: previous().literal, - type: FloatType + type: FloatType, + sourceLexeme: previous().lexeme }); } if (match2(4 /* NUMBER_HEX */)) { @@ -1942,6 +2069,9 @@ ${ptr}` : snippet; type: ByteType }); } + if (checkStructPtrCast()) { + return castPrimary(); + } if (match2(0 /* IDENTIFIER */)) { return variableExpr({ name: previous() @@ -1983,31 +2113,10 @@ ${ptr}` : snippet; consume(12 /* RIGHT_BRACKET */, "Expect ']' after list literal."); return listExpr({ bracket, values }); } - if (check(48 /* INT */) || check(46 /* FLOAT */) || check(38 /* BYTE */) || check(39 /* BOOL */)) { - const castType = type(); - switch (castType.category) { - case 5 /* INT */: - case 4 /* FLOAT */: - case 2 /* BYTE */: - case 1 /* BOOL */: - case 6 /* POINTER */: { - break; - } - default: { - throw parseError("Cannot cast to this type."); - } - } - consume(7 /* LEFT_PAREN */, "Expect '(' after type in cast expression."); - const paren = previous(); - const value = expression(); - consume(8 /* RIGHT_PAREN */, "Expect ')' after cast expression."); - return castExpr({ - token: paren, - type: castType, - value - }); + if (check(50 /* INT */) || check(47 /* FLOAT */) || check(38 /* BYTE */) || check(39 /* BOOL */)) { + return castPrimary(); } - if (match2(49 /* LEN */)) { + if (match2(51 /* LEN */)) { consume(7 /* LEFT_PAREN */, "Expect '(' before len expression."); const value = expression(); consume(8 /* RIGHT_PAREN */, "Expect ')' after len expression."); @@ -2097,7 +2206,7 @@ ${ptr}` : snippet; const canCoerce2 = canCoerce(node.resolvedType, type) || isNumberLiteral(node) && canCoerceNumberLiteral(node.value, type); if (canCoerce2) { out = castExpr({ - token: fakeToken2(57 /* EOF */, ""), + token: fakeToken2(59 /* EOF */, ""), type, value: node }); @@ -2144,13 +2253,17 @@ ${ptr}` : snippet; } case "!=": case "==": { - const lct = getLowestCommonNumeric(op.left.resolvedType, op.right.resolvedType); - if (lct) { - op.left = resolveNodeWithCoercion(op.left, isLiveAtEnd, lct, op.operator); - op.right = resolveNodeWithCoercion(op.right, isLiveAtEnd, lct, op.operator); - } else if (!isEqual(op.left.resolvedType, op.right.resolvedType)) { - const leftTypeStr = typeToString(op.left.resolvedType); - const rightTypeStr = typeToString(op.right.resolvedType); + const leftTypeStr = typeToString(op.left.resolvedType); + const rightTypeStr = typeToString(op.right.resolvedType); + if (isScalar(op.left.resolvedType) && isScalar(op.right.resolvedType)) { + const lct = getLowestCommonNumeric(op.left.resolvedType, op.right.resolvedType); + if (lct) { + op.left = resolveNodeWithCoercion(op.left, isLiveAtEnd, lct, op.operator); + op.right = resolveNodeWithCoercion(op.right, isLiveAtEnd, lct, op.operator); + } else if (!isEqual(op.left.resolvedType, op.right.resolvedType)) { + resolveError(op.operator, `Cannot compare ${leftTypeStr} to ${rightTypeStr}.`); + } + } else { resolveError(op.operator, `Cannot compare ${leftTypeStr} to ${rightTypeStr}.`); } op.resolvedType = BoolType; @@ -2254,6 +2367,7 @@ ${ptr}` : snippet; } case 3 /* CAST_EXPR */: { const op = node; + op.type = resolveType(op.type); resolveNode(op.value, isLiveAtEnd); if (!canCast(op.value.resolvedType, op.type)) { resolveError(op.token, `Cannot cast from ${typeToString(op.value.resolvedType)} to ${typeToString(op.type)}.`); @@ -2431,7 +2545,7 @@ ${ptr}` : snippet; category: 6 /* POINTER */, elementType: op.value.resolvedType }; - } else if (op.value.kind === 7 /* INDEX_EXPR */) { + } else if (op.value.kind === 7 /* INDEX_EXPR */ || op.value.kind === 5 /* DOT_EXPR */ || op.value.kind === 4 /* DEREF_EXPR */) { op.resolvedType = { category: 6 /* POINTER */, elementType: op.value.resolvedType @@ -2732,12 +2846,29 @@ ${cyclicVar.name.lineStr()}`); // src/backend.ts var codec2 = new UTF8Codec(); - var STACK_TOP_BYTE_OFFSET = 512 * 1024; - var DATA_TOP_BYTE_OFFSET = 1024 * 1024; + var STACK_TOP_BYTE_OFFSET = 4 * 1024 * 1024; + var DATA_TOP_BYTE_OFFSET = 8 * 1024 * 1024; var INITIAL_PAGES = 8 * 1024 * 1024 / (64 * 1024); function escapeString(str) { return str.replace(/'/g, "'").replace(/\\/g, "\\\\").replace(/"/g, '"'); } + function formatFloatLexeme(lexeme) { + let out = lexeme; + if (out.indexOf(".") >= 0) { + let end = out.length; + while (end > 0 && out.charAt(end - 1) === "0") { + end--; + } + if (end > 0 && out.charAt(end - 1) === ".") { + end--; + } + out = out.substring(0, end); + } + if (out.length === 0) { + out = "0"; + } + return out; + } function registerType(type) { switch (type.category) { case 0 /* ARRAY */: @@ -2774,7 +2905,7 @@ ${cyclicVar.name.lineStr()}`); const type = symbol.kind === 2 /* PARAM */ ? symbol.param.type : symbol.node.type; return type !== null && isScalar(type); } - var DEBUG_COMMENTS = true; + var DEBUG_COMMENTS = false; function emit(context) { var _a, _b; const globalLocs = /* @__PURE__ */ new Map(); @@ -3025,6 +3156,9 @@ ${cyclicVar.name.lineStr()}`); } } function emitDebugComments(node) { + if (!DEBUG_COMMENTS) { + return; + } switch (node.kind) { case 1 /* BINARY_EXPR */: case 3 /* CAST_EXPR */: @@ -3043,7 +3177,7 @@ ${cyclicVar.name.lineStr()}`); debugLine(``); } function visit(node, exprMode = 1 /* RVALUE */) { - var _a2, _b2, _c, _d, _e, _f; + var _a2, _b2, _c, _d, _e, _f, _g; if (skip.has(node)) { return; } @@ -3051,10 +3185,12 @@ ${cyclicVar.name.lineStr()}`); switch (node.kind) { case 0 /* ASSIGN_EXPR */: { const op = node; - op.operator.lineStr(true).split("\n").forEach((l) => { - debugLine(`;; ${l}`); - }); - debugLine(``); + if (DEBUG_COMMENTS) { + op.operator.lineStr(true).split("\n").forEach((l) => { + debugLine(`;; ${l}`); + }); + debugLine(``); + } if (op.left.kind === 13 /* VARIABLE_EXPR */) { const symbol = op.left.resolvedSymbol; visit(op.right); @@ -3292,7 +3428,8 @@ ${cyclicVar.name.lineStr()}`); break; } case 1 /* BOOL */: - case 5 /* INT */: { + case 5 /* INT */: + case 6 /* POINTER */: { break; } case 4 /* FLOAT */: { @@ -3323,7 +3460,7 @@ ${cyclicVar.name.lineStr()}`); break; } case 6 /* POINTER */: { - if (((_e = op.value.resolvedType) == null ? void 0 : _e.category) === 6 /* POINTER */) { + if (((_e = op.value.resolvedType) == null ? void 0 : _e.category) === 6 /* POINTER */ || ((_f = op.value.resolvedType) == null ? void 0 : _f.category) === 5 /* INT */) { } else { throw new Error(`Unexpected type ${typeToString(op.type)} for cast source`); } @@ -3351,10 +3488,12 @@ ${cyclicVar.name.lineStr()}`); } case 5 /* DOT_EXPR */: { const op = node; - op.dot.lineStr(true).split("\n").forEach((l) => { - debugLine(`;; ${l}`); - }); - debugLine(``); + if (DEBUG_COMMENTS) { + op.dot.lineStr(true).split("\n").forEach((l) => { + debugLine(`;; ${l}`); + }); + debugLine(``); + } const memberType = op.resolvedType; const structType = op.callee.resolvedType; if ((structType == null ? void 0 : structType.category) !== 7 /* STRUCT */) { @@ -3389,10 +3528,12 @@ ${cyclicVar.name.lineStr()}`); } case 7 /* INDEX_EXPR */: { const op = node; - op.bracket.lineStr(true).split("\n").forEach((l) => { - debugLine(`;; ${l}`); - }); - debugLine(``); + if (DEBUG_COMMENTS) { + op.bracket.lineStr(true).split("\n").forEach((l) => { + debugLine(`;; ${l}`); + }); + debugLine(``); + } const elementType = op.resolvedType; visit(op.callee, 0 /* LVALUE */); visit(op.index); @@ -3483,7 +3624,11 @@ ${cyclicVar.name.lineStr()}`); break; } case 4 /* FLOAT */: { - line(`f32.const ${op.value}`); + if (op.sourceLexeme !== null) { + line(`f32.const ${formatFloatLexeme(op.sourceLexeme)}`); + } else { + line(`f32.const ${op.value}`); + } break; } default: { @@ -3549,7 +3694,9 @@ ${cyclicVar.name.lineStr()}`); } break; } - case 7 /* INDEX_EXPR */: { + case 7 /* INDEX_EXPR */: + case 5 /* DOT_EXPR */: + case 4 /* DEREF_EXPR */: { visit(op.value, 0 /* LVALUE */); break; } @@ -3604,8 +3751,8 @@ ${cyclicVar.name.lineStr()}`); break; } localLocs = /* @__PURE__ */ new Map(); - if (op.name.lexeme === "main") { - line(`(func ${wasmId("main")} (export "main")`); + if (op.name.lexeme === "main" || op.isExported) { + line(`(func ${wasmId(op.name.lexeme)} (export "${op.name.lexeme}")`); } else { line(`(func ${wasmId(op.name.lexeme)}`); } @@ -3640,7 +3787,7 @@ ${cyclicVar.name.lineStr()}`); } }; op.body.scope.forEach((_, local) => allocateRegisterOrStackLoc(local)); - (_f = op.hoistedLocals) == null ? void 0 : _f.forEach((local) => allocateRegisterOrStackLoc(local)); + (_g = op.hoistedLocals) == null ? void 0 : _g.forEach((local) => allocateRegisterOrStackLoc(local)); line(`global.get ${wasmId("__stack_ptr__")}`); line(`local.set ${wasmId("__base_ptr__")}`); line(`global.get ${wasmId("__stack_ptr__")}`); @@ -3796,6 +3943,21 @@ ${cyclicVar.name.lineStr()}`); line(`(import "io" "putf" (func ${wasmId("__putf__")} (param f32)))`); line(`(import "io" "puti" (func ${wasmId("__puti__")} (param i32)))`); line(`(import "io" "flush" (func ${wasmId("__flush__")}))`); + context.topLevelStatements.forEach((statement) => { + if (statement.kind === 16 /* FUNCTION_STMT */) { + const fn = statement; + if (fn.hostModule !== null) { + let sig = ""; + fn.params.forEach((param) => { + sig += ` (param ${registerType(param.type)})`; + }); + if (!isEqual(fn.returnType, VoidType)) { + sig += ` (result ${registerType(fn.returnType)})`; + } + line(`(import "${fn.hostModule}" "${fn.name.lexeme}" (func ${wasmId(fn.name.lexeme)}${sig}))`); + } + } + }); line(`(memory $memory ${INITIAL_PAGES})`); line(`(global ${wasmId("__stack_ptr__")} (mut i32) i32.const ${STACK_TOP_BYTE_OFFSET})`); let globalByteOffset = DATA_TOP_BYTE_OFFSET; @@ -3888,6 +4050,19 @@ ${cyclicVar.name.lineStr()}`); line(`f32.sqrt`); } line(`)`); + line(`(func ${wasmId("__heap_end__")} (result i32)`); + { + line(`memory.size`); + line(`i32.const 65536`); + line(`i32.mul`); + } + line(`)`); + line(`(func ${wasmId("__grow_heap__")} (param $numPages i32) (result i32)`); + { + line(`local.get $numPages`); + line(`memory.grow`); + } + line(`)`); context.topLevelStatements.forEach((statement) => { visit(statement); }); @@ -3959,10 +4134,8 @@ ${cyclicVar.name.lineStr()}`); clearTimeout(timeoutID); } timeoutID = setTimeout(wrapped, wait); - console.log("debounce"); return; } - console.log("exec"); clearTimeout(timeoutID); timeoutID = -1; cb(arguments); diff --git a/package.json b/package.json index 84c1349..3b942d1 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,8 @@ "build": "tsc && node dist/index.js", "bundle-demo": "esbuild --bundle --outfile=demo.js --platform=browser demo.ts", "test": "tsc && jest dist/", - "test-debug": "tsc && node --inspect-brk node_modules/.bin/jest dist/" + "test-debug": "tsc && node --inspect-brk node_modules/.bin/jest dist/", + "bootstrap": "tsc && tools/bootstrap.sh" }, "author": "andrewkchan", "license": "ISC", diff --git a/test.ts b/test.ts index 6403f5e..a96033c 100644 --- a/test.ts +++ b/test.ts @@ -3115,4 +3115,203 @@ done }) }) +describe("self-hosting", () => { + const SELFHOST_SOURCES = [ + "selfhost/util.puff", + "selfhost/scanner.puff", + "selfhost/ast.puff", + "selfhost/sexpr.puff", + "selfhost/parser.puff", + "selfhost/resolver.puff", + "selfhost/backend.puff", + "selfhost/main.puff", + ] + + function compileWithReference(source: string): string { + const errors: string[] = [] + const reportError: ReportError = (line, msg) => { + errors.push(`${line}: ${msg}`) + } + const tokens = scanTokens(source, reportError) + expect(errors).toEqual([]) + const context = parse(tokens, reportError) + expect(errors).toEqual([]) + resolve(context, reportError) + expect(errors).toEqual([]) + return emit(context) + } + + // Runs a compiled puffscript compiler (WASM) on the given source text, + // returning its stdout (the generated WAT), stderr, and exit code. + async function runCompilerWasm(wasmFile: string, source: string): Promise<{ out: string, err: string, exitCode: number }> { + const codec = new UTF8Codec() + const input = codec.encodeString(source) + let inputPos = 0 + const outChunks: number[] = [] + const errChunks: number[] = [] + let ioBuffer = "" + let extraOut = "" + const instance = await WebAssembly.instantiate(fs.readFileSync(wasmFile), { + io: { + log: (x: any) => { extraOut += x + "\n" }, + putchar: (x: number) => { ioBuffer += codec.decodeASCIIChar(x) }, + putf: (x: number) => { ioBuffer += x }, + puti: (x: number) => { ioBuffer += x }, + flush: () => { extraOut += ioBuffer + "\n"; ioBuffer = "" } + }, + env: { + getchar: (): number => inputPos < input.length ? input[inputPos++] : -1, + putchar: (c: number) => { outChunks.push(c & 0xFF) }, + puterr: (c: number) => { errChunks.push(c & 0xFF) }, + exit: (code: number) => { throw new ExitCalled(code) } + } + }) + const exports = instance.instance.exports as any + let exitCode = 0 + try { + exports.__init_globals__() + exports.main() + } catch (e) { + if (e instanceof ExitCalled) { + exitCode = e.code + } else { + throw e + } + } + expect(extraOut).toBe("") + return { + out: Buffer.from(outChunks).toString("utf8"), + err: Buffer.from(errChunks).toString("utf8"), + exitCode + } + } + + test("bootstrap: the compiler compiles itself (fixpoint)", async () => { + const compilerSource = SELFHOST_SOURCES.map((f) => fs.readFileSync(f, "utf8")).join("\n") + + // Stage 1: compile the self-hosted compiler with the reference compiler. + const stage1Wat = compileWithReference(compilerSource) + fs.writeFileSync("test/selfhost-stage1.wat", stage1Wat) + child_process.execSync(`npx -p wabt wat2wasm test/selfhost-stage1.wat -o test/selfhost-stage1.wasm`) + + // Stage 2: the self-hosted compiler compiles its own source. + // Its output must be byte-identical to the reference compiler's. + const stage2 = await runCompilerWasm("test/selfhost-stage1.wasm", compilerSource) + expect(stage2.err).toBe("") + expect(stage2.exitCode).toBe(0) + expect(stage2.out).toBe(stage1Wat) + fs.writeFileSync("test/selfhost-stage2.wat", stage2.out) + child_process.execSync(`npx -p wabt wat2wasm test/selfhost-stage2.wat -o test/selfhost-stage2.wasm`) + + // Stage 3: the self-compiled compiler compiles its own source again; + // the output must reach a fixpoint. + const stage3 = await runCompilerWasm("test/selfhost-stage2.wasm", compilerSource) + expect(stage3.err).toBe("") + expect(stage3.exitCode).toBe(0) + expect(stage3.out).toBe(stage2.out) + }, 120000) + + test("self-hosted compiler compiles programs identically to the reference", async () => { + const compilerSource = SELFHOST_SOURCES.map((f) => fs.readFileSync(f, "utf8")).join("\n") + const stage1Wat = compileWithReference(compilerSource) + fs.writeFileSync("test/selfhost-stage1.wat", stage1Wat) + child_process.execSync(`npx -p wabt wat2wasm test/selfhost-stage1.wat -o test/selfhost-stage1.wasm`) + + const programs = [ + ` + def fib(n int) int { + if (n <= 1) { return 1; } + return fib(n-1) + fib(n-2); + } + struct Point { x float, y float } + def main() { + var p = Point{1.5, 2.5}; + var arr = [1, 2, 3]; + var s = "hello"; + for (var i = 0; i < len(arr); i += 1) { + print fib(arr[i]); + } + print p.x + p.y; + print s; + } + `, + ` + import def getchar() int; + import def putchar(c int); + def main() { + var c = getchar(); + while (c >= 0) { + putchar(c); + c = getchar(); + } + } + `, + ] + for (const program of programs) { + const source = program.trim() + "\n" + const expected = compileWithReference(source) + const result = await runCompilerWasm("test/selfhost-stage1.wasm", source) + expect(result.err).toBe("") + expect(result.exitCode).toBe(0) + expect(result.out).toBe(expected) + } + + // Error reporting must also match the reference compiler. + const badSource = ` + def main() { + var x int = true; + undefinedFn(); + } + `.trim() + "\n" + const errors: string[] = [] + const reportError: ReportError = (line, msg) => { errors.push(`${line}: ${msg}`) } + const tokens = scanTokens(badSource, reportError) + const context = parse(tokens, reportError) + resolve(context, reportError) + expect(errors.length).toBeGreaterThan(0) + const result = await runCompilerWasm("test/selfhost-stage1.wasm", badSource) + expect(result.exitCode).toBe(1) + expect(result.err).toBe(errors.map((e) => e + "\n").join("")) + }, 120000) + + test("self-hosted compiler output runs correctly", async () => { + const compilerSource = SELFHOST_SOURCES.map((f) => fs.readFileSync(f, "utf8")).join("\n") + const stage1Wat = compileWithReference(compilerSource) + fs.writeFileSync("test/selfhost-stage1.wat", stage1Wat) + child_process.execSync(`npx -p wabt wat2wasm test/selfhost-stage1.wat -o test/selfhost-stage1.wasm`) + + const program = ` + def main() { + var total = 0; + for (var i = 1; i <= 10; i += 1) { + total += i; + } + print total; + print "compiled by puffscript"; + } + `.trim() + "\n" + const compiled = await runCompilerWasm("test/selfhost-stage1.wasm", program) + expect(compiled.exitCode).toBe(0) + fs.writeFileSync("test/selfhost-out.wat", compiled.out) + child_process.execSync(`npx -p wabt wat2wasm test/selfhost-out.wat -o test/selfhost-out.wasm`) + + const codec = new UTF8Codec() + let ioBuffer = "" + let output = "" + const instance = await WebAssembly.instantiate(fs.readFileSync("test/selfhost-out.wasm"), { + io: { + log: (x: any) => { output += x + "\n" }, + putchar: (x: number) => { ioBuffer += codec.decodeASCIIChar(x) }, + putf: (x: number) => { ioBuffer += x }, + puti: (x: number) => { ioBuffer += x }, + flush: () => { output += ioBuffer + "\n"; ioBuffer = "" } + } + }) + const exports = instance.instance.exports as any + exports.__init_globals__() + exports.main() + expect(output).toBe("55\ncompiled by puffscript\n") + }, 120000) +}) + // TODO: string literals with non-ascii UTF-8 chars \ No newline at end of file diff --git a/tools/bootstrap.sh b/tools/bootstrap.sh new file mode 100755 index 0000000..70cc9c8 --- /dev/null +++ b/tools/bootstrap.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Bootstraps the self-hosted puffscript compiler and verifies the fixpoint: +# stage1: selfhost compiler sources compiled by the TypeScript compiler +# stage2: stage1 compiling its own source (must match the reference output) +# stage3: stage2 compiling its own source (must match stage2's output) +# Artifacts are written to test/ (gitignored). +set -euo pipefail +cd "$(dirname "$0")/.." + +SOURCES=( + selfhost/util.puff + selfhost/scanner.puff + selfhost/ast.puff + selfhost/sexpr.puff + selfhost/parser.puff + selfhost/resolver.puff + selfhost/backend.puff + selfhost/main.puff +) + +mkdir -p test +cat "${SOURCES[@]}" > test/puffc.puff + +echo "[stage1] compiling the self-hosted compiler with the TypeScript compiler..." +node dist/tools/puffc.js "${SOURCES[@]}" -o test/stage1.wat +node dist/tools/wat2wasm.js test/stage1.wat -o test/stage1.wasm + +echo "[stage2] self-hosted compiler compiling its own source..." +node dist/tools/run.js test/stage1.wasm --stdin test/puffc.puff --stdout test/stage2.wat +if ! cmp -s test/stage1.wat test/stage2.wat; then + echo "FAIL: stage2 output differs from the reference compiler's output" + exit 1 +fi +node dist/tools/wat2wasm.js test/stage2.wat -o test/stage2.wasm + +echo "[stage3] self-compiled compiler compiling its own source..." +node dist/tools/run.js test/stage2.wasm --stdin test/puffc.puff --stdout test/stage3.wat +if ! cmp -s test/stage2.wat test/stage3.wat; then + echo "FAIL: bootstrap did not reach a fixpoint" + exit 1 +fi + +echo "OK: bootstrap fixpoint reached." +echo " - self-hosted compiler binary: test/stage2.wasm" +echo " - compile a program with it:" +echo " node dist/tools/run.js test/stage2.wasm --stdin program.puff --stdout program.wat"