add build.zig.zon

.gitea/workflows/test.yml

@@ -1,14 +1,13 @@
-name: Run all tests
-run-name: Run all tests
+name: Run unit tests
 on: [push]
 
 jobs:
   test:
-    name: zig test
+    name: zig build test
     runs-on: ubuntu-latest
     steps:
       - uses: mlugg/setup-zig@v2
       - uses: actions/checkout@v6
         with:
           token: ${{ secrets.ACCESS_TOKEN }}
-      - run: find src/ -type f -regex '.*\.zig' -exec zig test '{}' ';'
+      - run: zig build test --summary all

.gitignore

@@ -1 +1,2 @@
-examples/
+zig-out/
+.zig-cache/

README.md

@@ -2,7 +2,7 @@
 
 [![license](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) [![test workflow](https://git.svecs.cz/svecs/neon/actions/workflows/test.yml/badge.svg?branch=main)](https://git.svecs.cz/svecs/neon/actions/?workflow=test.yml)
 
-> a simple scripting language (hopefully one day)
+> a simple programming language (hopefully one day)
 
 ## features
 
@@ -12,3 +12,13 @@
 > for now the language uses an s-expression syntax (even though it's not a lisp) to simplify parsing. this syntax WILL change in the future!
 
 - currently none lol
+
+## quick start
+
+required tools:
+
+- [zig](https://ziglang.org/download/)
+
+```shell
+zig build run -- examples/first.ne
+```

build.zig

@@ -0,0 +1,51 @@
+const std = @import("std");
+
+pub fn build(b: *std.Build) void {
+    const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
+
+    const lib = b.addLibrary(.{
+        .name = "neon",
+        .root_module = b.createModule(.{
+            .root_source_file = b.path("lib/root.zig"),
+            .target = target,
+            .optimize = optimize,
+        }),
+    });
+
+    const exe = b.addExecutable(.{
+        .name = "neon",
+        .root_module = b.createModule(.{
+            .root_source_file = b.path("src/main.zig"),
+            .target = target,
+            .optimize = optimize,
+            .imports = &.{.{ .name = "libneon", .module = lib.root_module }},
+        }),
+    });
+
+    b.installArtifact(exe);
+
+    const run_exe = b.addRunArtifact(exe);
+
+    if (b.args) |args| {
+        run_exe.addArgs(args);
+    }
+
+    const run_step = b.step("run", "Run the application");
+    run_step.dependOn(&run_exe.step);
+
+    const tests = b.addTest(.{
+        .name = "all",
+        .root_module = b.createModule(.{
+            .root_source_file = b.path("tests/all.zig"),
+            .target = target,
+            .optimize = optimize,
+            .imports = &.{.{ .name = "libneon", .module = lib.root_module }},
+        }),
+    });
+
+    const run_tests = b.addRunArtifact(tests);
+
+    const test_step = b.step("test", "Run the tests");
+    test_step.dependOn(&run_tests.step);
+}

build.zig.zon

@@ -0,0 +1,7 @@
+.{
+    .name = .neon,
+    .fingerprint = 0xfcce6dd0bc6abcf3,
+    .version = "0.0.1",
+    .minimum_zig_version = "0.15.2",
+    .paths = .{""},
+}

examples/first.ne

@@ -0,0 +1,14 @@
+(func print_fibs [limit] {
+    (let a 1)
+    (let b 1)
+
+    (while (< a limit) {
+        (printf "%d\n" a)
+        (set b (+ b a))
+        (set a (- b a))
+    })
+})
+
+(func main [] {
+    (print_fibs 100)
+})

lib/lexer.zig

@@ -11,17 +11,17 @@ pub const Token = union(enum) {
     identifier: []const u8,
     string: []const u8,
 
-    pub fn format(self: @This(), writer: anytype) !void {
+    pub fn format(self: @This(), w: *std.Io.Writer) !void {
         try switch (self) {
-            .lparen => writer.print("lparen", .{}),
-            .rparen => writer.print("rparen", .{}),
-            .lbrace => writer.print("lbrace", .{}),
-            .rbrace => writer.print("rbrace", .{}),
-            .lbracket => writer.print("lbracket", .{}),
-            .rbracket => writer.print("rbracket", .{}),
-            .integer => |int| writer.print("int: {d}", .{int}),
-            .identifier => |id| writer.print("id: {s}", .{id}),
-            .string => |str| writer.print("str: {s}", .{str}),
+            .lparen => w.print("lparen", .{}),
+            .rparen => w.print("rparen", .{}),
+            .lbrace => w.print("lbrace", .{}),
+            .rbrace => w.print("rbrace", .{}),
+            .lbracket => w.print("lbracket", .{}),
+            .rbracket => w.print("rbracket", .{}),
+            .integer => |int| w.print("int: {d}", .{int}),
+            .identifier => |id| w.print("id: {s}", .{id}),
+            .string => |str| w.print("str: {s}", .{str}),
         };
     }
 };
@@ -71,7 +71,7 @@ pub fn lex(gpa: std.mem.Allocator, src: []const u8) ![]const Token {
                 break :sw .{ .string = src[start..idx] };
             },
             else => {
-                std.debug.print("invalid character `{c}'\n", .{src[idx]});
+                std.debug.print("invalid character `{c}' at position {d}\n", .{ src[idx], idx });
                 return error.InvalidCharacter;
             },
         };
@@ -85,47 +85,3 @@ pub fn lex(gpa: std.mem.Allocator, src: []const u8) ![]const Token {
 
     return result.toOwnedSlice(gpa);
 }
-
-test "lexer" {
-    const t = std.testing;
-
-    var gpa = std.heap.DebugAllocator(.{}).init;
-
-    const src =
-        \\(let input (fs.read_file "input.txt")) # comment test
-        \\{
-        \\  (!_~special ch4racter test')
-        \\  (print [1 2 3 input])
-        \\}
-    ;
-
-    const tokens = try lex(gpa.allocator(), src);
-    defer gpa.allocator().free(tokens);
-
-    try t.expectEqualDeep(tokens, &[_]Token{
-        .lparen,
-        .{ .identifier = "let" },
-        .{ .identifier = "input" },
-        .lparen,
-        .{ .identifier = "fs.read_file" },
-        .{ .string = "input.txt" },
-        .rparen,
-        .rparen,
-        .lbrace,
-        .lparen,
-        .{ .identifier = "!_~special" },
-        .{ .identifier = "ch4racter" },
-        .{ .identifier = "test'" },
-        .rparen,
-        .lparen,
-        .{ .identifier = "print" },
-        .lbracket,
-        .{ .integer = 1 },
-        .{ .integer = 2 },
-        .{ .integer = 3 },
-        .{ .identifier = "input" },
-        .rbracket,
-        .rparen,
-        .rbrace,
-    });
-}

lib/parser.zig

@@ -0,0 +1,114 @@
+const std = @import("std");
+const lexer = @import("lexer.zig");
+
+pub const SExpr = union(enum) {
+    integer: u64, // 42
+    identifier: []const u8, // module.name
+    string: []const u8, // "foo bar"
+    call: []const SExpr, // (+ 1 2 3)
+    block: []const SExpr, // { (print "hi") }
+    list: []const SExpr, // [1 2 3]
+
+    fn format_(self: @This(), w: *std.Io.Writer, depth: usize) !void {
+        for (0..depth) |_| {
+            try w.print("  ", .{});
+        }
+        switch (self) {
+            .integer => |int| try w.print("int: {d}\n", .{int}),
+            .identifier => |id| try w.print("id: {s}\n", .{id}),
+            .string => |str| try w.print("str: {s}\n", .{str}),
+            .call => |call| {
+                try w.print("call: {s}\n", .{if (call.len == 0) "()" else ""});
+                for (call) |expr| {
+                    try expr.format_(w, depth + 1);
+                }
+            },
+            .block => |block| {
+                try w.print("block: {s}\n", .{if (block.len == 0) "{}" else ""});
+                for (block) |expr| {
+                    try expr.format_(w, depth + 1);
+                }
+            },
+            .list => |list| {
+                try w.print("list: {s}\n", .{if (list.len == 0) "[]" else ""});
+                for (list) |expr| {
+                    try expr.format_(w, depth + 1);
+                }
+            },
+        }
+    }
+
+    pub fn format(self: @This(), w: *std.Io.Writer) !void {
+        try self.format_(w, 0);
+    }
+};
+
+fn parse_expr(gpa: std.mem.Allocator, tokens: []const lexer.Token) !struct { SExpr, usize } {
+    if (tokens.len == 0) {
+        return error.UnexpectedEOF;
+    }
+
+    switch (tokens[0]) {
+        .integer => |int| return .{ .{ .integer = int }, 1 },
+        .identifier => |id| return .{ .{ .identifier = id }, 1 },
+        .string => |str| return .{ .{ .string = str }, 1 },
+        .lparen => {
+            var call = std.ArrayList(SExpr).empty;
+            var tokens_rest = tokens[1..];
+            while (tokens_rest.len > 0 and tokens_rest[0] != .rparen) {
+                const expr, const consumed = try parse_expr(gpa, tokens_rest);
+                try call.append(gpa, expr);
+                tokens_rest = tokens_rest[consumed..];
+            }
+            if (tokens_rest.len == 0) {
+                return error.UnmatchedParen;
+            }
+            const consumed = tokens_rest[1..].ptr - tokens.ptr;
+            return .{ .{ .call = try call.toOwnedSlice(gpa) }, consumed };
+        },
+        .lbrace => {
+            var block = std.ArrayList(SExpr).empty;
+            var tokens_rest = tokens[1..];
+            while (tokens_rest.len > 0 and tokens_rest[0] != .rbrace) {
+                const expr, const consumed = try parse_expr(gpa, tokens_rest);
+                try block.append(gpa, expr);
+                tokens_rest = tokens_rest[consumed..];
+            }
+            if (tokens_rest.len == 0) {
+                return error.UnmatchedBrace;
+            }
+            const consumed = tokens_rest[1..].ptr - tokens.ptr;
+            return .{ .{ .block = try block.toOwnedSlice(gpa) }, consumed };
+        },
+        .lbracket => {
+            var list = std.ArrayList(SExpr).empty;
+            var tokens_rest = tokens[1..];
+            while (tokens_rest.len > 0 and tokens_rest[0] != .rbracket) {
+                const expr, const consumed = try parse_expr(gpa, tokens_rest);
+                try list.append(gpa, expr);
+                tokens_rest = tokens_rest[consumed..];
+            }
+            if (tokens_rest.len == 0) {
+                return error.UnmatchedBracket;
+            }
+            const consumed = tokens_rest[1..].ptr - tokens.ptr;
+            return .{ .{ .list = try list.toOwnedSlice(gpa) }, consumed };
+        },
+        else => {
+            return error.UnexpectedToken;
+        },
+    }
+}
+
+pub fn parse_sexprs(gpa: std.mem.Allocator, tokens: []const lexer.Token) ![]const SExpr {
+    var result = std.ArrayList(SExpr).empty;
+    var idx: usize = 0;
+
+    while (idx < tokens.len) {
+        const expr, const consumed = try parse_expr(gpa, tokens[idx..]);
+        try result.append(gpa, expr);
+        idx += consumed;
+    }
+
+    return result.toOwnedSlice(gpa);
+}

lib/root.zig

@@ -0,0 +1,2 @@
+pub const lexer = @import("lexer.zig");
+pub const parser = @import("parser.zig");

src/main.zig

@@ -1,22 +1,75 @@
 const std = @import("std");
 
-const lexer = @import("lexer.zig");
-const parser = @import("parser.zig");
+const lib = @import("libneon");
+
+fn usage(w: *std.Io.Writer, program: []const u8) error{WriteFailed}!void {
+    try w.print("Usage: {s} <input>\n", .{program});
+}
+
+fn entry(init: struct {
+    arena: *std.heap.ArenaAllocator,
+    args: []const []const u8,
+    stdout: *std.Io.Writer,
+    stderr: *std.Io.Writer,
+}) !void {
+    const program_name = init.args[0];
+    var input_path: ?[]const u8 = null;
+
+    for (init.args[1..]) |arg| {
+        if (std.mem.eql(u8, arg, "--help")) {
+            try usage(init.stdout, program_name);
+            try init.stdout.flush();
+            return;
+        }
+        if (input_path == null) input_path = arg;
+    }
+
+    if (input_path == null) {
+        try usage(init.stderr, program_name);
+        try init.stderr.print("Error: no input file\n", .{});
+        try init.stderr.flush();
+        return;
+    }
+
+    const input_stat = std.fs.cwd().statFile(input_path.?) catch {
+        try init.stderr.print("Error: could not read file {s}\n", .{input_path.?});
+        try init.stderr.flush();
+        return;
+    };
+
+    const src = try std.fs.cwd().readFileAlloc(init.arena.allocator(), input_path.?, input_stat.size);
+
+    const tokens = try lib.lexer.lex(init.arena.allocator(), src);
+
+    const exprs = try lib.parser.parse_sexprs(init.arena.allocator(), tokens);
+
+    for (exprs) |expr| {
+        try init.stdout.print("{f}", .{expr});
+        try init.stdout.flush();
+    }
+}
 
 pub fn main() !void {
-    var gpa = std.heap.DebugAllocator(.{}).init;
+    var debug_allocator = std.heap.DebugAllocator(.{}).init;
+    defer std.debug.assert(debug_allocator.deinit() == .ok);
 
-    const src: []const u8 =
-        \\(func main [] {
-        \\  (let input (fs.read_file "input.txt"))
-        \\  (print input)
-        \\})
-    ;
+    var arena = std.heap.ArenaAllocator.init(debug_allocator.allocator());
+    defer arena.deinit();
 
-    const tokens = try lexer.lex(gpa.allocator(), src);
-    defer gpa.allocator().free(tokens);
+    var stdout_buffer: [1024]u8 = undefined;
+    var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer);
+    const stdout = &stdout_writer.interface;
 
-    const expr = try parser.parse_expr(gpa.allocator(), tokens);
+    var stderr_buffer: [1024]u8 = undefined;
+    var stderr_writer = std.fs.File.stderr().writer(&stderr_buffer);
+    const stderr = &stderr_writer.interface;
 
-    std.debug.print("{f}", .{expr});
+    const args = try std.process.argsAlloc(arena.allocator());
+
+    return entry(.{
+        .arena = &arena,
+        .args = args,
+        .stdout = stdout,
+        .stderr = stderr,
+    });
 }

src/parser.zig

@@ -1,178 +0,0 @@
-const std = @import("std");
-const lexer = @import("lexer.zig");
-
-pub const Expr = union(enum) {
-    integer: u64, // 42
-    identifier: []const u8, // module.name
-    string: []const u8, // "foo bar"
-    call: []const Expr, // (+ 1 2 3)
-    block: []const Expr, // { (print "hi") }
-    list: []const Expr, // [1 2 3]
-
-    fn format_(self: @This(), writer: anytype, depth: usize) !void {
-        for (0..depth) |_| {
-            try writer.print("  ", .{});
-        }
-        switch (self) {
-            .integer => |int| try writer.print("int: {d}\n", .{int}),
-            .identifier => |id| try writer.print("id: {s}\n", .{id}),
-            .string => |str| try writer.print("str: {s}\n", .{str}),
-            .call => |call| {
-                try writer.print("call: {s}\n", .{if (call.len == 0) "()" else ""});
-                for (call) |expr| {
-                    try expr.format_(writer, depth + 1);
-                }
-            },
-            .block => |block| {
-                try writer.print("block: {s}\n", .{if (block.len == 0) "{}" else ""});
-                for (block) |expr| {
-                    try expr.format_(writer, depth + 1);
-                }
-            },
-            .list => |list| {
-                try writer.print("list: {s}\n", .{if (list.len == 0) "[]" else ""});
-                for (list) |expr| {
-                    try expr.format_(writer, depth + 1);
-                }
-            },
-        }
-    }
-
-    pub fn format(self: @This(), writer: anytype) !void {
-        try self.format_(writer, 0);
-    }
-};
-
-fn parse_expr_(gpa: std.mem.Allocator, tokens: []const lexer.Token) !struct { Expr, usize } {
-    if (tokens.len == 0) {
-        return error.UnexpectedEOF;
-    }
-
-    switch (tokens[0]) {
-        .integer => |int| return .{ .{ .integer = int }, 1 },
-        .identifier => |id| return .{ .{ .identifier = id }, 1 },
-        .string => |str| return .{ .{ .string = str }, 1 },
-        .lparen => {
-            var call = std.ArrayList(Expr).empty;
-            var tokens_rest = tokens[1..];
-            while (tokens_rest.len > 0 and tokens_rest[0] != .rparen) {
-                const expr, const consumed = try parse_expr_(gpa, tokens_rest);
-                try call.append(gpa, expr);
-                tokens_rest = tokens_rest[consumed..];
-            }
-            if (tokens_rest.len == 0) {
-                return error.UnmatchedParen;
-            }
-            const consumed = tokens_rest[1..].ptr - tokens.ptr;
-            return .{ .{ .call = try call.toOwnedSlice(gpa) }, consumed };
-        },
-        .lbrace => {
-            var block = std.ArrayList(Expr).empty;
-            var tokens_rest = tokens[1..];
-            while (tokens_rest.len > 0 and tokens_rest[0] != .rbrace) {
-                const expr, const consumed = try parse_expr_(gpa, tokens_rest);
-                try block.append(gpa, expr);
-                tokens_rest = tokens_rest[consumed..];
-            }
-            if (tokens_rest.len == 0) {
-                return error.UnmatchedBrace;
-            }
-            const consumed = tokens_rest[1..].ptr - tokens.ptr;
-            return .{ .{ .block = try block.toOwnedSlice(gpa) }, consumed };
-        },
-        .lbracket => {
-            var list = std.ArrayList(Expr).empty;
-            var tokens_rest = tokens[1..];
-            while (tokens_rest.len > 0 and tokens_rest[0] != .rbracket) {
-                const expr, const consumed = try parse_expr_(gpa, tokens_rest);
-                try list.append(gpa, expr);
-                tokens_rest = tokens_rest[consumed..];
-            }
-            if (tokens_rest.len == 0) {
-                return error.UnmatchedBracket;
-            }
-            const consumed = tokens_rest[1..].ptr - tokens.ptr;
-            return .{ .{ .list = try list.toOwnedSlice(gpa) }, consumed };
-        },
-        else => {
-            return error.UnexpectedToken;
-        },
-    }
-}
-
-pub fn parse_expr(gpa: std.mem.Allocator, tokens: []const lexer.Token) !Expr {
-    const expr, const consumed = try parse_expr_(gpa, tokens);
-
-    if (tokens[consumed..].len > 0) return error.ExtraInput;
-
-    return expr;
-}
-
-test "parser" {
-    const t = std.testing;
-
-    var gpa = std.heap.DebugAllocator(.{}).init;
-
-    try t.expectEqualDeep(
-        Expr{ .call = &.{
-            .{ .identifier = "foo" },
-            .{ .identifier = "bar" },
-            .{ .string = "baz" },
-            .{ .list = &.{
-                .{ .integer = 1 },
-                .{ .integer = 2 },
-                .{ .integer = 3 },
-            } },
-            .{ .block = &.{} },
-        } },
-        try parse_expr(gpa.allocator(), &.{
-            .lparen,
-            .{ .identifier = "foo" },
-            .{ .identifier = "bar" },
-            .{ .string = "baz" },
-            .lbracket,
-            .{ .integer = 1 },
-            .{ .integer = 2 },
-            .{ .integer = 3 },
-            .rbracket,
-            .lbrace,
-            .rbrace,
-            .rparen,
-        }),
-    );
-
-    try t.expectError(
-        error.UnexpectedEOF,
-        parse_expr(gpa.allocator(), &.{}),
-    );
-
-    try t.expectError(
-        error.ExtraInput,
-        parse_expr(gpa.allocator(), &.{
-            .lparen,
-            .{ .integer = 123 },
-            .rparen,
-            .{ .identifier = "extra" },
-        }),
-    );
-
-    try t.expectError(
-        error.UnmatchedParen,
-        parse_expr(gpa.allocator(), &.{
-            .lbracket,
-            .{ .integer = 34 },
-            .lparen,
-            .{ .identifier = "name" },
-        }),
-    );
-
-    try t.expectError(
-        error.UnmatchedBracket,
-        parse_expr(gpa.allocator(), &.{
-            .lparen,
-            .{ .integer = 34 },
-            .lbracket,
-            .{ .identifier = "name" },
-        }),
-    );
-}

tests/all.zig

@@ -0,0 +1,4 @@
+test {
+    _ = @import("lexer_test.zig");
+    _ = @import("parser_test.zig");
+}

tests/lexer_test.zig

@@ -0,0 +1,45 @@
+const std = @import("std");
+const t = std.testing;
+
+const lib = @import("libneon");
+
+test "lexer" {
+    const tokens = try lib.lexer.lex(t.allocator,
+        \\(let input (fs.read_file "input.txt")) # comment test
+        \\{
+        \\  (!_~special ch4racter test')
+        \\  (print [1 2 3 input])
+        \\}
+    );
+    defer t.allocator.free(tokens);
+
+    try t.expectEqualDeep(
+        &[_]lib.lexer.Token{
+            .lparen,
+            .{ .identifier = "let" },
+            .{ .identifier = "input" },
+            .lparen,
+            .{ .identifier = "fs.read_file" },
+            .{ .string = "input.txt" },
+            .rparen,
+            .rparen,
+            .lbrace,
+            .lparen,
+            .{ .identifier = "!_~special" },
+            .{ .identifier = "ch4racter" },
+            .{ .identifier = "test'" },
+            .rparen,
+            .lparen,
+            .{ .identifier = "print" },
+            .lbracket,
+            .{ .integer = 1 },
+            .{ .integer = 2 },
+            .{ .integer = 3 },
+            .{ .identifier = "input" },
+            .rbracket,
+            .rparen,
+            .rbrace,
+        },
+        tokens,
+    );
+}

tests/parser_test.zig

@@ -0,0 +1,77 @@
+const std = @import("std");
+const t = std.testing;
+
+const lib = @import("libneon");
+
+test "large tree" {
+    var arena = std.heap.ArenaAllocator.init(t.allocator);
+    defer arena.deinit();
+    const gpa = arena.allocator();
+
+    try t.expectEqualDeep(
+        &[_]lib.parser.SExpr{
+            .{ .call = &.{
+                .{ .identifier = "foo" },
+                .{ .identifier = "bar" },
+                .{ .string = "baz" },
+                .{ .list = &.{
+                    .{ .integer = 1 },
+                    .{ .integer = 2 },
+                    .{ .integer = 3 },
+                } },
+                .{ .block = &.{} },
+            } },
+        },
+        try lib.parser.parse_sexprs(gpa, &.{
+            .lparen,
+            .{ .identifier = "foo" },
+            .{ .identifier = "bar" },
+            .{ .string = "baz" },
+            .lbracket,
+            .{ .integer = 1 },
+            .{ .integer = 2 },
+            .{ .integer = 3 },
+            .rbracket,
+            .lbrace,
+            .rbrace,
+            .rparen,
+        }),
+    );
+}
+
+test "empty" {
+    var arena = std.heap.ArenaAllocator.init(t.allocator);
+    defer arena.deinit();
+    const gpa = arena.allocator();
+
+    try t.expectEqualDeep(
+        &.{},
+        lib.parser.parse_sexprs(gpa, &.{}),
+    );
+}
+
+test "unmatched delimiters" {
+    var arena = std.heap.ArenaAllocator.init(t.allocator);
+    defer arena.deinit();
+    const gpa = arena.allocator();
+
+    try t.expectError(
+        error.UnmatchedParen,
+        lib.parser.parse_sexprs(gpa, &.{
+            .lbracket,
+            .{ .integer = 34 },
+            .lparen,
+            .{ .identifier = "name" },
+        }),
+    );
+
+    try t.expectError(
+        error.UnmatchedBracket,
+        lib.parser.parse_sexprs(gpa, &.{
+            .lparen,
+            .{ .integer = 34 },
+            .lbracket,
+            .{ .identifier = "name" },
+        }),
+    );
+}