From fec006aaa5ee3683457ebfc2ba1755b077e14c79 Mon Sep 17 00:00:00 2001 From: Anhgelus Morhtuuzh Date: Sat, 18 Apr 2026 19:21:47 +0200 Subject: fix(lexer): invalid acc for titles --- src/ast.zig | 62 ----------------------------------------------------- src/lexer/Lexer.zig | 31 ++++++++++++++++++++------- src/parser.zig | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/root.zig | 2 +- 4 files changed, 86 insertions(+), 71 deletions(-) delete mode 100644 src/ast.zig create mode 100644 src/parser.zig (limited to 'src') diff --git a/src/ast.zig b/src/ast.zig deleted file mode 100644 index e5a9ea2..0000000 --- a/src/ast.zig +++ /dev/null @@ -1,62 +0,0 @@ -const std = @import("std"); -const Lexed = @import("lexer/Lexed.zig"); -const Lexer = @import("lexer/Lexer.zig"); -const Element = @import("dom/Element.zig"); -const Allocator = std.mem.Allocator; -const paragraph = @import("paragraph.zig"); - -pub const Error = error{ - InvalidSequence, - UnclosedModifier, - FeatureNotSupported, -} || Lexer.Error; - -pub fn parse(parent: Allocator, content: []const u8) Error![]const u8 { - var arena = std.heap.ArenaAllocator.init(parent); - defer arena.deinit(); - const alloc = arena.allocator(); - - var elements = try std.ArrayList(Element).initCapacity(alloc, 2); - - var l = try Lexer.init(content); - while (l.nextKind()) |it| { - switch (it) { - .literal, .bold, .italic, .code => try elements.append(alloc, try paragraph.parseParagraph(alloc, &l)), - else => return Error.FeatureNotSupported, - } - } - - var res = try std.ArrayList(u8).initCapacity(parent, elements.items.len); - for (elements.items) |it| { - var v = it; - try res.appendSlice(parent, try v.render(alloc)); - } - return res.toOwnedSlice(parent); -} - -fn doTest(alloc: Allocator, t: []const u8, v: []const u8) !void { - const g = try parse(alloc, t); - defer alloc.free(g); - std.testing.expect(std.mem.eql(u8, g, v)) catch |err| { - std.debug.print("{s}\n", .{g}); - return err; - }; -} - -test "parse paragraphs" { - var arena = std.heap.DebugAllocator(.{}).init; - defer if (arena.deinit() == .leak) std.debug.print("leaking!\n", .{}); - const alloc = arena.allocator(); - - try doTest(alloc, "hello world", "

hello world

"); - try doTest(alloc, "*hello* world", "

hello world

"); - try doTest(alloc, "*he_ll_o* world", "

hello world

"); - - try doTest(alloc, - \\hello - \\world - \\ - \\foo bar - \\in new paragraph - , "

hello world

foo bar in new paragraph

"); -} diff --git a/src/lexer/Lexer.zig b/src/lexer/Lexer.zig index 7524479..318e422 100644 --- a/src/lexer/Lexer.zig +++ b/src/lexer/Lexer.zig @@ -45,7 +45,8 @@ pub fn next(self: *Self, alloc: Allocator) Error!?Lexed { } // conds here to avoid creating complex condition in while const next_rune = self.iter.peek(1); - if (requiresSpace(current_kind.?)) { + const next_kind = self.getCurrentKind(current_kind, next_rune, acc.items).kind; + if (requiresSpace(current_kind.?) and next_kind != current_kind.?) { if (eql(u8, next_rune, " ")) { // consume next space _ = self.iter.nextCodepoint(); @@ -57,7 +58,7 @@ pub fn next(self: *Self, alloc: Allocator) Error!?Lexed { }; } if (next_rune.len > 0 and - self.getCurrentKind(current_kind, next_rune, acc.items).kind != current_kind.? and + next_kind != current_kind.? and (override_if == null or !eql(u8, override_if.?, next_rune))) break; } @@ -82,7 +83,7 @@ const kindRes = struct { }; fn requiresDelimiter(before: ?Lexed.Kind, knd: Lexed.Kind) Lexed.Kind { - return if (before == null or before.?.isDelimiter()) knd else .literal; + return if (before == null or before.?.isDelimiter() or before.? == knd) knd else .literal; } fn getCurrentKind(self: *Self, before: ?Lexed.Kind, rune: []const u8, acc: []const u8) kindRes { @@ -154,9 +155,7 @@ fn isOneOrThree(op: []const u8, rune: []const u8, p: []const u8, one: Lexed.Kind fn requiresSpace(k: Lexed.Kind) bool { return switch (k) { - .title => true, - .list_ordored => true, - .list_unordored => true, + .title, .list_ordored, .list_unordored => true, else => false, }; } @@ -185,6 +184,22 @@ test "one or three" { try expect(isOneOrThree(":", "a", ":", .ref, .callout) == null); } +test "is" { + const expect = std.testing.expect; + + // valid + try expect(is('#', 6, "#", "")); + try expect(is('#', 6, "#", "#")); + try expect(is('#', 6, "#", "##")); + try expect(is('#', 6, "#", "###")); + try expect(is('#', 6, "#", "####")); + try expect(is('#', 6, "#", "#####")); + + // invalid + try expect(!is('#', 6, "#", "######")); + try expect(!is('#', 6, "u", "##")); +} + test "lexer common" { const expect = std.testing.expect; @@ -192,9 +207,9 @@ test "lexer common" { defer if (arena.deinit() == .leak) std.debug.print("leaking!\n", .{}); const alloc = arena.allocator(); - var l = try init("# hello world :)"); + var l = try init("## hello world :)"); - try doTest(alloc, &l, .title, "#"); + try doTest(alloc, &l, .title, "##"); try doTest(alloc, &l, .literal, "hello world "); try doTest(alloc, &l, .ref, ":"); try doTest(alloc, &l, .link, ")"); diff --git a/src/parser.zig b/src/parser.zig new file mode 100644 index 0000000..e5a9ea2 --- /dev/null +++ b/src/parser.zig @@ -0,0 +1,62 @@ +const std = @import("std"); +const Lexed = @import("lexer/Lexed.zig"); +const Lexer = @import("lexer/Lexer.zig"); +const Element = @import("dom/Element.zig"); +const Allocator = std.mem.Allocator; +const paragraph = @import("paragraph.zig"); + +pub const Error = error{ + InvalidSequence, + UnclosedModifier, + FeatureNotSupported, +} || Lexer.Error; + +pub fn parse(parent: Allocator, content: []const u8) Error![]const u8 { + var arena = std.heap.ArenaAllocator.init(parent); + defer arena.deinit(); + const alloc = arena.allocator(); + + var elements = try std.ArrayList(Element).initCapacity(alloc, 2); + + var l = try Lexer.init(content); + while (l.nextKind()) |it| { + switch (it) { + .literal, .bold, .italic, .code => try elements.append(alloc, try paragraph.parseParagraph(alloc, &l)), + else => return Error.FeatureNotSupported, + } + } + + var res = try std.ArrayList(u8).initCapacity(parent, elements.items.len); + for (elements.items) |it| { + var v = it; + try res.appendSlice(parent, try v.render(alloc)); + } + return res.toOwnedSlice(parent); +} + +fn doTest(alloc: Allocator, t: []const u8, v: []const u8) !void { + const g = try parse(alloc, t); + defer alloc.free(g); + std.testing.expect(std.mem.eql(u8, g, v)) catch |err| { + std.debug.print("{s}\n", .{g}); + return err; + }; +} + +test "parse paragraphs" { + var arena = std.heap.DebugAllocator(.{}).init; + defer if (arena.deinit() == .leak) std.debug.print("leaking!\n", .{}); + const alloc = arena.allocator(); + + try doTest(alloc, "hello world", "

hello world

"); + try doTest(alloc, "*hello* world", "

hello world

"); + try doTest(alloc, "*he_ll_o* world", "

hello world

"); + + try doTest(alloc, + \\hello + \\world + \\ + \\foo bar + \\in new paragraph + , "

hello world

foo bar in new paragraph

"); +} diff --git a/src/root.zig b/src/root.zig index 98b0274..2bc3125 100644 --- a/src/root.zig +++ b/src/root.zig @@ -1,5 +1,5 @@ const std = @import("std"); -pub const ast = @import("ast.zig"); +pub const parser = @import("parser.zig"); pub fn bufferedPrint() !void { // Stdout is for the actual output of your application, for example if you -- cgit v1.2.3