From 7b1b855ed68b4fc72d01de170226ea0fcc74a512 Mon Sep 17 00:00:00 2001 From: Anhgelus Morhtuuzh Date: Tue, 28 Apr 2026 19:56:22 +0200 Subject: feat(lexer): trim spaces when required --- example.td | 2 +- src/callout.zig | 2 +- src/lexer/Lexer.zig | 20 +++++++++++++++++++- src/lexer/Token.zig | 7 +++++++ 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/example.td b/example.td index 83762f9..e4fab16 100644 --- a/example.td +++ b/example.td @@ -18,7 +18,7 @@ Existing blocks: > yay with an attribution :D -:::info +::: info A beautiful callout ::: diff --git a/src/callout.zig b/src/callout.zig index 7badcfc..9c5bab5 100644 --- a/src/callout.zig +++ b/src/callout.zig @@ -64,7 +64,7 @@ test "callout" { \\::: , "

hey

"); try doTest(parse, alloc, - \\:::info Title + \\::: info Title \\hey \\::: , "

hey

"); diff --git a/src/lexer/Lexer.zig b/src/lexer/Lexer.zig index 4137b40..c14ce01 100644 --- a/src/lexer/Lexer.zig +++ b/src/lexer/Lexer.zig @@ -52,7 +52,7 @@ pub fn next(self: *Self) ?Token { } self.new_line = current_kind.?.isDelimiter(); // conds here to avoid creating complex condition in while - const next_rune = self.iter.peek(1); + var next_rune = self.iter.peek(1); const next_kind = self.getCurrentKind(current_kind, next_rune, self.content[beg..end]).kind; if (current_kind.?.requiresSpace() and next_kind != current_kind.?) { if (eql(u8, next_rune, " ")) { @@ -65,6 +65,9 @@ pub fn next(self: *Self) ?Token { else => .literal, }; } + if (current_kind.?.trimSpace()) { + while (eql(u8, next_rune, " ")) : (next_rune = self.iter.peek(1)) _ = self.iter.nextCodepoint(); + } if (next_rune.len > 0 and next_kind != current_kind.? and (override_if == null or !eql(u8, override_if.?, next_rune))) @@ -286,3 +289,18 @@ test "peek and consume" { try expect(l.next().?.equals(.bold, "*")); try expect(l.next() == null); } + +test "trim space" { + const expect = std.testing.expect; + + var l = try init( + \\> hello + \\> world + ); + + try expect(l.next().?.equals(.quote, ">")); + try expect(l.next().?.equals(.literal, "hello")); + try expect(l.next().?.equals(.weak_delimiter, "\n")); + try expect(l.next().?.equals(.quote, ">")); + try expect(l.next().?.equals(.literal, "world")); +} diff --git a/src/lexer/Token.zig b/src/lexer/Token.zig index bd0bdc2..3162869 100644 --- a/src/lexer/Token.zig +++ b/src/lexer/Token.zig @@ -41,6 +41,13 @@ pub const Kind = enum { else => false, }; } + + pub inline fn trimSpace(self: @This()) bool { + return switch (self) { + .quote, .callout, .code_block => true, + else => return self.requiresSpace(), + }; + } }; kind: Kind, -- cgit v1.2.3