diff options
Diffstat (limited to 'src/lexer')
| -rw-r--r-- | src/lexer/Lexer.zig | 20 | ||||
| -rw-r--r-- | src/lexer/Token.zig | 7 |
2 files changed, 26 insertions, 1 deletions
diff --git a/src/lexer/Lexer.zig b/src/lexer/Lexer.zig index 4137b40..c14ce01 100644 --- a/src/lexer/Lexer.zig +++ b/src/lexer/Lexer.zig @@ -52,7 +52,7 @@ pub fn next(self: *Self) ?Token { } self.new_line = current_kind.?.isDelimiter(); // conds here to avoid creating complex condition in while - const next_rune = self.iter.peek(1); + var next_rune = self.iter.peek(1); const next_kind = self.getCurrentKind(current_kind, next_rune, self.content[beg..end]).kind; if (current_kind.?.requiresSpace() and next_kind != current_kind.?) { if (eql(u8, next_rune, " ")) { @@ -65,6 +65,9 @@ pub fn next(self: *Self) ?Token { else => .literal, }; } + if (current_kind.?.trimSpace()) { + while (eql(u8, next_rune, " ")) : (next_rune = self.iter.peek(1)) _ = self.iter.nextCodepoint(); + } if (next_rune.len > 0 and next_kind != current_kind.? and (override_if == null or !eql(u8, override_if.?, next_rune))) @@ -286,3 +289,18 @@ test "peek and consume" { try expect(l.next().?.equals(.bold, "*")); try expect(l.next() == null); } + +test "trim space" { + const expect = std.testing.expect; + + var l = try init( + \\> hello + \\> world + ); + + try expect(l.next().?.equals(.quote, ">")); + try expect(l.next().?.equals(.literal, "hello")); + try expect(l.next().?.equals(.weak_delimiter, "\n")); + try expect(l.next().?.equals(.quote, ">")); + try expect(l.next().?.equals(.literal, "world")); +} diff --git a/src/lexer/Token.zig b/src/lexer/Token.zig index bd0bdc2..3162869 100644 --- a/src/lexer/Token.zig +++ b/src/lexer/Token.zig @@ -41,6 +41,13 @@ pub const Kind = enum { else => false, }; } + + pub inline fn trimSpace(self: @This()) bool { + return switch (self) { + .quote, .callout, .code_block => true, + else => return self.requiresSpace(), + }; + } }; kind: Kind, |