diff options
| -rw-r--r-- | src/content.zig | 8 | ||||
| -rw-r--r-- | src/lexer/Lexer.zig | 40 | ||||
| -rw-r--r-- | src/link.zig | 6 | ||||
| -rw-r--r-- | src/paragraph.zig | 16 | ||||
| -rw-r--r-- | src/parser.zig | 8 | ||||
| -rw-r--r-- | src/root.zig | 2 |
6 files changed, 52 insertions, 28 deletions
diff --git a/src/content.zig b/src/content.zig index e91576d..72acbc7 100644 --- a/src/content.zig +++ b/src/content.zig @@ -31,13 +31,13 @@ pub fn parse(alloc: Allocator, l: *Lexer) Error!Element { fn parseModifier(alloc: Allocator, l: *Lexer, knd: Token.Kind, tag: []const u8) Error!Element { var el = try Element.init(alloc, .content, tag); errdefer el.deinit(); - while (l.nextKind()) |it| { - if (it == knd) { + while (l.peek()) |next| { + if (next.kind == knd) { // consuming the finisher - _ = l.next(); + l.consume(); return el; } - if (it.isDelimiter()) return Error.ModifierNotClosed; + if (next.kind.isDelimiter()) return Error.ModifierNotClosed; try el.appendContent(try parse(alloc, l)); } return Error.ModifierNotClosed; diff --git a/src/lexer/Lexer.zig b/src/lexer/Lexer.zig index fa45b5a..c2528b5 100644 --- a/src/lexer/Lexer.zig +++ b/src/lexer/Lexer.zig @@ -7,6 +7,7 @@ const Token = @import("Token.zig"); content: []const u8, iter: unicode.Utf8Iterator, force_lit: bool = false, +before: ?Token = null, const Self = @This(); @@ -26,13 +27,11 @@ pub fn initReader(alloc: Allocator, r: *std.io.Reader) !Self { return init(try acc.toOwnedSlice(alloc)); } -pub fn nextKind(self: *Self) ?Token.Kind { - const next_rune = self.iter.peek(1); - if (next_rune.len == 0) return null; - return self.getCurrentKind(null, next_rune, &[0]u8{}).kind; -} - pub fn next(self: *Self) ?Token { + if (self.before) |it| { + self.consume(); + return it; + } const beg = self.iter.i; var end = self.iter.i; var current_kind: ?Token.Kind = null; @@ -69,8 +68,16 @@ pub fn next(self: *Self) ?Token { (override_if == null or !eql(u8, override_if.?, next_rune))) break; } - const kind = current_kind orelse return null; - return .{ .kind = kind, .content = self.content[beg..end] }; + return .{ .kind = current_kind orelse return null, .content = self.content[beg..end] }; +} + +pub fn peek(self: *Self) ?Token { + self.before = self.next(); + return self.before; +} + +pub fn consume(self: *Self) void { + self.before = null; } const kindRes = struct { @@ -244,3 +251,20 @@ test "lexer multiline" { try std.testing.expect(l.next() == null); } + +test "peek and consume" { + const expect = std.testing.expect; + + var l = try init("hello *world*"); + + try expect(l.peek().?.equals(.literal, "hello ")); + try expect(l.peek().?.equals(.literal, "hello ")); + l.consume(); + try expect(l.peek().?.equals(.bold, "*")); + try expect(l.peek().?.equals(.bold, "*")); + l.consume(); + try expect(l.next().?.equals(.literal, "world")); + try expect(l.peek().?.equals(.bold, "*")); + try expect(l.next().?.equals(.bold, "*")); + try expect(l.next() == null); +} diff --git a/src/link.zig b/src/link.zig index d5ffe3c..eb13447 100644 --- a/src/link.zig +++ b/src/link.zig @@ -37,11 +37,11 @@ pub fn parseData(alloc: Allocator, l: *Lexer) Error!Data { } var el = Element.initEmpty(alloc); errdefer el.deinit(); - while (l.nextKind()) |kind| { - switch (kind) { + while (l.peek()) |next| { + switch (next.kind) { .weak_delimiter, .strong_delimiter => return Error.InvalidLink, .link => { - const next = l.next().?; + l.consume(); if (!eql(u8, next.content, "](")) return Error.InvalidLink; break; }, diff --git a/src/paragraph.zig b/src/paragraph.zig index c8c6798..0e0f188 100644 --- a/src/paragraph.zig +++ b/src/paragraph.zig @@ -15,14 +15,14 @@ pub const Error = content.Error || link.Error || Lexer.Error || Allocator.Error; pub fn parse(alloc: Allocator, l: *Lexer) Error!Element { var el = try Element.init(alloc, .content, "p"); errdefer el.deinit(); - while (l.nextKind()) |kind| { - switch (kind) { + while (l.peek()) |next| { + switch (next.kind) { // because nextKind returns only an hint for the next rune + .strong_delimiter => return el, .weak_delimiter => { - const v = l.next().?; - if (v.kind == .strong_delimiter) return el; - const next = l.nextKind() orelse return el; - switch (next) { + l.consume(); + const future = l.peek() orelse return el; + switch (future.kind) { .literal, .italic, .code, .bold, .link => try el.appendContent(try Element.initLit(alloc, " ")), else => return el, } @@ -36,8 +36,8 @@ pub fn parse(alloc: Allocator, l: *Lexer) Error!Element { pub fn parseLine(alloc: Allocator, l: *Lexer) Error!Element { var line = Element.initEmpty(alloc); errdefer line.deinit(); - while (l.nextKind()) |kind| { - switch (kind) { + while (l.peek()) |next| { + switch (next.kind) { .weak_delimiter, .strong_delimiter => return line, .link => { var el = try link.parse(alloc, l); diff --git a/src/parser.zig b/src/parser.zig index 90f16fb..07722d8 100644 --- a/src/parser.zig +++ b/src/parser.zig @@ -29,15 +29,15 @@ fn gen(parent: Allocator, l: *Lexer) Error![]const u8 { var elements = try std.ArrayList(Element).initCapacity(alloc, 2); - while (l.nextKind()) |it| { - try elements.append(alloc, switch (it) { + base: while (l.peek()) |it| { + try elements.append(alloc, switch (it.kind) { // block paragraph .literal, .bold, .italic, .code, .link => try paragraph.parse(alloc, l), // other blocks .title => try title.parse(alloc, l), .weak_delimiter, .strong_delimiter => { - _ = l.next(); - continue; + l.consume(); + continue :base; }, else => return Error.FeatureNotSupported, }); diff --git a/src/root.zig b/src/root.zig index ac95f31..82ac3b1 100644 --- a/src/root.zig +++ b/src/root.zig @@ -56,7 +56,7 @@ pub fn parse(alloc: std.mem.Allocator, content: []const u8) Error![]const u8 { return parser.parse(alloc, content); } -pub fn zigParse(alloc: std.mem.Allocator, r: *std.io.Reader) ![]const u8 { +pub fn parseReader(alloc: std.mem.Allocator, r: *std.io.Reader) (Error || std.io.Reader.Error)![]const u8 { return parser.parseReader(alloc, r); } |