Re-generate parser with new tree-sitter

src/tree_sitter/parser.h

@@ -9,11 +9,9 @@ extern "C" {
 #include <stdbool.h>
 #include "tree_sitter/runtime.h"
 
-#define ts_lex_state_error 0
+#define TS_STATE_ERROR 0
 #define TS_DEBUG_BUFFER_SIZE 512
 
-typedef struct TSTree TSTree;
-
 typedef unsigned short TSStateId;
 
 typedef struct {
@@ -30,24 +28,30 @@ typedef struct {
   bool structural : 1;
 } TSSymbolMetadata;
 
-typedef struct TSLexer {
-  void (*start_fn)(struct TSLexer *, TSStateId);
-  void (*start_token_fn)(struct TSLexer *);
-  bool (*advance_fn)(struct TSLexer *, TSStateId);
-  TSTree *(*accept_fn)(struct TSLexer *, TSSymbol, TSSymbolMetadata,
-                       const char *, bool fragile);
+typedef enum {
+  TSTransitionTypeMain,
+  TSTransitionTypeSeparator,
+  TSTransitionTypeError,
+} TSTransitionType;
 
-  const char *chunk;
-  size_t chunk_start;
-  size_t chunk_size;
+typedef struct TSLexer {
+  void (*advance)(struct TSLexer *, TSStateId, TSTransitionType);
 
   TSLength current_position;
   TSLength token_end_position;
   TSLength token_start_position;
+  TSLength error_end_position;
+
+  const char *chunk;
+  size_t chunk_start;
+  size_t chunk_size;
 
   size_t lookahead_size;
   int32_t lookahead;
   TSStateId starting_state;
+  TSSymbol result_symbol;
+  bool result_follows_error;
+  int32_t first_unexpected_character;
 
   TSInput input;
   TSDebugger debugger;
@@ -55,10 +59,10 @@ typedef struct TSLexer {
 } TSLexer;
 
 typedef enum {
-  TSParseActionTypeError,
   TSParseActionTypeShift,
   TSParseActionTypeReduce,
   TSParseActionTypeAccept,
+  TSParseActionTypeRecover,
 } TSParseActionType;
 
 typedef struct {
@@ -68,16 +72,19 @@ typedef struct {
       TSSymbol symbol;
       unsigned short child_count;
     };
-  } data;
-  TSParseActionType type : 3;
+  };
+  TSParseActionType type : 4;
   bool extra : 1;
   bool fragile : 1;
-  bool can_hide_split : 1;
 } TSParseAction;
 
 typedef union {
   TSParseAction action;
-  unsigned int count;
+  struct {
+    unsigned short count;
+    bool reusable : 1;
+    bool depends_on_lookahead : 1;
+  };
 } TSParseActionEntry;
 
 struct TSLanguage {
@@ -87,71 +94,63 @@ struct TSLanguage {
   const unsigned short *parse_table;
   const TSParseActionEntry *parse_actions;
   const TSStateId *lex_states;
-  TSTree *(*lex_fn)(TSLexer *, TSStateId, bool);
+  bool (*lex_fn)(TSLexer *, TSStateId, bool);
 };
 
 /*
  *  Lexer Macros
  */
 
-#define START_LEXER()            \
-  lexer->start_fn(lexer, state); \
-  int32_t lookahead;             \
-  next_state:                    \
+#define START_LEXER() \
+  int32_t lookahead;  \
+  next_state:         \
   lookahead = lexer->lookahead;
 
-#define START_TOKEN() lexer->start_token_fn(lexer);
-
 #define GO_TO_STATE(state_value) \
   {                              \
     state = state_value;         \
     goto next_state;             \
   }
 
-#define ADVANCE(state_value)               \
-  {                                        \
-    lexer->advance_fn(lexer, state_value); \
-    GO_TO_STATE(state_value);              \
+#define ADVANCE(state_value)                                  \
+  {                                                           \
+    lexer->advance(lexer, state_value, TSTransitionTypeMain); \
+    GO_TO_STATE(state_value);                                 \
+  }
+
+#define SKIP(state_value)                                          \
+  {                                                                \
+    lexer->advance(lexer, state_value, TSTransitionTypeSeparator); \
+    GO_TO_STATE(state_value);                                      \
+  }
+
+#define ACCEPT_TOKEN(symbol_value)       \
+  {                                      \
+    lexer->result_symbol = symbol_value; \
+    return true;                         \
   }
 
-#define ACCEPT_FRAGILE_TOKEN(symbol)                                 \
-  return lexer->accept_fn(lexer, symbol, ts_symbol_metadata[symbol], \
-                          ts_symbol_names[symbol], true);
-
-#define ACCEPT_TOKEN(symbol)                                         \
-  return lexer->accept_fn(lexer, symbol, ts_symbol_metadata[symbol], \
-                          ts_symbol_names[symbol], false);
-
-#define LEX_ERROR()                    \
-  if (error_mode) {                    \
-    if (state == ts_lex_state_error)   \
-      lexer->advance_fn(lexer, state); \
-    GO_TO_STATE(ts_lex_state_error)    \
-  } else {                             \
-    ACCEPT_TOKEN(ts_builtin_sym_error) \
+#define LEX_ERROR()                                        \
+  if (error_mode) {                                        \
+    if (state == TS_STATE_ERROR)                           \
+      lexer->advance(lexer, state, TSTransitionTypeError); \
+    GO_TO_STATE(TS_STATE_ERROR);                           \
+  } else {                                                 \
+    return false;                                          \
   }
 
 /*
  *  Parse Table Macros
  */
 
-enum {
-  FRAGILE = 1,
-  CAN_HIDE_SPLIT = 2,
-};
-
-#define ERROR()                        \
-  {                                    \
-    { .type = TSParseActionTypeError } \
+#define SHIFT(to_state_value)                                      \
+  {                                                                \
+    { .type = TSParseActionTypeShift, .to_state = to_state_value } \
   }
 
-#define SHIFT(to_state_value, flags)                   \
-  {                                                    \
-    {                                                  \
-      .type = TSParseActionTypeShift,                  \
-      .can_hide_split = (flags & CAN_HIDE_SPLIT) != 0, \
-      .data = {.to_state = to_state_value }            \
-    }                                                  \
+#define RECOVER(to_state_value)                                      \
+  {                                                                  \
+    { .type = TSParseActionTypeRecover, .to_state = to_state_value } \
   }
 
 #define SHIFT_EXTRA()                                 \
@@ -159,21 +158,28 @@ enum {
     { .type = TSParseActionTypeShift, .extra = true } \
   }
 
-#define REDUCE_EXTRA(symbol_val)                        \
-  {                                                     \
-    {                                                   \
-      .type = TSParseActionTypeReduce, .extra = true,   \
-      .data = {.symbol = symbol_val, .child_count = 1 } \
-    }                                                   \
+#define REDUCE_EXTRA(symbol_val)                                               \
+  {                                                                            \
+    {                                                                          \
+      .type = TSParseActionTypeReduce, .symbol = symbol_val, .child_count = 1, \
+      .extra = true,                                                           \
+    }                                                                          \
+  }
+
+#define REDUCE(symbol_val, child_count_val)                  \
+  {                                                          \
+    {                                                        \
+      .type = TSParseActionTypeReduce, .symbol = symbol_val, \
+      .child_count = child_count_val,                        \
+    }                                                        \
   }
 
-#define REDUCE(symbol_val, child_count_val, flags)                        \
-  {                                                                       \
-    {                                                                     \
-      .type = TSParseActionTypeReduce, .fragile = (flags & FRAGILE) != 0, \
-      .can_hide_split = (flags & CAN_HIDE_SPLIT) != 0,                    \
-      .data = {.symbol = symbol_val, .child_count = child_count_val }     \
-    }                                                                     \
+#define REDUCE_FRAGILE(symbol_val, child_count_val)          \
+  {                                                          \
+    {                                                        \
+      .type = TSParseActionTypeReduce, .symbol = symbol_val, \
+      .child_count = child_count_val, .fragile = true,       \
+    }                                                        \
   }
 
 #define ACCEPT_INPUT()                  \

src/tree_sitter/runtime.h

@@ -91,6 +91,7 @@ void ts_document_set_input(TSDocument *, TSInput);
 void ts_document_set_input_string(TSDocument *, const char *);
 TSDebugger ts_document_debugger(const TSDocument *);
 void ts_document_set_debugger(TSDocument *, TSDebugger);
+void ts_document_print_debugging_graphs(TSDocument *, bool);
 void ts_document_edit(TSDocument *, TSInputEdit);
 int ts_document_parse(TSDocument *);
 void ts_document_invalidate(TSDocument *);
@@ -100,9 +101,9 @@ size_t ts_document_parse_count(const TSDocument *);
 size_t ts_language_symbol_count(const TSLanguage *);
 const char *ts_language_symbol_name(const TSLanguage *, TSSymbol);
 
-#define ts_builtin_sym_error 0
-#define ts_builtin_sym_end 1
-#define ts_builtin_sym_start 2
+#define ts_builtin_sym_error ((TSSymbol)-1)
+#define ts_builtin_sym_end 0
+#define ts_builtin_sym_start 1
 
 #ifdef __cplusplus
 }