diff --git a/grammar.js b/grammar.js index 7cdde6e..91b2928 100644 --- a/grammar.js +++ b/grammar.js @@ -35,6 +35,11 @@ const PREC = { module.exports = grammar({ name: 'c', + externals: $ => [ + $.gnu_raw_string_delimiter, + $.gnu_raw_string_content, + ], + conflicts: $ => [ [$.type_specifier, $._declarator], [$.type_specifier, $._declarator, $.macro_type_specifier], @@ -977,9 +982,25 @@ module.exports = grammar({ _string: $ => prec.left(choice( $.string_literal, + $.gnu_raw_string_literal, $.concatenated_string, )), + gnu_raw_string_literal: $ => seq( + choice('R"', 'LR"', 'uR"', 'UR"', 'u8R"'), + choice( + seq( + field('delimiter', $.gnu_raw_string_delimiter), + '(', + $.gnu_raw_string_content, + ')', + $.gnu_raw_string_delimiter, + ), + seq('(', $.gnu_raw_string_content, ')'), + ), + '"', + ), + comma_expression: $ => seq( field('left', $.expression), ',', @@ -1298,11 +1319,11 @@ module.exports = grammar({ // Identifier is added to parse macros that are strings, like PRIu64. concatenated_string: $ => prec.right(seq( choice( - seq($.identifier, $.string_literal), - seq($.string_literal, $.string_literal), - seq($.string_literal, $.identifier), + seq($.identifier, choice($.string_literal, $.gnu_raw_string_literal)), + seq(choice($.string_literal, $.gnu_raw_string_literal), choice($.string_literal, $.gnu_raw_string_literal)), + seq(choice($.string_literal, $.gnu_raw_string_literal), $.identifier), ), - repeat(choice($.string_literal, $.identifier)), + repeat(choice($.string_literal, $.gnu_raw_string_literal, $.identifier)), )), string_literal: $ => seq( diff --git a/src/scanner.c b/src/scanner.c new file mode 100644 index 0000000..9044c0a --- /dev/null +++ b/src/scanner.c @@ -0,0 +1,148 @@ +#include "tree_sitter/alloc.h" +#include "tree_sitter/parser.h" + +#include +#include +#include + +enum TokenType { GNU_RAW_STRING_DELIMITER, GNU_RAW_STRING_CONTENT }; + +/// The spec limits delimiters to 16 chars +#define MAX_DELIMITER_LENGTH 16 + +typedef struct { + uint8_t delimiter_length; + wchar_t delimiter[MAX_DELIMITER_LENGTH]; +} Scanner; + +static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); } + +static inline void reset(Scanner *scanner) { + scanner->delimiter_length = 0; + memset(scanner->delimiter, 0, sizeof scanner->delimiter); +} + +/// Scan the raw string delimiter in R"delimiter(content)delimiter" +static bool scan_gnu_raw_string_delimiter(Scanner *scanner, TSLexer *lexer) { + if (scanner->delimiter_length > 0) { + // Closing delimiter: must exactly match the opening delimiter. + // We already checked this when scanning content, but this is how we + // know when to stop. We can't stop at ", because R"""hello""" is valid. + for (int i = 0; i < scanner->delimiter_length; ++i) { + if (lexer->lookahead != scanner->delimiter[i]) { + return false; + } + advance(lexer); + } + reset(scanner); + return true; + } + + // Opening delimiter: record the d-char-sequence up to (. + // d-char is any basic character except parens, backslashes, and spaces. + for (;;) { + if (scanner->delimiter_length >= MAX_DELIMITER_LENGTH || lexer->eof(lexer) || lexer->lookahead == '\\' || + iswspace(lexer->lookahead)) { + return false; + } + if (lexer->lookahead == '(') { + // Rather than create a token for an empty delimiter, we fail and + // let the grammar fall back to a delimiter-less rule. + return scanner->delimiter_length > 0; + } + scanner->delimiter[scanner->delimiter_length++] = lexer->lookahead; + advance(lexer); + } +} + +/// Scan the raw string content in R"delimiter(content)delimiter" +static bool scan_gnu_raw_string_content(Scanner *scanner, TSLexer *lexer) { + // The progress made through the delimiter since the last ')'. + // The delimiter may not contain ')' so a single counter suffices. + for (int delimiter_index = -1;;) { + // If we hit EOF, consider the content to terminate there. + // This forms an incomplete gnu_raw_string_literal, and models the code + // well. + if (lexer->eof(lexer)) { + lexer->mark_end(lexer); + return true; + } + + if (delimiter_index >= 0) { + if (delimiter_index == scanner->delimiter_length) { + if (lexer->lookahead == '"') { + return true; + } + delimiter_index = -1; + } else { + if (lexer->lookahead == scanner->delimiter[delimiter_index]) { + delimiter_index += 1; + } else { + delimiter_index = -1; + } + } + } + + if (delimiter_index == -1 && lexer->lookahead == ')') { + // The content doesn't include the )delimiter" part. + // We must still scan through it, but exclude it from the token. + lexer->mark_end(lexer); + delimiter_index = 0; + } + + advance(lexer); + } +} + +void *tree_sitter_c_external_scanner_create() { + Scanner *scanner = (Scanner *)ts_calloc(1, sizeof(Scanner)); + memset(scanner, 0, sizeof(Scanner)); + return scanner; +} + +bool tree_sitter_c_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) { + Scanner *scanner = (Scanner *)payload; + + if (valid_symbols[GNU_RAW_STRING_DELIMITER] && valid_symbols[GNU_RAW_STRING_CONTENT]) { + // we're in error recovery + return false; + } + + // No skipping leading whitespace: raw-string grammar is space-sensitive. + if (valid_symbols[GNU_RAW_STRING_DELIMITER]) { + lexer->result_symbol = GNU_RAW_STRING_DELIMITER; + return scan_gnu_raw_string_delimiter(scanner, lexer); + } + + if (valid_symbols[GNU_RAW_STRING_CONTENT]) { + lexer->result_symbol = GNU_RAW_STRING_CONTENT; + return scan_gnu_raw_string_content(scanner, lexer); + } + + return false; +} + +unsigned tree_sitter_c_external_scanner_serialize(void *payload, char *buffer) { + static_assert(MAX_DELIMITER_LENGTH * sizeof(wchar_t) < TREE_SITTER_SERIALIZATION_BUFFER_SIZE, + "Serialized delimiter is too long!"); + + Scanner *scanner = (Scanner *)payload; + size_t size = scanner->delimiter_length * sizeof(wchar_t); + memcpy(buffer, scanner->delimiter, size); + return (unsigned)size; +} + +void tree_sitter_c_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) { + assert(length % sizeof(wchar_t) == 0 && "Can't decode serialized delimiter!"); + + Scanner *scanner = (Scanner *)payload; + scanner->delimiter_length = length / sizeof(wchar_t); + if (length > 0) { + memcpy(&scanner->delimiter[0], buffer, length); + } +} + +void tree_sitter_c_external_scanner_destroy(void *payload) { + Scanner *scanner = (Scanner *)payload; + ts_free(scanner); +} diff --git a/test/corpus/expressions.txt b/test/corpus/expressions.txt index 3cbed5c..6beb4b0 100644 --- a/test/corpus/expressions.txt +++ b/test/corpus/expressions.txt @@ -1481,3 +1481,195 @@ unsigned const char *c; (primitive_type)) (pointer_declarator (identifier)))) + +================================================================================ +Raw string literals +================================================================================ + +const char *s1 = R"( + This is a string. It ends with ')' and a quote. +)"; + +const char *s2 = R"FOO( + This is a string. It ends with ')FOO' and a quote. +)FOO"; + +const char *s3 = uR"FOO( + This is a string. It ends with ')FOO' and a quote. +)FOO"; + +const char *s4 = UR"FOO( + This is a string. It ends with ')FOO' and a quote. +)FOO"; + +const char *s5 = u8R"FOO( + This is a string. It ends with ')FOO' and a quote. +)FOO"; + +const char *s6 = LR"FOO( + This is a string. It ends with ')FOO' and a quote. +)FOO"; + +-------------------------------------------------------------------------------- + +(translation_unit + (declaration + (type_qualifier) + (primitive_type) + (init_declarator + (pointer_declarator + (identifier)) + (gnu_raw_string_literal + (gnu_raw_string_content)))) + (declaration + (type_qualifier) + (primitive_type) + (init_declarator + (pointer_declarator + (identifier)) + (gnu_raw_string_literal + (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)))) + (declaration + (type_qualifier) + (primitive_type) + (init_declarator + (pointer_declarator + (identifier)) + (gnu_raw_string_literal + (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)))) + (declaration + (type_qualifier) + (primitive_type) + (init_declarator + (pointer_declarator + (identifier)) + (gnu_raw_string_literal + (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)))) + (declaration + (type_qualifier) + (primitive_type) + (init_declarator + (pointer_declarator + (identifier)) + (gnu_raw_string_literal + (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)))) + (declaration + (type_qualifier) + (primitive_type) + (init_declarator + (pointer_declarator + (identifier)) + (gnu_raw_string_literal + (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter))))) + +================================================================================ +Concatenated raw strings +================================================================================ + +foo(R"RAW( +hello)RAW" PRI R"RAW(world)RAW"); +foo(R"RAW(hello)RAW" PRI); +foo("hello" R"RAW( +world)RAW"); +foo(R"RAW(hello)RAW" "world" PRI); +foo(PRI R"RAW(hello +)RAW" PRI); +foo(PRI R"RAW( +hello)RAW"); +foo("foo" R"RAW( +hello)RAW" L"world"); + +-------------------------------------------------------------------------------- + +(translation_unit + (expression_statement + (call_expression + function: (identifier) + arguments: (argument_list + (concatenated_string + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)) + (identifier) + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)))))) + (expression_statement + (call_expression + function: (identifier) + arguments: (argument_list + (concatenated_string + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)) + (identifier))))) + (expression_statement + (call_expression + function: (identifier) + arguments: (argument_list + (concatenated_string + (string_literal + (string_content)) + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)))))) + (expression_statement + (call_expression + function: (identifier) + arguments: (argument_list + (concatenated_string + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)) + (string_literal + (string_content)) + (identifier))))) + (expression_statement + (call_expression + function: (identifier) + arguments: (argument_list + (concatenated_string + (identifier) + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)) + (identifier))))) + (expression_statement + (call_expression + function: (identifier) + arguments: (argument_list + (concatenated_string + (identifier) + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)))))) + (expression_statement + (call_expression + function: (identifier) + arguments: (argument_list + (concatenated_string + (string_literal + (string_content)) + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)) + (string_literal + (string_content)))))))