From 5fd395ecaf526ea1362173dd91299679c0714213 Mon Sep 17 00:00:00 2001 From: WolodiaM Date: Sun, 14 Jun 2026 15:18:07 +0300 Subject: [PATCH 1/3] Ported support for raw string literals from c++ parser. It is a features of GNU C and supported by gcc and clang. Support is copied from tree-sitter/tree-sitter-cpp, scanner.c was copied directly and I applied minimal changes on top of it to work with C parser. And I ported pieces related to raw string literals from c++ grammar.js to C grammar.js with minimal changes to file. --- grammar.js | 29 ++++++++-- src/scanner.c | 148 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 173 insertions(+), 4 deletions(-) create mode 100644 src/scanner.c diff --git a/grammar.js b/grammar.js index 7cdde6e..99e979d 100644 --- a/grammar.js +++ b/grammar.js @@ -35,6 +35,11 @@ const PREC = { module.exports = grammar({ name: 'c', + externals: $ => [ + $.raw_string_delimiter, + $.raw_string_content, + ], + conflicts: $ => [ [$.type_specifier, $._declarator], [$.type_specifier, $._declarator, $.macro_type_specifier], @@ -977,9 +982,25 @@ module.exports = grammar({ _string: $ => prec.left(choice( $.string_literal, + $.raw_string_literal, $.concatenated_string, )), + raw_string_literal: $ => seq( + choice('R"', 'LR"', 'uR"', 'UR"', 'u8R"'), + choice( + seq( + field('delimiter', $.raw_string_delimiter), + '(', + $.raw_string_content, + ')', + $.raw_string_delimiter, + ), + seq('(', $.raw_string_content, ')'), + ), + '"', + ), + comma_expression: $ => seq( field('left', $.expression), ',', @@ -1298,11 +1319,11 @@ module.exports = grammar({ // Identifier is added to parse macros that are strings, like PRIu64. concatenated_string: $ => prec.right(seq( choice( - seq($.identifier, $.string_literal), - seq($.string_literal, $.string_literal), - seq($.string_literal, $.identifier), + seq($.identifier, choice($.string_literal, $.raw_string_literal)), + seq(choice($.string_literal, $.raw_string_literal), choice($.string_literal, $.raw_string_literal)), + seq(choice($.string_literal, $.raw_string_literal), $.identifier), ), - repeat(choice($.string_literal, $.identifier)), + repeat(choice($.string_literal, $.raw_string_literal, $.identifier)), )), string_literal: $ => seq( diff --git a/src/scanner.c b/src/scanner.c new file mode 100644 index 0000000..78ef5db --- /dev/null +++ b/src/scanner.c @@ -0,0 +1,148 @@ +#include "tree_sitter/alloc.h" +#include "tree_sitter/parser.h" + +#include +#include +#include + +enum TokenType { RAW_STRING_DELIMITER, RAW_STRING_CONTENT }; + +/// The spec limits delimiters to 16 chars +#define MAX_DELIMITER_LENGTH 16 + +typedef struct { + uint8_t delimiter_length; + wchar_t delimiter[MAX_DELIMITER_LENGTH]; +} Scanner; + +static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); } + +static inline void reset(Scanner *scanner) { + scanner->delimiter_length = 0; + memset(scanner->delimiter, 0, sizeof scanner->delimiter); +} + +/// Scan the raw string delimiter in R"delimiter(content)delimiter" +static bool scan_raw_string_delimiter(Scanner *scanner, TSLexer *lexer) { + if (scanner->delimiter_length > 0) { + // Closing delimiter: must exactly match the opening delimiter. + // We already checked this when scanning content, but this is how we + // know when to stop. We can't stop at ", because R"""hello""" is valid. + for (int i = 0; i < scanner->delimiter_length; ++i) { + if (lexer->lookahead != scanner->delimiter[i]) { + return false; + } + advance(lexer); + } + reset(scanner); + return true; + } + + // Opening delimiter: record the d-char-sequence up to (. + // d-char is any basic character except parens, backslashes, and spaces. + for (;;) { + if (scanner->delimiter_length >= MAX_DELIMITER_LENGTH || lexer->eof(lexer) || lexer->lookahead == '\\' || + iswspace(lexer->lookahead)) { + return false; + } + if (lexer->lookahead == '(') { + // Rather than create a token for an empty delimiter, we fail and + // let the grammar fall back to a delimiter-less rule. + return scanner->delimiter_length > 0; + } + scanner->delimiter[scanner->delimiter_length++] = lexer->lookahead; + advance(lexer); + } +} + +/// Scan the raw string content in R"delimiter(content)delimiter" +static bool scan_raw_string_content(Scanner *scanner, TSLexer *lexer) { + // The progress made through the delimiter since the last ')'. + // The delimiter may not contain ')' so a single counter suffices. + for (int delimiter_index = -1;;) { + // If we hit EOF, consider the content to terminate there. + // This forms an incomplete raw_string_literal, and models the code + // well. + if (lexer->eof(lexer)) { + lexer->mark_end(lexer); + return true; + } + + if (delimiter_index >= 0) { + if (delimiter_index == scanner->delimiter_length) { + if (lexer->lookahead == '"') { + return true; + } + delimiter_index = -1; + } else { + if (lexer->lookahead == scanner->delimiter[delimiter_index]) { + delimiter_index += 1; + } else { + delimiter_index = -1; + } + } + } + + if (delimiter_index == -1 && lexer->lookahead == ')') { + // The content doesn't include the )delimiter" part. + // We must still scan through it, but exclude it from the token. + lexer->mark_end(lexer); + delimiter_index = 0; + } + + advance(lexer); + } +} + +void *tree_sitter_c_external_scanner_create() { + Scanner *scanner = (Scanner *)ts_calloc(1, sizeof(Scanner)); + memset(scanner, 0, sizeof(Scanner)); + return scanner; +} + +bool tree_sitter_c_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) { + Scanner *scanner = (Scanner *)payload; + + if (valid_symbols[RAW_STRING_DELIMITER] && valid_symbols[RAW_STRING_CONTENT]) { + // we're in error recovery + return false; + } + + // No skipping leading whitespace: raw-string grammar is space-sensitive. + if (valid_symbols[RAW_STRING_DELIMITER]) { + lexer->result_symbol = RAW_STRING_DELIMITER; + return scan_raw_string_delimiter(scanner, lexer); + } + + if (valid_symbols[RAW_STRING_CONTENT]) { + lexer->result_symbol = RAW_STRING_CONTENT; + return scan_raw_string_content(scanner, lexer); + } + + return false; +} + +unsigned tree_sitter_c_external_scanner_serialize(void *payload, char *buffer) { + static_assert(MAX_DELIMITER_LENGTH * sizeof(wchar_t) < TREE_SITTER_SERIALIZATION_BUFFER_SIZE, + "Serialized delimiter is too long!"); + + Scanner *scanner = (Scanner *)payload; + size_t size = scanner->delimiter_length * sizeof(wchar_t); + memcpy(buffer, scanner->delimiter, size); + return (unsigned)size; +} + +void tree_sitter_c_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) { + assert(length % sizeof(wchar_t) == 0 && "Can't decode serialized delimiter!"); + + Scanner *scanner = (Scanner *)payload; + scanner->delimiter_length = length / sizeof(wchar_t); + if (length > 0) { + memcpy(&scanner->delimiter[0], buffer, length); + } +} + +void tree_sitter_c_external_scanner_destroy(void *payload) { + Scanner *scanner = (Scanner *)payload; + ts_free(scanner); +} From d7f6648cc4e54f7e1c38be4f7d1367c8bce5b748 Mon Sep 17 00:00:00 2001 From: WolodiaM Date: Sun, 14 Jun 2026 19:42:24 +0300 Subject: [PATCH 2/3] Added tests --- test/corpus/expressions.txt | 192 ++++++++++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) diff --git a/test/corpus/expressions.txt b/test/corpus/expressions.txt index 3cbed5c..b3e74b6 100644 --- a/test/corpus/expressions.txt +++ b/test/corpus/expressions.txt @@ -1481,3 +1481,195 @@ unsigned const char *c; (primitive_type)) (pointer_declarator (identifier)))) + +================================================================================ +Raw string literals +================================================================================ + +const char *s1 = R"( + This is a string. It ends with ')' and a quote. +)"; + +const char *s2 = R"FOO( + This is a string. It ends with ')FOO' and a quote. +)FOO"; + +const char *s3 = uR"FOO( + This is a string. It ends with ')FOO' and a quote. +)FOO"; + +const char *s4 = UR"FOO( + This is a string. It ends with ')FOO' and a quote. +)FOO"; + +const char *s5 = u8R"FOO( + This is a string. It ends with ')FOO' and a quote. +)FOO"; + +const char *s6 = LR"FOO( + This is a string. It ends with ')FOO' and a quote. +)FOO"; + +-------------------------------------------------------------------------------- + +(translation_unit + (declaration + (type_qualifier) + (primitive_type) + (init_declarator + (pointer_declarator + (identifier)) + (raw_string_literal + (raw_string_content)))) + (declaration + (type_qualifier) + (primitive_type) + (init_declarator + (pointer_declarator + (identifier)) + (raw_string_literal + (raw_string_delimiter) + (raw_string_content) + (raw_string_delimiter)))) + (declaration + (type_qualifier) + (primitive_type) + (init_declarator + (pointer_declarator + (identifier)) + (raw_string_literal + (raw_string_delimiter) + (raw_string_content) + (raw_string_delimiter)))) + (declaration + (type_qualifier) + (primitive_type) + (init_declarator + (pointer_declarator + (identifier)) + (raw_string_literal + (raw_string_delimiter) + (raw_string_content) + (raw_string_delimiter)))) + (declaration + (type_qualifier) + (primitive_type) + (init_declarator + (pointer_declarator + (identifier)) + (raw_string_literal + (raw_string_delimiter) + (raw_string_content) + (raw_string_delimiter)))) + (declaration + (type_qualifier) + (primitive_type) + (init_declarator + (pointer_declarator + (identifier)) + (raw_string_literal + (raw_string_delimiter) + (raw_string_content) + (raw_string_delimiter))))) + +================================================================================ +Concatenated raw strings +================================================================================ + +foo(R"RAW( +hello)RAW" PRI R"RAW(world)RAW"); +foo(R"RAW(hello)RAW" PRI); +foo("hello" R"RAW( +world)RAW"); +foo(R"RAW(hello)RAW" "world" PRI); +foo(PRI R"RAW(hello +)RAW" PRI); +foo(PRI R"RAW( +hello)RAW"); +foo("foo" R"RAW( +hello)RAW" L"world"); + +-------------------------------------------------------------------------------- + +(translation_unit + (expression_statement + (call_expression + function: (identifier) + arguments: (argument_list + (concatenated_string + (raw_string_literal + delimiter: (raw_string_delimiter) + (raw_string_content) + (raw_string_delimiter)) + (identifier) + (raw_string_literal + delimiter: (raw_string_delimiter) + (raw_string_content) + (raw_string_delimiter)))))) + (expression_statement + (call_expression + function: (identifier) + arguments: (argument_list + (concatenated_string + (raw_string_literal + delimiter: (raw_string_delimiter) + (raw_string_content) + (raw_string_delimiter)) + (identifier))))) + (expression_statement + (call_expression + function: (identifier) + arguments: (argument_list + (concatenated_string + (string_literal + (string_content)) + (raw_string_literal + delimiter: (raw_string_delimiter) + (raw_string_content) + (raw_string_delimiter)))))) + (expression_statement + (call_expression + function: (identifier) + arguments: (argument_list + (concatenated_string + (raw_string_literal + delimiter: (raw_string_delimiter) + (raw_string_content) + (raw_string_delimiter)) + (string_literal + (string_content)) + (identifier))))) + (expression_statement + (call_expression + function: (identifier) + arguments: (argument_list + (concatenated_string + (identifier) + (raw_string_literal + delimiter: (raw_string_delimiter) + (raw_string_content) + (raw_string_delimiter)) + (identifier))))) + (expression_statement + (call_expression + function: (identifier) + arguments: (argument_list + (concatenated_string + (identifier) + (raw_string_literal + delimiter: (raw_string_delimiter) + (raw_string_content) + (raw_string_delimiter)))))) + (expression_statement + (call_expression + function: (identifier) + arguments: (argument_list + (concatenated_string + (string_literal + (string_content)) + (raw_string_literal + delimiter: (raw_string_delimiter) + (raw_string_content) + (raw_string_delimiter)) + (string_literal + (string_content))))))) From 37dc8d34fc2b99526c2eca6407b35399a5288c0b Mon Sep 17 00:00:00 2001 From: WolodiaM Date: Wed, 17 Jun 2026 19:28:21 +0300 Subject: [PATCH 3/3] Renamed raw_string_* to gnu_raw_string_* This is GNU extension and GNU extensions seems to be prefixed with 'gnu_' in other places of source code. --- grammar.js | 24 ++++---- src/scanner.c | 22 ++++---- test/corpus/expressions.txt | 108 ++++++++++++++++++------------------ 3 files changed, 77 insertions(+), 77 deletions(-) diff --git a/grammar.js b/grammar.js index 99e979d..91b2928 100644 --- a/grammar.js +++ b/grammar.js @@ -36,8 +36,8 @@ module.exports = grammar({ name: 'c', externals: $ => [ - $.raw_string_delimiter, - $.raw_string_content, + $.gnu_raw_string_delimiter, + $.gnu_raw_string_content, ], conflicts: $ => [ @@ -982,21 +982,21 @@ module.exports = grammar({ _string: $ => prec.left(choice( $.string_literal, - $.raw_string_literal, + $.gnu_raw_string_literal, $.concatenated_string, )), - raw_string_literal: $ => seq( + gnu_raw_string_literal: $ => seq( choice('R"', 'LR"', 'uR"', 'UR"', 'u8R"'), choice( seq( - field('delimiter', $.raw_string_delimiter), + field('delimiter', $.gnu_raw_string_delimiter), '(', - $.raw_string_content, + $.gnu_raw_string_content, ')', - $.raw_string_delimiter, + $.gnu_raw_string_delimiter, ), - seq('(', $.raw_string_content, ')'), + seq('(', $.gnu_raw_string_content, ')'), ), '"', ), @@ -1319,11 +1319,11 @@ module.exports = grammar({ // Identifier is added to parse macros that are strings, like PRIu64. concatenated_string: $ => prec.right(seq( choice( - seq($.identifier, choice($.string_literal, $.raw_string_literal)), - seq(choice($.string_literal, $.raw_string_literal), choice($.string_literal, $.raw_string_literal)), - seq(choice($.string_literal, $.raw_string_literal), $.identifier), + seq($.identifier, choice($.string_literal, $.gnu_raw_string_literal)), + seq(choice($.string_literal, $.gnu_raw_string_literal), choice($.string_literal, $.gnu_raw_string_literal)), + seq(choice($.string_literal, $.gnu_raw_string_literal), $.identifier), ), - repeat(choice($.string_literal, $.raw_string_literal, $.identifier)), + repeat(choice($.string_literal, $.gnu_raw_string_literal, $.identifier)), )), string_literal: $ => seq( diff --git a/src/scanner.c b/src/scanner.c index 78ef5db..9044c0a 100644 --- a/src/scanner.c +++ b/src/scanner.c @@ -5,7 +5,7 @@ #include #include -enum TokenType { RAW_STRING_DELIMITER, RAW_STRING_CONTENT }; +enum TokenType { GNU_RAW_STRING_DELIMITER, GNU_RAW_STRING_CONTENT }; /// The spec limits delimiters to 16 chars #define MAX_DELIMITER_LENGTH 16 @@ -23,7 +23,7 @@ static inline void reset(Scanner *scanner) { } /// Scan the raw string delimiter in R"delimiter(content)delimiter" -static bool scan_raw_string_delimiter(Scanner *scanner, TSLexer *lexer) { +static bool scan_gnu_raw_string_delimiter(Scanner *scanner, TSLexer *lexer) { if (scanner->delimiter_length > 0) { // Closing delimiter: must exactly match the opening delimiter. // We already checked this when scanning content, but this is how we @@ -56,12 +56,12 @@ static bool scan_raw_string_delimiter(Scanner *scanner, TSLexer *lexer) { } /// Scan the raw string content in R"delimiter(content)delimiter" -static bool scan_raw_string_content(Scanner *scanner, TSLexer *lexer) { +static bool scan_gnu_raw_string_content(Scanner *scanner, TSLexer *lexer) { // The progress made through the delimiter since the last ')'. // The delimiter may not contain ')' so a single counter suffices. for (int delimiter_index = -1;;) { // If we hit EOF, consider the content to terminate there. - // This forms an incomplete raw_string_literal, and models the code + // This forms an incomplete gnu_raw_string_literal, and models the code // well. if (lexer->eof(lexer)) { lexer->mark_end(lexer); @@ -103,20 +103,20 @@ void *tree_sitter_c_external_scanner_create() { bool tree_sitter_c_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) { Scanner *scanner = (Scanner *)payload; - if (valid_symbols[RAW_STRING_DELIMITER] && valid_symbols[RAW_STRING_CONTENT]) { + if (valid_symbols[GNU_RAW_STRING_DELIMITER] && valid_symbols[GNU_RAW_STRING_CONTENT]) { // we're in error recovery return false; } // No skipping leading whitespace: raw-string grammar is space-sensitive. - if (valid_symbols[RAW_STRING_DELIMITER]) { - lexer->result_symbol = RAW_STRING_DELIMITER; - return scan_raw_string_delimiter(scanner, lexer); + if (valid_symbols[GNU_RAW_STRING_DELIMITER]) { + lexer->result_symbol = GNU_RAW_STRING_DELIMITER; + return scan_gnu_raw_string_delimiter(scanner, lexer); } - if (valid_symbols[RAW_STRING_CONTENT]) { - lexer->result_symbol = RAW_STRING_CONTENT; - return scan_raw_string_content(scanner, lexer); + if (valid_symbols[GNU_RAW_STRING_CONTENT]) { + lexer->result_symbol = GNU_RAW_STRING_CONTENT; + return scan_gnu_raw_string_content(scanner, lexer); } return false; diff --git a/test/corpus/expressions.txt b/test/corpus/expressions.txt index b3e74b6..6beb4b0 100644 --- a/test/corpus/expressions.txt +++ b/test/corpus/expressions.txt @@ -1519,58 +1519,58 @@ const char *s6 = LR"FOO( (init_declarator (pointer_declarator (identifier)) - (raw_string_literal - (raw_string_content)))) + (gnu_raw_string_literal + (gnu_raw_string_content)))) (declaration (type_qualifier) (primitive_type) (init_declarator (pointer_declarator (identifier)) - (raw_string_literal - (raw_string_delimiter) - (raw_string_content) - (raw_string_delimiter)))) + (gnu_raw_string_literal + (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)))) (declaration (type_qualifier) (primitive_type) (init_declarator (pointer_declarator (identifier)) - (raw_string_literal - (raw_string_delimiter) - (raw_string_content) - (raw_string_delimiter)))) + (gnu_raw_string_literal + (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)))) (declaration (type_qualifier) (primitive_type) (init_declarator (pointer_declarator (identifier)) - (raw_string_literal - (raw_string_delimiter) - (raw_string_content) - (raw_string_delimiter)))) + (gnu_raw_string_literal + (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)))) (declaration (type_qualifier) (primitive_type) (init_declarator (pointer_declarator (identifier)) - (raw_string_literal - (raw_string_delimiter) - (raw_string_content) - (raw_string_delimiter)))) + (gnu_raw_string_literal + (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)))) (declaration (type_qualifier) (primitive_type) (init_declarator (pointer_declarator (identifier)) - (raw_string_literal - (raw_string_delimiter) - (raw_string_content) - (raw_string_delimiter))))) + (gnu_raw_string_literal + (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter))))) ================================================================================ Concatenated raw strings @@ -1597,24 +1597,24 @@ hello)RAW" L"world"); function: (identifier) arguments: (argument_list (concatenated_string - (raw_string_literal - delimiter: (raw_string_delimiter) - (raw_string_content) - (raw_string_delimiter)) + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)) (identifier) - (raw_string_literal - delimiter: (raw_string_delimiter) - (raw_string_content) - (raw_string_delimiter)))))) + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)))))) (expression_statement (call_expression function: (identifier) arguments: (argument_list (concatenated_string - (raw_string_literal - delimiter: (raw_string_delimiter) - (raw_string_content) - (raw_string_delimiter)) + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)) (identifier))))) (expression_statement (call_expression @@ -1623,19 +1623,19 @@ hello)RAW" L"world"); (concatenated_string (string_literal (string_content)) - (raw_string_literal - delimiter: (raw_string_delimiter) - (raw_string_content) - (raw_string_delimiter)))))) + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)))))) (expression_statement (call_expression function: (identifier) arguments: (argument_list (concatenated_string - (raw_string_literal - delimiter: (raw_string_delimiter) - (raw_string_content) - (raw_string_delimiter)) + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)) (string_literal (string_content)) (identifier))))) @@ -1645,10 +1645,10 @@ hello)RAW" L"world"); arguments: (argument_list (concatenated_string (identifier) - (raw_string_literal - delimiter: (raw_string_delimiter) - (raw_string_content) - (raw_string_delimiter)) + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)) (identifier))))) (expression_statement (call_expression @@ -1656,10 +1656,10 @@ hello)RAW" L"world"); arguments: (argument_list (concatenated_string (identifier) - (raw_string_literal - delimiter: (raw_string_delimiter) - (raw_string_content) - (raw_string_delimiter)))))) + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)))))) (expression_statement (call_expression function: (identifier) @@ -1667,9 +1667,9 @@ hello)RAW" L"world"); (concatenated_string (string_literal (string_content)) - (raw_string_literal - delimiter: (raw_string_delimiter) - (raw_string_content) - (raw_string_delimiter)) + (gnu_raw_string_literal + delimiter: (gnu_raw_string_delimiter) + (gnu_raw_string_content) + (gnu_raw_string_delimiter)) (string_literal (string_content)))))))