// This file is part of GCC. // GCC is free software; you can redistribute it and/or modify it under // the terms of the GNU General Public License as published by the Free // Software Foundation; either version 3, or (at your option) any later // version. // GCC is distributed in the hope that it will be useful, but WITHOUT ANY // WARRANTY; without even the implied warranty of MERCHANTABILITY or // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License // for more details. // You should have received a copy of the GNU General Public License // along with GCC; see the file COPYING3. If not see // . #include "rust-lex.h" #include "rust-token-converter.h" #include "bi-map.h" #include "line-map.h" #include namespace Rust { static const BiMap suffixes = {{{CORETYPE_F32, "f32"}, {CORETYPE_F64, "f64"}, {CORETYPE_U8, "u8"}, {CORETYPE_U16, "u16"}, {CORETYPE_U32, "u32"}, {CORETYPE_U64, "u64"}, {CORETYPE_U128, "u128"}, {CORETYPE_I8, "i8"}, {CORETYPE_I16, "i16"}, {CORETYPE_I32, "i32"}, {CORETYPE_I64, "i64"}, {CORETYPE_I128, "i128"}, {CORETYPE_ISIZE, "isize"}, {CORETYPE_USIZE, "usize"}}}; static void pop_group (std::vector &streams, ProcMacro::Delimiter delim) { auto g = ProcMacro::Group::make_group (streams.back (), delim); streams.pop_back (); auto tt = ProcMacro::TokenTree::make_tokentree (g); streams.back ().push (tt); } static ProcMacro::Span convert (location_t location) { return ProcMacro::Span::make_span (location, 0); } static location_t convert (ProcMacro::Span span) { return span.start; } static ProcMacro::Literal handle_suffix (const const_TokenPtr &token, ProcMacro::LitKind kind) { auto str = token->as_string (); auto lookup = suffixes.lookup (token->get_type_hint ()); auto suffix = suffixes.is_iter_ok (lookup) ? lookup->second : ""; return ProcMacro::Literal::make_literal (kind, convert (token->get_locus ()), str, suffix); } ProcMacro::Literal convert_literal (const_TokenPtr lit) { auto loc = convert (lit->get_locus ()); switch (lit->get_id ()) { case FLOAT_LITERAL: return handle_suffix (lit, ProcMacro::LitKind::make_float ()); case INT_LITERAL: return handle_suffix (lit, ProcMacro::LitKind::make_integer ()); case CHAR_LITERAL: return ProcMacro::Literal::make_literal (ProcMacro::LitKind::make_char (), loc, lit->as_string ()); case STRING_LITERAL: return ProcMacro::Literal::make_literal (ProcMacro::LitKind::make_str (), loc, lit->as_string ()); case BYTE_CHAR_LITERAL: return ProcMacro::Literal::make_literal (ProcMacro::LitKind::make_byte (), loc, lit->as_string ()); case BYTE_STRING_LITERAL: return ProcMacro::Literal::make_literal ( ProcMacro::LitKind::make_byte_str (), loc, lit->as_string ()); default: rust_unreachable (); } } ProcMacro::TokenStream convert (const std::vector &tokens) { std::vector trees; trees.push_back (ProcMacro::TokenStream::make_tokenstream ()); for (auto &token : tokens) { auto loc = convert (token->get_locus ()); switch (token->get_id ()) { // Literals case FLOAT_LITERAL: case INT_LITERAL: case CHAR_LITERAL: case STRING_LITERAL: case BYTE_CHAR_LITERAL: case BYTE_STRING_LITERAL: trees.back ().push ( ProcMacro::TokenTree::make_tokentree (convert_literal (token))); break; // Ident case IDENTIFIER: case ABSTRACT: case AS: case ASYNC: case AUTO: case BECOME: case BOX: case BREAK: case CONST: case CONTINUE: case CRATE: case DO: case DYN: case ELSE: case ENUM_KW: case EXTERN_KW: case FINAL_KW: case FN_KW: case FOR: case IF: case IMPL: case IN: case LET: case LOOP: case MACRO: case MATCH_KW: case MOD: case MOVE: case MUT: case OVERRIDE_KW: case PRIV: case PUB: case REF: case RETURN_KW: case SELF_ALIAS: case SELF: case STATIC_KW: case STRUCT_KW: case SUPER: case TRAIT: case TRY: case TYPE: case TYPEOF: case UNSAFE: case UNSIZED: case USE: case VIRTUAL: case WHERE: case WHILE: case YIELD: // Underscore is not a Punct, considered as an Ident case UNDERSCORE: // True and false are idents, not literals // (https://doc.rust-lang.org/proc_macro/struct.Literal.html) case FALSE_LITERAL: case TRUE_LITERAL: trees.back ().push (ProcMacro::TokenTree::make_tokentree ( ProcMacro::Ident::make_ident (token->as_string (), loc))); break; // Joint punct case OR: case PIPE_EQ: case CARET_EQ: case RIGHT_SHIFT_EQ: case RIGHT_SHIFT: case GREATER_OR_EQUAL: case MATCH_ARROW: case LESS_OR_EQUAL: case LEFT_SHIFT_EQ: case LEFT_SHIFT: case DIV_EQ: case ELLIPSIS: case DOT_DOT_EQ: case DOT_DOT: case RETURN_TYPE: case MINUS_EQ: case PLUS_EQ: case ASTERISK_EQ: case LOGICAL_AND: case AMP_EQ: case PERCENT_EQ: case SCOPE_RESOLUTION: case NOT_EQUAL: case EQUAL_EQUAL: { auto str = token->as_string (); auto it = str.cbegin (); for (; it != str.cend () - 1; it++) trees.back ().push (ProcMacro::TokenTree::make_tokentree ( ProcMacro::Punct::make_punct (*it, loc, ProcMacro::JOINT))); trees.back ().push (ProcMacro::TokenTree::make_tokentree ( ProcMacro::Punct::make_punct (*it, loc, ProcMacro::ALONE))); } break; // Alone punct tokens case EQUAL: case RIGHT_ANGLE: case LEFT_ANGLE: case EXCLAM: case TILDE: case PLUS: case MINUS: case ASTERISK: case DIV: case PERCENT: case CARET: case AMP: case PIPE: case PATTERN_BIND: case DOT: case COMMA: case SEMICOLON: case COLON: case HASH: case DOLLAR_SIGN: case QUESTION_MARK: case SINGLE_QUOTE: trees.back ().push (ProcMacro::TokenTree::make_tokentree ( ProcMacro::Punct::make_punct (token->as_string ()[0], loc, ProcMacro::ALONE))); break; case RIGHT_PAREN: pop_group (trees, ProcMacro::PARENTHESIS); break; case RIGHT_CURLY: pop_group (trees, ProcMacro::BRACE); break; case RIGHT_SQUARE: pop_group (trees, ProcMacro::BRACKET); break; case LEFT_SQUARE: case LEFT_CURLY: case LEFT_PAREN: trees.push_back (ProcMacro::TokenStream::make_tokenstream ()); break; default: rust_unreachable (); } } return trees.back (); } static void from_tokenstream (const ProcMacro::TokenStream &ts, std::vector &result); /** * Append the token corresponding to a given Ident to a vector. * * @param literal Reference to the Ident to convert. * @param result Reference to the output vector. */ static void from_ident (const ProcMacro::Ident &ident, std::vector &result) { std::string value (ident.value.to_string ()); if (ident.is_raw) value = "r#" + value; Lexer lexer (value, nullptr); auto token = lexer.build_token (); token->set_locus (convert (ident.span)); result.push_back (token); } /** * Append the token corresponding to a given Literal to a vector. * * @param literal Reference to the Literal to convert. * @param result Reference to the vector tokens should be appended to. */ static void from_literal (const ProcMacro::Literal &literal, std::vector &result) { auto lookup = suffixes.lookup (literal.suffix.to_string ()); auto loc = convert (literal.span); auto suffix = suffixes.is_iter_ok (lookup) ? lookup->second : CORETYPE_UNKNOWN; // FIXME: Add spans instead of empty locations switch (literal.kind.tag) { case ProcMacro::BYTE: result.push_back ( Token::make_byte_char (loc, literal.text.to_string ()[0])); break; case ProcMacro::CHAR: result.push_back (Token::make_char (loc, literal.text.to_string ()[0])); break; case ProcMacro::INTEGER: result.push_back ( Token::make_int (loc, literal.text.to_string (), suffix)); break; case ProcMacro::FLOAT: result.push_back ( Token::make_float (loc, literal.text.to_string (), suffix)); break; case ProcMacro::STR: result.push_back (Token::make_string (loc, literal.text.to_string ())); break; case ProcMacro::BYTE_STR: result.push_back ( Token::make_byte_string (loc, literal.text.to_string ())); break; // FIXME: Handle raw string case ProcMacro::STR_RAW: case ProcMacro::BYTE_STR_RAW: default: rust_unreachable (); } } /** * Accumulate through successive calls multiple Punct until one is tagged * "Alone", then append the formed token to a given result vector. * * @param punct Reference to the Punct to convert. * @param acc Reference to an accumulator for joined Punct. * @param result Reference to the output token vector. */ static void from_punct (const ProcMacro::Punct &punct, std::vector &acc, std::vector &result) { acc.push_back (punct.ch); if (ProcMacro::ALONE == punct.spacing) /* Last punct of a chain */ { // TODO: UTF-8 string std::string whole (acc.begin (), acc.end ()); auto lexer = Lexer (whole, nullptr); auto token = lexer.build_token (); token->set_locus (convert (punct.span)); result.push_back (token); acc.clear (); } } /** * Iterate over a Group and append all inner tokens to a vector enclosed by its * delimiters. * * @param g Reference to the Group to convert. * @param result Reference to the vector tokens should be appended to. */ static void from_group (const ProcMacro::Group &g, std::vector &result) { auto loc = convert (g.span); switch (g.delimiter) { case ProcMacro::PARENTHESIS: result.push_back (Token::make (LEFT_PAREN, loc)); from_tokenstream (g.stream, result); result.push_back (Token::make (RIGHT_PAREN, loc)); break; case ProcMacro::BRACE: result.push_back (Token::make (LEFT_CURLY, loc)); from_tokenstream (g.stream, result); result.push_back (Token::make (RIGHT_CURLY, loc)); break; case ProcMacro::BRACKET: result.push_back (Token::make (LEFT_SQUARE, loc)); from_tokenstream (g.stream, result); result.push_back (Token::make (RIGHT_SQUARE, loc)); break; case ProcMacro::NONE: from_tokenstream (g.stream, result); break; default: rust_unreachable (); } } /** * Dispatch TokenTree's conversion to its inner type depending on its tag. * * @param tt Reference to the TokenTree. * @param punct_accumulator Reference to an accumulator for joined Punct. * @param result Reference to the vector tokens should be appended to. */ static void from_tokentree (const ProcMacro::TokenTree &tt, std::vector &punct_accumulator, std::vector &result) { switch (tt.tag) { case ProcMacro::GROUP: from_group (tt.payload.group, result); break; case ProcMacro::IDENT: from_ident (tt.payload.ident, result); break; case ProcMacro::PUNCT: from_punct (tt.payload.punct, punct_accumulator, result); break; case ProcMacro::LITERAL: from_literal (tt.payload.literal, result); break; default: rust_unreachable (); } } /** * Iterate over a TokenStream and append all inner tokens to a vector. * * @param ts Reference to the TokenStream. * @param result Reference to the vector tokens should be appended to. */ static void from_tokenstream (const ProcMacro::TokenStream &ts, std::vector &result) { std::vector punct_accumulator; for (std::uint64_t i = 0; i < ts.size; i++) { from_tokentree (ts.data[i], punct_accumulator, result); } } std::vector convert (const ProcMacro::TokenStream &ts) { std::vector result; from_tokenstream (ts, result); return result; } } // namespace Rust