1 // Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
4 //
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
10
11 use ast;
12 use ast::{P, Ident, Name, Mrk};
13 use ast_util;
14 use ext::mtwt;
15 use parse::token;
16 use util::interner::{RcStr, StrInterner};
17 use util::interner;
18
19 use serialize::{Decodable, Decoder, Encodable, Encoder};
20 use std::cast;
21 use std::fmt;
22 use std::path::BytesContainer;
23 use std::rc::Rc;
24 use std::strbuf::StrBuf;
25
26 #[allow(non_camel_case_types)]
27 #[deriving(Clone, Encodable, Decodable, Eq, TotalEq, Hash, Show)]
28 pub enum BinOp {
29 PLUS,
30 MINUS,
31 STAR,
32 SLASH,
33 PERCENT,
34 CARET,
35 AND,
36 OR,
37 SHL,
38 SHR,
39 }
40
41 #[allow(non_camel_case_types)]
42 #[deriving(Clone, Encodable, Decodable, Eq, TotalEq, Hash, Show)]
43 pub enum Token {
44 /* Expression-operator symbols. */
45 EQ,
46 LT,
47 LE,
48 EQEQ,
49 NE,
50 GE,
51 GT,
52 ANDAND,
53 OROR,
54 NOT,
55 TILDE,
56 BINOP(BinOp),
57 BINOPEQ(BinOp),
58
59 /* Structural symbols */
60 AT,
61 DOT,
62 DOTDOT,
63 DOTDOTDOT,
64 COMMA,
65 SEMI,
66 COLON,
67 MOD_SEP,
68 RARROW,
69 LARROW,
70 DARROW,
71 FAT_ARROW,
72 LPAREN,
73 RPAREN,
74 LBRACKET,
75 RBRACKET,
76 LBRACE,
77 RBRACE,
78 POUND,
79 DOLLAR,
80
81 /* Literals */
82 LIT_CHAR(char),
83 LIT_INT(i64, ast::IntTy),
84 LIT_UINT(u64, ast::UintTy),
85 LIT_INT_UNSUFFIXED(i64),
86 LIT_FLOAT(ast::Ident, ast::FloatTy),
87 LIT_FLOAT_UNSUFFIXED(ast::Ident),
88 LIT_STR(ast::Ident),
89 LIT_STR_RAW(ast::Ident, uint), /* raw str delimited by n hash symbols */
90
91 /* Name components */
92 // an identifier contains an "is_mod_name" boolean,
93 // indicating whether :: follows this token with no
94 // whitespace in between.
95 IDENT(ast::Ident, bool),
96 UNDERSCORE,
97 LIFETIME(ast::Ident),
98
99 /* For interpolation */
100 INTERPOLATED(Nonterminal),
101
102 DOC_COMMENT(ast::Ident),
103 EOF,
104 }
105
106 #[deriving(Clone, Encodable, Decodable, Eq, TotalEq, Hash)]
107 /// For interpolation during macro expansion.
108 pub enum Nonterminal {
109 NtItem(@ast::Item),
110 NtBlock(P<ast::Block>),
111 NtStmt(@ast::Stmt),
112 NtPat( @ast::Pat),
113 NtExpr(@ast::Expr),
114 NtTy( P<ast::Ty>),
115 NtIdent(Box<ast::Ident>, bool),
116 NtMeta(@ast::MetaItem), // stuff inside brackets for attributes
117 NtPath(Box<ast::Path>),
118 NtTT( @ast::TokenTree), // needs @ed to break a circularity
119 NtMatchers(Vec<ast::Matcher> )
120 }
121
122 impl fmt::Show for Nonterminal {
123 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
124 match *self {
125 NtItem(..) => f.pad("NtItem(..)"),
126 NtBlock(..) => f.pad("NtBlock(..)"),
127 NtStmt(..) => f.pad("NtStmt(..)"),
128 NtPat(..) => f.pad("NtPat(..)"),
129 NtExpr(..) => f.pad("NtExpr(..)"),
130 NtTy(..) => f.pad("NtTy(..)"),
131 NtIdent(..) => f.pad("NtIdent(..)"),
132 NtMeta(..) => f.pad("NtMeta(..)"),
133 NtPath(..) => f.pad("NtPath(..)"),
134 NtTT(..) => f.pad("NtTT(..)"),
135 NtMatchers(..) => f.pad("NtMatchers(..)"),
136 }
137 }
138 }
139
140 pub fn binop_to_str(o: BinOp) -> StrBuf {
141 match o {
142 PLUS => "+".to_strbuf(),
143 MINUS => "-".to_strbuf(),
144 STAR => "*".to_strbuf(),
145 SLASH => "/".to_strbuf(),
146 PERCENT => "%".to_strbuf(),
147 CARET => "^".to_strbuf(),
148 AND => "&".to_strbuf(),
149 OR => "|".to_strbuf(),
150 SHL => "<<".to_strbuf(),
151 SHR => ">>".to_strbuf()
152 }
153 }
154
155 pub fn to_str(t: &Token) -> StrBuf {
156 match *t {
157 EQ => "=".to_strbuf(),
158 LT => "<".to_strbuf(),
159 LE => "<=".to_strbuf(),
160 EQEQ => "==".to_strbuf(),
161 NE => "!=".to_strbuf(),
162 GE => ">=".to_strbuf(),
163 GT => ">".to_strbuf(),
164 NOT => "!".to_strbuf(),
165 TILDE => "~".to_strbuf(),
166 OROR => "||".to_strbuf(),
167 ANDAND => "&&".to_strbuf(),
168 BINOP(op) => binop_to_str(op),
169 BINOPEQ(op) => {
170 let mut s = binop_to_str(op);
171 s.push_str("=");
172 s
173 }
174
175 /* Structural symbols */
176 AT => "@".to_strbuf(),
177 DOT => ".".to_strbuf(),
178 DOTDOT => "..".to_strbuf(),
179 DOTDOTDOT => "...".to_strbuf(),
180 COMMA => ",".to_strbuf(),
181 SEMI => ";".to_strbuf(),
182 COLON => ":".to_strbuf(),
183 MOD_SEP => "::".to_strbuf(),
184 RARROW => "->".to_strbuf(),
185 LARROW => "<-".to_strbuf(),
186 DARROW => "<->".to_strbuf(),
187 FAT_ARROW => "=>".to_strbuf(),
188 LPAREN => "(".to_strbuf(),
189 RPAREN => ")".to_strbuf(),
190 LBRACKET => "[".to_strbuf(),
191 RBRACKET => "]".to_strbuf(),
192 LBRACE => "{".to_strbuf(),
193 RBRACE => "}".to_strbuf(),
194 POUND => "#".to_strbuf(),
195 DOLLAR => "$".to_strbuf(),
196
197 /* Literals */
198 LIT_CHAR(c) => {
199 let mut res = StrBuf::from_str("'");
200 c.escape_default(|c| {
201 res.push_char(c);
202 });
203 res.push_char('\'');
204 res
205 }
206 LIT_INT(i, t) => ast_util::int_ty_to_str(t, Some(i)),
207 LIT_UINT(u, t) => ast_util::uint_ty_to_str(t, Some(u)),
208 LIT_INT_UNSUFFIXED(i) => { i.to_str().to_strbuf() }
209 LIT_FLOAT(s, t) => {
210 let mut body = StrBuf::from_str(get_ident(s).get());
211 if body.as_slice().ends_with(".") {
212 body.push_char('0'); // `10.f` is not a float literal
213 }
214 body.push_str(ast_util::float_ty_to_str(t).as_slice());
215 body
216 }
217 LIT_FLOAT_UNSUFFIXED(s) => {
218 let mut body = StrBuf::from_str(get_ident(s).get());
219 if body.as_slice().ends_with(".") {
220 body.push_char('0'); // `10.f` is not a float literal
221 }
222 body
223 }
224 LIT_STR(s) => {
225 (format!("\"{}\"", get_ident(s).get().escape_default())).to_strbuf()
226 }
227 LIT_STR_RAW(s, n) => {
228 (format!("r{delim}\"{string}\"{delim}",
229 delim="#".repeat(n), string=get_ident(s))).to_strbuf()
230 }
231
232 /* Name components */
233 IDENT(s, _) => get_ident(s).get().to_strbuf(),
234 LIFETIME(s) => {
235 (format!("'{}", get_ident(s))).to_strbuf()
236 }
237 UNDERSCORE => "_".to_strbuf(),
238
239 /* Other */
240 DOC_COMMENT(s) => get_ident(s).get().to_strbuf(),
241 EOF => "<eof>".to_strbuf(),
242 INTERPOLATED(ref nt) => {
243 match nt {
244 &NtExpr(e) => ::print::pprust::expr_to_str(e),
245 &NtMeta(e) => ::print::pprust::meta_item_to_str(e),
246 _ => {
247 let mut s = "an interpolated ".to_strbuf();
248 match *nt {
249 NtItem(..) => s.push_str("item"),
250 NtBlock(..) => s.push_str("block"),
251 NtStmt(..) => s.push_str("statement"),
252 NtPat(..) => s.push_str("pattern"),
253 NtMeta(..) => fail!("should have been handled"),
254 NtExpr(..) => fail!("should have been handled above"),
255 NtTy(..) => s.push_str("type"),
256 NtIdent(..) => s.push_str("identifier"),
257 NtPath(..) => s.push_str("path"),
258 NtTT(..) => s.push_str("tt"),
259 NtMatchers(..) => s.push_str("matcher sequence")
260 };
261 s
262 }
263 }
264 }
265 }
266 }
267
268 pub fn can_begin_expr(t: &Token) -> bool {
269 match *t {
270 LPAREN => true,
271 LBRACE => true,
272 LBRACKET => true,
273 IDENT(_, _) => true,
274 UNDERSCORE => true,
275 TILDE => true,
276 LIT_CHAR(_) => true,
277 LIT_INT(_, _) => true,
278 LIT_UINT(_, _) => true,
279 LIT_INT_UNSUFFIXED(_) => true,
280 LIT_FLOAT(_, _) => true,
281 LIT_FLOAT_UNSUFFIXED(_) => true,
282 LIT_STR(_) => true,
283 LIT_STR_RAW(_, _) => true,
284 POUND => true,
285 AT => true,
286 NOT => true,
287 BINOP(MINUS) => true,
288 BINOP(STAR) => true,
289 BINOP(AND) => true,
290 BINOP(OR) => true, // in lambda syntax
291 OROR => true, // in lambda syntax
292 MOD_SEP => true,
293 INTERPOLATED(NtExpr(..))
294 | INTERPOLATED(NtIdent(..))
295 | INTERPOLATED(NtBlock(..))
296 | INTERPOLATED(NtPath(..)) => true,
297 _ => false
298 }
299 }
300
301 /// Returns the matching close delimiter if this is an open delimiter,
302 /// otherwise `None`.
303 pub fn close_delimiter_for(t: &Token) -> Option<Token> {
304 match *t {
305 LPAREN => Some(RPAREN),
306 LBRACE => Some(RBRACE),
307 LBRACKET => Some(RBRACKET),
308 _ => None
309 }
310 }
311
312 pub fn is_lit(t: &Token) -> bool {
313 match *t {
314 LIT_CHAR(_) => true,
315 LIT_INT(_, _) => true,
316 LIT_UINT(_, _) => true,
317 LIT_INT_UNSUFFIXED(_) => true,
318 LIT_FLOAT(_, _) => true,
319 LIT_FLOAT_UNSUFFIXED(_) => true,
320 LIT_STR(_) => true,
321 LIT_STR_RAW(_, _) => true,
322 _ => false
323 }
324 }
325
326 pub fn is_ident(t: &Token) -> bool {
327 match *t { IDENT(_, _) => true, _ => false }
328 }
329
330 pub fn is_ident_or_path(t: &Token) -> bool {
331 match *t {
332 IDENT(_, _) | INTERPOLATED(NtPath(..)) => true,
333 _ => false
334 }
335 }
336
337 pub fn is_plain_ident(t: &Token) -> bool {
338 match *t { IDENT(_, false) => true, _ => false }
339 }
340
341 pub fn is_bar(t: &Token) -> bool {
342 match *t { BINOP(OR) | OROR => true, _ => false }
343 }
344
345 // Get the first "argument"
346 macro_rules! first {
347 ( $first:expr, $( $remainder:expr, )* ) => ( $first )
348 }
349
350 // Get the last "argument" (has to be done recursively to avoid phoney local ambiguity error)
351 macro_rules! last {
352 ( $first:expr, $( $remainder:expr, )+ ) => ( last!( $( $remainder, )+ ) );
353 ( $first:expr, ) => ( $first )
354 }
355
356 // In this macro, there is the requirement that the name (the number) must be monotonically
357 // increasing by one in the special identifiers, starting at 0; the same holds for the keywords,
358 // except starting from the next number instead of zero, and with the additional exception that
359 // special identifiers are *also* allowed (they are deduplicated in the important place, the
360 // interner), an exception which is demonstrated by "static" and "self".
361 macro_rules! declare_special_idents_and_keywords {(
362 // So now, in these rules, why is each definition parenthesised?
363 // Answer: otherwise we get a spurious local ambiguity bug on the "}"
364 pub mod special_idents {
365 $( ($si_name:expr, $si_static:ident, $si_str:expr); )*
366 }
367
368 pub mod keywords {
369 'strict:
370 $( ($sk_name:expr, $sk_variant:ident, $sk_str:expr); )*
371 'reserved:
372 $( ($rk_name:expr, $rk_variant:ident, $rk_str:expr); )*
373 }
374 ) => {
375 static STRICT_KEYWORD_START: Name = first!($( $sk_name, )*);
376 static STRICT_KEYWORD_FINAL: Name = last!($( $sk_name, )*);
377 static RESERVED_KEYWORD_START: Name = first!($( $rk_name, )*);
378 static RESERVED_KEYWORD_FINAL: Name = last!($( $rk_name, )*);
379
380 pub mod special_idents {
381 use ast::Ident;
382 $( pub static $si_static: Ident = Ident { name: $si_name, ctxt: 0 }; )*
383 }
384
385 /**
386 * All the valid words that have meaning in the Rust language.
387 *
388 * Rust keywords are either 'strict' or 'reserved'. Strict keywords may not
389 * appear as identifiers at all. Reserved keywords are not used anywhere in
390 * the language and may not appear as identifiers.
391 */
392 pub mod keywords {
393 use ast::Ident;
394
395 pub enum Keyword {
396 $( $sk_variant, )*
397 $( $rk_variant, )*
398 }
399
400 impl Keyword {
401 pub fn to_ident(&self) -> Ident {
402 match *self {
403 $( $sk_variant => Ident { name: $sk_name, ctxt: 0 }, )*
404 $( $rk_variant => Ident { name: $rk_name, ctxt: 0 }, )*
405 }
406 }
407 }
408 }
409
410 fn mk_fresh_ident_interner() -> IdentInterner {
411 // The indices here must correspond to the numbers in
412 // special_idents, in Keyword to_ident(), and in static
413 // constants below.
414 let mut init_vec = Vec::new();
415 $(init_vec.push($si_str);)*
416 $(init_vec.push($sk_str);)*
417 $(init_vec.push($rk_str);)*
418 interner::StrInterner::prefill(init_vec.as_slice())
419 }
420 }}
421
422 // If the special idents get renumbered, remember to modify these two as appropriate
423 static SELF_KEYWORD_NAME: Name = 1;
424 static STATIC_KEYWORD_NAME: Name = 2;
425
426 declare_special_idents_and_keywords! {
427 pub mod special_idents {
428 // These ones are statics
429 (0, invalid, "");
430 (super::SELF_KEYWORD_NAME, self_, "self");
431 (super::STATIC_KEYWORD_NAME, statik, "static");
432
433 // for matcher NTs
434 (3, tt, "tt");
435 (4, matchers, "matchers");
436
437 // outside of libsyntax
438 (5, clownshoe_abi, "__rust_abi");
439 (6, opaque, "<opaque>");
440 (7, unnamed_field, "<unnamed_field>");
441 (8, type_self, "Self");
442 }
443
444 pub mod keywords {
445 // These ones are variants of the Keyword enum
446
447 'strict:
448 (9, As, "as");
449 (10, Break, "break");
450 (11, Const, "const");
451 (12, Crate, "crate");
452 (13, Else, "else");
453 (14, Enum, "enum");
454 (15, Extern, "extern");
455 (16, False, "false");
456 (17, Fn, "fn");
457 (18, For, "for");
458 (19, If, "if");
459 (20, Impl, "impl");
460 (21, In, "in");
461 (22, Let, "let");
462 (23, Loop, "loop");
463 (24, Match, "match");
464 (25, Mod, "mod");
465 (26, Mut, "mut");
466 (27, Once, "once");
467 (28, Pub, "pub");
468 (29, Ref, "ref");
469 (30, Return, "return");
470 // Static and Self are also special idents (prefill de-dupes)
471 (super::STATIC_KEYWORD_NAME, Static, "static");
472 (super::SELF_KEYWORD_NAME, Self, "self");
473 (31, Struct, "struct");
474 (32, Super, "super");
475 (33, True, "true");
476 (34, Trait, "trait");
477 (35, Type, "type");
478 (36, Unsafe, "unsafe");
479 (37, Use, "use");
480 (38, Virtual, "virtual");
481 (39, While, "while");
482 (40, Continue, "continue");
483 (41, Proc, "proc");
484 (42, Box, "box");
485
486 'reserved:
487 (43, Alignof, "alignof");
488 (44, Be, "be");
489 (45, Offsetof, "offsetof");
490 (46, Priv, "priv");
491 (47, Pure, "pure");
492 (48, Sizeof, "sizeof");
493 (49, Typeof, "typeof");
494 (50, Unsized, "unsized");
495 (51, Yield, "yield");
496 (52, Do, "do");
497 }
498 }
499
500 /**
501 * Maps a token to a record specifying the corresponding binary
502 * operator
503 */
504 pub fn token_to_binop(tok: &Token) -> Option<ast::BinOp> {
505 match *tok {
506 BINOP(STAR) => Some(ast::BiMul),
507 BINOP(SLASH) => Some(ast::BiDiv),
508 BINOP(PERCENT) => Some(ast::BiRem),
509 BINOP(PLUS) => Some(ast::BiAdd),
510 BINOP(MINUS) => Some(ast::BiSub),
511 BINOP(SHL) => Some(ast::BiShl),
512 BINOP(SHR) => Some(ast::BiShr),
513 BINOP(AND) => Some(ast::BiBitAnd),
514 BINOP(CARET) => Some(ast::BiBitXor),
515 BINOP(OR) => Some(ast::BiBitOr),
516 LT => Some(ast::BiLt),
517 LE => Some(ast::BiLe),
518 GE => Some(ast::BiGe),
519 GT => Some(ast::BiGt),
520 EQEQ => Some(ast::BiEq),
521 NE => Some(ast::BiNe),
522 ANDAND => Some(ast::BiAnd),
523 OROR => Some(ast::BiOr),
524 _ => None
525 }
526 }
527
528 // looks like we can get rid of this completely...
529 pub type IdentInterner = StrInterner;
530
531 // if an interner exists in TLS, return it. Otherwise, prepare a
532 // fresh one.
533 // FIXME(eddyb) #8726 This should probably use a task-local reference.
534 pub fn get_ident_interner() -> Rc<IdentInterner> {
535 local_data_key!(key: Rc<::parse::token::IdentInterner>)
536 match key.get() {
537 Some(interner) => interner.clone(),
538 None => {
539 let interner = Rc::new(mk_fresh_ident_interner());
540 key.replace(Some(interner.clone()));
541 interner
542 }
543 }
544 }
545
546 /// Represents a string stored in the task-local interner. Because the
547 /// interner lives for the life of the task, this can be safely treated as an
548 /// immortal string, as long as it never crosses between tasks.
549 ///
550 /// FIXME(pcwalton): You must be careful about what you do in the destructors
551 /// of objects stored in TLS, because they may run after the interner is
552 /// destroyed. In particular, they must not access string contents. This can
553 /// be fixed in the future by just leaking all strings until task death
554 /// somehow.
555 #[deriving(Clone, Eq, Hash, Ord, TotalEq, TotalOrd)]
556 pub struct InternedString {
557 string: RcStr,
558 }
559
560 impl InternedString {
561 #[inline]
562 pub fn new(string: &'static str) -> InternedString {
563 InternedString {
564 string: RcStr::new(string),
565 }
566 }
567
568 #[inline]
569 fn new_from_rc_str(string: RcStr) -> InternedString {
570 InternedString {
571 string: string,
572 }
573 }
574
575 #[inline]
576 pub fn get<'a>(&'a self) -> &'a str {
577 self.string.as_slice()
578 }
579 }
580
581 impl BytesContainer for InternedString {
582 fn container_as_bytes<'a>(&'a self) -> &'a [u8] {
583 // FIXME(pcwalton): This is a workaround for the incorrect signature
584 // of `BytesContainer`, which is itself a workaround for the lack of
585 // DST.
586 unsafe {
587 let this = self.get();
588 cast::transmute(this.container_as_bytes())
589 }
590 }
591 }
592
593 impl fmt::Show for InternedString {
594 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
595 write!(f.buf, "{}", self.string.as_slice())
596 }
597 }
598
599 impl<'a> Equiv<&'a str> for InternedString {
600 fn equiv(&self, other: & &'a str) -> bool {
601 (*other) == self.string.as_slice()
602 }
603 }
604
605 impl<D:Decoder<E>, E> Decodable<D, E> for InternedString {
606 fn decode(d: &mut D) -> Result<InternedString, E> {
607 Ok(get_name(get_ident_interner().intern(try!(d.read_str()))))
608 }
609 }
610
611 impl<S:Encoder<E>, E> Encodable<S, E> for InternedString {
612 fn encode(&self, s: &mut S) -> Result<(), E> {
613 s.emit_str(self.string.as_slice())
614 }
615 }
616
617 /// Returns the string contents of a name, using the task-local interner.
618 #[inline]
619 pub fn get_name(name: Name) -> InternedString {
620 let interner = get_ident_interner();
621 InternedString::new_from_rc_str(interner.get(name))
622 }
623
624 /// Returns the string contents of an identifier, using the task-local
625 /// interner.
626 #[inline]
627 pub fn get_ident(ident: Ident) -> InternedString {
628 get_name(ident.name)
629 }
630
631 /// Interns and returns the string contents of an identifier, using the
632 /// task-local interner.
633 #[inline]
634 pub fn intern_and_get_ident(s: &str) -> InternedString {
635 get_name(intern(s))
636 }
637
638 /// Maps a string to its interned representation.
639 #[inline]
640 pub fn intern(s: &str) -> Name {
641 get_ident_interner().intern(s)
642 }
643
644 /// gensym's a new uint, using the current interner.
645 #[inline]
646 pub fn gensym(s: &str) -> Name {
647 get_ident_interner().gensym(s)
648 }
649
650 /// Maps a string to an identifier with an empty syntax context.
651 #[inline]
652 pub fn str_to_ident(s: &str) -> ast::Ident {
653 ast::Ident::new(intern(s))
654 }
655
656 /// Maps a string to a gensym'ed identifier.
657 #[inline]
658 pub fn gensym_ident(s: &str) -> ast::Ident {
659 ast::Ident::new(gensym(s))
660 }
661
662 // create a fresh name that maps to the same string as the old one.
663 // note that this guarantees that str_ptr_eq(ident_to_str(src),interner_get(fresh_name(src)));
664 // that is, that the new name and the old one are connected to ptr_eq strings.
665 pub fn fresh_name(src: &ast::Ident) -> Name {
666 let interner = get_ident_interner();
667 interner.gensym_copy(src.name)
668 // following: debug version. Could work in final except that it's incompatible with
669 // good error messages and uses of struct names in ambiguous could-be-binding
670 // locations. Also definitely destroys the guarantee given above about ptr_eq.
671 /*let num = rand::task_rng().gen_uint_range(0,0xffff);
672 gensym(format!("{}_{}",ident_to_str(src),num))*/
673 }
674
675 // create a fresh mark.
676 pub fn fresh_mark() -> Mrk {
677 gensym("mark")
678 }
679
680 // See the macro above about the types of keywords
681
682 pub fn is_keyword(kw: keywords::Keyword, tok: &Token) -> bool {
683 match *tok {
684 token::IDENT(sid, false) => { kw.to_ident().name == sid.name }
685 _ => { false }
686 }
687 }
688
689 pub fn is_any_keyword(tok: &Token) -> bool {
690 match *tok {
691 token::IDENT(sid, false) => match sid.name {
692 SELF_KEYWORD_NAME | STATIC_KEYWORD_NAME |
693 STRICT_KEYWORD_START .. RESERVED_KEYWORD_FINAL => true,
694 _ => false,
695 },
696 _ => false
697 }
698 }
699
700 pub fn is_strict_keyword(tok: &Token) -> bool {
701 match *tok {
702 token::IDENT(sid, false) => match sid.name {
703 SELF_KEYWORD_NAME | STATIC_KEYWORD_NAME |
704 STRICT_KEYWORD_START .. STRICT_KEYWORD_FINAL => true,
705 _ => false,
706 },
707 _ => false,
708 }
709 }
710
711 pub fn is_reserved_keyword(tok: &Token) -> bool {
712 match *tok {
713 token::IDENT(sid, false) => match sid.name {
714 RESERVED_KEYWORD_START .. RESERVED_KEYWORD_FINAL => true,
715 _ => false,
716 },
717 _ => false,
718 }
719 }
720
721 pub fn mtwt_token_eq(t1 : &Token, t2 : &Token) -> bool {
722 match (t1,t2) {
723 (&IDENT(id1,_),&IDENT(id2,_)) | (&LIFETIME(id1),&LIFETIME(id2)) =>
724 mtwt::resolve(id1) == mtwt::resolve(id2),
725 _ => *t1 == *t2
726 }
727 }
728
729
730 #[cfg(test)]
731 mod test {
732 use super::*;
733 use ast;
734 use ext::mtwt;
735
736 fn mark_ident(id : ast::Ident, m : ast::Mrk) -> ast::Ident {
737 ast::Ident{name:id.name,ctxt:mtwt::new_mark(m,id.ctxt)}
738 }
739
740 #[test] fn mtwt_token_eq_test() {
741 assert!(mtwt_token_eq(>,>));
742 let a = str_to_ident("bac");
743 let a1 = mark_ident(a,92);
744 assert!(mtwt_token_eq(&IDENT(a,true),&IDENT(a1,false)));
745 }
746 }
libsyntax/parse/token.rs:42:66-42:66 -enum- definition:
pub enum Token {
/* Expression-operator symbols. */
EQ,
references:- 88libsyntax/parse/common.rs:
libsyntax/ext/quote.rs:
libsyntax/ext/tt/transcribe.rs:
libsyntax/ext/tt/macro_parser.rs:
libsyntax/ast.rs:
libsyntax/fold.rs:
libsyntax/parse/lexer.rs:
libsyntax/parse/parser.rs:
libsyntax/parse/token.rs:639:10-639:10 -fn- definition:
pub fn intern(s: &str) -> Name {
get_ident_interner().intern(s)
}
references:- 33libsyntax/ext/base.rs:
libsyntax/ext/expand.rs:
libsyntax/ext/base.rs:
libsyntax/parse/token.rs:336:1-336:1 -fn- definition:
pub fn is_plain_ident(t: &Token) -> bool {
match *t { IDENT(_, false) => true, _ => false }
}
references:- 7libsyntax/parse/parser.rs:
3012: -> ast::Pat_ {
3013: if !is_plain_ident(&self.token) {
3014: self.span_fatal(self.last_span,
--
3075: let lo = self.span.lo;
3076: if !is_plain_ident(&self.token) {
3077: self.fatal("expected ident");
--
4749: && self.look_ahead(1, |t| *t == token::NOT)
4750: && (self.look_ahead(2, |t| is_plain_ident(t))
4751: || self.look_ahead(2, |t| *t == token::LPAREN)
--
4761: // and remove this.
4762: let id = if is_plain_ident(&self.token) {
4763: self.parse_ident()
libsyntax/parse/token.rs:681:1-681:1 -fn- definition:
pub fn is_keyword(kw: keywords::Keyword, tok: &Token) -> bool {
match *tok {
token::IDENT(sid, false) => { kw.to_ident().name == sid.name }
references:- 24libsyntax/ext/trace_macros.rs:
22: match tt {
23: [ast::TTTok(_, ref tok)] if is_keyword(keywords::True, tok) => {
24: cx.set_trace_macros(true);
25: }
26: [ast::TTTok(_, ref tok)] if is_keyword(keywords::False, tok) => {
27: cx.set_trace_macros(false);
libsyntax/parse/lexer.rs:
829: if token::is_keyword(token::keywords::Self, tok) {
830: fatal_span(rdr, start, rdr.last_pos,
--
833: } else if token::is_any_keyword(tok) &&
834: !token::is_keyword(token::keywords::Static, tok) {
835: fatal_span(rdr, start, rdr.last_pos,
libsyntax/parse/parser.rs:
3581: this.look_ahead(2,
3582: |t| token::is_keyword(keywords::Self,
3583: t)) {
--
3589: this.look_ahead(2,
3590: |t| token::is_keyword(keywords::Self,
3591: t)) {
--
3599: }) &&
3600: this.look_ahead(3, |t| token::is_keyword(keywords::Self,
3601: t)) {
--
3649: _ if Parser::token_is_mutability(&self.token) &&
3650: self.look_ahead(1, |t| token::is_keyword(keywords::Self, t)) => {
3651: mutbl_self = self.parse_mutability();
--
3656: self.look_ahead(1, |t| *t == token::TILDE) &&
3657: self.look_ahead(2, |t| token::is_keyword(keywords::Self, t)) => {
3658: mutbl_self = self.parse_mutability();
libsyntax/parse/token.rs:302:22-302:22 -fn- definition:
/// otherwise `None`.
pub fn close_delimiter_for(t: &Token) -> Option<Token> {
match *t {
references:- 6libsyntax/parse/parser.rs:
3139: // consuming more tokens).
3140: let (bra, ket) = match token::close_delimiter_for(&self.token) {
3141: Some(ket) => (self.token.clone(), ket),
--
4767: // eat a matched-delimiter token tree:
4768: let tts = match token::close_delimiter_for(&self.token) {
4769: Some(ket) => {
libsyntax/parse/token.rs:657:10-657:10 -fn- definition:
pub fn gensym_ident(s: &str) -> ast::Ident {
ast::Ident::new(gensym(s))
}
references:- 4libsyntax/ext/expand.rs:
159: let local_ident = token::gensym_ident("i");
160: let next_ident = fld.cx.ident_of("next");
libsyntax/ext/tt/macro_rules.rs:
205: let lhs_nm = gensym_ident("lhs");
206: let rhs_nm = gensym_ident("rhs");
libsyntax/ast_util.rs:
266: }
267: token::gensym_ident(pretty.as_slice())
268: }
libsyntax/ext/tt/macro_rules.rs:
205: let lhs_nm = gensym_ident("lhs");
206: let rhs_nm = gensym_ident("rhs");
libsyntax/parse/token.rs:154:1-154:1 -fn- definition:
pub fn to_str(t: &Token) -> StrBuf {
match *t {
EQ => "=".to_strbuf(),
references:- 6libsyntax/parse/comments.rs:
404: } else {
405: debug!("tok: {}", token::to_str(&tok));
406: }
libsyntax/print/pprust.rs:
834: ast::TTTok(_, ref tk) => {
835: word(&mut self.s, parse::token::to_str(tk).as_slice())
836: }
--
845: try!(word(&mut self.s,
846: parse::token::to_str(tk).as_slice()));
847: }
libsyntax/ext/tt/macro_parser.rs:
441: _ => {
442: let token_str = token::to_str(&p.token);
443: p.fatal((format!("expected ident, found {}",
libsyntax/parse/parser.rs:
348: pub fn token_to_str(token: &token::Token) -> StrBuf {
349: token::to_str(token)
350: }
libsyntax/parse/token.rs:626:10-626:10 -fn- definition:
pub fn get_ident(ident: Ident) -> InternedString {
get_name(ident.name)
}
references:- 52libsyntax/parse/obsolete.rs:
libsyntax/print/pprust.rs:
libsyntax/ext/expand.rs:
libsyntax/ext/quote.rs:
libsyntax/ext/deriving/encodable.rs:
libsyntax/ext/deriving/decodable.rs:
libsyntax/ext/deriving/show.rs:
libsyntax/ext/tt/transcribe.rs:
libsyntax/ext/tt/macro_parser.rs:
libsyntax/ext/tt/macro_rules.rs:
libsyntax/ext/format.rs:
libsyntax/ext/concat_idents.rs:
libsyntax/ext/source_util.rs:
libsyntax/ast.rs:
libsyntax/ast_util.rs:
libsyntax/ast_map.rs:
libsyntax/parse/parser.rs:
libsyntax/ext/base.rs:
libsyntax/parse/token.rs:395:8-395:8 -enum- definition:
pub enum Keyword {
$( $sk_variant, )*
$( $rk_variant, )*
references:- 5400: impl Keyword {
401: pub fn to_ident(&self) -> Ident {
libsyntax/parse/parser.rs:
511: pub fn is_keyword(&mut self, kw: keywords::Keyword) -> bool {
512: token::is_keyword(kw, &self.token)
--
528: // otherwise, eat it.
529: pub fn expect_keyword(&mut self, kw: keywords::Keyword) {
530: if !self.eat_keyword(kw) {
libsyntax/parse/token.rs:
682: pub fn is_keyword(kw: keywords::Keyword, tok: &Token) -> bool {
683: match *tok {
libsyntax/parse/token.rs:107:46-107:46 -enum- definition:
/// For interpolation during macro expansion.
pub enum Nonterminal {
NtItem(@ast::Item),
references:- 1499: /* For interpolation */
100: INTERPOLATED(Nonterminal),
--
122: impl fmt::Show for Nonterminal {
123: fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
libsyntax/ext/tt/macro_parser.rs:
427: pub fn parse_nt(p: &mut Parser, name: &str) -> Nonterminal {
428: match name {
libsyntax/parse/token.rs:633:10-633:10 -fn- definition:
pub fn intern_and_get_ident(s: &str) -> InternedString {
get_name(intern(s))
}
references:- 20libsyntax/ext/asm.rs:
218: asm_str_style: asm_str_style.unwrap(),
219: clobbers: token::intern_and_get_ident(cons),
220: inputs: inputs,
libsyntax/ext/quote.rs:
131: let lit = dummy_spanned(ast::LitStr(
132: token::intern_and_get_ident(*self), ast::CookedStr));
133: pprust::lit_to_str(&lit)
libsyntax/ext/deriving/encodable.rs:
156: None => {
157: token::intern_and_get_ident(format!("_field{}", i))
158: }
libsyntax/ext/deriving/decodable.rs:
169: getarg(cx, span,
170: token::intern_and_get_ident(format!("_field{}",
171: i)),
libsyntax/ext/deriving/show.rs:
134: let s = token::intern_and_get_ident(format_string.as_slice());
135: let format_string = cx.expr_str(span, s);
libsyntax/ext/build.rs:
641: self.expr_str(span,
642: token::intern_and_get_ident(loc.file
643: .name
libsyntax/ext/format.rs:
439: }).collect();
440: let s = token::intern_and_get_ident(arm.selector);
441: let selector = self.ecx.expr_str(sp, s);
--
514: parse::String(s) => {
515: let s = token::intern_and_get_ident(s);
516: self.ecx.expr_call_global(sp,
libsyntax/ext/env.rs:
101: }
102: Some(s) => cx.expr_str(sp, token::intern_and_get_ident(s))
103: };
libsyntax/ext/concat.rs:
61: sp,
62: token::intern_and_get_ident(accumulator.into_owned())))
63: }
libsyntax/ext/source_util.rs:
121: let filename = file.display().to_str().to_strbuf();
122: let interned = token::intern_and_get_ident(src);
123: cx.codemap().new_filemap(filename, src.to_strbuf());
libsyntax/attr.rs:
126: InternedString::new("doc"),
127: token::intern_and_get_ident(strip_doc_comment_decoration(
128: comment.get()).as_slice()));
libsyntax/ext/source_util.rs:
59: let loc = cx.codemap().lookup_char_pos(topmost.call_site.lo);
60: let filename = token::intern_and_get_ident(loc.file.name.as_slice());
61: base::MacExpr::new(cx.expr_str(topmost.call_site, filename))
libsyntax/parse/token.rs:618:10-618:10 -fn- definition:
pub fn get_name(name: Name) -> InternedString {
let interner = get_ident_interner();
InternedString::new_from_rc_str(interner.get(name))
references:- 6606: fn decode(d: &mut D) -> Result<InternedString, E> {
607: Ok(get_name(get_ident_interner().intern(try!(d.read_str()))))
608: }
--
627: pub fn get_ident(ident: Ident) -> InternedString {
628: get_name(ident.name)
629: }
--
634: pub fn intern_and_get_ident(s: &str) -> InternedString {
635: get_name(intern(s))
636: }
libsyntax/print/pprust.rs:
1873: Some(ref lt) => {
1874: let token = token::get_name(lt.name);
1875: if token.get() != "static" {
libsyntax/ast_map.rs:
42: fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
43: let slot = token::get_name(self.name());
44: write!(f.buf, "{}", slot)
libsyntax/parse/token.rs:27:66-27:66 -enum- definition:
pub enum BinOp {
PLUS,
MINUS,
references:- 1628: pub enum BinOp {
--
55: TILDE,
56: BINOP(BinOp),
57: BINOPEQ(BinOp),
libsyntax/ext/quote.rs:
395: fn mk_binop(cx: &ExtCtxt, sp: Span, bop: token::BinOp) -> @ast::Expr {
396: let name = match bop {
libsyntax/parse/lexer.rs:
712: }
713: fn binop(rdr: &mut StringReader, op: token::BinOp) -> token::Token {
714: bump(rdr);
libsyntax/parse/token.rs:
140: pub fn binop_to_str(o: BinOp) -> StrBuf {
141: match o {
libsyntax/parse/token.rs:139:1-139:1 -fn- definition:
pub fn binop_to_str(o: BinOp) -> StrBuf {
match o {
PLUS => "+".to_strbuf(),
references:- 2167: ANDAND => "&&".to_strbuf(),
168: BINOP(op) => binop_to_str(op),
169: BINOPEQ(op) => {
170: let mut s = binop_to_str(op);
171: s.push_str("=");
libsyntax/parse/token.rs:329:1-329:1 -fn- definition:
pub fn is_ident_or_path(t: &Token) -> bool {
match *t {
IDENT(_, _) | INTERPOLATED(NtPath(..)) => true,
references:- 5libsyntax/parse/parser.rs:
761: // Stash token for error recovery (sometimes; clone is not necessarily cheap).
762: self.last_token = if is_ident_or_path(&self.token) {
763: Some(box self.token.clone())
--
1293: } else if self.token == token::MOD_SEP
1294: || is_ident_or_path(&self.token) {
1295: // NAMED TYPE
--
2878: if self.eat(&token::DOTDOT) {
2879: let end = if is_ident_or_path(&self.token) {
2880: let path = self.parse_path(LifetimeAndTypesWithColons)
libsyntax/parse/token.rs:555:53-555:53 -struct- definition:
pub struct InternedString {
string: RcStr,
}
references:- 101libsyntax/ext/base.rs:
libsyntax/ext/deriving/decodable.rs:
libsyntax/ext/build.rs:
libsyntax/attr.rs:
libsyntax/ast.rs:
libsyntax/parse/parser.rs:
libsyntax/parse/token.rs:
libsyntax/parse/token.rs:528:51-528:51 -NK_AS_STR_TODO- definition:
// looks like we can get rid of this completely...
pub type IdentInterner = StrInterner;
// if an interner exists in TLS, return it. Otherwise, prepare a
references:- 5410: fn mk_fresh_ident_interner() -> IdentInterner {
411: // The indices here must correspond to the numbers in
--
534: pub fn get_ident_interner() -> Rc<IdentInterner> {
535: local_data_key!(key: Rc<::parse::token::IdentInterner>)
536: match key.get() {
libsyntax/parse/parser.rs:
331: pub reader: Box<Reader:>,
332: pub interner: Rc<token::IdentInterner>,
333: /// The set of seen errors about obsolete syntax. Used to suppress
libsyntax/print/pprust.rs:
59: cm: Option<&'a CodeMap>,
60: intr: Rc<token::IdentInterner>,
61: comments: Option<Vec<comments::Comment> >,
libsyntax/parse/token.rs:664:79-664:79 -fn- definition:
// that is, that the new name and the old one are connected to ptr_eq strings.
pub fn fresh_name(src: &ast::Ident) -> Name {
let interner = get_ident_interner();
references:- 2libsyntax/ext/expand.rs:
222: Some(label) => {
223: let new_label = fresh_name(&label);
224: let rename = (label, new_label);
--
670: for ident in name_finder.ident_accumulator.iter() {
671: let new_name = fresh_name(ident);
672: new_pending_renames.push((*ident,new_name));
libsyntax/parse/token.rs:533:71-533:71 -fn- definition:
// FIXME(eddyb) #8726 This should probably use a task-local reference.
pub fn get_ident_interner() -> Rc<IdentInterner> {
local_data_key!(key: Rc<::parse::token::IdentInterner>)
references:- 9640: pub fn intern(s: &str) -> Name {
641: get_ident_interner().intern(s)
642: }
--
646: pub fn gensym(s: &str) -> Name {
647: get_ident_interner().gensym(s)
648: }
--
665: pub fn fresh_name(src: &ast::Ident) -> Name {
666: let interner = get_ident_interner();
667: interner.gensym_copy(src.name)
libsyntax/print/pprust.rs:
112: cm: Some(cm),
113: intr: token::get_ident_interner(),
114: comments: Some(cmnts),
libsyntax/ast_map.rs:
82: pub fn path_to_str<PI: Iterator<PathElem>>(mut path: PI) -> StrBuf {
83: let itr = token::get_ident_interner();
libsyntax/parse/parser.rs:
289: reader: rdr,
290: interner: token::get_ident_interner(),
291: sess: sess,
libsyntax/parse/token.rs:
606: fn decode(d: &mut D) -> Result<InternedString, E> {
607: Ok(get_name(get_ident_interner().intern(try!(d.read_str()))))
608: }
libsyntax/parse/token.rs:325:1-325:1 -fn- definition:
pub fn is_ident(t: &Token) -> bool {
match *t { IDENT(_, _) => true, _ => false }
}
references:- 4libsyntax/ext/format.rs:
100: if p.token == token::EOF { break } // accept trailing commas
101: if named || (token::is_ident(&p.token) &&
102: p.look_ahead(1, |t| *t == token::EQ)) {
libsyntax/parse/parser.rs:
1867: } else if self.token == token::MOD_SEP ||
1868: is_ident(&self.token) && !self.is_keyword(keywords::True) &&
1869: !self.is_keyword(keywords::False) {
--
3107: return @spanned(lo, decl.span.hi, StmtDecl(decl, ast::DUMMY_NODE_ID));
3108: } else if is_ident(&self.token)
3109: && !token::is_any_keyword(&self.token)
libsyntax/parse/token.rs:675:24-675:24 -fn- definition:
// create a fresh mark.
pub fn fresh_mark() -> Mrk {
gensym("mark")
references:- 3libsyntax/ext/expand.rs:
78: });
79: let fm = fresh_mark();
80: // mark before:
--
373: let extnamestr = token::get_ident(extname);
374: let fm = fresh_mark();
375: let expanded = match fld.extsbox.find(&extname.name) {
--
585: });
586: let fm = fresh_mark();
587: // mark before expansion:
libsyntax/parse/token.rs:645:10-645:10 -fn- definition:
pub fn gensym(s: &str) -> Name {
get_ident_interner().gensym(s)
}
references:- 2676: pub fn fresh_mark() -> Mrk {
677: gensym("mark")
678: }
libsyntax/parse/token.rs:688:1-688:1 -fn- definition:
pub fn is_any_keyword(tok: &Token) -> bool {
match *tok {
token::IDENT(sid, false) => match sid.name {
references:- 3libsyntax/parse/parser.rs:
4747: ) -> ItemOrViewItem {
4748: if macros_allowed && !token::is_any_keyword(&self.token)
4749: && self.look_ahead(1, |t| *t == token::NOT)
libsyntax/parse/lexer.rs:
832: is no longer a special lifetime".to_strbuf());
833: } else if token::is_any_keyword(tok) &&
834: !token::is_keyword(token::keywords::Static, tok) {
libsyntax/parse/token.rs:651:10-651:10 -fn- definition:
pub fn str_to_ident(s: &str) -> ast::Ident {
ast::Ident::new(intern(s))
}
references:- 16libsyntax/ext/base.rs:
501: pub fn ident_of(&self, st: &str) -> ast::Ident {
502: str_to_ident(st)
503: }
libsyntax/ext/quote.rs:
382: fn id_ext(str: &str) -> ast::Ident {
383: str_to_ident(str)
384: }
libsyntax/ext/concat_idents.rs:
44: }
45: let res = str_to_ident(res_str.into_owned());
libsyntax/ast.rs:
108: fn decode(d: &mut D) -> Result<Ident, E> {
109: Ok(str_to_ident(try!(d.read_str())))
110: }
libsyntax/parse/lexer.rs:
347: Some(TokenAndSpan{
348: tok: token::DOC_COMMENT(str_to_ident(string)),
349: sp: codemap::mk_sp(start_bpos, rdr.pos)
--
990: content_end_bpos,
991: str_to_ident);
992: return token::LIT_STR_RAW(str_content, hash_count);