123456789_123456789_123456789_123456789_123456789_

Class: Ripper::Lexer

Do not use. This class is for internal use only.
Relationships & Source Files
Namespace Children
Classes:
Elem, State
Super Chains via Extension / Inclusion / Inheritance
Class Chain:
self, ::Ripper
Instance Chain:
self, ::Ripper
Inherits: Ripper
Defined in: ext/ripper/lib/ripper/lexer.rb

Overview

internal use only

Constant Summary

::Ripper - Inherited

EVENTS, PARSER_EVENTS, SCANNER_EVENTS, Version

Class Method Summary

::Ripper - Inherited

.dedent_string

USE OF RIPPER LIBRARY ONLY.
.lex

Tokenizes the Ruby program and returns an array of an array, which is formatted like [[lineno, column], type, token, state].
.lex_state_name

Returns a string representation of lex_state.
.new

Create a new ::Ripper object.
.parse

Parses the given Ruby program read from src.
.sexp
EXPERIMENTAL

Parses src and create S-exp tree.
.sexp_raw
EXPERIMENTAL

Parses src and create S-exp tree.
.slice
EXPERIMENTAL

Parses src and return a string which was matched to pattern.
.tokenize

Tokenizes the Ruby program and returns an array of strings.
.token_match

Instance Attribute Summary

::Ripper - Inherited

#debug_output

Get debug output.
#debug_output=

Set debug output.
#end_seen?

Return true if parsed source ended by _END_.
#error?

Return true if parsed source has errors.
#yydebug

Get yydebug.
#yydebug=

Set yydebug.

Instance Method Summary

::Ripper - Inherited

#column

Return column number of current parsing line.
#encoding

Return encoding of the source.
#filename

Return current parsing filename.
#lineno

Return line number of current parsing line.
#parse

Start parsing and returns the value of the root action.
#state

Return scanner state of current token.
#token

Return the current token string.
#compile_error

This method is called when the parser found syntax error.
#dedent_string

Alias for dedent_string.
#warn

This method is called when weak warning is produced by the parser.
#warning

This method is called when strong warning is produced by the parser.
#assert_Qundef, #rawVALUE, #validate_object, #_dispatch_0, #_dispatch_1, #_dispatch_2, #_dispatch_3, #_dispatch_4, #_dispatch_5, #_dispatch_6, #_dispatch_7

Constructor Details

This class inherits a constructor from Ripper

Instance Attribute Details

#errors (readonly)

[ GitHub ] 

  


# File 'ext/ripper/lib/ripper/lexer.rb', line 96



attr_reader :errors


  







Instance Method Details



  

    #_push_token(tok)   (private)
  

  [ GitHub ]


  

  


# File 'ext/ripper/lib/ripper/lexer.rb', line 174



def _push_token(tok)
  @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
end


  








  

    #compile_error(mesg)   (private)
  



  

    

Alias for #on_error.


  



  [ GitHub ]


  

  


# File 'ext/ripper/lib/ripper/lexer.rb', line 182



alias compile_error on_error


  








  

    #lex  
  

  [ GitHub ]


  

  


# File 'ext/ripper/lib/ripper/lexer.rb', line 102



def lex
  parse().sort_by(&:pos).map(&:to_a)
end


  








  

    #on_error(mesg)   (private)
    Also known as: #on_parse_error, #compile_error
  

  [ GitHub ]


  

  


# File 'ext/ripper/lib/ripper/lexer.rb', line 178



def on_error(mesg)
  @errors.push Elem.new([lineno(), column()], __callee__, token(), state(), mesg)
end


  








  

    #on_heredoc_beg(tok)   (private)
  

  [ GitHub ]


  

  


# File 'ext/ripper/lib/ripper/lexer.rb', line 161



def on_heredoc_beg(tok)
  @stack.push @buf
  buf = []
  @buf.push buf
  @buf = buf
  @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
end


  








  

    #on_heredoc_dedent(v, w)   (private)
  

  [ GitHub ]


  

  


# File 'ext/ripper/lib/ripper/lexer.rb', line 138



def on_heredoc_dedent(v, w)
  ignored_sp = []
  heredoc = @buf.last
  heredoc.each_with_index do |e, i|
    if Elem === e and e.event == :on_tstring_content and e.pos[1].zero?
      tok = e.tok.dup if w > 0 and /\A\s/ =~ e.tok
      if (n = dedent_string(e.tok, w)) > 0
        if e.tok.empty?
          e.tok = tok[0, n]
          e.event = :on_ignored_sp
          next
        end
        ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n], e.state)]
        e.pos[1] += n
      end
    end
  end
  ignored_sp.reverse_each do |i, e|
    heredoc[i, 0] = [e]
  end
  v
end


  








  

    #on_heredoc_end(tok)   (private)
  

  [ GitHub ]


  

  


# File 'ext/ripper/lib/ripper/lexer.rb', line 169



def on_heredoc_end(tok)
  @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
  @buf = @stack.pop
end


  








  

    #on_parse_error(mesg)   (private)
  



  

    

Alias for #on_error.


  



  [ GitHub ]


  

  


# File 'ext/ripper/lib/ripper/lexer.rb', line 181



alias on_parse_error on_error


  








  

    #parse  
  

  [ GitHub ]


  

  


# File 'ext/ripper/lib/ripper/lexer.rb', line 121



def parse
  @errors = []
  @buf = []
  @stack = []
  super
  @buf.flatten!
  @buf
end


  








  

    #scan  
  



  

    

parse the code and returns elements including errors.


  



  [ GitHub ]


  

  


# File 'ext/ripper/lib/ripper/lexer.rb', line 107



def scan
  result = (parse() + errors + @stack.flatten).uniq.sort_by {|e| [*e.pos, (e.message ? -1 : 0)]}
  result.each_with_index do |e, i|
    if e.event == :on_parse_error and e.tok.empty? and (pre = result[i-1]) and
      pre.pos[0] == e.pos[0] and (pre.pos[1] + pre.tok.size) == e.pos[1]
      e.tok = pre.tok
      e.pos[1] = pre.pos[1]
      result[i-1] = e
      result[i] = pre
    end
  end
  result
end


  








  

    #tokenize  
  

  [ GitHub ]


  

  


# File 'ext/ripper/lib/ripper/lexer.rb', line 98



def tokenize
  parse().sort_by(&:pos).map(&:tok)
end