123456789_123456789_123456789_123456789_123456789_

Class: Ripper::Lexer

Do not use. This class is for internal use only.
Relationships & Source Files
Namespace Children
Classes:
Super Chains via Extension / Inclusion / Inheritance
Class Chain:
self, ::Ripper
Instance Chain:
self, ::Ripper
Inherits: Ripper
Defined in: ext/ripper/lib/ripper/lexer.rb

Overview

internal use only

Constant Summary

::Ripper - Inherited

EVENTS, PARSER_EVENTS, SCANNER_EVENTS

Class Method Summary

::Ripper - Inherited

.lex

Tokenizes the Ruby program and returns an array of an array, which is formatted like [[lineno, column], type, token, state].

.parse

Parses the given Ruby program read from src.

.sexp
EXPERIMENTAL

Parses src and create S-exp tree.

.sexp_raw
EXPERIMENTAL

Parses src and create S-exp tree.

.slice
EXPERIMENTAL

Parses src and return a string which was matched to pattern.

.tokenize

Tokenizes the Ruby program and returns an array of strings.

.token_match

Instance Attribute Summary

Instance Method Summary

::Ripper - Inherited

#compile_error

This method is called when the parser found syntax error.

#warn

This method is called when weak warning is produced by the parser.

#warning

This method is called when strong warning is produced by the parser.

#_dispatch_0, #_dispatch_1, #_dispatch_2, #_dispatch_3, #_dispatch_4, #_dispatch_5, #_dispatch_6, #_dispatch_7

Instance Attribute Details

#errors (readonly)

[ GitHub ]

  
# File 'ext/ripper/lib/ripper/lexer.rb', line 147

attr_reader :errors

Instance Method Details

#_push_token(tok) (private)

[ GitHub ]

  
# File 'ext/ripper/lib/ripper/lexer.rb', line 234

def _push_token(tok)
  e = Elem.new([lineno(), column()], __callee__, tok, state())
  @buf.push(e)
  e
end

#compile_error(mesg) (private)

Alias for #on_error1.

[ GitHub ]

  
# File 'ext/ripper/lib/ripper/lexer.rb', line 251

alias compile_error on_error1

#lex(**kw)

[ GitHub ]

  
# File 'ext/ripper/lib/ripper/lexer.rb', line 153

def lex(**kw)
  parse(**kw).sort_by(&:pos).map(&:to_a)
end

#on_error1(mesg) (private) Also known as: #compile_error

[ GitHub ]

  
# File 'ext/ripper/lib/ripper/lexer.rb', line 240

def on_error1(mesg)
  @errors.push Elem.new([lineno(), column()], __callee__, token(), state(), mesg)
end

#on_error2(mesg, elem) (private)

[ GitHub ]

  
# File 'ext/ripper/lib/ripper/lexer.rb', line 244

def on_error2(mesg, elem)
  @errors.push Elem.new(elem.pos, __callee__, elem.tok, elem.state, mesg)
end

#on_heredoc_beg(tok) (private)

[ GitHub ]

  
# File 'ext/ripper/lib/ripper/lexer.rb', line 221

def on_heredoc_beg(tok)
  @stack.push @buf
  buf = []
  @buf.push buf
  @buf = buf
  @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
end

#on_heredoc_dedent(v, w) (private)

[ GitHub ]

  
# File 'ext/ripper/lib/ripper/lexer.rb', line 196

def on_heredoc_dedent(v, w)
  ignored_sp = []
  heredoc = @buf.last
  if Array === heredoc
    heredoc.each_with_index do |e, i|
      if Elem === e and e.event == :on_tstring_content and e.pos[1].zero?
        tok = e.tok.dup if w > 0 and /\A\s/ =~ e.tok
        if (n = dedent_string(e.tok, w)) > 0
          if e.tok.empty?
            e.tok = tok[0, n]
            e.event = :on_ignored_sp
            next
          end
          ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n], e.state)]
          e.pos[1] += n
        end
      end
    end
  end
  ignored_sp.reverse_each do |i, e|
    heredoc[i, 0] = [e]
  end
  v
end

#on_heredoc_end(tok) (private)

[ GitHub ]

  
# File 'ext/ripper/lib/ripper/lexer.rb', line 229

def on_heredoc_end(tok)
  @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
  @buf = @stack.pop unless @stack.empty?
end

#parse(raise_errors: false)

[ GitHub ]

  
# File 'ext/ripper/lib/ripper/lexer.rb', line 172

def parse(raise_errors: false)
  @errors = []
  @buf = []
  @stack = []
  super()
  @buf = @stack.pop unless @stack.empty?
  if raise_errors and !@errors.empty?
    raise SyntaxError, @errors.map(&:message).join(' ;')
  end
  @buf.flatten!
  unless (result = @buf).empty?
    result.concat(@buf) until (@buf = []; super(); @buf.flatten!; @buf.empty?)
  end
  result
end

#scan(**kw)

parse the code and returns elements including errors.

[ GitHub ]

  
# File 'ext/ripper/lib/ripper/lexer.rb', line 158

def scan(**kw)
  result = (parse(**kw) + errors + @stack.flatten).uniq.sort_by {|e| [*e.pos, (e.message ? -1 : 0)]}
  result.each_with_index do |e, i|
    if e.event == :on_parse_error and e.tok.empty? and (pre = result[i-1]) and
      pre.pos[0] == e.pos[0] and (pre.pos[1] + pre.tok.size) == e.pos[1]
      e.tok = pre.tok
      e.pos[1] = pre.pos[1]
      result[i-1] = e
      result[i] = pre
    end
  end
  result
end

#tokenize(**kw)

[ GitHub ]

  
# File 'ext/ripper/lib/ripper/lexer.rb', line 149

def tokenize(**kw)
  parse(**kw).sort_by(&:pos).map(&:tok)
end