Class: RuboCop::AST::ProcessedSource

Relationships & Source Files
Inherits:	Object
Defined in:	lib/rubocop/ast/processed_source.rb

Overview

ProcessedSource contains objects which are generated by Parser and other information such as disabled lines for cops. It also provides a convenient way to access source lines.

Constant Summary

INVALID_LEVELS = private
# File 'lib/rubocop/ast/processed_source.rb', line 35
```
%i[error fatal].freeze
```
PARSER_ENGINES = private
# File 'lib/rubocop/ast/processed_source.rb', line 38
```
%i[default parser_whitequark parser_prism].freeze
```
STRING_SOURCE_NAME = Internal use only
# File 'lib/rubocop/ast/processed_source.rb', line 33
```
'(string)'
```

Class Method Summary

.from_file(path, ruby_version, parser_engine: :default)
.new(source, ruby_version, path = nil, parser_engine: :default, prism_result: nil) ⇒ ProcessedSource constructor

Instance Attribute Summary

#ast readonly
#blank? ⇒ Boolean readonly
#buffer readonly
#comments readonly
#diagnostics readonly
#parser_engine readonly
#parser_error readonly
#path readonly
#raw_source readonly
#ruby_version readonly
#tokens readonly
#valid_syntax? ⇒ Boolean readonly

Instance Method Summary

#[](*args)
#ast_with_comments
#checksum

Raw source checksum for tracking infinite loops.
#comment_at_line(line) ⇒ Comment^?
#commented?(source_range)

Alias for #contains_comment?.
#comments_before_line(line) deprecated Deprecated.

Use #each_comment_in_lines
#contains_comment?(source_range) ⇒ Boolean (also: #commented?)

Consider using #each_comment_in_lines instead.
#current_line(token)
#each_comment(&block) deprecated Deprecated.

Use comments.each
#each_comment_in_lines(line_range)

Enumerates on the comments contained with the given line_range.
#each_token(&block) deprecated Deprecated.

Use tokens.each
#file_path
#find_comment(&block) deprecated Deprecated.

Use #comment_at_line, #each_comment_in_lines, or comments.find
#find_token(&block) deprecated Deprecated.

Use tokens.find
#first_token_of(range_or_node)
#following_line(token)
#last_token_of(range_or_node)
#line_indentation(line_number)
#line_with_comment?(line) ⇒ Boolean
#lines

Returns the source lines, line break characters removed, excluding a possible END and everything that comes after.
#preceding_line(token)
#sorted_tokens

The tokens list is always sorted by token position, except for cases when heredoc is passed as a method argument.
#start_with?(string) ⇒ Boolean
#tokens_within(range_or_node)
#builder_class(parser_engine) private

Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength.
#comment_index private
#create_parser(ruby_version, parser_engine, prism_result) private

Metrics/MethodLength.
#default_parser_engine(ruby_version) private

The Parser gem does not support Ruby 3.5 or later.
#first_token_index(range_or_node) private
#last_token_index(range_or_node) private
#normalize_parser_engine(parser_engine, ruby_version) private

Metrics/MethodLength.
#parse(source, ruby_version, parser_engine, prism_result) private
#parser_class(ruby_version, parser_engine) private

Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength.
#source_range(range_or_node) private
#tokenize(parser) private

Constructor Details

.new(source, ruby_version, path = nil, parser_engine: :default, prism_result: nil) ⇒ `ProcessedSource`

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 49


def initialize(
  source, ruby_version, path = nil, parser_engine: :default, prism_result: nil
)
  parser_engine = normalize_parser_engine(parser_engine, ruby_version)

  # Defaults source encoding to UTF-8, regardless of the encoding it has
  # been read with, which could be non-utf8 depending on the default
  # external encoding.
  (+source).force_encoding(Encoding::UTF_8) unless source.encoding == Encoding::UTF_8

  @raw_source = source
  @path = path
  @diagnostics = []
  @ruby_version = ruby_version
  @parser_engine = parser_engine
  @parser_error = nil

  parse(source, ruby_version, parser_engine, prism_result)
end

Class Method Details

.from_file(path, ruby_version, parser_engine: :default)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 44


def self.from_file(path, ruby_version, parser_engine: :default)
  file = File.read(path, mode: 'rb')
  new(file, ruby_version, path, parser_engine: parser_engine)
end

Instance Attribute Details

#ast (readonly)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 41


attr_reader :path, :buffer, :ast, :comments, :tokens, :diagnostics,
            :parser_error, :raw_source, :ruby_version, :parser_engine

#blank? ⇒ `Boolean` (readonly)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 130


def blank?
  ast.nil?
end

#buffer (readonly)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 41


attr_reader :path, :buffer, :ast, :comments, :tokens, :diagnostics,
            :parser_error, :raw_source, :ruby_version, :parser_engine

#comments (readonly)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 41


attr_reader :path, :buffer, :ast, :comments, :tokens, :diagnostics,
            :parser_error, :raw_source, :ruby_version, :parser_engine

#diagnostics (readonly)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 41


attr_reader :path, :buffer, :ast, :comments, :tokens, :diagnostics,
            :parser_error, :raw_source, :ruby_version, :parser_engine

#parser_engine (readonly)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 41


attr_reader :path, :buffer, :ast, :comments, :tokens, :diagnostics,
            :parser_error, :raw_source, :ruby_version, :parser_engine

#parser_error (readonly)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 41


attr_reader :path, :buffer, :ast, :comments, :tokens, :diagnostics,
            :parser_error, :raw_source, :ruby_version, :parser_engine

#path (readonly)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 41


attr_reader :path, :buffer, :ast, :comments, :tokens, :diagnostics,
            :parser_error, :raw_source, :ruby_version, :parser_engine

#raw_source (readonly)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 41


attr_reader :path, :buffer, :ast, :comments, :tokens, :diagnostics,
            :parser_error, :raw_source, :ruby_version, :parser_engine

#ruby_version (readonly)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 41


attr_reader :path, :buffer, :ast, :comments, :tokens, :diagnostics,
            :parser_error, :raw_source, :ruby_version, :parser_engine

#tokens (readonly)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 41


attr_reader :path, :buffer, :ast, :comments, :tokens, :diagnostics,
            :parser_error, :raw_source, :ruby_version, :parser_engine

#valid_syntax? ⇒ `Boolean` (readonly)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 95


def valid_syntax?
  return false if @parser_error

  @diagnostics.none? { |d| INVALID_LEVELS.include?(d.level) }
end

Instance Method Details

#[](*args)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 91


def [](*args)
  lines[*args]
end

#ast_with_comments

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 69


def ast_with_comments
  return if !ast || !comments

  @ast_with_comments ||= Parser::Source::Comment.associate_by_identity(ast, comments)
end

#builder_class(parser_engine) (private)

Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 329


def builder_class(parser_engine)
  case parser_engine
  when :parser_whitequark
    RuboCop::AST::Builder
  when :parser_prism
    RuboCop::AST::BuilderPrism
  end
end

#checksum

Raw source checksum for tracking infinite loops.

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 102


def checksum
  Digest::SHA1.hexdigest(@raw_source)
end

#comment_at_line(line) ⇒ `Comment`^?

Returns:

(Comment, nil) —

the comment at that line, if any.

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 135


def comment_at_line(line)
  comment_index[line]
end

#comment_index (private)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 218


def comment_index
  @comment_index ||= {}.tap do |hash|
    comments.each { |c| hash[c.location.line] = c }
  end
end

#commented?(source_range)

Alias for #contains_comment?.

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 161


alias commented? contains_comment?

#comments_before_line(line)

Deprecated.

Use #each_comment_in_lines

Should have been called comments_before_or_at_line. Doubtful it has of any valid use.

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 165


def comments_before_line(line)
  each_comment_in_lines(0..line).to_a
end

#contains_comment?(source_range) ⇒ `Boolean` Also known as: #commented?

Consider using #each_comment_in_lines instead

Returns:

(Boolean) —

if any of the lines in the given #source_range has a comment.

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 157


def contains_comment?(source_range)
  each_comment_in_lines(source_range.line..source_range.last_line).any?
end

#create_parser(ruby_version, parser_engine, prism_result) (private)

Metrics/MethodLength

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 339


def create_parser(ruby_version, parser_engine, prism_result)
  builder = builder_class(parser_engine).new

  parser_class = parser_class(ruby_version, parser_engine)

  parser_instance = if parser_engine == :parser_prism && prism_result
                      # NOTE: Since it is intended for use with Ruby LSP, it targets only Prism.
                      # If there is no reuse of a pre-parsed result, such as in Ruby LSP,
                      # regular parsing with Prism occurs, and `else` branch will be executed.
                      prism_reparsed = PrismPreparsed.new(prism_result)
                      parser_class.new(builder, parser: prism_reparsed)
                    else
                      parser_class.new(builder)
                    end

  parser_instance.tap do |parser|
    # On JRuby there's a risk that we hang in tokenize() if we
    # don't set the all errors as fatal flag. The problem is caused by a bug
    # in Racc that is discussed in issue #93 of the whitequark/parser
    # project on GitHub.
    parser.diagnostics.all_errors_are_fatal = (RUBY_ENGINE != 'ruby')
    parser.diagnostics.ignore_warnings = false
    parser.diagnostics.consumer = lambda do |diagnostic|
      @diagnostics << diagnostic
    end
  end
end

#current_line(token)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 179


def current_line(token)
  lines[token.line - 1]
end

#default_parser_engine(ruby_version) (private)

The Parser gem does not support Ruby 3.5 or later. It is also not fully compatible with Ruby 3.4 but for now respects using parser for backwards compatibility.

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 385


def default_parser_engine(ruby_version)
  if ruby_version >= 3.4
    :parser_prism
  else
    :parser_whitequark
  end
end

#each_comment(&block)

Deprecated.

Use comments.each

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 107


def each_comment(&block)
  comments.each(&block)
end

#each_comment_in_lines(line_range)

Enumerates on the comments contained with the given line_range

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 145


def each_comment_in_lines(line_range)
  return to_enum(:each_comment_in_lines, line_range) unless block_given?

  line_range.each do |line|
    if (comment = comment_index[line])
      yield comment
    end
  end
end

#each_token(&block)

Deprecated.

Use tokens.each

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 117


def each_token(&block)
  tokens.each(&block)
end

#file_path

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 126


def file_path
  buffer.name
end

#find_comment(&block)

Deprecated.

Use #comment_at_line, #each_comment_in_lines, or comments.find

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 112


def find_comment(&block)
  comments.find(&block)
end

#find_token(&block)

Deprecated.

Use tokens.find

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 122


def find_token(&block)
  tokens.find(&block)
end

#first_token_index(range_or_node) (private)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 393


def first_token_index(range_or_node)
  begin_pos = source_range(range_or_node).begin_pos
  sorted_tokens.bsearch_index { |token| token.begin_pos >= begin_pos }
end

#first_token_of(range_or_node)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 200


def first_token_of(range_or_node)
  sorted_tokens[first_token_index(range_or_node)]
end

#following_line(token)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 183


def following_line(token)
  lines[token.line]
end

#last_token_index(range_or_node) (private)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 398


def last_token_index(range_or_node)
  end_pos = source_range(range_or_node).end_pos
  sorted_tokens.bsearch_index { |token| token.end_pos >= end_pos }
end

#last_token_of(range_or_node)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 204


def last_token_of(range_or_node)
  sorted_tokens[last_token_index(range_or_node)]
end

#line_indentation(line_number)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 187


def line_indentation(line_number)
  lines[line_number - 1]
    .match(/^(\s*)/)[1]
    .to_s
    .length
end

#line_with_comment?(line) ⇒ `Boolean`

Returns:

(Boolean) —

if the given line number has a comment.

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 140


def line_with_comment?(line)
  comment_index.include?(line)
end

#lines

Returns the source lines, line break characters removed, excluding a possible END and everything that comes after.

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 77


def lines
  @lines ||= begin
    all_lines = @buffer.source_lines
    last_token_line = tokens.any? ? tokens.last.line : all_lines.size
    result = []
    all_lines.each_with_index do |line, ix|
      break if ix >= last_token_line && line == '__END__'

      result << line
    end
    result
  end
end

#normalize_parser_engine(parser_engine, ruby_version) (private)

Metrics/MethodLength

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 368


def normalize_parser_engine(parser_engine, ruby_version)
  parser_engine = parser_engine.to_sym
  unless PARSER_ENGINES.include?(parser_engine)
    raise ArgumentError, 'The keyword argument `parser_engine` accepts `default`, ' \
                         "`parser_whitequark`, or `parser_prism`, but `#{parser_engine}` " \
                         'was passed.'
  end
  if parser_engine == :default
    default_parser_engine(ruby_version)
  else
    parser_engine
  end
end

#parse(source, ruby_version, parser_engine, prism_result) (private)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 224


def parse(source, ruby_version, parser_engine, prism_result)
  buffer_name = @path || STRING_SOURCE_NAME
  @buffer = Parser::Source::Buffer.new(buffer_name, 1)

  begin
    @buffer.source = source
  rescue EncodingError, Parser::UnknownEncodingInMagicComment => e
    @parser_error = e
    @ast = nil
    @comments = []
    @tokens = []
    return
  end

  parser = create_parser(ruby_version, parser_engine, prism_result)

  @ast, @comments, @tokens = tokenize(parser)
end

#parser_class(ruby_version, parser_engine) (private)

Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 260


def parser_class(ruby_version, parser_engine)
  case parser_engine
  when :parser_whitequark
    case ruby_version
    when 1.9
      require 'parser/ruby19'
      Parser::Ruby19
    when 2.0
      require 'parser/ruby20'
      Parser::Ruby20
    when 2.1
      require 'parser/ruby21'
      Parser::Ruby21
    when 2.2
      require 'parser/ruby22'
      Parser::Ruby22
    when 2.3
      require 'parser/ruby23'
      Parser::Ruby23
    when 2.4
      require 'parser/ruby24'
      Parser::Ruby24
    when 2.5
      require 'parser/ruby25'
      Parser::Ruby25
    when 2.6
      require 'parser/ruby26'
      Parser::Ruby26
    when 2.7
      require 'parser/ruby27'
      Parser::Ruby27
    when 2.8, 3.0
      require 'parser/ruby30'
      Parser::Ruby30
    when 3.1
      require 'parser/ruby31'
      Parser::Ruby31
    when 3.2
      require 'parser/ruby32'
      Parser::Ruby32
    when 3.3
      require 'parser/ruby33'
      Parser::Ruby33
    when 3.4
      require 'parser/ruby34'
      Parser::Ruby34
    else
      raise ArgumentError, 'RuboCop supports target Ruby versions 3.4 and below with ' \
                           "`parser`. Specified target Ruby version: #{ruby_version.inspect}"
    end
  when :parser_prism
    case ruby_version
    when 3.3
      require 'prism/translation/parser33'
      Prism::Translation::Parser33
    when 3.4
      require 'prism/translation/parser34'
      Prism::Translation::Parser34
    when 3.5
      require 'prism/translation/parser35'
      Prism::Translation::Parser35
    else
      raise ArgumentError, 'RuboCop supports target Ruby versions 3.3 and above with Prism. ' \
                           "Specified target Ruby version: #{ruby_version.inspect}"
    end
  end
end

#preceding_line(token)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 175


def preceding_line(token)
  lines[token.line - 2]
end

#sorted_tokens

The tokens list is always sorted by token position, except for cases when heredoc is passed as a method argument. In this case tokens are interleaved by heredoc contents' tokens.

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 211


def sorted_tokens
  # Use stable sort.
  @sorted_tokens ||= tokens.sort_by.with_index { |token, i| [token.begin_pos, i] }
end

#source_range(range_or_node) (private)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 403


def source_range(range_or_node)
  if range_or_node.respond_to?(:source_range)
    range_or_node.source_range
  else
    range_or_node
  end
end

#start_with?(string) ⇒ `Boolean`

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 169


def start_with?(string)
  return false if self[0].nil?

  self[0].start_with?(string)
end

#tokenize(parser) (private)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 243


def tokenize(parser)
  begin
    ast, comments, tokens = parser.tokenize(@buffer)
    ast ||= nil # force `false` to `nil`, see https://github.com/whitequark/parser/pull/722
  rescue Parser::SyntaxError
    # All errors are in diagnostics. No need to handle exception.
    comments = []
    tokens = []
  end

  ast&.complete!
  tokens.map! { |t| Token.from_parser_token(t) }

  [ast, comments, tokens]
end

#tokens_within(range_or_node)

[ GitHub ]

# File 'lib/rubocop/ast/processed_source.rb', line 194


def tokens_within(range_or_node)
  begin_index = first_token_index(range_or_node)
  end_index = last_token_index(range_or_node)
  sorted_tokens[begin_index..end_index]
end

Class: RuboCop::AST::ProcessedSource

Overview

Constant Summary

Class Method Summary

Instance Attribute Summary

Instance Method Summary

Constructor Details

.new(source, ruby_version, path = nil, parser_engine: :default, prism_result: nil) ⇒ ProcessedSource

Class Method Details

.from_file(path, ruby_version, parser_engine: :default)

Instance Attribute Details

#ast (readonly)

#blank? ⇒ Boolean (readonly)

#buffer (readonly)

#comments (readonly)

#diagnostics (readonly)

#parser_engine (readonly)

#parser_error (readonly)

#path (readonly)

#raw_source (readonly)

#ruby_version (readonly)

#tokens (readonly)

#valid_syntax? ⇒ Boolean (readonly)

Instance Method Details

#[](*args)

#ast_with_comments

#builder_class(parser_engine) (private)

#checksum

#comment_at_line(line) ⇒ Comment?

#comment_index (private)

#commented?(source_range)

#comments_before_line(line)

#contains_comment?(source_range) ⇒ Boolean Also known as: #commented?

#create_parser(ruby_version, parser_engine, prism_result) (private)

#current_line(token)

#default_parser_engine(ruby_version) (private)

#each_comment(&block)

#each_comment_in_lines(line_range)

#each_token(&block)

#file_path

#find_comment(&block)

#find_token(&block)

#first_token_index(range_or_node) (private)

#first_token_of(range_or_node)

#following_line(token)

#last_token_index(range_or_node) (private)

#last_token_of(range_or_node)

#line_indentation(line_number)

#line_with_comment?(line) ⇒ Boolean

#lines

#normalize_parser_engine(parser_engine, ruby_version) (private)

#parse(source, ruby_version, parser_engine, prism_result) (private)

#parser_class(ruby_version, parser_engine) (private)

#preceding_line(token)

#sorted_tokens

#source_range(range_or_node) (private)

#start_with?(string) ⇒ Boolean

#tokenize(parser) (private)

#tokens_within(range_or_node)

.new(source, ruby_version, path = nil, parser_engine: :default, prism_result: nil) ⇒ `ProcessedSource`

#blank? ⇒ `Boolean` (readonly)

#valid_syntax? ⇒ `Boolean` (readonly)

#comment_at_line(line) ⇒ `Comment`^?

#contains_comment?(source_range) ⇒ `Boolean` Also known as: #commented?

#line_with_comment?(line) ⇒ `Boolean`

#start_with?(string) ⇒ `Boolean`