Class: YARD::Parser::SourceParser
| Relationships & Source Files | |
| Inherits: | Object |
| Defined in: | lib/yard/parser/source_parser.rb |
Overview
Responsible for parsing a source file into the namespace. Parsing also invokes handlers to process the parsed statements and generate any code objects that may be recognized.
=== Custom Parsers
SourceParser allows custom parsers to be registered and called when
a certain filetype is recognized. To register a parser and hook it
up to a set of file extensions, call .register_parser_type
Constant Summary
-
DEFAULT_PATH_GLOB =
# File 'lib/yard/parser/source_parser.rb', line 70
The default glob of files to be parsed.
["{lib,app}/**/*.rb", "ext/**/*.{c,cc,cxx,cpp,rb}"]
-
ENCODING_BYTE_ORDER_MARKS =
# File 'lib/yard/parser/source_parser.rb', line 74
Byte order marks for various encodings
{ 'utf-8' => String.new("\xEF\xBB\xBF"), # Not yet supported # 'utf-16be' => "\xFE\xFF", # 'utf-16le' => "\xFF\xFE", # 'utf-32be' => "\x00\x00\xFF\xFE", # 'utf-32le' => "\xFF\xFE", } -
ENCODING_LINE =
# File 'lib/yard/parser/source_parser.rb', line 65%r{\A(?:\s*#*!.*\r?\n)?\s*(?:#+|/\*+|//+).*coding\s*[:=]{1,2}\s*([a-z\d_\-]+)}i -
FROZEN_STRING_LINE =
# File 'lib/yard/parser/source_parser.rb', line 66/frozen(-|_)string(-|_)literal:\s+(true|false)/i -
SHEBANG_LINE =
# File 'lib/yard/parser/source_parser.rb', line 64/\A\s*#!\S+/
Parser Callbacks
-
.after_parse_file {|parser| ... } ⇒ Proc
Registers a callback to be called after an individual file is parsed.
- .after_parse_file_callbacks ⇒ Array<Proc>
-
.after_parse_list {|files, globals| ... } ⇒ Proc
Registers a callback to be called after a list of files is parsed via .parse.
- .after_parse_list_callbacks ⇒ Array<Proc>
-
.before_parse_file {|parser| ... } ⇒ Proc
Registers a callback to be called before an individual file is parsed.
- .before_parse_file_callbacks ⇒ Array<Proc>
-
.before_parse_list {|files, globals| ... } ⇒ Proc
Registers a callback to be called before a list of files is parsed via .parse.
- .before_parse_list_callbacks ⇒ Array<Proc>
- #contents ⇒ String readonly
-
#convert_encoding(content)
private
Searches for encoding line and forces encoding.
- #file ⇒ String rw
- #globals ⇒ OpenStruct readonly
- .new(parser_type = SourceParser.parser_type, globals = nil) ⇒ SourceParser constructor
-
#parse(content = __FILE__) ⇒ Object?
The main parser method.
-
.parse_in_order(*files) ⇒ void
private
Parses a list of files in a queue.
- #parser_class private
- #parser_type ⇒ Symbol rw
- #parser_type=(value) rw private
-
#parser_type_for_filename(filename) ⇒ Symbol
private
Guesses the parser type to use depending on the file extension.
-
#post_process ⇒ void
private
Runs a
::YARD::Handlers::Processorobject to post process the parsed statements. -
#tokenize(content) ⇒ Array
Tokenizes but does not parse the block of code using the current #parser_type
Class Attribute Summary
- .parser_type ⇒ Symbol rw
- .parser_type=(value) rw
- .parser_type_extensions ⇒ Hash rw Internal use only Internal use only
- .parser_type_extensions=(value) rw
- .parser_types ⇒ Hash{Symbol=>Object} rw Internal use only Internal use only
- .parser_types=(value) rw
Class Method Summary
-
.parse(paths = DEFAULT_PATH_GLOB, excluded = [], level = log.level) ⇒ void
Parses a path or set of paths.
-
.parse_string(content, ptype = parser_type) ⇒ Object
Parses a string
content -
.parser_type_for_extension(extension) ⇒ Symbol
Finds a parser type that is registered for the extension.
-
.register_parser_type(type, parser_klass, extensions = nil) ⇒ void
Registers a new parser type.
-
.tokenize(content, ptype = parser_type) ⇒ Array
Tokenizes but does not parse the block of code.
-
.validated_parser_type(type) ⇒ Symbol
Internal use only
Internal use only
Returns the validated parser type.
Constructor Details
.new(parser_type = SourceParser.parser_type, globals = nil) ⇒ SourceParser
# File 'lib/yard/parser/source_parser.rb', line 406
def initialize(parser_type = SourceParser.parser_type, globals1 = nil, globals2 = nil) globals = [true, false].include?(globals1) ? globals2 : globals1 @file = '(stdin)' @globals = globals || OpenStruct.new self.parser_type = parser_type end
Class Attribute Details
.parser_type ⇒ Symbol (rw)
# File 'lib/yard/parser/source_parser.rb', line 85
attr_reader :parser_type
.parser_type=(value) (rw)
[ GitHub ]# File 'lib/yard/parser/source_parser.rb', line 87
def parser_type=(value) @parser_type = validated_parser_type(value) end
.parser_type_extensions ⇒ Hash (rw)
# File 'lib/yard/parser/source_parser.rb', line 163
def parser_type_extensions; @@parser_type_extensions ||= {} end
.parser_type_extensions=(value) (rw)
[ GitHub ]# File 'lib/yard/parser/source_parser.rb', line 164
def parser_type_extensions=(value) @@parser_type_extensions = value end
.parser_types ⇒ Hash{Symbol=>Object} (rw)
# File 'lib/yard/parser/source_parser.rb', line 157
def parser_types; @@parser_types ||= {} end
.parser_types=(value) (rw)
[ GitHub ]# File 'lib/yard/parser/source_parser.rb', line 158
def parser_types=(value) @@parser_types = value end
Class Method Details
.after_parse_file {|parser| ... } ⇒ Proc
Registers a callback to be called after an individual file is parsed. The block passed to this method will be called on subsequent parse calls.
To register a callback that is called after the entire list of files is processed, see .after_parse_list.
# File 'lib/yard/parser/source_parser.rb', line 324
def after_parse_file(&block) after_parse_file_callbacks << block end
.after_parse_file_callbacks ⇒ Array<Proc>
# File 'lib/yard/parser/source_parser.rb', line 352
def after_parse_file_callbacks @after_parse_file_callbacks ||= [] end
.after_parse_list {|files, globals| ... } ⇒ Proc
Registers a callback to be called after a list of files is parsed via .parse. The block passed to this method will be called on subsequent parse calls.
# File 'lib/yard/parser/source_parser.rb', line 258
def after_parse_list(&block) after_parse_list_callbacks << block end
.after_parse_list_callbacks ⇒ Array<Proc>
# File 'lib/yard/parser/source_parser.rb', line 338
def after_parse_list_callbacks @after_parse_list_callbacks ||= [] end
.before_parse_file {|parser| ... } ⇒ Proc
Registers a callback to be called before an individual file is parsed. The block passed to this method will be called on subsequent parse calls.
To register a callback that is called before the entire list of files is processed, see .before_parse_list.
# File 'lib/yard/parser/source_parser.rb', line 295
def before_parse_file(&block) before_parse_file_callbacks << block end
.before_parse_file_callbacks ⇒ Array<Proc>
# File 'lib/yard/parser/source_parser.rb', line 345
def before_parse_file_callbacks @before_parse_file_callbacks ||= [] end
.before_parse_list {|files, globals| ... } ⇒ Proc
Registers a callback to be called before a list of files is parsed via .parse. The block passed to this method will be called on subsequent parse calls.
# File 'lib/yard/parser/source_parser.rb', line 234
def before_parse_list(&block) before_parse_list_callbacks << block end
.before_parse_list_callbacks ⇒ Array<Proc>
# File 'lib/yard/parser/source_parser.rb', line 331
def before_parse_list_callbacks @before_parse_list_callbacks ||= [] end
.parse(paths = DEFAULT_PATH_GLOB, excluded = [], level = log.level) ⇒ void
This method returns an undefined value.
Parses a path or set of paths
# File 'lib/yard/parser/source_parser.rb', line 99
def parse(paths = DEFAULT_PATH_GLOB, excluded = [], level = log.level) log.debug("Parsing #{paths.inspect} with `#{parser_type}` parser") excluded = excluded.map do |path| case path when Regexp; path else Regexp.new(path.to_s, Regexp::IGNORECASE) end end files = [paths].flatten. map {|p| File.directory?(p) ? "#{p}/**/*.{rb,c,cc,cxx,cpp}" : p }. map {|p| p.include?("*") ? Dir[p].sort_by {|d| [d.length, d] } : p }.flatten. reject {|p| !File.file?(p) || excluded.any? {|re| p =~ re } }. map {|p| p.encoding == Encoding.default_external ? p : p.dup.force_encoding(Encoding.default_external) } log.enter_level(level) do parse_in_order(*files.uniq) end end
.parse_in_order(*files) ⇒ void (private)
This method returns an undefined value.
Parses a list of files in a queue.
# File 'lib/yard/parser/source_parser.rb', line 364
def parse_in_order(*files) global_state = OpenStruct.new return if before_parse_list_callbacks.any? do |cb| cb.call(files, global_state) == false end OrderedParser.new(global_state, files).parse after_parse_list_callbacks.each do |cb| cb.call(files, global_state) end end
.parse_string(content, ptype = parser_type) ⇒ Object
Parses a string content
# File 'lib/yard/parser/source_parser.rb', line 123
def parse_string(content, ptype = parser_type) new(ptype).parse(StringIO.new(content)) end
.parser_type_for_extension(extension) ⇒ Symbol
Finds a parser type that is registered for the extension. If no
type is found, the default Ruby type is returned.
# File 'lib/yard/parser/source_parser.rb', line 171
def parser_type_for_extension(extension) type = parser_type_extensions.find do |_t, exts| [exts].flatten.any? {|ext| ext === extension } end validated_parser_type(type ? type.first : :ruby) end
.register_parser_type(type, parser_klass, extensions = nil) ⇒ void
This method returns an undefined value.
Registers a new parser type.
# File 'lib/yard/parser/source_parser.rb', line 146
def register_parser_type(type, parser_klass, extensions = nil) unless Base > parser_klass raise ArgumentError, "expecting parser_klass to be a subclass of YARD::Parser::Base" end parser_type_extensions[type.to_sym] = extensions if extensions parser_types[type.to_sym] = parser_klass end
.tokenize(content, ptype = parser_type) ⇒ Array
Tokenizes but does not parse the block of code
# File 'lib/yard/parser/source_parser.rb', line 132
def tokenize(content, ptype = parser_type) new(ptype).tokenize(content) end
.validated_parser_type(type) ⇒ Symbol
Returns the validated parser type. Basically, enforces that :ruby
type is never set if the Ripper library is not available
# File 'lib/yard/parser/source_parser.rb', line 184
def validated_parser_type(type) !defined?(::Ripper) && type == :ruby ? :ruby18 : type end
Instance Attribute Details
#contents ⇒ String (readonly)
# File 'lib/yard/parser/source_parser.rb', line 399
attr_reader :contents
#file ⇒ String (rw)
# File 'lib/yard/parser/source_parser.rb', line 386
attr_accessor :file
#globals ⇒ OpenStruct (readonly)
# File 'lib/yard/parser/source_parser.rb', line 395
attr_reader :globals
#parser_type ⇒ Symbol (rw)
# File 'lib/yard/parser/source_parser.rb', line 390
attr_reader :parser_type
#parser_type=(value) (rw, private)
[ GitHub ]# File 'lib/yard/parser/source_parser.rb', line 500
def parser_type=(value) @parser_type = self.class.validated_parser_type(value) end
Instance Method Details
#convert_encoding(content) (private)
Searches for encoding line and forces encoding
# File 'lib/yard/parser/source_parser.rb', line 471
def convert_encoding(content) return content unless content.respond_to?(:force_encoding) if content =~ ENCODING_LINE content.force_encoding($1) else content.force_encoding('binary') ENCODING_BYTE_ORDER_MARKS.each do |encoding, bom| bom.force_encoding('binary') if content.start_with?(bom) return content.sub(bom, '').force_encoding(encoding) end end content.force_encoding('utf-8') # UTF-8 is default encoding content end end
#parse(content = __FILE__) ⇒ Object?
The main parser method. This should not be called directly. Instead,
use the class methods parse and .parse_string.
# File 'lib/yard/parser/source_parser.rb', line 418
def parse(content = __FILE__) case content when String @file = File.cleanpath(content) content = convert_encoding(String.new(File.read_binary(file))) checksum = Registry.checksum_for(content) return if Registry.checksums[file] == checksum if Registry.checksums.key?(file) log.info "File '#{file}' was modified, re-processing..." end Registry.checksums[@file] = checksum self.parser_type = parser_type_for_filename(file) else content = content.read if content.respond_to? :read end @contents = content @parser = parser_class.new(content, file) self.class.before_parse_file_callbacks.each do |cb| return @parser if cb.call(self) == false end @parser.parse post_process self.class.after_parse_file_callbacks.each do |cb| cb.call(self) end @parser rescue ArgumentError, NotImplementedError => e log.warn("Cannot parse `#{file}': #{e.}") log.backtrace(e, :warn) rescue ParserSyntaxError => e log.warn(e..capitalize) log.backtrace(e, :warn) end
#parser_class (private)
# File 'lib/yard/parser/source_parser.rb', line 515
def parser_class klass = self.class.parser_types[parser_type] unless klass raise ArgumentError, "invalid parser type '#{parser_type}' or unrecognized file", caller[1..-1] end klass end
#parser_type_for_filename(filename) ⇒ Symbol (private)
Guesses the parser type to use depending on the file extension.
# File 'lib/yard/parser/source_parser.rb', line 508
def parser_type_for_filename(filename) ext = (File.extname(filename)[1..-1] || "").downcase type = self.class.parser_type_for_extension(ext) parser_type == :ruby18 && type == :ruby ? :ruby18 : type end
#post_process ⇒ void (private)
This method returns an undefined value.
Runs a ::YARD::Handlers::Processor object to post process the parsed statements.
#tokenize(content) ⇒ Array
Tokenizes but does not parse the block of code using the current #parser_type
# File 'lib/yard/parser/source_parser.rb', line 462
def tokenize(content) @parser = parser_class.new(content, file) @parser.tokenize end