Class: YARD::Parser::SourceParser
Relationships & Source Files | |
Inherits: | Object |
Defined in: | lib/yard/parser/source_parser.rb |
Overview
Responsible for parsing a source file into the namespace. Parsing also invokes handlers to process the parsed statements and generate any code objects that may be recognized.
Custom Parsers
SourceParser
allows custom parsers to be registered and called when
a certain filetype is recognized. To register a parser and hook it
up to a set of file extensions, call .register_parser_type
Constant Summary
-
DEFAULT_PATH_GLOB =
The default glob of files to be parsed.
["{lib,app}/**/*.rb", "ext/**/*.{c,cc,cxx,cpp,rb}"]
-
ENCODING_BYTE_ORDER_MARKS =
Byte order marks for various encodings
{ 'utf-8' => String.new("\xEF\xBB\xBF"), # Not yet supported # 'utf-16be' => "\xFE\xFF", # 'utf-16le' => "\xFF\xFE", # 'utf-32be' => "\x00\x00\xFF\xFE", # 'utf-32le' => "\xFF\xFE", }
-
ENCODING_LINE =
# File 'lib/yard/parser/source_parser.rb', line 65%r{\A(?:\s*#*!.*\r?\n)?\s*(?:#+|/\*+|//+).*coding\s*[:=]{1,2}\s*([a-z\d_\-]+)}i
-
FROZEN_STRING_LINE =
# File 'lib/yard/parser/source_parser.rb', line 66/frozen(-|_)string(-|_)literal:\s+(true|false)/i
-
SHEBANG_LINE =
# File 'lib/yard/parser/source_parser.rb', line 64/\A\s*#!\S+/
Parser Callbacks
-
.after_parse_file {|parser| ... } ⇒ Proc
Registers a callback to be called after an individual file is parsed.
- .after_parse_file_callbacks ⇒ Array<Proc>
-
.after_parse_list {|files, globals| ... } ⇒ Proc
Registers a callback to be called after a list of files is parsed via .parse.
- .after_parse_list_callbacks ⇒ Array<Proc>
-
.before_parse_file {|parser| ... } ⇒ Proc
Registers a callback to be called before an individual file is parsed.
- .before_parse_file_callbacks ⇒ Array<Proc>
-
.before_parse_list {|files, globals| ... } ⇒ Proc
Registers a callback to be called before a list of files is parsed via .parse.
- .before_parse_list_callbacks ⇒ Array<Proc>
- #contents ⇒ String readonly
-
#convert_encoding(content)
private
Searches for encoding line and forces encoding.
- #file ⇒ String rw
- #globals ⇒ OpenStruct readonly
- .new(parser_type = SourceParser.parser_type, globals = nil) ⇒ SourceParser constructor
-
#parse(content = __FILE__) ⇒ Object?
The main parser method.
-
.parse_in_order(*files) ⇒ void
private
Parses a list of files in a queue.
- #parser_class private
- #parser_type ⇒ Symbol rw
- #parser_type=(value) rw private
-
#parser_type_for_filename(filename) ⇒ Symbol
private
Guesses the parser type to use depending on the file extension.
-
#post_process ⇒ void
private
Runs a
::YARD::Handlers::Processor
object to post process the parsed statements. -
#tokenize(content) ⇒ Array
Tokenizes but does not parse the block of code using the current #parser_type
Class Attribute Summary
- .parser_type ⇒ Symbol rw
- .parser_type=(value) rw
- .parser_type_extensions ⇒ Hash rw Internal use only Internal use only
- .parser_type_extensions=(value) rw
- .parser_types ⇒ Hash{Symbol=>Object} rw Internal use only Internal use only
- .parser_types=(value) rw
Class Method Summary
-
.parse(paths = DEFAULT_PATH_GLOB, excluded = [], level = log.level) ⇒ void
Parses a path or set of paths.
-
.parse_string(content, ptype = parser_type) ⇒ Object
Parses a string
content
-
.parser_type_for_extension(extension) ⇒ Symbol
Finds a parser type that is registered for the extension.
-
.register_parser_type(type, parser_klass, extensions = nil) ⇒ void
Registers a new parser type.
-
.tokenize(content, ptype = parser_type) ⇒ Array
Tokenizes but does not parse the block of code.
-
.validated_parser_type(type) ⇒ Symbol
Internal use only
Internal use only
Returns the validated parser type.
Constructor Details
.new(parser_type = SourceParser.parser_type, globals = nil) ⇒ SourceParser
# File 'lib/yard/parser/source_parser.rb', line 406
def initialize(parser_type = SourceParser.parser_type, globals1 = nil, globals2 = nil) globals = [true, false].include?(globals1) ? globals2 : globals1 @file = '(stdin)' @globals = globals || OpenStruct.new self.parser_type = parser_type end
Class Attribute Details
.parser_type ⇒ Symbol
(rw)
# File 'lib/yard/parser/source_parser.rb', line 85
attr_reader :parser_type
.parser_type=(value) (rw)
[ GitHub ]# File 'lib/yard/parser/source_parser.rb', line 87
def parser_type=(value) @parser_type = validated_parser_type(value) end
.parser_type_extensions ⇒ Hash (rw)
# File 'lib/yard/parser/source_parser.rb', line 163
def parser_type_extensions; @@parser_type_extensions ||= {} end
.parser_type_extensions=(value) (rw)
[ GitHub ]# File 'lib/yard/parser/source_parser.rb', line 164
def parser_type_extensions=(value) @@parser_type_extensions = value end
.parser_types ⇒ Hash{Symbol
=>Object
} (rw)
# File 'lib/yard/parser/source_parser.rb', line 157
def parser_types; @@parser_types ||= {} end
.parser_types=(value) (rw)
[ GitHub ]# File 'lib/yard/parser/source_parser.rb', line 158
def parser_types=(value) @@parser_types = value end
Class Method Details
.after_parse_file {|parser| ... } ⇒ Proc
Registers a callback to be called after an individual file is parsed. The block passed to this method will be called on subsequent parse calls.
To register a callback that is called after the entire list of files is processed, see .after_parse_list.
# File 'lib/yard/parser/source_parser.rb', line 324
def after_parse_file(&block) after_parse_file_callbacks << block end
.after_parse_file_callbacks ⇒ Array<Proc
>
# File 'lib/yard/parser/source_parser.rb', line 352
def after_parse_file_callbacks @after_parse_file_callbacks ||= [] end
.after_parse_list {|files, globals| ... } ⇒ Proc
Registers a callback to be called after a list of files is parsed via .parse. The block passed to this method will be called on subsequent parse calls.
# File 'lib/yard/parser/source_parser.rb', line 258
def after_parse_list(&block) after_parse_list_callbacks << block end
.after_parse_list_callbacks ⇒ Array<Proc
>
# File 'lib/yard/parser/source_parser.rb', line 338
def after_parse_list_callbacks @after_parse_list_callbacks ||= [] end
.before_parse_file {|parser| ... } ⇒ Proc
Registers a callback to be called before an individual file is parsed. The block passed to this method will be called on subsequent parse calls.
To register a callback that is called before the entire list of files is processed, see .before_parse_list.
# File 'lib/yard/parser/source_parser.rb', line 295
def before_parse_file(&block) before_parse_file_callbacks << block end
.before_parse_file_callbacks ⇒ Array<Proc
>
# File 'lib/yard/parser/source_parser.rb', line 345
def before_parse_file_callbacks @before_parse_file_callbacks ||= [] end
.before_parse_list {|files, globals| ... } ⇒ Proc
Registers a callback to be called before a list of files is parsed via .parse. The block passed to this method will be called on subsequent parse calls.
# File 'lib/yard/parser/source_parser.rb', line 234
def before_parse_list(&block) before_parse_list_callbacks << block end
.before_parse_list_callbacks ⇒ Array<Proc
>
# File 'lib/yard/parser/source_parser.rb', line 331
def before_parse_list_callbacks @before_parse_list_callbacks ||= [] end
.parse(paths = DEFAULT_PATH_GLOB, excluded = [], level = log.level) ⇒ void
This method returns an undefined value.
Parses a path or set of paths
# File 'lib/yard/parser/source_parser.rb', line 99
def parse(paths = DEFAULT_PATH_GLOB, excluded = [], level = log.level) log.debug("Parsing #{paths.inspect} with `#{parser_type}` parser") excluded = excluded.map do |path| case path when Regexp; path else Regexp.new(path.to_s, Regexp::IGNORECASE) end end files = [paths].flatten. map {|p| File.directory?(p) ? "#{p}/**/*.{rb,c,cc,cxx,cpp}" : p }. map {|p| p.include?("*") ? Dir[p].sort_by {|d| [d.length, d] } : p }.flatten. reject {|p| !File.file?(p) || excluded.any? {|re| p =~ re } }. map {|p| p.encoding == Encoding.default_external ? p : p.dup.force_encoding(Encoding.default_external) } log.enter_level(level) do parse_in_order(*files.uniq) end end
.parse_in_order(*files) ⇒ void
(private)
This method returns an undefined value.
Parses a list of files in a queue.
# File 'lib/yard/parser/source_parser.rb', line 364
def parse_in_order(*files) global_state = OpenStruct.new return if before_parse_list_callbacks.any? do |cb| cb.call(files, global_state) == false end OrderedParser.new(global_state, files).parse after_parse_list_callbacks.each do |cb| cb.call(files, global_state) end end
.parse_string(content, ptype = parser_type) ⇒ Object
Parses a string content
# File 'lib/yard/parser/source_parser.rb', line 123
def parse_string(content, ptype = parser_type) new(ptype).parse(StringIO.new(content)) end
.parser_type_for_extension(extension) ⇒ Symbol
Finds a parser type that is registered for the extension. If no
type is found, the default Ruby
type is returned.
# File 'lib/yard/parser/source_parser.rb', line 171
def parser_type_for_extension(extension) type = parser_type_extensions.find do |_t, exts| [exts].flatten.any? {|ext| ext === extension } end validated_parser_type(type ? type.first : :ruby) end
.register_parser_type(type, parser_klass, extensions = nil) ⇒ void
This method returns an undefined value.
Registers a new parser type.
# File 'lib/yard/parser/source_parser.rb', line 146
def register_parser_type(type, parser_klass, extensions = nil) unless Base > parser_klass raise ArgumentError, "expecting parser_klass to be a subclass of YARD::Parser::Base" end parser_type_extensions[type.to_sym] = extensions if extensions parser_types[type.to_sym] = parser_klass end
.tokenize(content, ptype = parser_type) ⇒ Array
Tokenizes but does not parse the block of code
# File 'lib/yard/parser/source_parser.rb', line 132
def tokenize(content, ptype = parser_type) new(ptype).tokenize(content) end
.validated_parser_type(type) ⇒ Symbol
Returns the validated parser type. Basically, enforces that :ruby
type is never set if the Ripper library is not available
# File 'lib/yard/parser/source_parser.rb', line 184
def validated_parser_type(type) !defined?(::Ripper) && type == :ruby ? :ruby18 : type end
Instance Attribute Details
#contents ⇒ String (readonly)
# File 'lib/yard/parser/source_parser.rb', line 399
attr_reader :contents
#file ⇒ String (rw)
# File 'lib/yard/parser/source_parser.rb', line 386
attr_accessor :file
#globals ⇒ OpenStruct (readonly)
# File 'lib/yard/parser/source_parser.rb', line 395
attr_reader :globals
#parser_type ⇒ Symbol
(rw)
# File 'lib/yard/parser/source_parser.rb', line 390
attr_reader :parser_type
#parser_type=(value) (rw, private)
[ GitHub ]# File 'lib/yard/parser/source_parser.rb', line 500
def parser_type=(value) @parser_type = self.class.validated_parser_type(value) end
Instance Method Details
#convert_encoding(content) (private)
Searches for encoding line and forces encoding
# File 'lib/yard/parser/source_parser.rb', line 471
def convert_encoding(content) return content unless content.respond_to?(:force_encoding) if content =~ ENCODING_LINE content.force_encoding($1) else content.force_encoding('binary') ENCODING_BYTE_ORDER_MARKS.each do |encoding, bom| bom.force_encoding('binary') if content.start_with?(bom) return content.sub(bom, '').force_encoding(encoding) end end content.force_encoding('utf-8') # UTF-8 is default encoding content end end
#parse(content = __FILE__) ⇒ Object
?
The main parser method. This should not be called directly. Instead,
use the class methods parse
and .parse_string.
# File 'lib/yard/parser/source_parser.rb', line 418
def parse(content = __FILE__) case content when String @file = File.cleanpath(content) content = convert_encoding(String.new(File.read_binary(file))) checksum = Registry.checksum_for(content) return if Registry.checksums[file] == checksum if Registry.checksums.key?(file) log.info "File '#{file}' was modified, re-processing..." end Registry.checksums[@file] = checksum self.parser_type = parser_type_for_filename(file) else content = content.read if content.respond_to? :read end @contents = content @parser = parser_class.new(content, file) self.class.before_parse_file_callbacks.each do |cb| return @parser if cb.call(self) == false end @parser.parse post_process self.class.after_parse_file_callbacks.each do |cb| cb.call(self) end @parser rescue ArgumentError, NotImplementedError => e log.warn("Cannot parse `#{file}': #{e.}") log.backtrace(e, :warn) rescue ParserSyntaxError => e log.warn(e. .capitalize) log.backtrace(e, :warn) end
#parser_class (private)
# File 'lib/yard/parser/source_parser.rb', line 515
def parser_class klass = self.class.parser_types[parser_type] unless klass raise ArgumentError, "invalid parser type '#{parser_type}' or unrecognized file", caller[1..-1] end klass end
#parser_type_for_filename(filename) ⇒ Symbol
(private)
Guesses the parser type to use depending on the file extension.
# File 'lib/yard/parser/source_parser.rb', line 508
def parser_type_for_filename(filename) ext = (File.extname(filename)[1..-1] || "").downcase type = self.class.parser_type_for_extension(ext) parser_type == :ruby18 && type == :ruby ? :ruby18 : type end
#post_process ⇒ void
(private)
This method returns an undefined value.
Runs a ::YARD::Handlers::Processor
object to post process the parsed statements.
#tokenize(content) ⇒ Array
Tokenizes but does not parse the block of code using the current #parser_type
# File 'lib/yard/parser/source_parser.rb', line 462
def tokenize(content) @parser = parser_class.new(content, file) @parser.tokenize end