123456789_123456789_123456789_123456789_123456789_

Class: RDoc::Markup::Parser

Relationships & Source Files
Namespace Children
Classes:
MyStringScanner
Exceptions:
Error, ParseError
Extension / Inclusion / Inheritance Descendants
Subclasses:
TomDoc
Super Chains via Extension / Inclusion / Inheritance
Instance Chain:
self, ::RDoc::Text
Inherits: Object
Defined in: lib/rdoc/markup/parser.rb

Overview

A recursive-descent parser for ::RDoc::RDoc markup.

The parser tokenizes an input string then parses the tokens into a Document. Documents can be converted into output formats by writing a visitor like RDoc::Markup::ToHTML.

The parser only handles the block-level constructs Paragraph, List, ListItem, Heading, Verbatim, BlankLine, Rule and BlockQuote. Inline markup such as +blah+ is handled separately by InlineParser.

To see what markup the Parser implements read ::RDoc::RDoc. To see how to use ::RDoc::RDoc markup to format text in your program read ::RDoc::Markup.

Constant Summary

::RDoc::Text - Included

MARKUP_FORMAT, SPACE_SEPARATED_LETTER_CLASS

Class Method Summary

Instance Attribute Summary

  • #debug rw

    Enables display of debugging information.

  • #tokens readonly

    Token accessor.

::RDoc::Text - Included

#language

The language for this text.

Instance Method Summary

::RDoc::Text - Included

#flush_left

Flush text left based on the shortest line.
#markup

Convert a string in markup format into HTML.
#normalize_comment

Strips hashes, expands tabs then flushes text to the left.
#parse

Normalizes text then builds a Document from it.
#snippet

The first limit characters of text as HTML.
#strip_hashes

Strips leading # characters from text
#strip_newlines

Strips leading and trailing \n characters from text
#strip_stars

Strips /* */ style comments.
#wrap

Wraps txt to line_len

Constructor Details

.new ⇒ Parser

Creates a new Parser. See also .parse

[ GitHub ] 

  


# File 'lib/rdoc/markup/parser.rb', line 81



def initialize
  @binary_input   = nil
  @current_token  = nil
  @debug          = false
  @s              = nil
  @tokens         = []
end


  







Class Method Details



  

    .parse(str)  
  



  

    
Parses str into a Document.


Use RDoc::Markup#parse instead of this method.

  



  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 62



def self.parse(str)
  parser = new
  parser.tokenize str
  doc = Markup::Document.new
  parser.parse doc
end


  








  

    .tokenize(str)  
  



  

    
Returns a token stream for str, for testing

  



  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 72



def self.tokenize(str)
  parser = new
  parser.tokenize str
  parser.tokens
end


  







Instance Attribute Details



  

    #debug   (rw)
  



  

    
Enables display of debugging information

  



  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 50



attr_accessor :debug


  








  

    #tokens   (readonly)
  



  

    
Token accessor

  



  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 55



attr_reader :tokens


  







Instance Method Details



  

    #build_heading(level)  
  



  

    
Builds a Heading of level

  



  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 92



def build_heading(level)
  type, text, = get

  text = case type
         when :TEXT
           skip :NEWLINE
           text
         else
           unget
           ''
         end

  Markup::Heading.new level, text
end


  








  

    #build_list(margin)  
  



  

    
Builds a List flush to margin

  



  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 110



def build_list(margin)
  p :list_start => margin if @debug

  list = Markup::List.new
  label = nil

  until @tokens.empty? do
    type, data, column, = get

    case type
    when *LIST_TOKENS
      if column < margin || (list.type && list.type != type)
        unget
        break
      end

      list.type = type
      peek_type, _, column, = peek_token

      case type
      when :NOTE, :LABEL
        label = [] unless label

        if peek_type == :NEWLINE
          # description not on the same line as LABEL/NOTE
          # skip the trailing newline & any blank lines below
          while peek_type == :NEWLINE
            get
            peek_type, _, column, = peek_token
          end

          # we may be:
          #   - at end of stream
          #   - at a column < margin:
          #         [text]
          #       blah blah blah
          #   - at the same column, but with a different type of list item
          #       [text]
          #       * blah blah
          #   - at the same column, with the same type of list item
          #       [one]
          #       [two]
          # In all cases, we have an empty description.
          # In the last case only, we continue.
          if peek_type.nil? || column < margin
            empty = true
          elsif column == margin
            case peek_type
            when type
              empty = :continue
            when *LIST_TOKENS
              empty = true
            else
              empty = false
            end
          else
            empty = false
          end

          if empty
            label << data
            next if empty == :continue
            break
          end
        end
      else
        data = nil
      end

      if label
        data = label << data
        label = nil
      end

      list_item = Markup::ListItem.new data
      parse list_item, column
      list << list_item

    else
      unget
      break
    end
  end

  p :list_end => margin if @debug

  if list.empty?
    return nil unless label
    return nil unless [:LABEL, :NOTE].include? list.type

    list_item = Markup::ListItem.new label, Markup::BlankLine.new
    list << list_item
  end

  list
end


  








  

    #build_paragraph(margin)  
  



  

    
Builds a Paragraph that is flush to margin

  



  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 210



def build_paragraph(margin)
  p :paragraph_start => margin if @debug

  paragraph = Markup::Paragraph.new

  until @tokens.empty? do
    type, data, column, = get

    if type == :TEXT and column == margin
      paragraph << data

      break if peek_token.first == :BREAK

      data << ' ' if skip :NEWLINE and /#{SPACE_SEPARATED_LETTER_CLASS}\z/o.match?(data)
    else
      unget
      break
    end
  end

  paragraph.parts.last.sub!(/ \z/, '') # cleanup

  p :paragraph_end => margin if @debug

  paragraph
end


  








  

    #build_verbatim(margin)  
  



  

    
Builds a Verbatim that is indented from margin.


The verbatim block is shifted left (the least indented lines start in
column 0).  Each part of the verbatim is one line of text, always
terminated by a newline.  Blank lines always consist of a single newline
character, and there is never a single newline at the end of the verbatim.

  



  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 245



def build_verbatim(margin)
  p :verbatim_begin => margin if @debug
  verbatim = Markup::Verbatim.new

  min_indent = nil
  generate_leading_spaces = true
  line = ''.dup

  until @tokens.empty? do
    type, data, column, = get

    if type == :NEWLINE
      line << data
      verbatim << line
      line = ''.dup
      generate_leading_spaces = true
      next
    end

    if column <= margin
      unget
      break
    end

    if generate_leading_spaces
      indent = column - margin
      line << ' ' * indent
      min_indent = indent if min_indent.nil? || indent < min_indent
      generate_leading_spaces = false
    end

    case type
    when :HEADER
      line << '=' * data
      _, _, peek_column, = peek_token
      peek_column ||= column + data
      indent = peek_column - column - data
      line << ' ' * indent
    when :RULE
      width = 2 + data
      line << '-' * width
      _, _, peek_column, = peek_token
      peek_column ||= column + width
      indent = peek_column - column - width
      line << ' ' * indent
    when :BREAK, :TEXT
      line << data
    when :BLOCKQUOTE
      line << '>>>'
      peek_type, _, peek_column = peek_token
      if peek_type != :NEWLINE and peek_column
        line << ' ' * (peek_column - column - 3)
      end
    else # *LIST_TOKENS
      list_marker = case type
                    when :BULLET then data
                    when :LABEL  then "[#{data}]"
                    when :NOTE   then "#{data}::"
                    else # :LALPHA, :NUMBER, :UALPHA
                      "#{data}."
                    end
      line << list_marker
      peek_type, _, peek_column = peek_token
      unless peek_type == :NEWLINE
        peek_column ||= column + list_marker.length
        indent = peek_column - column - list_marker.length
        line << ' ' * indent
      end
    end

  end

  verbatim << line << "\n" unless line.empty?
  verbatim.parts.each { |p| p.slice!(0, min_indent) unless p == "\n" } if min_indent > 0
  verbatim.normalize

  p :verbatim_end => margin if @debug

  verbatim
end


  








  

    #get  
  



  

    
Pulls the next token from the stream.

  



  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 329



def get
  @current_token = @tokens.shift
  p :get => @current_token if @debug
  @current_token
end


  








  

    #parse(parent, indent = 0)  
  



  

    
Parses the tokens into an array of RDoc::Markup::XXX objects,
and appends them to the passed parent RDoc::Markup::YYY object.


Exits at the end of the token stream, or when it encounters a token
in a column less than indent (unless it is a NEWLINE).


Returns parent.

  



  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 344



def parse(parent, indent = 0)
  p :parse_start => indent if @debug

  until @tokens.empty? do
    type, data, column, = get

    case type
    when :BREAK
      parent << Markup::BlankLine.new
      skip :NEWLINE, false
      next
    when :NEWLINE
      # trailing newlines are skipped below, so this is a blank line
      parent << Markup::BlankLine.new
      skip :NEWLINE, false
      next
    end

    # indentation change: break or verbatim
    if column < indent
      unget
      break
    elsif column > indent
      unget
      parent << build_verbatim(indent)
      next
    end

    # indentation is the same
    case type
    when :HEADER
      parent << build_heading(data)
    when :RULE
      parent << Markup::Rule.new(data)
      skip :NEWLINE
    when :TEXT
      unget
      parse_text parent, indent
    when :BLOCKQUOTE
      nil while (type, = get; type) and type != :NEWLINE
      _, _, column, = peek_token
      bq = Markup::BlockQuote.new
      p :blockquote_start => [data, column] if @debug
      parse bq, column
      p :blockquote_end => indent if @debug
      parent << bq
    when *LIST_TOKENS
      unget
      parent << build_list(indent)
    else
      type, data, column, line = @current_token
      raise ParseError, "Unhandled token #{type} (#{data.inspect}) at #{line}:#{column}"
    end
  end

  p :parse_end => indent if @debug

  parent

end


  








  

    #parse_text(parent, indent)  
  

  

    This method is for internal use only.
  



  

    
Small hook that is overridden by ::RDoc::TomDoc

  



  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 408



def parse_text(parent, indent) # :nodoc:
  parent << build_paragraph(indent)
end


  








  

    #peek_token  
  



  

    
Returns the next token on the stream without modifying the stream

  



  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 415



def peek_token
  token = @tokens.first || []
  p :peek => token if @debug
  token
end


  








  

    #setup_scanner(input)  
  



  

    
Creates the StringScanner

  



  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 470



def setup_scanner(input)
  @s = MyStringScanner.new input
end


  








  

    #skip(token_type, error = true)  
  



  

    
Skips the next token if its type is token_type.


Optionally raises an error if the next token is not of the expected type.

  





  
Raises:





  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 479



def skip(token_type, error = true)
  type, = get
  return unless type # end of stream
  return @current_token if token_type == type
  unget
  raise ParseError, "expected #{token_type} got #{@current_token.inspect}" if error
end


  








  

    #tokenize(input)  
  



  

    
Turns text input into a stream of tokens

  



  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 490



def tokenize(input)
  setup_scanner input

  until @s.eos? do
    pos = @s.pos

    # leading spaces will be reflected by the column of the next token
    # the only thing we loose are trailing spaces at the end of the file
    next if @s.scan(/ +/)

    # note: after BULLET, LABEL, etc.,
    # indent will be the column of the next non-newline token

    @tokens << case
               # [CR]LF => :NEWLINE
               when @s.scan(/\r?\n/)
                 token = [:NEWLINE, @s.matched, *pos]
                 @s.newline!
                 token
               # === text => :HEADER then :TEXT
               when @s.scan(/(=+)(\s*)/)
                 level = @s[1].length
                 header = [:HEADER, level, *pos]

                 if @s[2] =~ /^\r?\n/
                   @s.unscan(@s[2])
                   header
                 else
                   pos = @s.pos
                   @s.scan(/.*/)
                   @tokens << header
                   [:TEXT, @s.matched.sub(/\r$/, ''), *pos]
                 end
               # --- (at least 3) and nothing else on the line => :RULE
               when @s.scan(/(-{3,}) *\r?$/)
                 [:RULE, @s[1].length - 2, *pos]
               # * or - followed by white space and text => :BULLET
               when @s.scan(/([*-]) +(\S)/)
                 @s.unscan(@s[2])
                 [:BULLET, @s[1], *pos]
               # A. text, a. text, 12. text => :UALPHA, :LALPHA, :NUMBER
               when @s.scan(/([a-z]|\d+)\. +(\S)/i)
                 # FIXME if tab(s), the column will be wrong
                 # either support tabs everywhere by first expanding them to
                 # spaces, or assume that they will have been replaced
                 # before (and provide a check for that at least in debug
                 # mode)
                 list_label = @s[1]
                 @s.unscan(@s[2])
                 list_type =
                   case list_label
                   when /[a-z]/ then :LALPHA
                   when /[A-Z]/ then :UALPHA
                   when /\d/    then :NUMBER
                   else
                     raise ParseError, "BUG token #{list_label}"
                   end
                 [list_type, list_label, *pos]
               # [text] followed by spaces or end of line => :LABEL
               when @s.scan(/\[(.*?)\]( +|\r?$)/)
                 [:LABEL, @s[1], *pos]
               # text:: followed by spaces or end of line => :NOTE
               when @s.scan(/(.*?)::( +|\r?$)/)
                 [:NOTE, @s[1], *pos]
               # >>> followed by end of line => :BLOCKQUOTE
               when @s.scan(/>>> *(\w+)?$/)
                 if word = @s[1]
                   @s.unscan(word)
                 end
                 [:BLOCKQUOTE, word, *pos]
               # anything else: :TEXT
               else
                 @s.scan(/(.*?)(  )?\r?$/)
                 token = [:TEXT, @s[1], *pos]

                 if @s[2]
                   @tokens << token
                   [:BREAK, @s[2], pos[0] + @s[1].length, pos[1]]
                 else
                   token
                 end
               end
  end

  self
end


  








  

    #unget  
  



  

    
Returns the current token to the token stream

  





  
Raises:





  [ GitHub ]


  

  


# File 'lib/rdoc/markup/parser.rb', line 580



def unget
  token = @current_token
  p :unget => token if @debug
  raise Error, 'too many #ungets' if token == @tokens.first
  @tokens.unshift token if token
end