Class: Rack::Multipart::Parser
| Relationships & Source Files | |
| Namespace Children | |
|
Classes:
| |
| Extension / Inclusion / Inheritance Descendants | |
|
Subclasses:
|
|
| Inherits: | Object |
| Defined in: | lib/rack/multipart/parser.rb |
Overview
Parser handles parsing of multipart/form-data requests.
File Parameter Contents
When processing file uploads, the parser returns a hash containing
information about uploaded files. For file parameters, the hash includes:
:filename- The original filename, already URL decoded by the parser:type- The content type of the uploaded file:name- The parameter name from the form:tempfile- A Tempfile object containing the uploaded data:head- The raw header content for this part
Constant Summary
-
BOUNDARY_START_LIMIT =
private
# File 'lib/rack/multipart/parser.rb', line 6816 * 1024
-
BUFFERED_UPLOAD_BYTESIZE_LIMIT =
private
# File 'lib/rack/multipart/parser.rb', line 88env_int.call("RACK_MULTIPART_BUFFERED_UPLOAD_BYTESIZE_LIMIT", 16 * 1024 * 1024)
-
BUFSIZE =
# File 'lib/rack/multipart/parser.rb', line 601_048_576 -
CHARSET =
# File 'lib/rack/multipart/parser.rb', line 605"charset" -
CONTENT_DISPOSITION_MAX_BYTES =
# File 'lib/rack/multipart/parser.rb', line 4191536 -
CONTENT_DISPOSITION_MAX_PARAMS =
# File 'lib/rack/multipart/parser.rb', line 41816 -
CONTENT_DISPOSITION_QUOTED_ESCAPES_LIMIT =
private
# File 'lib/rack/multipart/parser.rb', line 95env_int.call("RACK_MULTIPART_CONTENT_DISPOSITION_QUOTED_ESCAPES_LIMIT", 8 * 1024)
-
EMPTY =
# File 'lib/rack/multipart/parser.rb', line 128MultipartInfo.new(nil, [].freeze).freeze
-
MIME_HEADER_BYTESIZE_LIMIT =
private
# File 'lib/rack/multipart/parser.rb', line 7164 * 1024
-
OBS_UNFOLD =
private
# File 'lib/rack/multipart/parser.rb', line 420/\r\n([ \t])/ -
PARSER_BYTESIZE_LIMIT =
private
# File 'lib/rack/multipart/parser.rb', line 92
simplecov:disable
bytesize_limit > 0 ? bytesize_limit : nil
-
REENCODE_DUMMY_ENCODINGS =
# File 'lib/rack/multipart/parser.rb', line 648{ # ISO-2022-JP is a legacy but still widely used encoding in Japan # Here we convert ISO-2022-JP to UTF-8 so that it can be handled. Encoding::ISO_2022_JP => true # Other dummy encodings are rarely used and have not been supported yet. # Adding support for them will require careful considerations. } -
TEMPFILE_FACTORY =
# File 'lib/rack/multipart/parser.rb', line 62lambda { |filename, content_type| extension = ::File.extname(filename.gsub("\0", '%00'))[0, 129] Tempfile.new(["RackMultipart", extension]) }
-
TEXT_PLAIN =
# File 'lib/rack/multipart/parser.rb', line 61"text/plain"
Class Method Summary
Instance Attribute Summary
- #state readonly
Instance Method Summary
- #parse(io)
- #result
-
#consume_boundary
private
Scan until the we find the start or end of the boundary.
-
#find_encoding(enc)
private
Return the related Encoding object.
- #handle_consume_token private
- #handle_dummy_encoding(name, body) private
- #handle_empty_content!(content) private
-
#handle_fast_forward
private
This handles the initial parser state.
- #handle_mime_body private
- #handle_mime_head private
- #normalize_filename(filename) private
- #read_data(io, outbuf) private
- #tag_multipart_encoding(filename, content_type, name, body) private
- #update_retained_size(size) private
Constructor Details
.new(boundary, tempfile, bufsize, query_parser) ⇒ Parser
# File 'lib/rack/multipart/parser.rb', line 304
def initialize(boundary, tempfile, bufsize, query_parser) @query_parser = query_parser @params = query_parser.make_params @bufsize = bufsize @state = :FAST_FORWARD @mime_index = 0 @body_retained = nil @retained_size = 0 @total_bytes_read = (0 if PARSER_BYTESIZE_LIMIT) @content_disposition_quoted_escapes = 0 @collector = Collector.new tempfile @sbuf = StringScanner.new("".b) @body_regex = /(?:#{EOL}|\A)--#{Regexp.quote(boundary)}(?:#{EOL}|--)/m @body_regex_at_end = /#{@body_regex}\z/m @end_boundary_size = boundary.bytesize + 4 # (-- at start, -- at finish) @rx_max_size = boundary.bytesize + 6 # (\r\n-- at start, either \r\n or -- at finish) @head_regex = /(.*?#{EOL})#{EOL}/m end
Class Method Details
.default_bufsize
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 153
def self.default_bufsize BUFSIZE end
.default_query_parser
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 157
def self.default_query_parser Rack::Utils.default_query_parser end
.default_tempfile_factory
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 149
def self.default_tempfile_factory TEMPFILE_FACTORY end
.parse(io, content_length, content_type, tmpfile, bufsize, qp)
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 181
def self.parse(io, content_length, content_type, tmpfile, bufsize, qp) return EMPTY if 0 == content_length boundary = parse_boundary content_type return EMPTY unless boundary if PARSER_BYTESIZE_LIMIT && content_length && content_length > PARSER_BYTESIZE_LIMIT raise Error, "multipart Content-Length #{content_length} exceeds limit of #{PARSER_BYTESIZE_LIMIT} bytes" end if boundary.length > 70 # RFC 1521 Section 7.2.1 imposes a 70 character maximum for the boundary. # Most clients use no more than 55 characters. raise BoundaryTooLongError, "multipart boundary size too large (#{boundary.length} characters)" end io = BoundedIO.new(io, content_length) if content_length parser = new(boundary, tmpfile, bufsize, qp) parser.parse(io) parser.result end
.parse_boundary(content_type)
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 130
def self.parse_boundary(content_type) return unless content_type data = content_type.match(MULTIPART) return unless data unless data[1].empty? raise Error, "whitespace between boundary parameter name and equal sign" end # Count across the whole header rather than only after the match: the # lazy prefix in MULTIPART can skip a first boundary parameter whose # value is empty or starts with a quote, semicolon, or comma. Also # count RFC 2231 boundary* parameters, which some parsers prefer. if content_type.scan(/#{NOT_AFTER_TOKEN_CHAR}boundary(?:\*\d*\*?)?\s*=/i).size > 1 raise BoundaryTooLongError, "multiple boundary parameters found in multipart content type" end data[2] end
.parse_from_env(env, query_parser: default_query_parser)
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 161
def self.parse_from_env(env, query_parser: default_query_parser) unless io = env[RACK_INPUT] raise MissingInputError, "Missing input stream!" end if content_length = env['CONTENT_LENGTH'] content_length = content_length.to_i end content_type = env['CONTENT_TYPE'] tempfile = env[RACK_MULTIPART_TEMPFILE_FACTORY] || default_tempfile_factory bufsize = env[RACK_MULTIPART_BUFFER_SIZE] || default_bufsize info = parse(io, content_length, content_type, tempfile, bufsize, query_parser) env[RACK_TEMPFILES] = info.tmp_files return info.params end
Instance Attribute Details
#state (readonly)
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 302
attr_reader :state
Instance Method Details
#consume_boundary (private)
Scan until the we find the start or end of the boundary. If we find it, return the appropriate symbol for the start or end of the boundary. If we don't find the start or end of the boundary, clear the buffer and return nil.
# File 'lib/rack/multipart/parser.rb', line 582
def consume_boundary if read_buffer = @sbuf.scan_until(@body_regex) read_buffer.end_with?(EOL) ? :BOUNDARY : :END_BOUNDARY else @sbuf.terminate nil end end
#find_encoding(enc) (private)
Return the related Encoding object. However, because enc is submitted by the user, it may be invalid, so use a binary encoding in that case.
# File 'lib/rack/multipart/parser.rb', line 642
def find_encoding(enc) Encoding.find enc rescue ArgumentError Encoding::BINARY end
#handle_consume_token (private)
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 408
def handle_consume_token tok = consume_boundary # break if we're at the end of a buffer, but not if it is the end of a field @state = if tok == :END_BOUNDARY || (@sbuf.eos? && tok != :BOUNDARY) :DONE else :MIME_HEAD end end
#handle_dummy_encoding(name, body) (private)
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 657
def handle_dummy_encoding(name, body) # A string object with a 'dummy' encoding does not have full functionality and can cause errors. # So here we covert it to UTF-8 so that it can be handled properly. if name.encoding.dummy? && REENCODE_DUMMY_ENCODINGS[name.encoding] name = name.encode(Encoding::UTF_8) body = body.encode(Encoding::UTF_8) end return name, body end
#handle_empty_content!(content) (private)
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 667
def handle_empty_content!(content) if content.nil? || content.empty? raise EmptyContentError end end
#handle_fast_forward (private)
This handles the initial parser state. We read until we find the starting boundary, then we can transition to the next state. If we find the ending boundary, this is an invalid multipart upload, but keep scanning for opening boundary in that case. If no boundary found, we need to keep reading data and retry. It's highly unlikely the initial read will not consume the boundary. The client would have to deliberately craft a response with the opening boundary beyond the buffer size for that to happen.
# File 'lib/rack/multipart/parser.rb', line 381
def handle_fast_forward while true case consume_boundary when :BOUNDARY # found opening boundary, transition to next state @state = :MIME_HEAD return when :END_BOUNDARY # invalid multipart upload if @sbuf.pos == @end_boundary_size && @sbuf.rest == EOL # stop parsing a buffer if a buffer is only an end boundary. @state = :DONE return end # retry for opening boundary else # We raise if we don't find the multipart boundary, to avoid unbounded memory # buffering. Note that the actual limit is the higher of 16KB and the buffer size (1MB by default) raise Error, "multipart boundary not found within limit" if @sbuf.string.bytesize > BOUNDARY_START_LIMIT # no boundary found, keep reading data return :want_read end end end
#handle_mime_body (private)
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 549
def handle_mime_body if (body_with_boundary = @sbuf.check_until(@body_regex)) # check but do not advance the pointer yet body = body_with_boundary.sub(@body_regex_at_end, '') # remove the boundary from the string update_retained_size(body.bytesize) if @body_retained @collector.on_mime_body @mime_index, body @sbuf.pos += body.length + 2 # skip \r\n after the content @state = :CONSUME_TOKEN @mime_index += 1 else # Save what we have so far if @rx_max_size < @sbuf.rest_size delta = @sbuf.rest_size - @rx_max_size body = @sbuf.peek(delta) update_retained_size(body.bytesize) if @body_retained @collector.on_mime_body @mime_index, body @sbuf.pos += delta end @sbuf.string = @sbuf.rest :want_read end end
#handle_mime_head (private)
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 423
def handle_mime_head if @sbuf.scan_until(@head_regex) head = @sbuf[1] content_type = head[MULTIPART_CONTENT_TYPE, 1] content_type.gsub!(OBS_UNFOLD, '\1') if content_type if (disposition = head[MULTIPART_CONTENT_DISPOSITION, 1]) && disposition.bytesize <= CONTENT_DISPOSITION_MAX_BYTES # Implement OBS unfolding (RFC 5322 Section 2.2.3) disposition.gsub!(OBS_UNFOLD, '\1') # ignore actual content-disposition value (should always be form-data) if i = disposition.index(';') disposition.slice!(0, i + 1) else disposition = '' end param = nil num_params = 0 # Parse parameter list while i = disposition.index('=') # Only parse up to max parameters, to avoid potential denial of service num_params += 1 break if num_params > CONTENT_DISPOSITION_MAX_PARAMS # Found end of parameter name, ensure forward progress in loop param = disposition.slice!(0, i+1) # Remove ending equals and preceding whitespace from parameter name param.chomp!('=') param.lstrip! if disposition[0] == '"' # Parameter value is quoted, parse it, handling backslash escapes disposition.slice!(0, 1) value = String.new while i = disposition.index(/(["\\])/) c = $1 # Append all content until ending quote or escape value << disposition.slice!(0, i) # Remove either backslash or ending quote, # ensures forward progress in loop disposition.slice!(0, 1) # stop parsing parameter value if found ending quote break if c == '"' @content_disposition_quoted_escapes += 1 if @content_disposition_quoted_escapes > CONTENT_DISPOSITION_QUOTED_ESCAPES_LIMIT raise Error, "number of quoted escapes during content disposition parsing exceeds limit" end escaped_char = disposition.slice!(0, 1) if param == 'filename' && escaped_char != '"' # Possible IE uploaded filename, append both escape backslash and value value << c << escaped_char else # Other only append escaped value value << escaped_char end end else if i = disposition.index(';') # Parameter value unquoted (which may be invalid), value ends at semicolon value = disposition.slice!(0, i) else # If no ending semicolon, assume remainder of line is value and stop # parsing disposition.strip! value = disposition disposition = '' end end case param when 'name' name = value when 'filename' filename = value # else # ignore other parameters end # skip trailing semicolon, to proceed to next parameter if i = disposition.index(';') disposition.slice!(0, i+1) end end else name = head[MULTIPART_CONTENT_ID, 1] end if filename filename = normalize_filename(filename) end if name.nil? || name.empty? name = filename || "#{content_type || TEXT_PLAIN}[]".dup end # Mime part head data is retained for both TempfilePart and BufferPart # for the entireity of the parse, even though it isn't used for BufferPart. update_retained_size(head.bytesize) # If a filename is given, a TempfilePart will be used, so the body will # not be buffered in memory. However, if a filename is not given, a BufferPart # will be used, and the body will be buffered in memory. @body_retained = !filename @collector.on_mime_head @mime_index, head, filename, content_type, name @state = :MIME_BODY else # We raise if the mime part header is too large, to avoid unbounded memory # buffering. Note that the actual limit is the higher of 64KB and the buffer size (1MB by default) raise Error, "multipart mime part header too large" if @sbuf.rest.bytesize > MIME_HEADER_BYTESIZE_LIMIT @sbuf.string = @sbuf.rest return :want_read end end
#normalize_filename(filename) (private)
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 591
def normalize_filename(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) } filename = Utils.unescape_path(filename) end # Interpret as UTF-8 if it's binary and contains valid UTF-8 bytes # (handles "commonly deployed systems" that send UTF-8 directly) filename.force_encoding(Encoding::UTF_8) filename.scrub! filename.split(/[\/\\]/).last || String.new end
#parse(io)
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 325
def parse(io) @total_bytes_read &&= nil if io.is_a?(BoundedIO) outbuf = String.new read_data(io, outbuf) loop do status = case @state when :FAST_FORWARD handle_fast_forward when :CONSUME_TOKEN handle_consume_token when :MIME_HEAD handle_mime_head when :MIME_BODY handle_mime_body else # when :DONE return end read_data(io, outbuf) if status == :want_read end end
#read_data(io, outbuf) (private)
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 362
def read_data(io, outbuf) content = io.read(@bufsize, outbuf) handle_empty_content!(content) if @total_bytes_read @total_bytes_read += content.bytesize if @total_bytes_read > PARSER_BYTESIZE_LIMIT raise Error, "multipart upload exceeds limit of #{PARSER_BYTESIZE_LIMIT} bytes" end end @sbuf.concat(content) end
#result
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 349
def result @collector.each do |part| part.get_data do |data| tag_multipart_encoding(part.filename, part.content_type, part.name, data) name, data = handle_dummy_encoding(part.name, data) @query_parser.normalize_params(@params, name, data) end end MultipartInfo.new @params.to_params_hash, @collector.find_all(&:file?).map(&:body) end
#tag_multipart_encoding(filename, content_type, name, body) (private)
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 608
def tag_multipart_encoding(filename, content_type, name, body) name = name.to_s encoding = Encoding::UTF_8 name.force_encoding(encoding) return if filename if content_type list = content_type.split(';') type_subtype = list.first type_subtype.strip! if TEXT_PLAIN == type_subtype rest = list.drop 1 rest.each do |param| k, v = param.split('=', 2) k.strip! next unless v v.strip! v = v[1..-2] if v.start_with?('"') && v.end_with?('"') if k == "charset" encoding = find_encoding(v) end end end end name.force_encoding(encoding) body.force_encoding(encoding) end
#update_retained_size(size) (private)
[ GitHub ]# File 'lib/rack/multipart/parser.rb', line 571
def update_retained_size(size) @retained_size += size if @retained_size > BUFFERED_UPLOAD_BYTESIZE_LIMIT raise Error, "multipart data over retained size limit" end end