482: def fetch_page(uri, request, cur_page=current_page(), request_data=[])
483: raise "unsupported scheme: #{uri.scheme}" unless ['http', 'https'].include?(uri.scheme.downcase)
484:
485: log.info("#{ request.class }: #{ request.path }") if log
486:
487: page = nil
488:
489: cache_obj = (@connection_cache["#{uri.host}:#{uri.port}"] ||= {
490: :connection => nil,
491: :keep_alive_options => {},
492: })
493: http_obj = cache_obj[:connection]
494: if http_obj.nil? || ! http_obj.started?
495: http_obj = cache_obj[:connection] =
496: Net::HTTP.new( uri.host,
497: uri.port,
498: @proxy_addr,
499: @proxy_port,
500: @proxy_user,
501: @proxy_pass
502: )
503: cache_obj[:keep_alive_options] = {}
504:
505:
506: http_obj.open_timeout = @open_timeout if @open_timeout
507: http_obj.read_timeout = @read_timeout if @read_timeout
508: end
509:
510: if uri.scheme == 'https' && ! http_obj.started?
511: http_obj.use_ssl = true
512: http_obj.verify_mode = OpenSSL::SSL::VERIFY_NONE
513: if @ca_file
514: http_obj.ca_file = @ca_file
515: http_obj.verify_mode = OpenSSL::SSL::VERIFY_PEER
516: http_obj.verify_callback = @verify_callback if @verify_callback
517: end
518: if @cert && @key
519: http_obj.cert = OpenSSL::X509::Certificate.new(::File.read(@cert))
520: http_obj.key = OpenSSL::PKey::RSA.new(::File.read(@key), @pass)
521: end
522: end
523:
524:
525:
526:
527: if @keep_alive && http_obj.started?
528: opts = cache_obj[:keep_alive_options]
529: if((opts[:timeout] &&
530: Time.now.to_i - cache_obj[:last_request_time] > opts[:timeout].to_i) ||
531: opts[:max] && opts[:max].to_i == 1)
532:
533: log.debug('Finishing stale connection') if log
534: http_obj.finish
535:
536: end
537: end
538:
539: http_obj.start unless http_obj.started?
540:
541: request = set_headers(uri, request, cur_page)
542:
543:
544: if log
545: request.each_header do |k, v|
546: log.debug("request-header: #{ k } => #{ v }")
547: end
548: end
549:
550: cache_obj[:last_request_time] = Time.now.to_i
551:
552:
553: begin
554: response = http_obj.request(request, *request_data) {|response|
555:
556: body = StringIO.new
557: total = 0
558: response.read_body { |part|
559: total += part.length
560: body.write(part)
561: log.debug("Read #{total} bytes") if log
562: }
563:
564: raise EOFError if response.content_length() && response.content_length() != total
565: body.rewind
566:
567: response.each_header { |k,v|
568: log.debug("response-header: #{ k } => #{ v }")
569: } if log
570:
571: content_type = nil
572: unless response['Content-Type'].nil?
573: data = response['Content-Type'].match(/^([^;]*)/)
574: content_type = data[1].downcase unless data.nil?
575: end
576:
577: response_body =
578: if encoding = response['Content-Encoding']
579: case encoding.downcase
580: when 'gzip'
581: log.debug('gunzip body') if log
582: if response['Content-Length'].to_i > 0 || body.length > 0
583: begin
584: Zlib::GzipReader.new(body).read
585: rescue Zlib::BufError => e
586: log.error('Caught a Zlib::BufError') if log
587: body.rewind
588: body.read(10)
589: Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(body.read)
590: end
591: else
592: ''
593: end
594: when 'x-gzip'
595: body.read
596: else
597: raise 'Unsupported content encoding'
598: end
599: else
600: body.read
601: end
602:
603:
604: page = @pluggable_parser.parser(content_type).new(
605: uri,
606: response,
607: response_body,
608: response.code
609: ) { |parser|
610: parser.mech = self if parser.respond_to? :mech=
611: if parser.respond_to?(:watch_for_set=) && @watch_for_set
612: parser.watch_for_set = @watch_for_set
613: end
614: }
615:
616: }
617: rescue EOFError, Errno::ECONNRESET, Errno::EPIPE
618: log.error("Rescuing EOF error") if log
619: http_obj.finish
620: request.body = nil
621: http_obj.start
622: retry
623: end
624:
625:
626: if keep_alive_info = response['keep-alive']
627: keep_alive_info.split(/,\s*/).each do |option|
628: k, v = option.split(/=/)
629: cache_obj[:keep_alive_options] ||= {}
630: cache_obj[:keep_alive_options][k.intern] = v
631: end
632: end
633:
634: (response.get_fields('Set-Cookie')||[]).each do |cookie|
635: Cookie::parse(uri, cookie, log) { |c|
636: log.debug("saved cookie: #{c}") if log
637: @cookie_jar.add(uri, c)
638: }
639: end
640:
641: log.info("status: #{ page.code }") if log
642:
643: res_klass = Net::HTTPResponse::CODE_TO_OBJ[page.code.to_s]
644:
645: if follow_meta_refresh && page.respond_to?(:meta) &&
646: (redirect = page.meta.first)
647: return redirect.click
648: end
649:
650: return page if res_klass <= Net::HTTPSuccess
651:
652: if res_klass == Net::HTTPNotModified
653: log.debug("Got cached page") if log
654: return visited_page(uri)
655: elsif res_klass <= Net::HTTPRedirection
656: return page unless follow_redirect?
657: log.info("follow redirect to: #{ response['Location'] }") if log
658: from_uri = page.uri
659: abs_uri = to_absolute_uri(response['Location'].to_s, page)
660: page = fetch_page(abs_uri, fetch_request(abs_uri), page)
661: @history.push(page, from_uri)
662: return page
663: elsif res_klass <= Net::HTTPUnauthorized
664: raise ResponseCodeError.new(page) unless @user || @password
665: raise ResponseCodeError.new(page) if @auth_hash.has_key?(uri.host)
666: if response['www-authenticate'] =~ /Digest/i
667: @auth_hash[uri.host] = :digest
668: @digest = response['www-authenticate']
669: else
670: @auth_hash[uri.host] = :basic
671: end
672:
673: req = fetch_request(uri, request.method.downcase.to_sym)
674: request.each_header do |k,v|
675: req[k] = v
676: end
677: return fetch_page(uri, req, cur_page, request_data)
678: end
679:
680: raise ResponseCodeError.new(page), "Unhandled response", caller
681: end
