470: def fetch_page(params)
471: options = {
472: :request => nil,
473: :response => nil,
474: :connection => nil,
475: :referer => current_page(),
476: :uri => nil,
477: :verb => :get,
478: :agent => self,
479: :redirects => 0,
480: :params => [],
481: :headers => {},
482: }.merge(params)
483:
484: before_connect = Chain.new([
485: Chain::URIResolver.new(@scheme_handlers),
486: Chain::ParameterResolver.new,
487: Chain::RequestResolver.new,
488: Chain::ConnectionResolver.new(
489: @connection_cache,
490: @keep_alive,
491: @proxy_addr,
492: @proxy_port,
493: @proxy_user,
494: @proxy_pass
495: ),
496: Chain::SSLResolver.new(@ca_file, @verify_callback, @cert, @key, @pass),
497: Chain::AuthHeaders.new(@auth_hash, @user, @password, @digest),
498: Chain::HeaderResolver.new(
499: @keep_alive,
500: @keep_alive_time,
501: @cookie_jar,
502: @user_agent,
503: @gzip_enabled,
504: @request_headers
505: ),
506: Chain::CustomHeaders.new,
507: @pre_connect_hook,
508: ])
509: before_connect.handle(options)
510:
511: uri = options[:uri]
512: request = options[:request]
513: cur_page = options[:referer]
514: request_data = options[:params]
515: redirects = options[:redirects]
516: http_obj = options[:connection]
517:
518:
519: if( (page = visited_page(uri)) && page.response['Last-Modified'] )
520: request['If-Modified-Since'] = page.response['Last-Modified']
521: end if(@conditional_requests)
522:
523: http_obj.mu_lock
524:
525: http_obj.open_timeout = @open_timeout if @open_timeout
526: http_obj.read_timeout = @read_timeout if @read_timeout
527: http_obj.start unless http_obj.started?
528:
529:
530: log.info("#{ request.class }: #{ request.path }") if log
531: request.each_header do |k, v|
532: log.debug("request-header: #{ k } => #{ v }")
533: end if log
534:
535:
536: attempts = 0
537: begin
538: response = http_obj.request(request, *request_data) { |r|
539: connection_chain = Chain.new([
540: Chain::ResponseReader.new(r),
541: Chain::BodyDecodingHandler.new,
542: ])
543: connection_chain.handle(options)
544: }
545: rescue EOFError, Errno::ECONNRESET, Errno::EPIPE => x
546: log.error("Rescuing EOF error") if log
547: http_obj.finish
548: raise x if attempts >= 2
549: request.body = nil
550: http_obj.start
551: attempts += 1
552: retry
553: end
554:
555: after_connect = Chain.new([
556: @post_connect_hook,
557: Chain::ResponseBodyParser.new(@pluggable_parser, @watch_for_set),
558: Chain::ResponseHeaderHandler.new(@cookie_jar, @connection_cache),
559: ])
560: after_connect.handle(options)
561: http_obj.mu_unlock
562:
563: res_klass = options[:res_klass]
564: response_body = options[:response_body]
565: page = options[:page]
566:
567: log.info("status: #{ page.code }") if log
568:
569: if follow_meta_refresh
570: redirect_uri = nil
571: referer = page
572: if (page.respond_to?(:meta) && (redirect = page.meta.first))
573: redirect_uri = redirect.uri.to_s
574: sleep redirect.node['delay'].to_f
575: referer = Page.new(nil, {'content-type'=>'text/html'})
576: elsif refresh = response['refresh']
577: delay, redirect_uri = Page::Meta.parse(refresh, uri)
578: raise StandardError, "Invalid refresh http header" unless delay
579: if redirects + 1 > redirection_limit
580: raise RedirectLimitReachedError.new(page, redirects)
581: end
582: sleep delay.to_f
583: end
584: if redirect_uri
585: @history.push(page, page.uri)
586: return fetch_page(
587: :uri => redirect_uri,
588: :referer => referer,
589: :params => [],
590: :verb => :get,
591: :redirects => redirects + 1
592: )
593: end
594: end
595:
596: return page if res_klass <= Net::HTTPSuccess
597:
598: if res_klass == Net::HTTPNotModified
599: log.debug("Got cached page") if log
600: return visited_page(uri) || page
601: elsif res_klass <= Net::HTTPRedirection
602: return page unless follow_redirect?
603: log.info("follow redirect to: #{ response['Location'] }") if log
604: from_uri = page.uri
605: raise RedirectLimitReachedError.new(page, redirects) if redirects + 1 > redirection_limit
606: redirect_verb = options[:verb] == :head ? :head : :get
607: page = fetch_page( :uri => response['Location'].to_s,
608: :referer => page,
609: :params => [],
610: :verb => redirect_verb,
611: :redirects => redirects + 1
612: )
613: @history.push(page, from_uri)
614: return page
615: elsif res_klass <= Net::HTTPUnauthorized
616: raise ResponseCodeError.new(page) unless @user || @password
617: raise ResponseCodeError.new(page) if @auth_hash.has_key?(uri.host)
618: if response['www-authenticate'] =~ /Digest/i
619: @auth_hash[uri.host] = :digest
620: if response['server'] =~ /Microsoft-IIS/
621: @auth_hash[uri.host] = :iis_digest
622: end
623: @digest = response['www-authenticate']
624: else
625: @auth_hash[uri.host] = :basic
626: end
627: return fetch_page( :uri => uri,
628: :referer => cur_page,
629: :verb => request.method.downcase.to_sym,
630: :params => request_data,
631: :headers => options[:headers]
632: )
633: end
634:
635: raise ResponseCodeError.new(page), "Unhandled response", caller
636: end
