Class | Gem::RemoteFetcher |
In: |
lib/rubygems/remote_fetcher.rb
|
Parent: | Object |
RemoteFetcher handles the details of fetching gems and gem information from a remote source.
Cached RemoteFetcher instance.
# File lib/rubygems/remote_fetcher.rb, line 40 40: def self.fetcher 41: @fetcher ||= self.new Gem.configuration[:http_proxy] 42: end
Initialize a remote fetcher using the source URI and possible proxy information.
proxy
variable setting
HTTP_PROXY_PASS)
# File lib/rubygems/remote_fetcher.rb, line 55 55: def initialize(proxy = nil) 56: require 'net/http' 57: require 'stringio' 58: require 'time' 59: require 'uri' 60: 61: Socket.do_not_reverse_lookup = true 62: 63: @connections = {} 64: @requests = Hash.new 0 65: @proxy_uri = 66: case proxy 67: when :no_proxy then nil 68: when nil then get_proxy_from_env 69: when URI::HTTP then proxy 70: else URI.parse(proxy) 71: end 72: @user_agent = user_agent 73: end
Creates or an HTTP connection based on uri, or retrieves an existing connection, using a proxy if needed.
# File lib/rubygems/remote_fetcher.rb, line 298 298: def connection_for(uri) 299: net_http_args = [uri.host, uri.port] 300: 301: if @proxy_uri then 302: net_http_args += [ 303: @proxy_uri.host, 304: @proxy_uri.port, 305: @proxy_uri.user, 306: @proxy_uri.password 307: ] 308: end 309: 310: connection_id = [Thread.current.object_id, *net_http_args].join ':' 311: @connections[connection_id] ||= Net::HTTP.new(*net_http_args) 312: connection = @connections[connection_id] 313: 314: if uri.scheme == 'https' and not connection.started? then 315: require 'net/https' 316: connection.use_ssl = true 317: connection.verify_mode = OpenSSL::SSL::VERIFY_NONE 318: end 319: 320: connection.start unless connection.started? 321: 322: connection 323: rescue Errno::EHOSTDOWN => e 324: raise FetchError.new(e.message, uri) 325: end
# File lib/rubygems/remote_fetcher.rb, line 327 327: def correct_for_windows_path(path) 328: if path[0].chr == '/' && path[1].chr =~ /[a-z]/i && path[2].chr == ':' 329: path = path[1..-1] 330: else 331: path 332: end 333: end
Moves the gem spec from source_uri to the cache dir unless it is already there. If the source_uri is local the gem cache dir copy is always replaced.
# File lib/rubygems/remote_fetcher.rb, line 97 97: def download(spec, source_uri, install_dir = Gem.dir) 98: Gem.ensure_gem_subdirectories(install_dir) rescue nil 99: 100: if File.writable?(install_dir) 101: cache_dir = Gem.cache_dir(install_dir) 102: else 103: cache_dir = Gem.cache_dir(Gem.user_dir) 104: end 105: 106: gem_file_name = spec.file_name 107: local_gem_path = File.join cache_dir, gem_file_name 108: 109: FileUtils.mkdir_p cache_dir rescue nil unless File.exist? cache_dir 110: 111: # Always escape URI's to deal with potential spaces and such 112: unless URI::Generic === source_uri 113: source_uri = URI.parse(URI.const_defined?(:DEFAULT_PARSER) ? 114: URI::DEFAULT_PARSER.escape(source_uri) : 115: URI.escape(source_uri)) 116: end 117: 118: scheme = source_uri.scheme 119: 120: # URI.parse gets confused by MS Windows paths with forward slashes. 121: scheme = nil if scheme =~ /^[a-z]$/i 122: 123: case scheme 124: when 'http', 'https' then 125: unless File.exist? local_gem_path then 126: begin 127: say "Downloading gem #{gem_file_name}" if 128: Gem.configuration.really_verbose 129: 130: remote_gem_path = source_uri + "gems/#{gem_file_name}" 131: 132: gem = self.fetch_path remote_gem_path 133: rescue Gem::RemoteFetcher::FetchError 134: raise if spec.original_platform == spec.platform 135: 136: alternate_name = "#{spec.original_name}.gem" 137: 138: say "Failed, downloading gem #{alternate_name}" if 139: Gem.configuration.really_verbose 140: 141: remote_gem_path = source_uri + "gems/#{alternate_name}" 142: 143: gem = self.fetch_path remote_gem_path 144: end 145: 146: File.open local_gem_path, 'wb' do |fp| 147: fp.write gem 148: end 149: end 150: when 'file' then 151: begin 152: path = source_uri.path 153: path = File.dirname(path) if File.extname(path) == '.gem' 154: 155: remote_gem_path = correct_for_windows_path(File.join(path, 'gems', gem_file_name)) 156: 157: FileUtils.cp(remote_gem_path, local_gem_path) 158: rescue Errno::EACCES 159: local_gem_path = source_uri.to_s 160: end 161: 162: say "Using local gem #{local_gem_path}" if 163: Gem.configuration.really_verbose 164: when nil then # TODO test for local overriding cache 165: source_path = if Gem.win_platform? && source_uri.scheme && 166: !source_uri.path.include?(':') then 167: "#{source_uri.scheme}:#{source_uri.path}" 168: else 169: source_uri.path 170: end 171: 172: source_path = unescape source_path 173: 174: begin 175: FileUtils.cp source_path, local_gem_path unless 176: File.expand_path(source_path) == File.expand_path(local_gem_path) 177: rescue Errno::EACCES 178: local_gem_path = source_uri.to_s 179: end 180: 181: say "Using local gem #{local_gem_path}" if 182: Gem.configuration.really_verbose 183: else 184: raise Gem::InstallError, "unsupported URI scheme #{source_uri.scheme}" 185: end 186: 187: local_gem_path 188: end
Given a name and requirement, downloads this gem into cache and returns the filename. Returns nil if the gem cannot be located.
# File lib/rubygems/remote_fetcher.rb, line 82 82: def download_to_cache dependency 83: found = Gem::SpecFetcher.fetcher.fetch dependency 84: 85: return if found.empty? 86: 87: spec, source_uri = found.first 88: 89: download spec, source_uri 90: end
# File lib/rubygems/remote_fetcher.rb, line 250 250: def escape(str) 251: return unless str 252: @uri_parser ||= uri_escaper 253: @uri_parser.escape str 254: end
File Fetcher. Dispatched by fetch_path. Use it instead.
# File lib/rubygems/remote_fetcher.rb, line 193 193: def fetch_file uri, *_ 194: Gem.read_binary correct_for_windows_path uri.path 195: end
HTTP Fetcher. Dispatched by fetch_path. Use it instead.
# File lib/rubygems/remote_fetcher.rb, line 200 200: def fetch_http uri, last_modified = nil, head = false, depth = 0 201: fetch_type = head ? Net::HTTP::Head : Net::HTTP::Get 202: response = request uri, fetch_type, last_modified 203: 204: case response 205: when Net::HTTPOK, Net::HTTPNotModified then 206: head ? response : response.body 207: when Net::HTTPMovedPermanently, Net::HTTPFound, Net::HTTPSeeOther, 208: Net::HTTPTemporaryRedirect then 209: raise FetchError.new('too many redirects', uri) if depth > 10 210: 211: location = URI.parse response['Location'] 212: fetch_http(location, last_modified, head, depth + 1) 213: else 214: raise FetchError.new("bad response #{response.message} #{response.code}", uri) 215: end 216: end
Downloads uri and returns it as a String.
# File lib/rubygems/remote_fetcher.rb, line 223 223: def fetch_path(uri, mtime = nil, head = false) 224: uri = URI.parse uri unless URI::Generic === uri 225: 226: raise ArgumentError, "bad uri: #{uri}" unless uri 227: raise ArgumentError, "uri scheme is invalid: #{uri.scheme.inspect}" unless 228: uri.scheme 229: 230: data = send "fetch_#{uri.scheme}", uri, mtime, head 231: data = Gem.gunzip data if data and not head and uri.to_s =~ /gz$/ 232: data 233: rescue FetchError 234: raise 235: rescue Timeout::Error 236: raise FetchError.new('timed out', uri.to_s) 237: rescue IOError, SocketError, SystemCallError => e 238: raise FetchError.new("#{e.class}: #{e}", uri.to_s) 239: end
Returns the size of uri in bytes.
# File lib/rubygems/remote_fetcher.rb, line 244 244: def fetch_size(uri) # TODO: phase this out 245: response = fetch_path(uri, nil, true) 246: 247: response['content-length'].to_i 248: end
Returns an HTTP proxy URI if one is set in the environment variables.
# File lib/rubygems/remote_fetcher.rb, line 271 271: def get_proxy_from_env 272: env_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY'] 273: 274: return nil if env_proxy.nil? or env_proxy.empty? 275: 276: uri = URI.parse(normalize_uri(env_proxy)) 277: 278: if uri and uri.user.nil? and uri.password.nil? then 279: # Probably we have http_proxy_* variables? 280: uri.user = escape(ENV['http_proxy_user'] || ENV['HTTP_PROXY_USER']) 281: uri.password = escape(ENV['http_proxy_pass'] || ENV['HTTP_PROXY_PASS']) 282: end 283: 284: uri 285: end
Normalize the URI by adding "http://" if it is missing.
# File lib/rubygems/remote_fetcher.rb, line 290 290: def normalize_uri(uri) 291: (uri =~ /^(https?|ftp|file):/) ? uri : "http://#{uri}" 292: end
Read the data from the (source based) URI, but if it is a file:// URI, read from the filesystem instead.
# File lib/rubygems/remote_fetcher.rb, line 339 339: def open_uri_or_path(uri, last_modified = nil, head = false, depth = 0) 340: raise "NO: Use fetch_path instead" 341: # TODO: deprecate for fetch_path 342: end
Performs a Net::HTTP request of type request_class on uri returning a Net::HTTP response object. request maintains a table of persistent connections to reduce connect overhead.
# File lib/rubygems/remote_fetcher.rb, line 349 349: def request(uri, request_class, last_modified = nil) 350: request = request_class.new uri.request_uri 351: 352: unless uri.nil? || uri.user.nil? || uri.user.empty? then 353: request.basic_auth uri.user, uri.password 354: end 355: 356: request.add_field 'User-Agent', @user_agent 357: request.add_field 'Connection', 'keep-alive' 358: request.add_field 'Keep-Alive', '30' 359: 360: if last_modified then 361: last_modified = last_modified.utc 362: request.add_field 'If-Modified-Since', last_modified.rfc2822 363: end 364: 365: yield request if block_given? 366: 367: connection = connection_for uri 368: 369: retried = false 370: bad_response = false 371: 372: begin 373: @requests[connection.object_id] += 1 374: 375: say "#{request.method} #{uri}" if 376: Gem.configuration.really_verbose 377: 378: file_name = File.basename(uri.path) 379: # perform download progress reporter only for gems 380: if request.response_body_permitted? && file_name =~ /\.gem$/ 381: reporter = ui.download_reporter 382: response = connection.request(request) do |incomplete_response| 383: if Net::HTTPOK === incomplete_response 384: reporter.fetch(file_name, incomplete_response.content_length) 385: downloaded = 0 386: data = '' 387: 388: incomplete_response.read_body do |segment| 389: data << segment 390: downloaded += segment.length 391: reporter.update(downloaded) 392: end 393: reporter.done 394: if incomplete_response.respond_to? :body= 395: incomplete_response.body = data 396: else 397: incomplete_response.instance_variable_set(:@body, data) 398: end 399: end 400: end 401: else 402: response = connection.request request 403: end 404: 405: say "#{response.code} #{response.message}" if 406: Gem.configuration.really_verbose 407: 408: rescue Net::HTTPBadResponse 409: say "bad response" if Gem.configuration.really_verbose 410: 411: reset connection 412: 413: raise FetchError.new('too many bad responses', uri) if bad_response 414: 415: bad_response = true 416: retry 417: # HACK work around EOFError bug in Net::HTTP 418: # NOTE Errno::ECONNABORTED raised a lot on Windows, and make impossible 419: # to install gems. 420: rescue EOFError, Timeout::Error, 421: Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE 422: 423: requests = @requests[connection.object_id] 424: say "connection reset after #{requests} requests, retrying" if 425: Gem.configuration.really_verbose 426: 427: raise FetchError.new('too many connection resets', uri) if retried 428: 429: reset connection 430: 431: retried = true 432: retry 433: end 434: 435: response 436: end
Resets HTTP connection connection.
# File lib/rubygems/remote_fetcher.rb, line 441 441: def reset(connection) 442: @requests.delete connection.object_id 443: 444: connection.finish 445: connection.start 446: end
# File lib/rubygems/remote_fetcher.rb, line 256 256: def unescape(str) 257: return unless str 258: @uri_parser ||= uri_escaper 259: @uri_parser.unescape str 260: end
# File lib/rubygems/remote_fetcher.rb, line 262 262: def uri_escaper 263: URI::Parser.new 264: rescue NameError 265: URI 266: end
# File lib/rubygems/remote_fetcher.rb, line 448 448: def user_agent 449: ua = "RubyGems/#{Gem::VERSION} #{Gem::Platform.local}" 450: 451: ruby_version = RUBY_VERSION 452: ruby_version += 'dev' if RUBY_PATCHLEVEL == -1 453: 454: ua << " Ruby/#{ruby_version} (#{RUBY_RELEASE_DATE}" 455: if RUBY_PATCHLEVEL >= 0 then 456: ua << " patchlevel #{RUBY_PATCHLEVEL}" 457: elsif defined?(RUBY_REVISION) then 458: ua << " revision #{RUBY_REVISION}" 459: end 460: ua << ")" 461: 462: ua << " #{RUBY_ENGINE}" if defined?(RUBY_ENGINE) and RUBY_ENGINE != 'ruby' 463: 464: ua 465: end