Class | Gem::Indexer |
In: |
lib/rubygems/indexer.rb
|
Parent: | Object |
Top level class for building the gem repository index.
dest_directory | [R] | Index install location |
directory | [R] | Index build directory |
Create an indexer that will index the gems in directory.
# File lib/rubygems/indexer.rb, line 33 33: def initialize(directory) 34: unless ''.respond_to? :to_xs then 35: fail "Gem::Indexer requires that the XML Builder library be installed:" \ 36: "\n\tgem install builder" 37: end 38: 39: @dest_directory = directory 40: @directory = File.join Dir.tmpdir, "gem_generate_index_#{$$}" 41: 42: marshal_name = "Marshal.#{Gem.marshal_version}" 43: 44: @master_index = File.join @directory, 'yaml' 45: @marshal_index = File.join @directory, marshal_name 46: 47: @quick_dir = File.join @directory, 'quick' 48: 49: @quick_marshal_dir = File.join @quick_dir, marshal_name 50: 51: @quick_index = File.join @quick_dir, 'index' 52: @latest_index = File.join @quick_dir, 'latest_index' 53: 54: @specs_index = File.join @directory, "specs.#{Gem.marshal_version}" 55: @latest_specs_index = File.join @directory, 56: "latest_specs.#{Gem.marshal_version}" 57: 58: files = [ 59: @specs_index, 60: "#{@specs_index}.gz", 61: @latest_specs_index, 62: "#{@latest_specs_index}.gz", 63: @quick_dir, 64: @master_index, 65: "#{@master_index}.Z", 66: @marshal_index, 67: "#{@marshal_index}.Z", 68: ] 69: 70: @files = files.map do |path| 71: path.sub @directory, '' 72: end 73: end
Abbreviate the spec for downloading. Abbreviated specs are only used for searching, downloading and related activities and do not need deployment specific information (e.g. list of files). So we abbreviate the spec, making it much smaller for quicker downloads.
# File lib/rubygems/indexer.rb, line 81 81: def abbreviate(spec) 82: spec.files = [] 83: spec.test_files = [] 84: spec.rdoc_options = [] 85: spec.extra_rdoc_files = [] 86: spec.cert_chain = [] 87: spec 88: end
Build various indicies
# File lib/rubygems/indexer.rb, line 93 93: def build_indicies(index) 94: progress = ui.progress_reporter index.size, 95: "Generating quick index gemspecs for #{index.size} gems", 96: "Complete" 97: 98: index.each do |original_name, spec| 99: spec_file_name = "#{original_name}.gemspec.rz" 100: yaml_name = File.join @quick_dir, spec_file_name 101: marshal_name = File.join @quick_marshal_dir, spec_file_name 102: 103: yaml_zipped = Gem.deflate spec.to_yaml 104: open yaml_name, 'wb' do |io| io.write yaml_zipped end 105: 106: marshal_zipped = Gem.deflate Marshal.dump(spec) 107: open marshal_name, 'wb' do |io| io.write marshal_zipped end 108: 109: progress.updated original_name 110: end 111: 112: progress.done 113: 114: say "Generating specs index" 115: 116: open @specs_index, 'wb' do |io| 117: specs = index.sort.map do |_, spec| 118: platform = spec.original_platform 119: platform = Gem::Platform::RUBY if platform.nil? 120: [spec.name, spec.version, platform] 121: end 122: 123: specs = compact_specs specs 124: 125: Marshal.dump specs, io 126: end 127: 128: say "Generating latest specs index" 129: 130: open @latest_specs_index, 'wb' do |io| 131: specs = index.latest_specs.sort.map do |spec| 132: [spec.name, spec.version, spec.original_platform] 133: end 134: 135: specs = compact_specs specs 136: 137: Marshal.dump specs, io 138: end 139: 140: say "Generating quick index" 141: 142: quick_index = File.join @quick_dir, 'index' 143: open quick_index, 'wb' do |io| 144: io.puts index.sort.map { |_, spec| spec.original_name } 145: end 146: 147: say "Generating latest index" 148: 149: latest_index = File.join @quick_dir, 'latest_index' 150: open latest_index, 'wb' do |io| 151: io.puts index.latest_specs.sort.map { |spec| spec.original_name } 152: end 153: 154: say "Generating Marshal master index" 155: 156: open @marshal_index, 'wb' do |io| 157: io.write index.dump 158: end 159: 160: progress = ui.progress_reporter index.size, 161: "Generating YAML master index for #{index.size} gems (this may take a while)", 162: "Complete" 163: 164: open @master_index, 'wb' do |io| 165: io.puts "--- !ruby/object:#{index.class}" 166: io.puts "gems:" 167: 168: gems = index.sort_by { |name, gemspec| gemspec.sort_obj } 169: gems.each do |original_name, gemspec| 170: yaml = gemspec.to_yaml.gsub(/^/, ' ') 171: yaml = yaml.sub(/\A ---/, '') # there's a needed extra ' ' here 172: io.print " #{original_name}:" 173: io.puts yaml 174: 175: progress.updated original_name 176: end 177: end 178: 179: progress.done 180: 181: say "Compressing indicies" 182: # use gzip for future files. 183: 184: compress quick_index, 'rz' 185: paranoid quick_index, 'rz' 186: 187: compress latest_index, 'rz' 188: paranoid latest_index, 'rz' 189: 190: compress @marshal_index, 'Z' 191: paranoid @marshal_index, 'Z' 192: 193: compress @master_index, 'Z' 194: paranoid @master_index, 'Z' 195: 196: gzip @specs_index 197: gzip @latest_specs_index 198: end
Collect specifications from .gem files from the gem directory.
# File lib/rubygems/indexer.rb, line 203 203: def collect_specs 204: index = Gem::SourceIndex.new 205: 206: progress = ui.progress_reporter gem_file_list.size, 207: "Loading #{gem_file_list.size} gems from #{@dest_directory}", 208: "Loaded all gems" 209: 210: gem_file_list.each do |gemfile| 211: if File.size(gemfile.to_s) == 0 then 212: alert_warning "Skipping zero-length gem: #{gemfile}" 213: next 214: end 215: 216: begin 217: spec = Gem::Format.from_file_by_path(gemfile).spec 218: 219: unless gemfile =~ /\/#{Regexp.escape spec.original_name}.*\.gem\z/i then 220: alert_warning "Skipping misnamed gem: #{gemfile} => #{spec.full_name} (#{spec.original_name})" 221: next 222: end 223: 224: abbreviate spec 225: sanitize spec 226: 227: index.gems[spec.original_name] = spec 228: 229: progress.updated spec.original_name 230: 231: rescue SignalException => e 232: alert_error "Received signal, exiting" 233: raise 234: rescue Exception => e 235: alert_error "Unable to process #{gemfile}\n#{e.message} (#{e.class})\n\t#{e.backtrace.join "\n\t"}" 236: end 237: end 238: 239: progress.done 240: 241: index 242: end
Compacts Marshal output for the specs index data source by using identical objects as much as possible.
# File lib/rubygems/indexer.rb, line 248 248: def compact_specs(specs) 249: names = {} 250: versions = {} 251: platforms = {} 252: 253: specs.map do |(name, version, platform)| 254: names[name] = name unless names.include? name 255: versions[version] = version unless versions.include? version 256: platforms[platform] = platform unless platforms.include? platform 257: 258: [names[name], versions[version], platforms[platform]] 259: end 260: end
Compress filename with extension.
# File lib/rubygems/indexer.rb, line 265 265: def compress(filename, extension) 266: data = Gem.read_binary filename 267: 268: zipped = Gem.deflate data 269: 270: open "#{filename}.#{extension}", 'wb' do |io| 271: io.write zipped 272: end 273: end
List of gem file names to index.
# File lib/rubygems/indexer.rb, line 278 278: def gem_file_list 279: Dir.glob(File.join(@dest_directory, "gems", "*.gem")) 280: end
Builds and installs indexicies.
# File lib/rubygems/indexer.rb, line 285 285: def generate_index 286: FileUtils.rm_rf @directory 287: FileUtils.mkdir_p @directory, :mode => 0700 288: FileUtils.mkdir_p @quick_marshal_dir 289: 290: index = collect_specs 291: build_indicies index 292: install_indicies 293: rescue SignalException 294: ensure 295: FileUtils.rm_rf @directory 296: end
Zlib::GzipWriter wrapper that gzips filename on disk.
# File lib/rubygems/indexer.rb, line 301 301: def gzip(filename) 302: Zlib::GzipWriter.open "#{filename}.gz" do |io| 303: io.write Gem.read_binary(filename) 304: end 305: end
Install generated indicies into the destination directory.
# File lib/rubygems/indexer.rb, line 310 310: def install_indicies 311: verbose = Gem.configuration.really_verbose 312: 313: say "Moving index into production dir #{@dest_directory}" if verbose 314: 315: @files.each do |file| 316: src_name = File.join @directory, file 317: dst_name = File.join @dest_directory, file 318: 319: FileUtils.rm_rf dst_name, :verbose => verbose 320: FileUtils.mv src_name, @dest_directory, :verbose => verbose, 321: :force => true 322: end 323: end
Ensure path and path with extension are identical.
# File lib/rubygems/indexer.rb, line 328 328: def paranoid(path, extension) 329: data = Gem.read_binary path 330: compressed_data = Gem.read_binary "#{path}.#{extension}" 331: 332: unless data == Gem.inflate(compressed_data) then 333: raise "Compressed file #{compressed_path} does not match uncompressed file #{path}" 334: end 335: end
Sanitize the descriptive fields in the spec. Sometimes non-ASCII characters will garble the site index. Non-ASCII characters will be replaced by their XML entity equivalent.
# File lib/rubygems/indexer.rb, line 342 342: def sanitize(spec) 343: spec.summary = sanitize_string(spec.summary) 344: spec.description = sanitize_string(spec.description) 345: spec.post_install_message = sanitize_string(spec.post_install_message) 346: spec.authors = spec.authors.collect { |a| sanitize_string(a) } 347: spec 348: end
Sanitize a single string.
# File lib/rubygems/indexer.rb, line 353 353: def sanitize_string(string) 354: # HACK the #to_s is in here because RSpec has an Array of Arrays of 355: # Strings for authors. Need a way to disallow bad values on gempsec 356: # generation. (Probably won't happen.) 357: string ? string.to_s.to_xs : string 358: end