Class | Gem::Indexer |
In: |
lib/rubygems/indexer.rb
|
Parent: | Object |
Top level class for building the gem repository index.
dest_directory | [R] | Index install location |
directory | [R] | Index build directory |
Create an indexer that will index the gems in directory.
# File lib/rubygems/indexer.rb, line 33 33: def initialize(directory) 34: unless ''.respond_to? :to_xs then 35: fail "Gem::Indexer requires that the XML Builder library be installed:" \ 36: "\n\tgem install builder" 37: end 38: 39: @dest_directory = directory 40: @directory = File.join Dir.tmpdir, "gem_generate_index_#{$$}" 41: 42: marshal_name = "Marshal.#{Gem.marshal_version}" 43: 44: @master_index = File.join @directory, 'yaml' 45: @marshal_index = File.join @directory, marshal_name 46: 47: @quick_dir = File.join @directory, 'quick' 48: 49: @quick_marshal_dir = File.join @quick_dir, marshal_name 50: 51: @quick_index = File.join @quick_dir, 'index' 52: @latest_index = File.join @quick_dir, 'latest_index' 53: 54: @specs_index = File.join @directory, "specs.#{Gem.marshal_version}" 55: @latest_specs_index = File.join @directory, 56: "latest_specs.#{Gem.marshal_version}" 57: 58: files = [ 59: @specs_index, 60: "#{@specs_index}.gz", 61: @latest_specs_index, 62: "#{@latest_specs_index}.gz", 63: @quick_dir, 64: @master_index, 65: "#{@master_index}.Z", 66: @marshal_index, 67: "#{@marshal_index}.Z", 68: ] 69: 70: @files = files.map do |path| 71: path.sub @directory, '' 72: end 73: end
Abbreviate the spec for downloading. Abbreviated specs are only used for searching, downloading and related activities and do not need deployment specific information (e.g. list of files). So we abbreviate the spec, making it much smaller for quicker downloads.
# File lib/rubygems/indexer.rb, line 81 81: def abbreviate(spec) 82: spec.files = [] 83: spec.test_files = [] 84: spec.rdoc_options = [] 85: spec.extra_rdoc_files = [] 86: spec.cert_chain = [] 87: spec 88: end
Build various indicies
# File lib/rubygems/indexer.rb, line 93 93: def build_indicies(index) 94: progress = ui.progress_reporter index.size, 95: "Generating quick index gemspecs for #{index.size} gems", 96: "Complete" 97: 98: index.each do |original_name, spec| 99: spec_file_name = "#{original_name}.gemspec.rz" 100: yaml_name = File.join @quick_dir, spec_file_name 101: marshal_name = File.join @quick_marshal_dir, spec_file_name 102: 103: yaml_zipped = Gem.deflate spec.to_yaml 104: open yaml_name, 'wb' do |io| io.write yaml_zipped end 105: 106: marshal_zipped = Gem.deflate Marshal.dump(spec) 107: open marshal_name, 'wb' do |io| io.write marshal_zipped end 108: 109: progress.updated original_name 110: end 111: 112: progress.done 113: 114: say "Generating specs index" 115: 116: open @specs_index, 'wb' do |io| 117: specs = index.sort.map do |_, spec| 118: platform = spec.original_platform 119: platform = Gem::Platform::RUBY if platform.nil? or platform.empty? 120: [spec.name, spec.version, platform] 121: end 122: 123: specs = compact_specs specs 124: 125: Marshal.dump specs, io 126: end 127: 128: say "Generating latest specs index" 129: 130: open @latest_specs_index, 'wb' do |io| 131: specs = index.latest_specs.sort.map do |spec| 132: platform = spec.original_platform 133: platform = Gem::Platform::RUBY if platform.nil? or platform.empty? 134: [spec.name, spec.version, platform] 135: end 136: 137: specs = compact_specs specs 138: 139: Marshal.dump specs, io 140: end 141: 142: say "Generating quick index" 143: 144: quick_index = File.join @quick_dir, 'index' 145: open quick_index, 'wb' do |io| 146: io.puts index.sort.map { |_, spec| spec.original_name } 147: end 148: 149: say "Generating latest index" 150: 151: latest_index = File.join @quick_dir, 'latest_index' 152: open latest_index, 'wb' do |io| 153: io.puts index.latest_specs.sort.map { |spec| spec.original_name } 154: end 155: 156: say "Generating Marshal master index" 157: 158: open @marshal_index, 'wb' do |io| 159: io.write index.dump 160: end 161: 162: progress = ui.progress_reporter index.size, 163: "Generating YAML master index for #{index.size} gems (this may take a while)", 164: "Complete" 165: 166: open @master_index, 'wb' do |io| 167: io.puts "--- !ruby/object:#{index.class}" 168: io.puts "gems:" 169: 170: gems = index.sort_by { |name, gemspec| gemspec.sort_obj } 171: gems.each do |original_name, gemspec| 172: yaml = gemspec.to_yaml.gsub(/^/, ' ') 173: yaml = yaml.sub(/\A ---/, '') # there's a needed extra ' ' here 174: io.print " #{original_name}:" 175: io.puts yaml 176: 177: progress.updated original_name 178: end 179: end 180: 181: progress.done 182: 183: say "Compressing indicies" 184: # use gzip for future files. 185: 186: compress quick_index, 'rz' 187: paranoid quick_index, 'rz' 188: 189: compress latest_index, 'rz' 190: paranoid latest_index, 'rz' 191: 192: compress @marshal_index, 'Z' 193: paranoid @marshal_index, 'Z' 194: 195: compress @master_index, 'Z' 196: paranoid @master_index, 'Z' 197: 198: gzip @specs_index 199: gzip @latest_specs_index 200: end
Collect specifications from .gem files from the gem directory.
# File lib/rubygems/indexer.rb, line 205 205: def collect_specs 206: index = Gem::SourceIndex.new 207: 208: progress = ui.progress_reporter gem_file_list.size, 209: "Loading #{gem_file_list.size} gems from #{@dest_directory}", 210: "Loaded all gems" 211: 212: gem_file_list.each do |gemfile| 213: if File.size(gemfile.to_s) == 0 then 214: alert_warning "Skipping zero-length gem: #{gemfile}" 215: next 216: end 217: 218: begin 219: spec = Gem::Format.from_file_by_path(gemfile).spec 220: 221: unless gemfile =~ /\/#{Regexp.escape spec.original_name}.*\.gem\z/i then 222: alert_warning "Skipping misnamed gem: #{gemfile} => #{spec.full_name} (#{spec.original_name})" 223: next 224: end 225: 226: abbreviate spec 227: sanitize spec 228: 229: index.gems[spec.original_name] = spec 230: 231: progress.updated spec.original_name 232: 233: rescue SignalException => e 234: alert_error "Received signal, exiting" 235: raise 236: rescue Exception => e 237: alert_error "Unable to process #{gemfile}\n#{e.message} (#{e.class})\n\t#{e.backtrace.join "\n\t"}" 238: end 239: end 240: 241: progress.done 242: 243: index 244: end
Compacts Marshal output for the specs index data source by using identical objects as much as possible.
# File lib/rubygems/indexer.rb, line 250 250: def compact_specs(specs) 251: names = {} 252: versions = {} 253: platforms = {} 254: 255: specs.map do |(name, version, platform)| 256: names[name] = name unless names.include? name 257: versions[version] = version unless versions.include? version 258: platforms[platform] = platform unless platforms.include? platform 259: 260: [names[name], versions[version], platforms[platform]] 261: end 262: end
Compress filename with extension.
# File lib/rubygems/indexer.rb, line 267 267: def compress(filename, extension) 268: data = Gem.read_binary filename 269: 270: zipped = Gem.deflate data 271: 272: open "#{filename}.#{extension}", 'wb' do |io| 273: io.write zipped 274: end 275: end
List of gem file names to index.
# File lib/rubygems/indexer.rb, line 280 280: def gem_file_list 281: Dir.glob(File.join(@dest_directory, "gems", "*.gem")) 282: end
Builds and installs indexicies.
# File lib/rubygems/indexer.rb, line 287 287: def generate_index 288: make_temp_directories 289: index = collect_specs 290: build_indicies index 291: install_indicies 292: rescue SignalException 293: ensure 294: FileUtils.rm_rf @directory 295: end
Zlib::GzipWriter wrapper that gzips filename on disk.
# File lib/rubygems/indexer.rb, line 300 300: def gzip(filename) 301: Zlib::GzipWriter.open "#{filename}.gz" do |io| 302: io.write Gem.read_binary(filename) 303: end 304: end
Install generated indicies into the destination directory.
# File lib/rubygems/indexer.rb, line 309 309: def install_indicies 310: verbose = Gem.configuration.really_verbose 311: 312: say "Moving index into production dir #{@dest_directory}" if verbose 313: 314: @files.each do |file| 315: src_name = File.join @directory, file 316: dst_name = File.join @dest_directory, file 317: 318: FileUtils.rm_rf dst_name, :verbose => verbose 319: FileUtils.mv src_name, @dest_directory, :verbose => verbose, 320: :force => true 321: end 322: end
Make directories for index generation
# File lib/rubygems/indexer.rb, line 327 327: def make_temp_directories 328: FileUtils.rm_rf @directory 329: FileUtils.mkdir_p @directory, :mode => 0700 330: FileUtils.mkdir_p @quick_marshal_dir 331: end
Ensure path and path with extension are identical.
# File lib/rubygems/indexer.rb, line 336 336: def paranoid(path, extension) 337: data = Gem.read_binary path 338: compressed_data = Gem.read_binary "#{path}.#{extension}" 339: 340: unless data == Gem.inflate(compressed_data) then 341: raise "Compressed file #{compressed_path} does not match uncompressed file #{path}" 342: end 343: end
Sanitize the descriptive fields in the spec. Sometimes non-ASCII characters will garble the site index. Non-ASCII characters will be replaced by their XML entity equivalent.
# File lib/rubygems/indexer.rb, line 350 350: def sanitize(spec) 351: spec.summary = sanitize_string(spec.summary) 352: spec.description = sanitize_string(spec.description) 353: spec.post_install_message = sanitize_string(spec.post_install_message) 354: spec.authors = spec.authors.collect { |a| sanitize_string(a) } 355: 356: spec 357: end
Sanitize a single string.
# File lib/rubygems/indexer.rb, line 362 362: def sanitize_string(string) 363: # HACK the #to_s is in here because RSpec has an Array of Arrays of 364: # Strings for authors. Need a way to disallow bad values on gempsec 365: # generation. (Probably won't happen.) 366: string ? string.to_s.to_xs : string 367: end