Package CedarBackup2 :: Package tools :: Module span
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup2.tools.span

  1  # -*- coding: iso-8859-1 -*- 
  2  # vim: set ft=python ts=3 sw=3 expandtab: 
  3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  4  # 
  5  #              C E D A R 
  6  #          S O L U T I O N S       "Software done right." 
  7  #           S O F T W A R E 
  8  # 
  9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 10  # 
 11  # Copyright (c) 2007-2008,2010 Kenneth J. Pronovici. 
 12  # All rights reserved. 
 13  # 
 14  # This program is free software; you can redistribute it and/or 
 15  # modify it under the terms of the GNU General Public License, 
 16  # Version 2, as published by the Free Software Foundation. 
 17  # 
 18  # This program is distributed in the hope that it will be useful, 
 19  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
 21  # 
 22  # Copies of the GNU General Public License are available from 
 23  # the Free Software Foundation website, http://www.gnu.org/. 
 24  # 
 25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 26  # 
 27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
 28  # Language : Python (>= 2.3) 
 29  # Project  : Cedar Backup, release 2 
 30  # Revision : $Id: span.py 956 2010-01-10 22:49:01Z pronovic $ 
 31  # Purpose  : Spans staged data among multiple discs 
 32  # 
 33  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 34   
 35  ######################################################################## 
 36  # Notes 
 37  ######################################################################## 
 38   
 39  """ 
 40  Spans staged data among multiple discs 
 41   
 42  This is the Cedar Backup span tool.  It is intended for use by people who stage 
 43  more data than can fit on a single disc.  It allows a user to split staged data 
 44  among more than one disc.  It can't be an extension because it requires user 
 45  input when switching media. 
 46   
 47  Most configuration is taken from the Cedar Backup configuration file, 
 48  specifically the store section.  A few pieces of configuration are taken 
 49  directly from the user. 
 50   
 51  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
 52  """ 
 53   
 54  ######################################################################## 
 55  # Imported modules and constants 
 56  ######################################################################## 
 57   
 58  # System modules 
 59  import sys 
 60  import os 
 61  import logging 
 62  import tempfile 
 63   
 64  # Cedar Backup modules  
 65  from CedarBackup2.release import AUTHOR, EMAIL, VERSION, DATE, COPYRIGHT 
 66  from CedarBackup2.util import displayBytes, convertSize, mount, unmount 
 67  from CedarBackup2.util import UNIT_SECTORS, UNIT_BYTES 
 68  from CedarBackup2.config import Config 
 69  from CedarBackup2.filesystem import FilesystemList, BackupFileList, compareDigestMaps, normalizeDir 
 70  from CedarBackup2.cli import Options, setupLogging, setupPathResolver 
 71  from CedarBackup2.cli import DEFAULT_CONFIG, DEFAULT_LOGFILE, DEFAULT_OWNERSHIP, DEFAULT_MODE 
 72  from CedarBackup2.actions.constants import STORE_INDICATOR 
 73  from CedarBackup2.actions.util import createWriter 
 74  from CedarBackup2.actions.store import consistencyCheck, writeIndicatorFile 
 75  from CedarBackup2.actions.util import findDailyDirs 
 76  from CedarBackup2.knapsack import firstFit, bestFit, worstFit, alternateFit 
 77   
 78   
 79  ######################################################################## 
 80  # Module-wide constants and variables 
 81  ######################################################################## 
 82   
 83  logger = logging.getLogger("CedarBackup2.log.tools.span") 
 84   
 85   
 86  ####################################################################### 
 87  # SpanOptions class 
 88  ####################################################################### 
 89   
90 -class SpanOptions(Options):
91 92 """ 93 Tool-specific command-line options. 94 95 Most of the cback command-line options are exactly what we need here -- 96 logfile path, permissions, verbosity, etc. However, we need to make a few 97 tweaks since we don't accept any actions. 98 99 Also, a few extra command line options that we accept are really ignored 100 underneath. I just don't care about that for a tool like this. 101 """ 102
103 - def validate(self):
104 """ 105 Validates command-line options represented by the object. 106 There are no validations here, because we don't use any actions. 107 @raise ValueError: If one of the validations fails. 108 """ 109 pass
110 111 112 ####################################################################### 113 # Public functions 114 ####################################################################### 115 116 ################# 117 # cli() function 118 ################# 119
120 -def cli():
121 """ 122 Implements the command-line interface for the C{cback-span} script. 123 124 Essentially, this is the "main routine" for the cback-span script. It does 125 all of the argument processing for the script, and then also implements the 126 tool functionality. 127 128 This function looks pretty similiar to C{CedarBackup2.cli.cli()}. It's not 129 easy to refactor this code to make it reusable and also readable, so I've 130 decided to just live with the duplication. 131 132 A different error code is returned for each type of failure: 133 134 - C{1}: The Python interpreter version is < 2.3 135 - C{2}: Error processing command-line arguments 136 - C{3}: Error configuring logging 137 - C{4}: Error parsing indicated configuration file 138 - C{5}: Backup was interrupted with a CTRL-C or similar 139 - C{6}: Error executing other parts of the script 140 141 @note: This script uses print rather than logging to the INFO level, because 142 it is interactive. Underlying Cedar Backup functionality uses the logging 143 mechanism exclusively. 144 145 @return: Error code as described above. 146 """ 147 try: 148 if map(int, [sys.version_info[0], sys.version_info[1]]) < [2, 3]: 149 sys.stderr.write("Python version 2.3 or greater required.\n") 150 return 1 151 except: 152 # sys.version_info isn't available before 2.0 153 sys.stderr.write("Python version 2.3 or greater required.\n") 154 return 1 155 156 try: 157 options = SpanOptions(argumentList=sys.argv[1:]) 158 except Exception, e: 159 _usage() 160 sys.stderr.write(" *** Error: %s\n" % e) 161 return 2 162 163 if options.help: 164 _usage() 165 return 0 166 if options.version: 167 _version() 168 return 0 169 170 try: 171 logfile = setupLogging(options) 172 except Exception, e: 173 sys.stderr.write("Error setting up logging: %s\n" % e) 174 return 3 175 176 logger.info("Cedar Backup 'span' utility run started.") 177 logger.info("Options were [%s]" % options) 178 logger.info("Logfile is [%s]" % logfile) 179 180 if options.config is None: 181 logger.debug("Using default configuration file.") 182 configPath = DEFAULT_CONFIG 183 else: 184 logger.debug("Using user-supplied configuration file.") 185 configPath = options.config 186 187 try: 188 logger.info("Configuration path is [%s]" % configPath) 189 config = Config(xmlPath=configPath) 190 setupPathResolver(config) 191 except Exception, e: 192 logger.error("Error reading or handling configuration: %s" % e) 193 logger.info("Cedar Backup 'span' utility run completed with status 4.") 194 return 4 195 196 if options.stacktrace: 197 _executeAction(options, config) 198 else: 199 try: 200 _executeAction(options, config) 201 except KeyboardInterrupt: 202 logger.error("Backup interrupted.") 203 logger.info("Cedar Backup 'span' utility run completed with status 5.") 204 return 5 205 except Exception, e: 206 logger.error("Error executing backup: %s" % e) 207 logger.info("Cedar Backup 'span' utility run completed with status 6.") 208 return 6 209 210 logger.info("Cedar Backup 'span' utility run completed with status 0.") 211 return 0
212 213 214 ####################################################################### 215 # Utility functions 216 ####################################################################### 217 218 #################### 219 # _usage() function 220 #################### 221
222 -def _usage(fd=sys.stderr):
223 """ 224 Prints usage information for the cback script. 225 @param fd: File descriptor used to print information. 226 @note: The C{fd} is used rather than C{print} to facilitate unit testing. 227 """ 228 fd.write("\n") 229 fd.write(" Usage: cback-span [switches]\n") 230 fd.write("\n") 231 fd.write(" Cedar Backup 'span' tool.\n") 232 fd.write("\n") 233 fd.write(" This Cedar Backup utility spans staged data between multiple discs.\n") 234 fd.write(" It is a utility, not an extension, and requires user interaction.\n") 235 fd.write("\n") 236 fd.write(" The following switches are accepted, mostly to set up underlying\n") 237 fd.write(" Cedar Backup functionality:\n") 238 fd.write("\n") 239 fd.write(" -h, --help Display this usage/help listing\n") 240 fd.write(" -V, --version Display version information\n") 241 fd.write(" -b, --verbose Print verbose output as well as logging to disk\n") 242 fd.write(" -c, --config Path to config file (default: %s)\n" % DEFAULT_CONFIG) 243 fd.write(" -l, --logfile Path to logfile (default: %s)\n" % DEFAULT_LOGFILE) 244 fd.write(" -o, --owner Logfile ownership, user:group (default: %s:%s)\n" % (DEFAULT_OWNERSHIP[0], DEFAULT_OWNERSHIP[1])) 245 fd.write(" -m, --mode Octal logfile permissions mode (default: %o)\n" % DEFAULT_MODE) 246 fd.write(" -O, --output Record some sub-command (i.e. tar) output to the log\n") 247 fd.write(" -d, --debug Write debugging information to the log (implies --output)\n") 248 fd.write(" -s, --stack Dump a Python stack trace instead of swallowing exceptions\n") 249 fd.write("\n")
250 251 252 ###################### 253 # _version() function 254 ###################### 255
256 -def _version(fd=sys.stdout):
257 """ 258 Prints version information for the cback script. 259 @param fd: File descriptor used to print information. 260 @note: The C{fd} is used rather than C{print} to facilitate unit testing. 261 """ 262 fd.write("\n") 263 fd.write(" Cedar Backup 'span' tool.\n") 264 fd.write(" Included with Cedar Backup version %s, released %s.\n" % (VERSION, DATE)) 265 fd.write("\n") 266 fd.write(" Copyright (c) %s %s <%s>.\n" % (COPYRIGHT, AUTHOR, EMAIL)) 267 fd.write(" See CREDITS for a list of included code and other contributors.\n") 268 fd.write(" This is free software; there is NO warranty. See the\n") 269 fd.write(" GNU General Public License version 2 for copying conditions.\n") 270 fd.write("\n") 271 fd.write(" Use the --help option for usage information.\n") 272 fd.write("\n")
273 274 275 ############################ 276 # _executeAction() function 277 ############################ 278
279 -def _executeAction(options, config):
280 """ 281 Implements the guts of the cback-span tool. 282 283 @param options: Program command-line options. 284 @type options: SpanOptions object. 285 286 @param config: Program configuration. 287 @type config: Config object. 288 289 @raise Exception: Under many generic error conditions 290 """ 291 print "" 292 print "================================================"; 293 print " Cedar Backup 'span' tool" 294 print "================================================"; 295 print "" 296 print "This the Cedar Backup span tool. It is used to split up staging" 297 print "data when that staging data does not fit onto a single disc." 298 print "" 299 print "This utility operates using Cedar Backup configuration. Configuration" 300 print "specifies which staging directory to look at and which writer device" 301 print "and media type to use." 302 print "" 303 if not _getYesNoAnswer("Continue?", default="Y"): 304 return 305 print "===" 306 307 print "" 308 print "Cedar Backup store configuration looks like this:" 309 print "" 310 print " Source Directory...: %s" % config.store.sourceDir 311 print " Media Type.........: %s" % config.store.mediaType 312 print " Device Type........: %s" % config.store.deviceType 313 print " Device Path........: %s" % config.store.devicePath 314 print " Device SCSI ID.....: %s" % config.store.deviceScsiId 315 print " Drive Speed........: %s" % config.store.driveSpeed 316 print " Check Data Flag....: %s" % config.store.checkData 317 print " No Eject Flag......: %s" % config.store.noEject 318 print "" 319 if not _getYesNoAnswer("Is this OK?", default="Y"): 320 return 321 print "===" 322 323 (writer, mediaCapacity) = _getWriter(config) 324 325 print "" 326 print "Please wait, indexing the source directory (this may take a while)..." 327 (dailyDirs, fileList) = _findDailyDirs(config.store.sourceDir) 328 print "===" 329 330 print "" 331 print "The following daily staging directories have not yet been written to disc:" 332 print "" 333 for dailyDir in dailyDirs: 334 print " %s" % dailyDir 335 336 totalSize = fileList.totalSize() 337 print "" 338 print "The total size of the data in these directories is %s." % displayBytes(totalSize) 339 print "" 340 if not _getYesNoAnswer("Continue?", default="Y"): 341 return 342 print "===" 343 344 print "" 345 print "Based on configuration, the capacity of your media is %s." % displayBytes(mediaCapacity) 346 347 print "" 348 print "Since estimates are not perfect and there is some uncertainly in" 349 print "media capacity calculations, it is good to have a \"cushion\"," 350 print "a percentage of capacity to set aside. The cushion reduces the" 351 print "capacity of your media, so a 1.5% cushion leaves 98.5% remaining." 352 print "" 353 cushion = _getFloat("What cushion percentage?", default=4.5) 354 print "===" 355 356 realCapacity = ((100.0 - cushion)/100.0) * mediaCapacity 357 minimumDiscs = (totalSize/realCapacity) + 1; 358 print "" 359 print "The real capacity, taking into account the %.2f%% cushion, is %s." % (cushion, displayBytes(realCapacity)) 360 print "It will take at least %d disc(s) to store your %s of data." % (minimumDiscs, displayBytes(totalSize)) 361 print "" 362 if not _getYesNoAnswer("Continue?", default="Y"): 363 return 364 print "===" 365 366 happy = False 367 while not happy: 368 print "" 369 print "Which algorithm do you want to use to span your data across" 370 print "multiple discs?" 371 print "" 372 print "The following algorithms are available:" 373 print "" 374 print " first....: The \"first-fit\" algorithm" 375 print " best.....: The \"best-fit\" algorithm" 376 print " worst....: The \"worst-fit\" algorithm" 377 print " alternate: The \"alternate-fit\" algorithm" 378 print "" 379 print "If you don't like the results you will have a chance to try a" 380 print "different one later." 381 print "" 382 algorithm = _getChoiceAnswer("Which algorithm?", "worst", [ "first", "best", "worst", "alternate",]) 383 print "===" 384 385 print "" 386 print "Please wait, generating file lists (this may take a while)..." 387 spanSet = fileList.generateSpan(capacity=realCapacity, algorithm="%s_fit" % algorithm) 388 print "===" 389 390 print "" 391 print "Using the \"%s-fit\" algorithm, Cedar Backup can split your data" % algorithm 392 print "into %d discs." % len(spanSet) 393 print "" 394 counter = 0 395 for item in spanSet: 396 counter += 1 397 print "Disc %d: %d files, %s, %.2f%% utilization" % (counter, len(item.fileList), 398 displayBytes(item.size), item.utilization) 399 print "" 400 if _getYesNoAnswer("Accept this solution?", default="Y"): 401 happy = True 402 print "===" 403 404 counter = 0 405 for spanItem in spanSet: 406 counter += 1 407 if counter == 1: 408 print "" 409 _getReturn("Please place the first disc in your backup device.\nPress return when ready.") 410 print "===" 411 else: 412 print "" 413 _getReturn("Please replace the disc in your backup device.\nPress return when ready.") 414 print "===" 415 _writeDisc(config, writer, spanItem) 416 417 _writeStoreIndicator(config, dailyDirs) 418 419 print "" 420 print "Completed writing all discs."
421 422 423 ############################ 424 # _findDailyDirs() function 425 ############################ 426
427 -def _findDailyDirs(stagingDir):
428 """ 429 Returns a list of all daily staging directories that have not yet been 430 stored. 431 432 The store indicator file C{cback.store} will be written to a daily staging 433 directory once that directory is written to disc. So, this function looks 434 at each daily staging directory within the configured staging directory, and 435 returns a list of those which do not contain the indicator file. 436 437 Returned is a tuple containing two items: a list of daily staging 438 directories, and a BackupFileList containing all files among those staging 439 directories. 440 441 @param stagingDir: Configured staging directory 442 443 @return: Tuple (staging dirs, backup file list) 444 """ 445 results = findDailyDirs(stagingDir, STORE_INDICATOR) 446 fileList = BackupFileList() 447 for item in results: 448 fileList.addDirContents(item) 449 return (results, fileList)
450 451 452 ################################## 453 # _writeStoreIndicator() function 454 ################################## 455
456 -def _writeStoreIndicator(config, dailyDirs):
457 """ 458 Writes a store indicator file into daily directories. 459 460 @param config: Config object. 461 @param dailyDirs: List of daily directories 462 """ 463 for dailyDir in dailyDirs: 464 writeIndicatorFile(dailyDir, STORE_INDICATOR, 465 config.options.backupUser, 466 config.options.backupGroup)
467 468 469 ######################## 470 # _getWriter() function 471 ######################## 472
473 -def _getWriter(config):
474 """ 475 Gets a writer and media capacity from store configuration. 476 Returned is a writer and a media capacity in bytes. 477 @param config: Cedar Backup configuration 478 @return: Tuple of (writer, mediaCapacity) 479 """ 480 writer = createWriter(config) 481 mediaCapacity = convertSize(writer.media.capacity, UNIT_SECTORS, UNIT_BYTES) 482 return (writer, mediaCapacity)
483 484 485 ######################## 486 # _writeDisc() function 487 ######################## 488
489 -def _writeDisc(config, writer, spanItem):
490 """ 491 Writes a span item to disc. 492 @param config: Cedar Backup configuration 493 @param writer: Writer to use 494 @param spanItem: Span item to write 495 """ 496 print "" 497 _discInitializeImage(config, writer, spanItem) 498 _discWriteImage(config, writer, spanItem) 499 _discConsistencyCheck(config, writer, spanItem) 500 print "Write process is complete." 501 print "==="
502
503 -def _discInitializeImage(config, writer, spanItem):
504 """ 505 Initialize an ISO image for a span item. 506 @param config: Cedar Backup configuration 507 @param writer: Writer to use 508 @param spanItem: Span item to write 509 """ 510 complete = False 511 while not complete: 512 try: 513 print "Initializing image..." 514 writer.initializeImage(newDisc=True, tmpdir=config.options.workingDir) 515 for path in spanItem.fileList: 516 graftPoint = os.path.dirname(path.replace(config.store.sourceDir, "", 1)) 517 writer.addImageEntry(path, graftPoint) 518 complete = True 519 except KeyboardInterrupt, e: 520 raise e 521 except Exception, e: 522 logger.error("Failed to initialize image: %s" % e) 523 if not _getYesNoAnswer("Retry initialization step?", default="Y"): 524 raise e 525 print "Ok, attempting retry." 526 print "===" 527 print "Completed initializing image."
528
529 -def _discWriteImage(config, writer, spanItem):
530 """ 531 Writes a ISO image for a span item. 532 @param config: Cedar Backup configuration 533 @param writer: Writer to use 534 @param spanItem: Span item to write 535 """ 536 complete = False 537 while not complete: 538 try: 539 print "Writing image to disc..." 540 writer.writeImage() 541 complete = True 542 except KeyboardInterrupt, e: 543 raise e 544 except Exception, e: 545 logger.error("Failed to write image: %s" % e) 546 if not _getYesNoAnswer("Retry this step?", default="Y"): 547 raise e 548 print "Ok, attempting retry." 549 _getReturn("Please replace media if needed.\nPress return when ready.") 550 print "===" 551 print "Completed writing image."
552
553 -def _discConsistencyCheck(config, writer, spanItem):
554 """ 555 Run a consistency check on an ISO image for a span item. 556 @param config: Cedar Backup configuration 557 @param writer: Writer to use 558 @param spanItem: Span item to write 559 """ 560 if config.store.checkData: 561 complete = False 562 while not complete: 563 try: 564 print "Running consistency check..." 565 _consistencyCheck(config, spanItem.fileList) 566 complete = True 567 except KeyboardInterrupt, e: 568 raise e 569 except Exception, e: 570 logger.error("Consistency check failed: %s" % e) 571 if not _getYesNoAnswer("Retry the consistency check?", default="Y"): 572 raise e 573 if _getYesNoAnswer("Rewrite the disc first?", default="N"): 574 print "Ok, attempting retry." 575 _getReturn("Please replace the disc in your backup device.\nPress return when ready.") 576 print "===" 577 _discWriteImage(config, writer, spanItem) 578 else: 579 print "Ok, attempting retry." 580 print "===" 581 print "Completed consistency check."
582 583 584 ############################### 585 # _consistencyCheck() function 586 ############################### 587
588 -def _consistencyCheck(config, fileList):
589 """ 590 Runs a consistency check against media in the backup device. 591 592 The function mounts the device at a temporary mount point in the working 593 directory, and then compares the passed-in file list's digest map with the 594 one generated from the disc. The two lists should be identical. 595 596 If no exceptions are thrown, there were no problems with the consistency 597 check. 598 599 @warning: The implementation of this function is very UNIX-specific. 600 601 @param config: Config object. 602 @param fileList: BackupFileList whose contents to check against 603 604 @raise ValueError: If the check fails 605 @raise IOError: If there is a problem working with the media. 606 """ 607 logger.debug("Running consistency check.") 608 mountPoint = tempfile.mkdtemp(dir=config.options.workingDir) 609 try: 610 mount(config.store.devicePath, mountPoint, "iso9660") 611 discList = BackupFileList() 612 discList.addDirContents(mountPoint) 613 sourceList = BackupFileList() 614 sourceList.extend(fileList) 615 discListDigest = discList.generateDigestMap(stripPrefix=normalizeDir(mountPoint)) 616 sourceListDigest = sourceList.generateDigestMap(stripPrefix=normalizeDir(config.store.sourceDir)) 617 compareDigestMaps(sourceListDigest, discListDigest, verbose=True) 618 logger.info("Consistency check completed. No problems found.") 619 finally: 620 unmount(mountPoint, True, 5, 1) # try 5 times, and remove mount point when done
621 622 623 ######################################################################### 624 # User interface utilities 625 ######################################################################## 626
627 -def _getYesNoAnswer(prompt, default):
628 """ 629 Get a yes/no answer from the user. 630 The default will be placed at the end of the prompt. 631 A "Y" or "y" is considered yes, anything else no. 632 A blank (empty) response results in the default. 633 @param prompt: Prompt to show. 634 @param default: Default to set if the result is blank 635 @return: Boolean true/false corresponding to Y/N 636 """ 637 if default == "Y": 638 prompt = "%s [Y/n]: " % prompt 639 else: 640 prompt = "%s [y/N]: " % prompt 641 answer = raw_input(prompt) 642 if answer in [ None, "", ]: 643 answer = default 644 if answer[0] in [ "Y", "y", ]: 645 return True 646 else: 647 return False
648
649 -def _getChoiceAnswer(prompt, default, validChoices):
650 """ 651 Get a particular choice from the user. 652 The default will be placed at the end of the prompt. 653 The function loops until getting a valid choice. 654 A blank (empty) response results in the default. 655 @param prompt: Prompt to show. 656 @param default: Default to set if the result is None or blank. 657 @param validChoices: List of valid choices (strings) 658 @return: Valid choice from user. 659 """ 660 prompt = "%s [%s]: " % (prompt, default) 661 answer = raw_input(prompt) 662 if answer in [ None, "", ]: 663 answer = default 664 while answer not in validChoices: 665 print "Choice must be one of %s" % validChoices 666 answer = raw_input(prompt) 667 return answer
668
669 -def _getFloat(prompt, default):
670 """ 671 Get a floating point number from the user. 672 The default will be placed at the end of the prompt. 673 The function loops until getting a valid floating point number. 674 A blank (empty) response results in the default. 675 @param prompt: Prompt to show. 676 @param default: Default to set if the result is None or blank. 677 @return: Floating point number from user 678 """ 679 prompt = "%s [%.2f]: " % (prompt, default) 680 while True: 681 answer = raw_input(prompt) 682 if answer in [ None, "" ]: 683 return default 684 else: 685 try: 686 return float(answer) 687 except ValueError: 688 print "Enter a floating point number."
689
690 -def _getReturn(prompt):
691 """ 692 Get a return key from the user. 693 @param prompt: Prompt to show. 694 """ 695 raw_input(prompt)
696 697 698 ######################################################################### 699 # Main routine 700 ######################################################################## 701 702 if __name__ == "__main__": 703 result = cli() 704 sys.exit(result) 705