1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39 """
40 Spans staged data among multiple discs
41
42 This is the Cedar Backup span tool. It is intended for use by people who stage
43 more data than can fit on a single disc. It allows a user to split staged data
44 among more than one disc. It can't be an extension because it requires user
45 input when switching media.
46
47 Most configuration is taken from the Cedar Backup configuration file,
48 specifically the store section. A few pieces of configuration are taken
49 directly from the user.
50
51 @author: Kenneth J. Pronovici <pronovic@ieee.org>
52 """
53
54
55
56
57
58
59 import sys
60 import os
61 import logging
62 import tempfile
63
64
65 from CedarBackup2.release import AUTHOR, EMAIL, VERSION, DATE, COPYRIGHT
66 from CedarBackup2.util import displayBytes, convertSize, mount, unmount
67 from CedarBackup2.util import UNIT_SECTORS, UNIT_BYTES
68 from CedarBackup2.config import Config
69 from CedarBackup2.filesystem import FilesystemList, BackupFileList, compareDigestMaps, normalizeDir
70 from CedarBackup2.cli import Options, setupLogging, setupPathResolver
71 from CedarBackup2.cli import DEFAULT_CONFIG, DEFAULT_LOGFILE, DEFAULT_OWNERSHIP, DEFAULT_MODE
72 from CedarBackup2.actions.constants import STORE_INDICATOR
73 from CedarBackup2.actions.util import createWriter
74 from CedarBackup2.actions.store import consistencyCheck, writeIndicatorFile
75 from CedarBackup2.actions.util import findDailyDirs
76 from CedarBackup2.knapsack import firstFit, bestFit, worstFit, alternateFit
77
78
79
80
81
82
83 logger = logging.getLogger("CedarBackup2.log.tools.span")
84
85
86
87
88
89
91
92 """
93 Tool-specific command-line options.
94
95 Most of the cback command-line options are exactly what we need here --
96 logfile path, permissions, verbosity, etc. However, we need to make a few
97 tweaks since we don't accept any actions.
98
99 Also, a few extra command line options that we accept are really ignored
100 underneath. I just don't care about that for a tool like this.
101 """
102
104 """
105 Validates command-line options represented by the object.
106 There are no validations here, because we don't use any actions.
107 @raise ValueError: If one of the validations fails.
108 """
109 pass
110
111
112
113
114
115
116
117
118
119
121 """
122 Implements the command-line interface for the C{cback-span} script.
123
124 Essentially, this is the "main routine" for the cback-span script. It does
125 all of the argument processing for the script, and then also implements the
126 tool functionality.
127
128 This function looks pretty similiar to C{CedarBackup2.cli.cli()}. It's not
129 easy to refactor this code to make it reusable and also readable, so I've
130 decided to just live with the duplication.
131
132 A different error code is returned for each type of failure:
133
134 - C{1}: The Python interpreter version is < 2.3
135 - C{2}: Error processing command-line arguments
136 - C{3}: Error configuring logging
137 - C{4}: Error parsing indicated configuration file
138 - C{5}: Backup was interrupted with a CTRL-C or similar
139 - C{6}: Error executing other parts of the script
140
141 @note: This script uses print rather than logging to the INFO level, because
142 it is interactive. Underlying Cedar Backup functionality uses the logging
143 mechanism exclusively.
144
145 @return: Error code as described above.
146 """
147 try:
148 if map(int, [sys.version_info[0], sys.version_info[1]]) < [2, 3]:
149 sys.stderr.write("Python version 2.3 or greater required.\n")
150 return 1
151 except:
152
153 sys.stderr.write("Python version 2.3 or greater required.\n")
154 return 1
155
156 try:
157 options = SpanOptions(argumentList=sys.argv[1:])
158 except Exception, e:
159 _usage()
160 sys.stderr.write(" *** Error: %s\n" % e)
161 return 2
162
163 if options.help:
164 _usage()
165 return 0
166 if options.version:
167 _version()
168 return 0
169
170 try:
171 logfile = setupLogging(options)
172 except Exception, e:
173 sys.stderr.write("Error setting up logging: %s\n" % e)
174 return 3
175
176 logger.info("Cedar Backup 'span' utility run started.")
177 logger.info("Options were [%s]" % options)
178 logger.info("Logfile is [%s]" % logfile)
179
180 if options.config is None:
181 logger.debug("Using default configuration file.")
182 configPath = DEFAULT_CONFIG
183 else:
184 logger.debug("Using user-supplied configuration file.")
185 configPath = options.config
186
187 try:
188 logger.info("Configuration path is [%s]" % configPath)
189 config = Config(xmlPath=configPath)
190 setupPathResolver(config)
191 except Exception, e:
192 logger.error("Error reading or handling configuration: %s" % e)
193 logger.info("Cedar Backup 'span' utility run completed with status 4.")
194 return 4
195
196 if options.stacktrace:
197 _executeAction(options, config)
198 else:
199 try:
200 _executeAction(options, config)
201 except KeyboardInterrupt:
202 logger.error("Backup interrupted.")
203 logger.info("Cedar Backup 'span' utility run completed with status 5.")
204 return 5
205 except Exception, e:
206 logger.error("Error executing backup: %s" % e)
207 logger.info("Cedar Backup 'span' utility run completed with status 6.")
208 return 6
209
210 logger.info("Cedar Backup 'span' utility run completed with status 0.")
211 return 0
212
213
214
215
216
217
218
219
220
221
223 """
224 Prints usage information for the cback script.
225 @param fd: File descriptor used to print information.
226 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
227 """
228 fd.write("\n")
229 fd.write(" Usage: cback-span [switches]\n")
230 fd.write("\n")
231 fd.write(" Cedar Backup 'span' tool.\n")
232 fd.write("\n")
233 fd.write(" This Cedar Backup utility spans staged data between multiple discs.\n")
234 fd.write(" It is a utility, not an extension, and requires user interaction.\n")
235 fd.write("\n")
236 fd.write(" The following switches are accepted, mostly to set up underlying\n")
237 fd.write(" Cedar Backup functionality:\n")
238 fd.write("\n")
239 fd.write(" -h, --help Display this usage/help listing\n")
240 fd.write(" -V, --version Display version information\n")
241 fd.write(" -b, --verbose Print verbose output as well as logging to disk\n")
242 fd.write(" -c, --config Path to config file (default: %s)\n" % DEFAULT_CONFIG)
243 fd.write(" -l, --logfile Path to logfile (default: %s)\n" % DEFAULT_LOGFILE)
244 fd.write(" -o, --owner Logfile ownership, user:group (default: %s:%s)\n" % (DEFAULT_OWNERSHIP[0], DEFAULT_OWNERSHIP[1]))
245 fd.write(" -m, --mode Octal logfile permissions mode (default: %o)\n" % DEFAULT_MODE)
246 fd.write(" -O, --output Record some sub-command (i.e. tar) output to the log\n")
247 fd.write(" -d, --debug Write debugging information to the log (implies --output)\n")
248 fd.write(" -s, --stack Dump a Python stack trace instead of swallowing exceptions\n")
249 fd.write("\n")
250
251
252
253
254
255
257 """
258 Prints version information for the cback script.
259 @param fd: File descriptor used to print information.
260 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
261 """
262 fd.write("\n")
263 fd.write(" Cedar Backup 'span' tool.\n")
264 fd.write(" Included with Cedar Backup version %s, released %s.\n" % (VERSION, DATE))
265 fd.write("\n")
266 fd.write(" Copyright (c) %s %s <%s>.\n" % (COPYRIGHT, AUTHOR, EMAIL))
267 fd.write(" See CREDITS for a list of included code and other contributors.\n")
268 fd.write(" This is free software; there is NO warranty. See the\n")
269 fd.write(" GNU General Public License version 2 for copying conditions.\n")
270 fd.write("\n")
271 fd.write(" Use the --help option for usage information.\n")
272 fd.write("\n")
273
274
275
276
277
278
280 """
281 Implements the guts of the cback-span tool.
282
283 @param options: Program command-line options.
284 @type options: SpanOptions object.
285
286 @param config: Program configuration.
287 @type config: Config object.
288
289 @raise Exception: Under many generic error conditions
290 """
291 print ""
292 print "================================================";
293 print " Cedar Backup 'span' tool"
294 print "================================================";
295 print ""
296 print "This the Cedar Backup span tool. It is used to split up staging"
297 print "data when that staging data does not fit onto a single disc."
298 print ""
299 print "This utility operates using Cedar Backup configuration. Configuration"
300 print "specifies which staging directory to look at and which writer device"
301 print "and media type to use."
302 print ""
303 if not _getYesNoAnswer("Continue?", default="Y"):
304 return
305 print "==="
306
307 print ""
308 print "Cedar Backup store configuration looks like this:"
309 print ""
310 print " Source Directory...: %s" % config.store.sourceDir
311 print " Media Type.........: %s" % config.store.mediaType
312 print " Device Type........: %s" % config.store.deviceType
313 print " Device Path........: %s" % config.store.devicePath
314 print " Device SCSI ID.....: %s" % config.store.deviceScsiId
315 print " Drive Speed........: %s" % config.store.driveSpeed
316 print " Check Data Flag....: %s" % config.store.checkData
317 print " No Eject Flag......: %s" % config.store.noEject
318 print ""
319 if not _getYesNoAnswer("Is this OK?", default="Y"):
320 return
321 print "==="
322
323 (writer, mediaCapacity) = _getWriter(config)
324
325 print ""
326 print "Please wait, indexing the source directory (this may take a while)..."
327 (dailyDirs, fileList) = _findDailyDirs(config.store.sourceDir)
328 print "==="
329
330 print ""
331 print "The following daily staging directories have not yet been written to disc:"
332 print ""
333 for dailyDir in dailyDirs:
334 print " %s" % dailyDir
335
336 totalSize = fileList.totalSize()
337 print ""
338 print "The total size of the data in these directories is %s." % displayBytes(totalSize)
339 print ""
340 if not _getYesNoAnswer("Continue?", default="Y"):
341 return
342 print "==="
343
344 print ""
345 print "Based on configuration, the capacity of your media is %s." % displayBytes(mediaCapacity)
346
347 print ""
348 print "Since estimates are not perfect and there is some uncertainly in"
349 print "media capacity calculations, it is good to have a \"cushion\","
350 print "a percentage of capacity to set aside. The cushion reduces the"
351 print "capacity of your media, so a 1.5% cushion leaves 98.5% remaining."
352 print ""
353 cushion = _getFloat("What cushion percentage?", default=4.5)
354 print "==="
355
356 realCapacity = ((100.0 - cushion)/100.0) * mediaCapacity
357 minimumDiscs = (totalSize/realCapacity) + 1;
358 print ""
359 print "The real capacity, taking into account the %.2f%% cushion, is %s." % (cushion, displayBytes(realCapacity))
360 print "It will take at least %d disc(s) to store your %s of data." % (minimumDiscs, displayBytes(totalSize))
361 print ""
362 if not _getYesNoAnswer("Continue?", default="Y"):
363 return
364 print "==="
365
366 happy = False
367 while not happy:
368 print ""
369 print "Which algorithm do you want to use to span your data across"
370 print "multiple discs?"
371 print ""
372 print "The following algorithms are available:"
373 print ""
374 print " first....: The \"first-fit\" algorithm"
375 print " best.....: The \"best-fit\" algorithm"
376 print " worst....: The \"worst-fit\" algorithm"
377 print " alternate: The \"alternate-fit\" algorithm"
378 print ""
379 print "If you don't like the results you will have a chance to try a"
380 print "different one later."
381 print ""
382 algorithm = _getChoiceAnswer("Which algorithm?", "worst", [ "first", "best", "worst", "alternate",])
383 print "==="
384
385 print ""
386 print "Please wait, generating file lists (this may take a while)..."
387 spanSet = fileList.generateSpan(capacity=realCapacity, algorithm="%s_fit" % algorithm)
388 print "==="
389
390 print ""
391 print "Using the \"%s-fit\" algorithm, Cedar Backup can split your data" % algorithm
392 print "into %d discs." % len(spanSet)
393 print ""
394 counter = 0
395 for item in spanSet:
396 counter += 1
397 print "Disc %d: %d files, %s, %.2f%% utilization" % (counter, len(item.fileList),
398 displayBytes(item.size), item.utilization)
399 print ""
400 if _getYesNoAnswer("Accept this solution?", default="Y"):
401 happy = True
402 print "==="
403
404 print ""
405 _getReturn("Please place the first disc in your backup device.\nPress return when ready.")
406 print "==="
407
408 counter = 0
409 for spanItem in spanSet:
410 counter += 1
411 _writeDisc(config, writer, spanItem)
412 print ""
413 _getReturn("Please replace the disc in your backup device.\nPress return when ready.")
414 print "==="
415
416 _writeStoreIndicator(config, dailyDirs)
417
418 print ""
419 print "Completed writing all discs."
420
421
422
423
424
425
427 """
428 Returns a list of all daily staging directories that have not yet been
429 stored.
430
431 The store indicator file C{cback.store} will be written to a daily staging
432 directory once that directory is written to disc. So, this function looks
433 at each daily staging directory within the configured staging directory, and
434 returns a list of those which do not contain the indicator file.
435
436 Returned is a tuple containing two items: a list of daily staging
437 directories, and a BackupFileList containing all files among those staging
438 directories.
439
440 @param stagingDir: Configured staging directory
441
442 @return: Tuple (staging dirs, backup file list)
443 """
444 results = findDailyDirs(stagingDir, STORE_INDICATOR)
445 fileList = BackupFileList()
446 for item in results:
447 fileList.addDirContents(item)
448 return (results, fileList)
449
450
451
452
453
454
466
467
468
469
470
471
482
483
484
485
486
487
507
508
509
510
511
512
514 """
515 Runs a consistency check against media in the backup device.
516
517 The function mounts the device at a temporary mount point in the working
518 directory, and then compares the passed-in file list's digest map with the
519 one generated from the disc. The two lists should be identical.
520
521 If no exceptions are thrown, there were no problems with the consistency
522 check.
523
524 @warning: The implementation of this function is very UNIX-specific.
525
526 @param config: Config object.
527 @param fileList: BackupFileList whose contents to check against
528
529 @raise ValueError: If the check fails
530 @raise IOError: If there is a problem working with the media.
531 """
532 logger.debug("Running consistency check.")
533 mountPoint = tempfile.mkdtemp(dir=config.options.workingDir)
534 try:
535 mount(config.store.devicePath, mountPoint, "iso9660")
536 discList = BackupFileList()
537 discList.addDirContents(mountPoint)
538 sourceList = BackupFileList()
539 sourceList.extend(fileList)
540 discListDigest = discList.generateDigestMap(stripPrefix=normalizeDir(mountPoint))
541 sourceListDigest = sourceList.generateDigestMap(stripPrefix=normalizeDir(config.store.sourceDir))
542 compareDigestMaps(sourceListDigest, discListDigest, verbose=True)
543 logger.info("Consistency check completed. No problems found.")
544 finally:
545 unmount(mountPoint, True, 5, 1)
546
547
548
549
550
551
553 """
554 Get a yes/no answer from the user.
555 The default will be placed at the end of the prompt.
556 A "Y" or "y" is considered yes, anything else no.
557 A blank (empty) response results in the default.
558 @param prompt: Prompt to show.
559 @param default: Default to set if the result is blank
560 @return: Boolean true/false corresponding to Y/N
561 """
562 if default == "Y":
563 prompt = "%s [Y/n]: " % prompt
564 else:
565 prompt = "%s [y/N]: " % prompt
566 answer = raw_input(prompt)
567 if answer in [ None, "", ]:
568 answer = default
569 if answer[0] in [ "Y", "y", ]:
570 return True
571 else:
572 return False
573
575 """
576 Get a particular choice from the user.
577 The default will be placed at the end of the prompt.
578 The function loops until getting a valid choice.
579 A blank (empty) response results in the default.
580 @param prompt: Prompt to show.
581 @param default: Default to set if the result is None or blank.
582 @param validChoices: List of valid choices (strings)
583 @return: Valid choice from user.
584 """
585 prompt = "%s [%s]: " % (prompt, default)
586 answer = raw_input(prompt)
587 if answer in [ None, "", ]:
588 answer = default
589 while answer not in validChoices:
590 print "Choice must be one of %s" % validChoices
591 answer = raw_input(prompt)
592 return answer
593
595 """
596 Get a floating point number from the user.
597 The default will be placed at the end of the prompt.
598 The function loops until getting a valid floating point number.
599 A blank (empty) response results in the default.
600 @param prompt: Prompt to show.
601 @param default: Default to set if the result is None or blank.
602 @return: Floating point number from user
603 """
604 prompt = "%s [%.2f]: " % (prompt, default)
605 while True:
606 answer = raw_input(prompt)
607 if answer in [ None, "" ]:
608 return default
609 else:
610 try:
611 return float(answer)
612 except ValueError:
613 print "Enter a floating point number."
614
616 """
617 Get a return key from the user.
618 @param prompt: Prompt to show.
619 """
620 raw_input(prompt)
621
622
623
624
625
626
627 if __name__ == "__main__":
628 result = cli()
629 sys.exit(result)
630