1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39 """
40 Provides backup peer-related objects and utility functions.
41
42 @sort: LocalPeer, Remote Peer
43
44 @var DEF_COLLECT_INDICATOR: Name of the default collect indicator file.
45 @var DEF_STAGE_INDICATOR: Name of the default stage indicator file.
46
47 @author: Kenneth J. Pronovici <pronovic@ieee.org>
48 """
49
50
51
52
53
54
55
56 import os
57 import logging
58 import shutil
59 import sets
60 import re
61
62
63 from CedarBackup2.filesystem import FilesystemList
64 from CedarBackup2.util import resolveCommand, executeCommand
65 from CedarBackup2.util import splitCommandLine, encodePath
66 from CedarBackup2.config import VALID_FAILURE_MODES
67
68
69
70
71
72
73 logger = logging.getLogger("CedarBackup2.log.peer")
74
75 DEF_RCP_COMMAND = [ "/usr/bin/scp", "-B", "-q", "-C" ]
76 DEF_RSH_COMMAND = [ "/usr/bin/ssh", ]
77 DEF_CBACK_COMMAND = "/usr/bin/cback"
78
79 DEF_COLLECT_INDICATOR = "cback.collect"
80 DEF_STAGE_INDICATOR = "cback.stage"
81
82 SU_COMMAND = [ "su" ]
83
84
85
86
87
88
90
91
92
93
94
95 """
96 Backup peer representing a local peer in a backup pool.
97
98 This is a class representing a local (non-network) peer in a backup pool.
99 Local peers are backed up by simple filesystem copy operations. A local
100 peer has associated with it a name (typically, but not necessarily, a
101 hostname) and a collect directory.
102
103 The public methods other than the constructor are part of a "backup peer"
104 interface shared with the C{RemotePeer} class.
105
106 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator,
107 _copyLocalDir, _copyLocalFile, name, collectDir
108 """
109
110
111
112
113
114 - def __init__(self, name, collectDir, ignoreFailureMode=None):
115 """
116 Initializes a local backup peer.
117
118 Note that the collect directory must be an absolute path, but does not
119 have to exist when the object is instantiated. We do a lazy validation
120 on this value since we could (potentially) be creating peer objects
121 before an ongoing backup completed.
122
123 @param name: Name of the backup peer
124 @type name: String, typically a hostname
125
126 @param collectDir: Path to the peer's collect directory
127 @type collectDir: String representing an absolute local path on disk
128
129 @param ignoreFailureMode: Ignore failure mode for this peer
130 @type ignoreFailureMode: One of VALID_FAILURE_MODES
131
132 @raise ValueError: If the name is empty.
133 @raise ValueError: If collect directory is not an absolute path.
134 """
135 self._name = None
136 self._collectDir = None
137 self._ignoreFailureMode = None
138 self.name = name
139 self.collectDir = collectDir
140 self.ignoreFailureMode = ignoreFailureMode
141
142
143
144
145
146
148 """
149 Property target used to set the peer name.
150 The value must be a non-empty string and cannot be C{None}.
151 @raise ValueError: If the value is an empty string or C{None}.
152 """
153 if value is None or len(value) < 1:
154 raise ValueError("Peer name must be a non-empty string.")
155 self._name = value
156
158 """
159 Property target used to get the peer name.
160 """
161 return self._name
162
164 """
165 Property target used to set the collect directory.
166 The value must be an absolute path and cannot be C{None}.
167 It does not have to exist on disk at the time of assignment.
168 @raise ValueError: If the value is C{None} or is not an absolute path.
169 @raise ValueError: If a path cannot be encoded properly.
170 """
171 if value is None or not os.path.isabs(value):
172 raise ValueError("Collect directory must be an absolute path.")
173 self._collectDir = encodePath(value)
174
176 """
177 Property target used to get the collect directory.
178 """
179 return self._collectDir
180
182 """
183 Property target used to set the ignoreFailure mode.
184 If not C{None}, the mode must be one of the values in L{VALID_FAILURE_MODES}.
185 @raise ValueError: If the value is not valid.
186 """
187 if value is not None:
188 if value not in VALID_FAILURE_MODES:
189 raise ValueError("Ignore failure mode must be one of %s." % VALID_FAILURE_MODES)
190 self._ignoreFailureMode = value
191
193 """
194 Property target used to get the ignoreFailure mode.
195 """
196 return self._ignoreFailureMode
197
198 name = property(_getName, _setName, None, "Name of the peer.")
199 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).")
200 ignoreFailureMode = property(_getIgnoreFailureMode, _setIgnoreFailureMode, None, "Ignore failure mode for peer.")
201
202
203
204
205
206
207 - def stagePeer(self, targetDir, ownership=None, permissions=None):
208 """
209 Stages data from the peer into the indicated local target directory.
210
211 The collect and target directories must both already exist before this
212 method is called. If passed in, ownership and permissions will be
213 applied to the files that are copied.
214
215 @note: The caller is responsible for checking that the indicator exists,
216 if they care. This function only stages the files within the directory.
217
218 @note: If you have user/group as strings, call the L{util.getUidGid} function
219 to get the associated uid/gid as an ownership tuple.
220
221 @param targetDir: Target directory to write data into
222 @type targetDir: String representing a directory on disk
223
224 @param ownership: Owner and group that the staged files should have
225 @type ownership: Tuple of numeric ids C{(uid, gid)}
226
227 @param permissions: Permissions that the staged files should have
228 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
229
230 @return: Number of files copied from the source directory to the target directory.
231
232 @raise ValueError: If collect directory is not a directory or does not exist
233 @raise ValueError: If target directory is not a directory, does not exist or is not absolute.
234 @raise ValueError: If a path cannot be encoded properly.
235 @raise IOError: If there were no files to stage (i.e. the directory was empty)
236 @raise IOError: If there is an IO error copying a file.
237 @raise OSError: If there is an OS error copying or changing permissions on a file
238 """
239 targetDir = encodePath(targetDir)
240 if not os.path.isabs(targetDir):
241 logger.debug("Target directory [%s] not an absolute path." % targetDir)
242 raise ValueError("Target directory must be an absolute path.")
243 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir):
244 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir)
245 raise ValueError("Collect directory is not a directory or does not exist on disk.")
246 if not os.path.exists(targetDir) or not os.path.isdir(targetDir):
247 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir)
248 raise ValueError("Target directory is not a directory or does not exist on disk.")
249 count = LocalPeer._copyLocalDir(self.collectDir, targetDir, ownership, permissions)
250 if count == 0:
251 raise IOError("Did not copy any files from local peer.")
252 return count
253
255 """
256 Checks the collect indicator in the peer's staging directory.
257
258 When a peer has completed collecting its backup files, it will write an
259 empty indicator file into its collect directory. This method checks to
260 see whether that indicator has been written. We're "stupid" here - if
261 the collect directory doesn't exist, you'll naturally get back C{False}.
262
263 If you need to, you can override the name of the collect indicator file
264 by passing in a different name.
265
266 @param collectIndicator: Name of the collect indicator file to check
267 @type collectIndicator: String representing name of a file in the collect directory
268
269 @return: Boolean true/false depending on whether the indicator exists.
270 @raise ValueError: If a path cannot be encoded properly.
271 """
272 collectIndicator = encodePath(collectIndicator)
273 if collectIndicator is None:
274 return os.path.exists(os.path.join(self.collectDir, DEF_COLLECT_INDICATOR))
275 else:
276 return os.path.exists(os.path.join(self.collectDir, collectIndicator))
277
279 """
280 Writes the stage indicator in the peer's staging directory.
281
282 When the master has completed collecting its backup files, it will write
283 an empty indicator file into the peer's collect directory. The presence
284 of this file implies that the staging process is complete.
285
286 If you need to, you can override the name of the stage indicator file by
287 passing in a different name.
288
289 @note: If you have user/group as strings, call the L{util.getUidGid}
290 function to get the associated uid/gid as an ownership tuple.
291
292 @param stageIndicator: Name of the indicator file to write
293 @type stageIndicator: String representing name of a file in the collect directory
294
295 @param ownership: Owner and group that the indicator file should have
296 @type ownership: Tuple of numeric ids C{(uid, gid)}
297
298 @param permissions: Permissions that the indicator file should have
299 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
300
301 @raise ValueError: If collect directory is not a directory or does not exist
302 @raise ValueError: If a path cannot be encoded properly.
303 @raise IOError: If there is an IO error creating the file.
304 @raise OSError: If there is an OS error creating or changing permissions on the file
305 """
306 stageIndicator = encodePath(stageIndicator)
307 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir):
308 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir)
309 raise ValueError("Collect directory is not a directory or does not exist on disk.")
310 if stageIndicator is None:
311 fileName = os.path.join(self.collectDir, DEF_STAGE_INDICATOR)
312 else:
313 fileName = os.path.join(self.collectDir, stageIndicator)
314 LocalPeer._copyLocalFile(None, fileName, ownership, permissions)
315
316
317
318
319
320
321 - def _copyLocalDir(sourceDir, targetDir, ownership=None, permissions=None):
322 """
323 Copies files from the source directory to the target directory.
324
325 This function is not recursive. Only the files in the directory will be
326 copied. Ownership and permissions will be left at their default values
327 if new values are not specified. The source and target directories are
328 allowed to be soft links to a directory, but besides that soft links are
329 ignored.
330
331 @note: If you have user/group as strings, call the L{util.getUidGid}
332 function to get the associated uid/gid as an ownership tuple.
333
334 @param sourceDir: Source directory
335 @type sourceDir: String representing a directory on disk
336
337 @param targetDir: Target directory
338 @type targetDir: String representing a directory on disk
339
340 @param ownership: Owner and group that the copied files should have
341 @type ownership: Tuple of numeric ids C{(uid, gid)}
342
343 @param permissions: Permissions that the staged files should have
344 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
345
346 @return: Number of files copied from the source directory to the target directory.
347
348 @raise ValueError: If source or target is not a directory or does not exist.
349 @raise ValueError: If a path cannot be encoded properly.
350 @raise IOError: If there is an IO error copying the files.
351 @raise OSError: If there is an OS error copying or changing permissions on a files
352 """
353 filesCopied = 0
354 sourceDir = encodePath(sourceDir)
355 targetDir = encodePath(targetDir)
356 for fileName in os.listdir(sourceDir):
357 sourceFile = os.path.join(sourceDir, fileName)
358 targetFile = os.path.join(targetDir, fileName)
359 LocalPeer._copyLocalFile(sourceFile, targetFile, ownership, permissions)
360 filesCopied += 1
361 return filesCopied
362 _copyLocalDir = staticmethod(_copyLocalDir)
363
364 - def _copyLocalFile(sourceFile=None, targetFile=None, ownership=None, permissions=None, overwrite=True):
365 """
366 Copies a source file to a target file.
367
368 If the source file is C{None} then the target file will be created or
369 overwritten as an empty file. If the target file is C{None}, this method
370 is a no-op. Attempting to copy a soft link or a directory will result in
371 an exception.
372
373 @note: If you have user/group as strings, call the L{util.getUidGid}
374 function to get the associated uid/gid as an ownership tuple.
375
376 @note: We will not overwrite a target file that exists when this method
377 is invoked. If the target already exists, we'll raise an exception.
378
379 @param sourceFile: Source file to copy
380 @type sourceFile: String representing a file on disk, as an absolute path
381
382 @param targetFile: Target file to create
383 @type targetFile: String representing a file on disk, as an absolute path
384
385 @param ownership: Owner and group that the copied should have
386 @type ownership: Tuple of numeric ids C{(uid, gid)}
387
388 @param permissions: Permissions that the staged files should have
389 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
390
391 @param overwrite: Indicates whether it's OK to overwrite the target file.
392 @type overwrite: Boolean true/false.
393
394 @raise ValueError: If the passed-in source file is not a regular file.
395 @raise ValueError: If a path cannot be encoded properly.
396 @raise IOError: If the target file already exists.
397 @raise IOError: If there is an IO error copying the file
398 @raise OSError: If there is an OS error copying or changing permissions on a file
399 """
400 targetFile = encodePath(targetFile)
401 sourceFile = encodePath(sourceFile)
402 if targetFile is None:
403 return
404 if not overwrite:
405 if os.path.exists(targetFile):
406 raise IOError("Target file [%s] already exists." % targetFile)
407 if sourceFile is None:
408 open(targetFile, "w").write("")
409 else:
410 if os.path.isfile(sourceFile) and not os.path.islink(sourceFile):
411 shutil.copy(sourceFile, targetFile)
412 else:
413 logger.debug("Source [%s] is not a regular file." % sourceFile)
414 raise ValueError("Source is not a regular file.")
415 if ownership is not None:
416 os.chown(targetFile, ownership[0], ownership[1])
417 if permissions is not None:
418 os.chmod(targetFile, permissions)
419 _copyLocalFile = staticmethod(_copyLocalFile)
420
421
422
423
424
425
427
428
429
430
431
432 """
433 Backup peer representing a remote peer in a backup pool.
434
435 This is a class representing a remote (networked) peer in a backup pool.
436 Remote peers are backed up using an rcp-compatible copy command. A remote
437 peer has associated with it a name (which must be a valid hostname), a
438 collect directory, a working directory and a copy method (an rcp-compatible
439 command).
440
441 You can also set an optional local user value. This username will be used
442 as the local user for any remote copies that are required. It can only be
443 used if the root user is executing the backup. The root user will C{su} to
444 the local user and execute the remote copies as that user.
445
446 The copy method is associated with the peer and not with the actual request
447 to copy, because we can envision that each remote host might have a
448 different connect method.
449
450 The public methods other than the constructor are part of a "backup peer"
451 interface shared with the C{LocalPeer} class.
452
453 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator,
454 executeRemoteCommand, executeManagedAction, _getDirContents,
455 _copyRemoteDir, _copyRemoteFile, _pushLocalFile, name, collectDir,
456 remoteUser, rcpCommand, rshCommand, cbackCommand
457 """
458
459
460
461
462
463 - def __init__(self, name=None, collectDir=None, workingDir=None, remoteUser=None,
464 rcpCommand=None, localUser=None, rshCommand=None, cbackCommand=None,
465 ignoreFailureMode=None):
466 """
467 Initializes a remote backup peer.
468
469 @note: If provided, each command will eventually be parsed into a list of
470 strings suitable for passing to C{util.executeCommand} in order to avoid
471 security holes related to shell interpolation. This parsing will be
472 done by the L{util.splitCommandLine} function. See the documentation for
473 that function for some important notes about its limitations.
474
475 @param name: Name of the backup peer
476 @type name: String, must be a valid DNS hostname
477
478 @param collectDir: Path to the peer's collect directory
479 @type collectDir: String representing an absolute path on the remote peer
480
481 @param workingDir: Working directory that can be used to create temporary files, etc.
482 @type workingDir: String representing an absolute path on the current host.
483
484 @param remoteUser: Name of the Cedar Backup user on the remote peer
485 @type remoteUser: String representing a username, valid via remote shell to the peer
486
487 @param localUser: Name of the Cedar Backup user on the current host
488 @type localUser: String representing a username, valid on the current host
489
490 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
491 @type rcpCommand: String representing a system command including required arguments
492
493 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer
494 @type rshCommand: String representing a system command including required arguments
495
496 @param cbackCommand: A chack-compatible command to use for executing managed actions
497 @type cbackCommand: String representing a system command including required arguments
498
499 @param ignoreFailureMode: Ignore failure mode for this peer
500 @type ignoreFailureMode: One of VALID_FAILURE_MODES
501
502 @raise ValueError: If collect directory is not an absolute path
503 """
504 self._name = None
505 self._collectDir = None
506 self._workingDir = None
507 self._remoteUser = None
508 self._localUser = None
509 self._rcpCommand = None
510 self._rcpCommandList = None
511 self._rshCommand = None
512 self._rshCommandList = None
513 self._cbackCommand = None
514 self._ignoreFailureMode = None
515 self.name = name
516 self.collectDir = collectDir
517 self.workingDir = workingDir
518 self.remoteUser = remoteUser
519 self.localUser = localUser
520 self.rcpCommand = rcpCommand
521 self.rshCommand = rshCommand
522 self.cbackCommand = cbackCommand
523 self.ignoreFailureMode = ignoreFailureMode
524
525
526
527
528
529
531 """
532 Property target used to set the peer name.
533 The value must be a non-empty string and cannot be C{None}.
534 @raise ValueError: If the value is an empty string or C{None}.
535 """
536 if value is None or len(value) < 1:
537 raise ValueError("Peer name must be a non-empty string.")
538 self._name = value
539
541 """
542 Property target used to get the peer name.
543 """
544 return self._name
545
547 """
548 Property target used to set the collect directory.
549 The value must be an absolute path and cannot be C{None}.
550 It does not have to exist on disk at the time of assignment.
551 @raise ValueError: If the value is C{None} or is not an absolute path.
552 @raise ValueError: If the value cannot be encoded properly.
553 """
554 if value is not None:
555 if not os.path.isabs(value):
556 raise ValueError("Collect directory must be an absolute path.")
557 self._collectDir = encodePath(value)
558
560 """
561 Property target used to get the collect directory.
562 """
563 return self._collectDir
564
566 """
567 Property target used to set the working directory.
568 The value must be an absolute path and cannot be C{None}.
569 @raise ValueError: If the value is C{None} or is not an absolute path.
570 @raise ValueError: If the value cannot be encoded properly.
571 """
572 if value is not None:
573 if not os.path.isabs(value):
574 raise ValueError("Working directory must be an absolute path.")
575 self._workingDir = encodePath(value)
576
578 """
579 Property target used to get the working directory.
580 """
581 return self._workingDir
582
584 """
585 Property target used to set the remote user.
586 The value must be a non-empty string and cannot be C{None}.
587 @raise ValueError: If the value is an empty string or C{None}.
588 """
589 if value is None or len(value) < 1:
590 raise ValueError("Peer remote user must be a non-empty string.")
591 self._remoteUser = value
592
594 """
595 Property target used to get the remote user.
596 """
597 return self._remoteUser
598
600 """
601 Property target used to set the local user.
602 The value must be a non-empty string if it is not C{None}.
603 @raise ValueError: If the value is an empty string.
604 """
605 if value is not None:
606 if len(value) < 1:
607 raise ValueError("Peer local user must be a non-empty string.")
608 self._localUser = value
609
611 """
612 Property target used to get the local user.
613 """
614 return self._localUser
615
617 """
618 Property target to set the rcp command.
619
620 The value must be a non-empty string or C{None}. Its value is stored in
621 the two forms: "raw" as provided by the client, and "parsed" into a list
622 suitable for being passed to L{util.executeCommand} via
623 L{util.splitCommandLine}.
624
625 However, all the caller will ever see via the property is the actual
626 value they set (which includes seeing C{None}, even if we translate that
627 internally to C{DEF_RCP_COMMAND}). Internally, we should always use
628 C{self._rcpCommandList} if we want the actual command list.
629
630 @raise ValueError: If the value is an empty string.
631 """
632 if value is None:
633 self._rcpCommand = None
634 self._rcpCommandList = DEF_RCP_COMMAND
635 else:
636 if len(value) >= 1:
637 self._rcpCommand = value
638 self._rcpCommandList = splitCommandLine(self._rcpCommand)
639 else:
640 raise ValueError("The rcp command must be a non-empty string.")
641
643 """
644 Property target used to get the rcp command.
645 """
646 return self._rcpCommand
647
649 """
650 Property target to set the rsh command.
651
652 The value must be a non-empty string or C{None}. Its value is stored in
653 the two forms: "raw" as provided by the client, and "parsed" into a list
654 suitable for being passed to L{util.executeCommand} via
655 L{util.splitCommandLine}.
656
657 However, all the caller will ever see via the property is the actual
658 value they set (which includes seeing C{None}, even if we translate that
659 internally to C{DEF_RSH_COMMAND}). Internally, we should always use
660 C{self._rshCommandList} if we want the actual command list.
661
662 @raise ValueError: If the value is an empty string.
663 """
664 if value is None:
665 self._rshCommand = None
666 self._rshCommandList = DEF_RSH_COMMAND
667 else:
668 if len(value) >= 1:
669 self._rshCommand = value
670 self._rshCommandList = splitCommandLine(self._rshCommand)
671 else:
672 raise ValueError("The rsh command must be a non-empty string.")
673
675 """
676 Property target used to get the rsh command.
677 """
678 return self._rshCommand
679
681 """
682 Property target to set the cback command.
683
684 The value must be a non-empty string or C{None}. Unlike the other
685 command, this value is only stored in the "raw" form provided by the
686 client.
687
688 @raise ValueError: If the value is an empty string.
689 """
690 if value is None:
691 self._cbackCommand = None
692 else:
693 if len(value) >= 1:
694 self._cbackCommand = value
695 else:
696 raise ValueError("The cback command must be a non-empty string.")
697
699 """
700 Property target used to get the cback command.
701 """
702 return self._cbackCommand
703
705 """
706 Property target used to set the ignoreFailure mode.
707 If not C{None}, the mode must be one of the values in L{VALID_FAILURE_MODES}.
708 @raise ValueError: If the value is not valid.
709 """
710 if value is not None:
711 if value not in VALID_FAILURE_MODES:
712 raise ValueError("Ignore failure mode must be one of %s." % VALID_FAILURE_MODES)
713 self._ignoreFailureMode = value
714
716 """
717 Property target used to get the ignoreFailure mode.
718 """
719 return self._ignoreFailureMode
720
721 name = property(_getName, _setName, None, "Name of the peer (a valid DNS hostname).")
722 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).")
723 workingDir = property(_getWorkingDir, _setWorkingDir, None, "Path to the peer's working directory (an absolute local path).")
724 remoteUser = property(_getRemoteUser, _setRemoteUser, None, "Name of the Cedar Backup user on the remote peer.")
725 localUser = property(_getLocalUser, _setLocalUser, None, "Name of the Cedar Backup user on the current host.")
726 rcpCommand = property(_getRcpCommand, _setRcpCommand, None, "An rcp-compatible copy command to use for copying files.")
727 rshCommand = property(_getRshCommand, _setRshCommand, None, "An rsh-compatible command to use for remote shells to the peer.")
728 cbackCommand = property(_getCbackCommand, _setCbackCommand, None, "A chack-compatible command to use for executing managed actions.")
729 ignoreFailureMode = property(_getIgnoreFailureMode, _setIgnoreFailureMode, None, "Ignore failure mode for peer.")
730
731
732
733
734
735
736 - def stagePeer(self, targetDir, ownership=None, permissions=None):
737 """
738 Stages data from the peer into the indicated local target directory.
739
740 The target directory must already exist before this method is called. If
741 passed in, ownership and permissions will be applied to the files that
742 are copied.
743
744 @note: The returned count of copied files might be inaccurate if some of
745 the copied files already existed in the staging directory prior to the
746 copy taking place. We don't clear the staging directory first, because
747 some extension might also be using it.
748
749 @note: If you have user/group as strings, call the L{util.getUidGid} function
750 to get the associated uid/gid as an ownership tuple.
751
752 @note: Unlike the local peer version of this method, an I/O error might
753 or might not be raised if the directory is empty. Since we're using a
754 remote copy method, we just don't have the fine-grained control over our
755 exceptions that's available when we can look directly at the filesystem,
756 and we can't control whether the remote copy method thinks an empty
757 directory is an error.
758
759 @param targetDir: Target directory to write data into
760 @type targetDir: String representing a directory on disk
761
762 @param ownership: Owner and group that the staged files should have
763 @type ownership: Tuple of numeric ids C{(uid, gid)}
764
765 @param permissions: Permissions that the staged files should have
766 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
767
768 @return: Number of files copied from the source directory to the target directory.
769
770 @raise ValueError: If target directory is not a directory, does not exist or is not absolute.
771 @raise ValueError: If a path cannot be encoded properly.
772 @raise IOError: If there were no files to stage (i.e. the directory was empty)
773 @raise IOError: If there is an IO error copying a file.
774 @raise OSError: If there is an OS error copying or changing permissions on a file
775 """
776 targetDir = encodePath(targetDir)
777 if not os.path.isabs(targetDir):
778 logger.debug("Target directory [%s] not an absolute path." % targetDir)
779 raise ValueError("Target directory must be an absolute path.")
780 if not os.path.exists(targetDir) or not os.path.isdir(targetDir):
781 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir)
782 raise ValueError("Target directory is not a directory or does not exist on disk.")
783 count = RemotePeer._copyRemoteDir(self.remoteUser, self.localUser, self.name,
784 self._rcpCommand, self._rcpCommandList,
785 self.collectDir, targetDir,
786 ownership, permissions)
787 if count == 0:
788 raise IOError("Did not copy any files from local peer.")
789 return count
790
792 """
793 Checks the collect indicator in the peer's staging directory.
794
795 When a peer has completed collecting its backup files, it will write an
796 empty indicator file into its collect directory. This method checks to
797 see whether that indicator has been written. If the remote copy command
798 fails, we return C{False} as if the file weren't there.
799
800 If you need to, you can override the name of the collect indicator file
801 by passing in a different name.
802
803 @note: Apparently, we can't count on all rcp-compatible implementations
804 to return sensible errors for some error conditions. As an example, the
805 C{scp} command in Debian 'woody' returns a zero (normal) status even when
806 it can't find a host or if the login or path is invalid. Because of
807 this, the implementation of this method is rather convoluted.
808
809 @param collectIndicator: Name of the collect indicator file to check
810 @type collectIndicator: String representing name of a file in the collect directory
811
812 @return: Boolean true/false depending on whether the indicator exists.
813 @raise ValueError: If a path cannot be encoded properly.
814 """
815 try:
816 if collectIndicator is None:
817 sourceFile = os.path.join(self.collectDir, DEF_COLLECT_INDICATOR)
818 targetFile = os.path.join(self.workingDir, DEF_COLLECT_INDICATOR)
819 else:
820 collectIndicator = encodePath(collectIndicator)
821 sourceFile = os.path.join(self.collectDir, collectIndicator)
822 targetFile = os.path.join(self.workingDir, collectIndicator)
823 logger.debug("Fetch remote [%s] into [%s]." % (sourceFile, targetFile))
824 if os.path.exists(targetFile):
825 try:
826 os.remove(targetFile)
827 except:
828 raise Exception("Error: collect indicator [%s] already exists!" % targetFile)
829 try:
830 RemotePeer._copyRemoteFile(self.remoteUser, self.localUser, self.name,
831 self._rcpCommand, self._rcpCommandList,
832 sourceFile, targetFile,
833 overwrite=False)
834 if os.path.exists(targetFile):
835 return True
836 else:
837 return False
838 except Exception, e:
839 logger.info("Failed looking for collect indicator: %s" % e)
840 return False
841 finally:
842 if os.path.exists(targetFile):
843 try:
844 os.remove(targetFile)
845 except: pass
846
848 """
849 Writes the stage indicator in the peer's staging directory.
850
851 When the master has completed collecting its backup files, it will write
852 an empty indicator file into the peer's collect directory. The presence
853 of this file implies that the staging process is complete.
854
855 If you need to, you can override the name of the stage indicator file by
856 passing in a different name.
857
858 @note: If you have user/group as strings, call the L{util.getUidGid} function
859 to get the associated uid/gid as an ownership tuple.
860
861 @param stageIndicator: Name of the indicator file to write
862 @type stageIndicator: String representing name of a file in the collect directory
863
864 @raise ValueError: If a path cannot be encoded properly.
865 @raise IOError: If there is an IO error creating the file.
866 @raise OSError: If there is an OS error creating or changing permissions on the file
867 """
868 stageIndicator = encodePath(stageIndicator)
869 if stageIndicator is None:
870 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR)
871 targetFile = os.path.join(self.collectDir, DEF_STAGE_INDICATOR)
872 else:
873 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR)
874 targetFile = os.path.join(self.collectDir, stageIndicator)
875 try:
876 if not os.path.exists(sourceFile):
877 open(sourceFile, "w").write("")
878 RemotePeer._pushLocalFile(self.remoteUser, self.localUser, self.name,
879 self._rcpCommand, self._rcpCommandList,
880 sourceFile, targetFile)
881 finally:
882 if os.path.exists(sourceFile):
883 try:
884 os.remove(sourceFile)
885 except: pass
886
888 """
889 Executes a command on the peer via remote shell.
890
891 @param command: Command to execute
892 @type command: String command-line suitable for use with rsh.
893
894 @raise IOError: If there is an error executing the command on the remote peer.
895 """
896 RemotePeer._executeRemoteCommand(self.remoteUser, self.localUser,
897 self.name, self._rshCommand,
898 self._rshCommandList, command)
899
901 """
902 Executes a managed action on this peer.
903
904 @param action: Name of the action to execute.
905 @param fullBackup: Whether a full backup should be executed.
906
907 @raise IOError: If there is an error executing the action on the remote peer.
908 """
909 try:
910 command = RemotePeer._buildCbackCommand(self.cbackCommand, action, fullBackup)
911 self.executeRemoteCommand(command)
912 except IOError, e:
913 logger.info(e)
914 raise IOError("Failed to execute action [%s] on managed client [%s]." % (action, self.name))
915
916
917
918
919
920
921 - def _getDirContents(path):
922 """
923 Returns the contents of a directory in terms of a Set.
924
925 The directory's contents are read as a L{FilesystemList} containing only
926 files, and then the list is converted into a C{sets.Set} object for later
927 use.
928
929 @param path: Directory path to get contents for
930 @type path: String representing a path on disk
931
932 @return: Set of files in the directory
933 @raise ValueError: If path is not a directory or does not exist.
934 """
935 contents = FilesystemList()
936 contents.excludeDirs = True
937 contents.excludeLinks = True
938 contents.addDirContents(path)
939 return sets.Set(contents)
940 _getDirContents = staticmethod(_getDirContents)
941
942 - def _copyRemoteDir(remoteUser, localUser, remoteHost, rcpCommand, rcpCommandList,
943 sourceDir, targetDir, ownership=None, permissions=None):
944 """
945 Copies files from the source directory to the target directory.
946
947 This function is not recursive. Only the files in the directory will be
948 copied. Ownership and permissions will be left at their default values
949 if new values are not specified. Behavior when copying soft links from
950 the collect directory is dependent on the behavior of the specified rcp
951 command.
952
953 @note: The returned count of copied files might be inaccurate if some of
954 the copied files already existed in the staging directory prior to the
955 copy taking place. We don't clear the staging directory first, because
956 some extension might also be using it.
957
958 @note: If you have user/group as strings, call the L{util.getUidGid} function
959 to get the associated uid/gid as an ownership tuple.
960
961 @note: We don't have a good way of knowing exactly what files we copied
962 down from the remote peer, unless we want to parse the output of the rcp
963 command (ugh). We could change permissions on everything in the target
964 directory, but that's kind of ugly too. Instead, we use Python's set
965 functionality to figure out what files were added while we executed the
966 rcp command. This isn't perfect - for instance, it's not correct if
967 someone else is messing with the directory at the same time we're doing
968 the remote copy - but it's about as good as we're going to get.
969
970 @note: Apparently, we can't count on all rcp-compatible implementations
971 to return sensible errors for some error conditions. As an example, the
972 C{scp} command in Debian 'woody' returns a zero (normal) status even
973 when it can't find a host or if the login or path is invalid. We try
974 to work around this by issuing C{IOError} if we don't copy any files from
975 the remote host.
976
977 @param remoteUser: Name of the Cedar Backup user on the remote peer
978 @type remoteUser: String representing a username, valid via the copy command
979
980 @param localUser: Name of the Cedar Backup user on the current host
981 @type localUser: String representing a username, valid on the current host
982
983 @param remoteHost: Hostname of the remote peer
984 @type remoteHost: String representing a hostname, accessible via the copy command
985
986 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
987 @type rcpCommand: String representing a system command including required arguments
988
989 @param rcpCommandList: An rcp-compatible copy command to use for copying files
990 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
991
992 @param sourceDir: Source directory
993 @type sourceDir: String representing a directory on disk
994
995 @param targetDir: Target directory
996 @type targetDir: String representing a directory on disk
997
998 @param ownership: Owner and group that the copied files should have
999 @type ownership: Tuple of numeric ids C{(uid, gid)}
1000
1001 @param permissions: Permissions that the staged files should have
1002 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
1003
1004 @return: Number of files copied from the source directory to the target directory.
1005
1006 @raise ValueError: If source or target is not a directory or does not exist.
1007 @raise IOError: If there is an IO error copying the files.
1008 """
1009 beforeSet = RemotePeer._getDirContents(targetDir)
1010 if localUser is not None:
1011 try:
1012 if os.getuid() != 0:
1013 raise IOError("Only root can remote copy as another user.")
1014 except AttributeError: pass
1015 actualCommand = "%s %s@%s:%s/* %s" % (rcpCommand, remoteUser, remoteHost, sourceDir, targetDir)
1016 command = resolveCommand(SU_COMMAND)
1017 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1018 if result != 0:
1019 raise IOError("Error (%d) copying files from remote host as local user [%s]." % (result, localUser))
1020 else:
1021 copySource = "%s@%s:%s/*" % (remoteUser, remoteHost, sourceDir)
1022 command = resolveCommand(rcpCommandList)
1023 result = executeCommand(command, [copySource, targetDir])[0]
1024 if result != 0:
1025 raise IOError("Error (%d) copying files from remote host." % result)
1026 afterSet = RemotePeer._getDirContents(targetDir)
1027 if len(afterSet) == 0:
1028 raise IOError("Did not copy any files from remote peer.")
1029 differenceSet = afterSet.difference(beforeSet)
1030 if len(differenceSet) == 0:
1031 raise IOError("Apparently did not copy any new files from remote peer.")
1032 for targetFile in differenceSet:
1033 if ownership is not None:
1034 os.chown(targetFile, ownership[0], ownership[1])
1035 if permissions is not None:
1036 os.chmod(targetFile, permissions)
1037 return len(differenceSet)
1038 _copyRemoteDir = staticmethod(_copyRemoteDir)
1039
1040 - def _copyRemoteFile(remoteUser, localUser, remoteHost,
1041 rcpCommand, rcpCommandList,
1042 sourceFile, targetFile, ownership=None,
1043 permissions=None, overwrite=True):
1044 """
1045 Copies a remote source file to a target file.
1046
1047 @note: Internally, we have to go through and escape any spaces in the
1048 source path with double-backslash, otherwise things get screwed up. It
1049 doesn't seem to be required in the target path. I hope this is portable
1050 to various different rcp methods, but I guess it might not be (all I have
1051 to test with is OpenSSH).
1052
1053 @note: If you have user/group as strings, call the L{util.getUidGid} function
1054 to get the associated uid/gid as an ownership tuple.
1055
1056 @note: We will not overwrite a target file that exists when this method
1057 is invoked. If the target already exists, we'll raise an exception.
1058
1059 @note: Apparently, we can't count on all rcp-compatible implementations
1060 to return sensible errors for some error conditions. As an example, the
1061 C{scp} command in Debian 'woody' returns a zero (normal) status even when
1062 it can't find a host or if the login or path is invalid. We try to work
1063 around this by issuing C{IOError} the target file does not exist when
1064 we're done.
1065
1066 @param remoteUser: Name of the Cedar Backup user on the remote peer
1067 @type remoteUser: String representing a username, valid via the copy command
1068
1069 @param remoteHost: Hostname of the remote peer
1070 @type remoteHost: String representing a hostname, accessible via the copy command
1071
1072 @param localUser: Name of the Cedar Backup user on the current host
1073 @type localUser: String representing a username, valid on the current host
1074
1075 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
1076 @type rcpCommand: String representing a system command including required arguments
1077
1078 @param rcpCommandList: An rcp-compatible copy command to use for copying files
1079 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
1080
1081 @param sourceFile: Source file to copy
1082 @type sourceFile: String representing a file on disk, as an absolute path
1083
1084 @param targetFile: Target file to create
1085 @type targetFile: String representing a file on disk, as an absolute path
1086
1087 @param ownership: Owner and group that the copied should have
1088 @type ownership: Tuple of numeric ids C{(uid, gid)}
1089
1090 @param permissions: Permissions that the staged files should have
1091 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
1092
1093 @param overwrite: Indicates whether it's OK to overwrite the target file.
1094 @type overwrite: Boolean true/false.
1095
1096 @raise IOError: If the target file already exists.
1097 @raise IOError: If there is an IO error copying the file
1098 @raise OSError: If there is an OS error changing permissions on the file
1099 """
1100 if not overwrite:
1101 if os.path.exists(targetFile):
1102 raise IOError("Target file [%s] already exists." % targetFile)
1103 if localUser is not None:
1104 try:
1105 if os.getuid() != 0:
1106 raise IOError("Only root can remote copy as another user.")
1107 except AttributeError: pass
1108 actualCommand = "%s %s@%s:%s %s" % (rcpCommand, remoteUser, remoteHost, sourceFile.replace(" ", "\\ "), targetFile)
1109 command = resolveCommand(SU_COMMAND)
1110 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1111 if result != 0:
1112 raise IOError("Error (%d) copying [%s] from remote host as local user [%s]." % (result, sourceFile, localUser))
1113 else:
1114 copySource = "%s@%s:%s" % (remoteUser, remoteHost, sourceFile.replace(" ", "\\ "))
1115 command = resolveCommand(rcpCommandList)
1116 result = executeCommand(command, [copySource, targetFile])[0]
1117 if result != 0:
1118 raise IOError("Error (%d) copying [%s] from remote host." % (result, sourceFile))
1119 if not os.path.exists(targetFile):
1120 raise IOError("Apparently unable to copy file from remote host.")
1121 if ownership is not None:
1122 os.chown(targetFile, ownership[0], ownership[1])
1123 if permissions is not None:
1124 os.chmod(targetFile, permissions)
1125 _copyRemoteFile = staticmethod(_copyRemoteFile)
1126
1127 - def _pushLocalFile(remoteUser, localUser, remoteHost,
1128 rcpCommand, rcpCommandList,
1129 sourceFile, targetFile, overwrite=True):
1130 """
1131 Copies a local source file to a remote host.
1132
1133 @note: We will not overwrite a target file that exists when this method
1134 is invoked. If the target already exists, we'll raise an exception.
1135
1136 @note: Internally, we have to go through and escape any spaces in the
1137 source and target paths with double-backslash, otherwise things get
1138 screwed up. I hope this is portable to various different rcp methods,
1139 but I guess it might not be (all I have to test with is OpenSSH).
1140
1141 @note: If you have user/group as strings, call the L{util.getUidGid} function
1142 to get the associated uid/gid as an ownership tuple.
1143
1144 @param remoteUser: Name of the Cedar Backup user on the remote peer
1145 @type remoteUser: String representing a username, valid via the copy command
1146
1147 @param localUser: Name of the Cedar Backup user on the current host
1148 @type localUser: String representing a username, valid on the current host
1149
1150 @param remoteHost: Hostname of the remote peer
1151 @type remoteHost: String representing a hostname, accessible via the copy command
1152
1153 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
1154 @type rcpCommand: String representing a system command including required arguments
1155
1156 @param rcpCommandList: An rcp-compatible copy command to use for copying files
1157 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
1158
1159 @param sourceFile: Source file to copy
1160 @type sourceFile: String representing a file on disk, as an absolute path
1161
1162 @param targetFile: Target file to create
1163 @type targetFile: String representing a file on disk, as an absolute path
1164
1165 @param overwrite: Indicates whether it's OK to overwrite the target file.
1166 @type overwrite: Boolean true/false.
1167
1168 @raise IOError: If there is an IO error copying the file
1169 @raise OSError: If there is an OS error changing permissions on the file
1170 """
1171 if not overwrite:
1172 if os.path.exists(targetFile):
1173 raise IOError("Target file [%s] already exists." % targetFile)
1174 if localUser is not None:
1175 try:
1176 if os.getuid() != 0:
1177 raise IOError("Only root can remote copy as another user.")
1178 except AttributeError: pass
1179 actualCommand = '%s "%s" "%s@%s:%s"' % (rcpCommand, sourceFile, remoteUser, remoteHost, targetFile)
1180 command = resolveCommand(SU_COMMAND)
1181 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1182 if result != 0:
1183 raise IOError("Error (%d) copying [%s] to remote host as local user [%s]." % (result, sourceFile, localUser))
1184 else:
1185 copyTarget = "%s@%s:%s" % (remoteUser, remoteHost, targetFile.replace(" ", "\\ "))
1186 command = resolveCommand(rcpCommandList)
1187 result = executeCommand(command, [sourceFile.replace(" ", "\\ "), copyTarget])[0]
1188 if result != 0:
1189 raise IOError("Error (%d) copying [%s] to remote host." % (result, sourceFile))
1190 _pushLocalFile = staticmethod(_pushLocalFile)
1191
1192 - def _executeRemoteCommand(remoteUser, localUser, remoteHost, rshCommand, rshCommandList, remoteCommand):
1193 """
1194 Executes a command on the peer via remote shell.
1195
1196 @param remoteUser: Name of the Cedar Backup user on the remote peer
1197 @type remoteUser: String representing a username, valid on the remote host
1198
1199 @param localUser: Name of the Cedar Backup user on the current host
1200 @type localUser: String representing a username, valid on the current host
1201
1202 @param remoteHost: Hostname of the remote peer
1203 @type remoteHost: String representing a hostname, accessible via the copy command
1204
1205 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer
1206 @type rshCommand: String representing a system command including required arguments
1207
1208 @param rshCommandList: An rsh-compatible copy command to use for remote shells to the peer
1209 @type rshCommandList: Command as a list to be passed to L{util.executeCommand}
1210
1211 @param remoteCommand: The command to be executed on the remote host
1212 @type remoteCommand: String command-line, with no special shell characters ($, <, etc.)
1213
1214 @raise IOError: If there is an error executing the remote command
1215 """
1216 actualCommand = "%s %s@%s '%s'" % (rshCommand, remoteUser, remoteHost, remoteCommand)
1217 if localUser is not None:
1218 try:
1219 if os.getuid() != 0:
1220 raise IOError("Only root can remote shell as another user.")
1221 except AttributeError: pass
1222 command = resolveCommand(SU_COMMAND)
1223 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1224 if result != 0:
1225 raise IOError("Command failed [su -c %s \"%s\"]" % (localUser, actualCommand))
1226 else:
1227 command = resolveCommand(rshCommandList)
1228 result = executeCommand(command, ["%s@%s" % (remoteUser, remoteHost), "%s" % remoteCommand])[0]
1229 if result != 0:
1230 raise IOError("Command failed [%s]" % (actualCommand))
1231 _executeRemoteCommand = staticmethod(_executeRemoteCommand)
1232
1234 """
1235 Builds a Cedar Backup command line for the named action.
1236
1237 @note: If the cback command is None, then DEF_CBACK_COMMAND is used.
1238
1239 @param cbackCommand: cback command to execute, including required options
1240 @param action: Name of the action to execute.
1241 @param fullBackup: Whether a full backup should be executed.
1242
1243 @return: String suitable for passing to L{_executeRemoteCommand} as remoteCommand.
1244 @raise ValueError: If action is None.
1245 """
1246 if action is None:
1247 raise ValueError("Action cannot be None.")
1248 if cbackCommand is None:
1249 cbackCommand = DEF_CBACK_COMMAND
1250 if fullBackup:
1251 return "%s --full %s" % (cbackCommand, action)
1252 else:
1253 return "%s %s" % (cbackCommand, action)
1254 _buildCbackCommand = staticmethod(_buildCbackCommand)
1255