1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39 """
40 Provides backup peer-related objects and utility functions.
41
42 @sort: LocalPeer, RemotePeer
43
44 @var DEF_COLLECT_INDICATOR: Name of the default collect indicator file.
45 @var DEF_STAGE_INDICATOR: Name of the default stage indicator file.
46
47 @author: Kenneth J. Pronovici <pronovic@ieee.org>
48 """
49
50
51
52
53
54
55
56 import os
57 import logging
58 import shutil
59 import re
60
61
62 from CedarBackup2.filesystem import FilesystemList
63 from CedarBackup2.util import resolveCommand, executeCommand
64 from CedarBackup2.util import splitCommandLine, encodePath
65 from CedarBackup2.config import VALID_FAILURE_MODES
66
67
68
69
70
71
72 logger = logging.getLogger("CedarBackup2.log.peer")
73
74 DEF_RCP_COMMAND = [ "/usr/bin/scp", "-B", "-q", "-C" ]
75 DEF_RSH_COMMAND = [ "/usr/bin/ssh", ]
76 DEF_CBACK_COMMAND = "/usr/bin/cback"
77
78 DEF_COLLECT_INDICATOR = "cback.collect"
79 DEF_STAGE_INDICATOR = "cback.stage"
80
81 SU_COMMAND = [ "su" ]
82
83
84
85
86
87
89
90
91
92
93
94 """
95 Backup peer representing a local peer in a backup pool.
96
97 This is a class representing a local (non-network) peer in a backup pool.
98 Local peers are backed up by simple filesystem copy operations. A local
99 peer has associated with it a name (typically, but not necessarily, a
100 hostname) and a collect directory.
101
102 The public methods other than the constructor are part of a "backup peer"
103 interface shared with the C{RemotePeer} class.
104
105 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator,
106 _copyLocalDir, _copyLocalFile, name, collectDir
107 """
108
109
110
111
112
113 - def __init__(self, name, collectDir, ignoreFailureMode=None):
114 """
115 Initializes a local backup peer.
116
117 Note that the collect directory must be an absolute path, but does not
118 have to exist when the object is instantiated. We do a lazy validation
119 on this value since we could (potentially) be creating peer objects
120 before an ongoing backup completed.
121
122 @param name: Name of the backup peer
123 @type name: String, typically a hostname
124
125 @param collectDir: Path to the peer's collect directory
126 @type collectDir: String representing an absolute local path on disk
127
128 @param ignoreFailureMode: Ignore failure mode for this peer
129 @type ignoreFailureMode: One of VALID_FAILURE_MODES
130
131 @raise ValueError: If the name is empty.
132 @raise ValueError: If collect directory is not an absolute path.
133 """
134 self._name = None
135 self._collectDir = None
136 self._ignoreFailureMode = None
137 self.name = name
138 self.collectDir = collectDir
139 self.ignoreFailureMode = ignoreFailureMode
140
141
142
143
144
145
147 """
148 Property target used to set the peer name.
149 The value must be a non-empty string and cannot be C{None}.
150 @raise ValueError: If the value is an empty string or C{None}.
151 """
152 if value is None or len(value) < 1:
153 raise ValueError("Peer name must be a non-empty string.")
154 self._name = value
155
157 """
158 Property target used to get the peer name.
159 """
160 return self._name
161
163 """
164 Property target used to set the collect directory.
165 The value must be an absolute path and cannot be C{None}.
166 It does not have to exist on disk at the time of assignment.
167 @raise ValueError: If the value is C{None} or is not an absolute path.
168 @raise ValueError: If a path cannot be encoded properly.
169 """
170 if value is None or not os.path.isabs(value):
171 raise ValueError("Collect directory must be an absolute path.")
172 self._collectDir = encodePath(value)
173
175 """
176 Property target used to get the collect directory.
177 """
178 return self._collectDir
179
181 """
182 Property target used to set the ignoreFailure mode.
183 If not C{None}, the mode must be one of the values in L{VALID_FAILURE_MODES}.
184 @raise ValueError: If the value is not valid.
185 """
186 if value is not None:
187 if value not in VALID_FAILURE_MODES:
188 raise ValueError("Ignore failure mode must be one of %s." % VALID_FAILURE_MODES)
189 self._ignoreFailureMode = value
190
192 """
193 Property target used to get the ignoreFailure mode.
194 """
195 return self._ignoreFailureMode
196
197 name = property(_getName, _setName, None, "Name of the peer.")
198 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).")
199 ignoreFailureMode = property(_getIgnoreFailureMode, _setIgnoreFailureMode, None, "Ignore failure mode for peer.")
200
201
202
203
204
205
206 - def stagePeer(self, targetDir, ownership=None, permissions=None):
207 """
208 Stages data from the peer into the indicated local target directory.
209
210 The collect and target directories must both already exist before this
211 method is called. If passed in, ownership and permissions will be
212 applied to the files that are copied.
213
214 @note: The caller is responsible for checking that the indicator exists,
215 if they care. This function only stages the files within the directory.
216
217 @note: If you have user/group as strings, call the L{util.getUidGid} function
218 to get the associated uid/gid as an ownership tuple.
219
220 @param targetDir: Target directory to write data into
221 @type targetDir: String representing a directory on disk
222
223 @param ownership: Owner and group that the staged files should have
224 @type ownership: Tuple of numeric ids C{(uid, gid)}
225
226 @param permissions: Permissions that the staged files should have
227 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
228
229 @return: Number of files copied from the source directory to the target directory.
230
231 @raise ValueError: If collect directory is not a directory or does not exist
232 @raise ValueError: If target directory is not a directory, does not exist or is not absolute.
233 @raise ValueError: If a path cannot be encoded properly.
234 @raise IOError: If there were no files to stage (i.e. the directory was empty)
235 @raise IOError: If there is an IO error copying a file.
236 @raise OSError: If there is an OS error copying or changing permissions on a file
237 """
238 targetDir = encodePath(targetDir)
239 if not os.path.isabs(targetDir):
240 logger.debug("Target directory [%s] not an absolute path." % targetDir)
241 raise ValueError("Target directory must be an absolute path.")
242 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir):
243 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir)
244 raise ValueError("Collect directory is not a directory or does not exist on disk.")
245 if not os.path.exists(targetDir) or not os.path.isdir(targetDir):
246 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir)
247 raise ValueError("Target directory is not a directory or does not exist on disk.")
248 count = LocalPeer._copyLocalDir(self.collectDir, targetDir, ownership, permissions)
249 if count == 0:
250 raise IOError("Did not copy any files from local peer.")
251 return count
252
254 """
255 Checks the collect indicator in the peer's staging directory.
256
257 When a peer has completed collecting its backup files, it will write an
258 empty indicator file into its collect directory. This method checks to
259 see whether that indicator has been written. We're "stupid" here - if
260 the collect directory doesn't exist, you'll naturally get back C{False}.
261
262 If you need to, you can override the name of the collect indicator file
263 by passing in a different name.
264
265 @param collectIndicator: Name of the collect indicator file to check
266 @type collectIndicator: String representing name of a file in the collect directory
267
268 @return: Boolean true/false depending on whether the indicator exists.
269 @raise ValueError: If a path cannot be encoded properly.
270 """
271 collectIndicator = encodePath(collectIndicator)
272 if collectIndicator is None:
273 return os.path.exists(os.path.join(self.collectDir, DEF_COLLECT_INDICATOR))
274 else:
275 return os.path.exists(os.path.join(self.collectDir, collectIndicator))
276
278 """
279 Writes the stage indicator in the peer's staging directory.
280
281 When the master has completed collecting its backup files, it will write
282 an empty indicator file into the peer's collect directory. The presence
283 of this file implies that the staging process is complete.
284
285 If you need to, you can override the name of the stage indicator file by
286 passing in a different name.
287
288 @note: If you have user/group as strings, call the L{util.getUidGid}
289 function to get the associated uid/gid as an ownership tuple.
290
291 @param stageIndicator: Name of the indicator file to write
292 @type stageIndicator: String representing name of a file in the collect directory
293
294 @param ownership: Owner and group that the indicator file should have
295 @type ownership: Tuple of numeric ids C{(uid, gid)}
296
297 @param permissions: Permissions that the indicator file should have
298 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
299
300 @raise ValueError: If collect directory is not a directory or does not exist
301 @raise ValueError: If a path cannot be encoded properly.
302 @raise IOError: If there is an IO error creating the file.
303 @raise OSError: If there is an OS error creating or changing permissions on the file
304 """
305 stageIndicator = encodePath(stageIndicator)
306 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir):
307 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir)
308 raise ValueError("Collect directory is not a directory or does not exist on disk.")
309 if stageIndicator is None:
310 fileName = os.path.join(self.collectDir, DEF_STAGE_INDICATOR)
311 else:
312 fileName = os.path.join(self.collectDir, stageIndicator)
313 LocalPeer._copyLocalFile(None, fileName, ownership, permissions)
314
315
316
317
318
319
320 - def _copyLocalDir(sourceDir, targetDir, ownership=None, permissions=None):
321 """
322 Copies files from the source directory to the target directory.
323
324 This function is not recursive. Only the files in the directory will be
325 copied. Ownership and permissions will be left at their default values
326 if new values are not specified. The source and target directories are
327 allowed to be soft links to a directory, but besides that soft links are
328 ignored.
329
330 @note: If you have user/group as strings, call the L{util.getUidGid}
331 function to get the associated uid/gid as an ownership tuple.
332
333 @param sourceDir: Source directory
334 @type sourceDir: String representing a directory on disk
335
336 @param targetDir: Target directory
337 @type targetDir: String representing a directory on disk
338
339 @param ownership: Owner and group that the copied files should have
340 @type ownership: Tuple of numeric ids C{(uid, gid)}
341
342 @param permissions: Permissions that the staged files should have
343 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
344
345 @return: Number of files copied from the source directory to the target directory.
346
347 @raise ValueError: If source or target is not a directory or does not exist.
348 @raise ValueError: If a path cannot be encoded properly.
349 @raise IOError: If there is an IO error copying the files.
350 @raise OSError: If there is an OS error copying or changing permissions on a files
351 """
352 filesCopied = 0
353 sourceDir = encodePath(sourceDir)
354 targetDir = encodePath(targetDir)
355 for fileName in os.listdir(sourceDir):
356 sourceFile = os.path.join(sourceDir, fileName)
357 targetFile = os.path.join(targetDir, fileName)
358 LocalPeer._copyLocalFile(sourceFile, targetFile, ownership, permissions)
359 filesCopied += 1
360 return filesCopied
361 _copyLocalDir = staticmethod(_copyLocalDir)
362
363 - def _copyLocalFile(sourceFile=None, targetFile=None, ownership=None, permissions=None, overwrite=True):
364 """
365 Copies a source file to a target file.
366
367 If the source file is C{None} then the target file will be created or
368 overwritten as an empty file. If the target file is C{None}, this method
369 is a no-op. Attempting to copy a soft link or a directory will result in
370 an exception.
371
372 @note: If you have user/group as strings, call the L{util.getUidGid}
373 function to get the associated uid/gid as an ownership tuple.
374
375 @note: We will not overwrite a target file that exists when this method
376 is invoked. If the target already exists, we'll raise an exception.
377
378 @param sourceFile: Source file to copy
379 @type sourceFile: String representing a file on disk, as an absolute path
380
381 @param targetFile: Target file to create
382 @type targetFile: String representing a file on disk, as an absolute path
383
384 @param ownership: Owner and group that the copied should have
385 @type ownership: Tuple of numeric ids C{(uid, gid)}
386
387 @param permissions: Permissions that the staged files should have
388 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
389
390 @param overwrite: Indicates whether it's OK to overwrite the target file.
391 @type overwrite: Boolean true/false.
392
393 @raise ValueError: If the passed-in source file is not a regular file.
394 @raise ValueError: If a path cannot be encoded properly.
395 @raise IOError: If the target file already exists.
396 @raise IOError: If there is an IO error copying the file
397 @raise OSError: If there is an OS error copying or changing permissions on a file
398 """
399 targetFile = encodePath(targetFile)
400 sourceFile = encodePath(sourceFile)
401 if targetFile is None:
402 return
403 if not overwrite:
404 if os.path.exists(targetFile):
405 raise IOError("Target file [%s] already exists." % targetFile)
406 if sourceFile is None:
407 open(targetFile, "w").write("")
408 else:
409 if os.path.isfile(sourceFile) and not os.path.islink(sourceFile):
410 shutil.copy(sourceFile, targetFile)
411 else:
412 logger.debug("Source [%s] is not a regular file." % sourceFile)
413 raise ValueError("Source is not a regular file.")
414 if ownership is not None:
415 os.chown(targetFile, ownership[0], ownership[1])
416 if permissions is not None:
417 os.chmod(targetFile, permissions)
418 _copyLocalFile = staticmethod(_copyLocalFile)
419
420
421
422
423
424
426
427
428
429
430
431 """
432 Backup peer representing a remote peer in a backup pool.
433
434 This is a class representing a remote (networked) peer in a backup pool.
435 Remote peers are backed up using an rcp-compatible copy command. A remote
436 peer has associated with it a name (which must be a valid hostname), a
437 collect directory, a working directory and a copy method (an rcp-compatible
438 command).
439
440 You can also set an optional local user value. This username will be used
441 as the local user for any remote copies that are required. It can only be
442 used if the root user is executing the backup. The root user will C{su} to
443 the local user and execute the remote copies as that user.
444
445 The copy method is associated with the peer and not with the actual request
446 to copy, because we can envision that each remote host might have a
447 different connect method.
448
449 The public methods other than the constructor are part of a "backup peer"
450 interface shared with the C{LocalPeer} class.
451
452 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator,
453 executeRemoteCommand, executeManagedAction, _getDirContents,
454 _copyRemoteDir, _copyRemoteFile, _pushLocalFile, name, collectDir,
455 remoteUser, rcpCommand, rshCommand, cbackCommand
456 """
457
458
459
460
461
462 - def __init__(self, name=None, collectDir=None, workingDir=None, remoteUser=None,
463 rcpCommand=None, localUser=None, rshCommand=None, cbackCommand=None,
464 ignoreFailureMode=None):
465 """
466 Initializes a remote backup peer.
467
468 @note: If provided, each command will eventually be parsed into a list of
469 strings suitable for passing to C{util.executeCommand} in order to avoid
470 security holes related to shell interpolation. This parsing will be
471 done by the L{util.splitCommandLine} function. See the documentation for
472 that function for some important notes about its limitations.
473
474 @param name: Name of the backup peer
475 @type name: String, must be a valid DNS hostname
476
477 @param collectDir: Path to the peer's collect directory
478 @type collectDir: String representing an absolute path on the remote peer
479
480 @param workingDir: Working directory that can be used to create temporary files, etc.
481 @type workingDir: String representing an absolute path on the current host.
482
483 @param remoteUser: Name of the Cedar Backup user on the remote peer
484 @type remoteUser: String representing a username, valid via remote shell to the peer
485
486 @param localUser: Name of the Cedar Backup user on the current host
487 @type localUser: String representing a username, valid on the current host
488
489 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
490 @type rcpCommand: String representing a system command including required arguments
491
492 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer
493 @type rshCommand: String representing a system command including required arguments
494
495 @param cbackCommand: A chack-compatible command to use for executing managed actions
496 @type cbackCommand: String representing a system command including required arguments
497
498 @param ignoreFailureMode: Ignore failure mode for this peer
499 @type ignoreFailureMode: One of VALID_FAILURE_MODES
500
501 @raise ValueError: If collect directory is not an absolute path
502 """
503 self._name = None
504 self._collectDir = None
505 self._workingDir = None
506 self._remoteUser = None
507 self._localUser = None
508 self._rcpCommand = None
509 self._rcpCommandList = None
510 self._rshCommand = None
511 self._rshCommandList = None
512 self._cbackCommand = None
513 self._ignoreFailureMode = None
514 self.name = name
515 self.collectDir = collectDir
516 self.workingDir = workingDir
517 self.remoteUser = remoteUser
518 self.localUser = localUser
519 self.rcpCommand = rcpCommand
520 self.rshCommand = rshCommand
521 self.cbackCommand = cbackCommand
522 self.ignoreFailureMode = ignoreFailureMode
523
524
525
526
527
528
530 """
531 Property target used to set the peer name.
532 The value must be a non-empty string and cannot be C{None}.
533 @raise ValueError: If the value is an empty string or C{None}.
534 """
535 if value is None or len(value) < 1:
536 raise ValueError("Peer name must be a non-empty string.")
537 self._name = value
538
540 """
541 Property target used to get the peer name.
542 """
543 return self._name
544
546 """
547 Property target used to set the collect directory.
548 The value must be an absolute path and cannot be C{None}.
549 It does not have to exist on disk at the time of assignment.
550 @raise ValueError: If the value is C{None} or is not an absolute path.
551 @raise ValueError: If the value cannot be encoded properly.
552 """
553 if value is not None:
554 if not os.path.isabs(value):
555 raise ValueError("Collect directory must be an absolute path.")
556 self._collectDir = encodePath(value)
557
559 """
560 Property target used to get the collect directory.
561 """
562 return self._collectDir
563
565 """
566 Property target used to set the working directory.
567 The value must be an absolute path and cannot be C{None}.
568 @raise ValueError: If the value is C{None} or is not an absolute path.
569 @raise ValueError: If the value cannot be encoded properly.
570 """
571 if value is not None:
572 if not os.path.isabs(value):
573 raise ValueError("Working directory must be an absolute path.")
574 self._workingDir = encodePath(value)
575
577 """
578 Property target used to get the working directory.
579 """
580 return self._workingDir
581
583 """
584 Property target used to set the remote user.
585 The value must be a non-empty string and cannot be C{None}.
586 @raise ValueError: If the value is an empty string or C{None}.
587 """
588 if value is None or len(value) < 1:
589 raise ValueError("Peer remote user must be a non-empty string.")
590 self._remoteUser = value
591
593 """
594 Property target used to get the remote user.
595 """
596 return self._remoteUser
597
599 """
600 Property target used to set the local user.
601 The value must be a non-empty string if it is not C{None}.
602 @raise ValueError: If the value is an empty string.
603 """
604 if value is not None:
605 if len(value) < 1:
606 raise ValueError("Peer local user must be a non-empty string.")
607 self._localUser = value
608
610 """
611 Property target used to get the local user.
612 """
613 return self._localUser
614
616 """
617 Property target to set the rcp command.
618
619 The value must be a non-empty string or C{None}. Its value is stored in
620 the two forms: "raw" as provided by the client, and "parsed" into a list
621 suitable for being passed to L{util.executeCommand} via
622 L{util.splitCommandLine}.
623
624 However, all the caller will ever see via the property is the actual
625 value they set (which includes seeing C{None}, even if we translate that
626 internally to C{DEF_RCP_COMMAND}). Internally, we should always use
627 C{self._rcpCommandList} if we want the actual command list.
628
629 @raise ValueError: If the value is an empty string.
630 """
631 if value is None:
632 self._rcpCommand = None
633 self._rcpCommandList = DEF_RCP_COMMAND
634 else:
635 if len(value) >= 1:
636 self._rcpCommand = value
637 self._rcpCommandList = splitCommandLine(self._rcpCommand)
638 else:
639 raise ValueError("The rcp command must be a non-empty string.")
640
642 """
643 Property target used to get the rcp command.
644 """
645 return self._rcpCommand
646
648 """
649 Property target to set the rsh command.
650
651 The value must be a non-empty string or C{None}. Its value is stored in
652 the two forms: "raw" as provided by the client, and "parsed" into a list
653 suitable for being passed to L{util.executeCommand} via
654 L{util.splitCommandLine}.
655
656 However, all the caller will ever see via the property is the actual
657 value they set (which includes seeing C{None}, even if we translate that
658 internally to C{DEF_RSH_COMMAND}). Internally, we should always use
659 C{self._rshCommandList} if we want the actual command list.
660
661 @raise ValueError: If the value is an empty string.
662 """
663 if value is None:
664 self._rshCommand = None
665 self._rshCommandList = DEF_RSH_COMMAND
666 else:
667 if len(value) >= 1:
668 self._rshCommand = value
669 self._rshCommandList = splitCommandLine(self._rshCommand)
670 else:
671 raise ValueError("The rsh command must be a non-empty string.")
672
674 """
675 Property target used to get the rsh command.
676 """
677 return self._rshCommand
678
680 """
681 Property target to set the cback command.
682
683 The value must be a non-empty string or C{None}. Unlike the other
684 command, this value is only stored in the "raw" form provided by the
685 client.
686
687 @raise ValueError: If the value is an empty string.
688 """
689 if value is None:
690 self._cbackCommand = None
691 else:
692 if len(value) >= 1:
693 self._cbackCommand = value
694 else:
695 raise ValueError("The cback command must be a non-empty string.")
696
698 """
699 Property target used to get the cback command.
700 """
701 return self._cbackCommand
702
704 """
705 Property target used to set the ignoreFailure mode.
706 If not C{None}, the mode must be one of the values in L{VALID_FAILURE_MODES}.
707 @raise ValueError: If the value is not valid.
708 """
709 if value is not None:
710 if value not in VALID_FAILURE_MODES:
711 raise ValueError("Ignore failure mode must be one of %s." % VALID_FAILURE_MODES)
712 self._ignoreFailureMode = value
713
715 """
716 Property target used to get the ignoreFailure mode.
717 """
718 return self._ignoreFailureMode
719
720 name = property(_getName, _setName, None, "Name of the peer (a valid DNS hostname).")
721 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).")
722 workingDir = property(_getWorkingDir, _setWorkingDir, None, "Path to the peer's working directory (an absolute local path).")
723 remoteUser = property(_getRemoteUser, _setRemoteUser, None, "Name of the Cedar Backup user on the remote peer.")
724 localUser = property(_getLocalUser, _setLocalUser, None, "Name of the Cedar Backup user on the current host.")
725 rcpCommand = property(_getRcpCommand, _setRcpCommand, None, "An rcp-compatible copy command to use for copying files.")
726 rshCommand = property(_getRshCommand, _setRshCommand, None, "An rsh-compatible command to use for remote shells to the peer.")
727 cbackCommand = property(_getCbackCommand, _setCbackCommand, None, "A chack-compatible command to use for executing managed actions.")
728 ignoreFailureMode = property(_getIgnoreFailureMode, _setIgnoreFailureMode, None, "Ignore failure mode for peer.")
729
730
731
732
733
734
735 - def stagePeer(self, targetDir, ownership=None, permissions=None):
736 """
737 Stages data from the peer into the indicated local target directory.
738
739 The target directory must already exist before this method is called. If
740 passed in, ownership and permissions will be applied to the files that
741 are copied.
742
743 @note: The returned count of copied files might be inaccurate if some of
744 the copied files already existed in the staging directory prior to the
745 copy taking place. We don't clear the staging directory first, because
746 some extension might also be using it.
747
748 @note: If you have user/group as strings, call the L{util.getUidGid} function
749 to get the associated uid/gid as an ownership tuple.
750
751 @note: Unlike the local peer version of this method, an I/O error might
752 or might not be raised if the directory is empty. Since we're using a
753 remote copy method, we just don't have the fine-grained control over our
754 exceptions that's available when we can look directly at the filesystem,
755 and we can't control whether the remote copy method thinks an empty
756 directory is an error.
757
758 @param targetDir: Target directory to write data into
759 @type targetDir: String representing a directory on disk
760
761 @param ownership: Owner and group that the staged files should have
762 @type ownership: Tuple of numeric ids C{(uid, gid)}
763
764 @param permissions: Permissions that the staged files should have
765 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
766
767 @return: Number of files copied from the source directory to the target directory.
768
769 @raise ValueError: If target directory is not a directory, does not exist or is not absolute.
770 @raise ValueError: If a path cannot be encoded properly.
771 @raise IOError: If there were no files to stage (i.e. the directory was empty)
772 @raise IOError: If there is an IO error copying a file.
773 @raise OSError: If there is an OS error copying or changing permissions on a file
774 """
775 targetDir = encodePath(targetDir)
776 if not os.path.isabs(targetDir):
777 logger.debug("Target directory [%s] not an absolute path." % targetDir)
778 raise ValueError("Target directory must be an absolute path.")
779 if not os.path.exists(targetDir) or not os.path.isdir(targetDir):
780 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir)
781 raise ValueError("Target directory is not a directory or does not exist on disk.")
782 count = RemotePeer._copyRemoteDir(self.remoteUser, self.localUser, self.name,
783 self._rcpCommand, self._rcpCommandList,
784 self.collectDir, targetDir,
785 ownership, permissions)
786 if count == 0:
787 raise IOError("Did not copy any files from local peer.")
788 return count
789
791 """
792 Checks the collect indicator in the peer's staging directory.
793
794 When a peer has completed collecting its backup files, it will write an
795 empty indicator file into its collect directory. This method checks to
796 see whether that indicator has been written. If the remote copy command
797 fails, we return C{False} as if the file weren't there.
798
799 If you need to, you can override the name of the collect indicator file
800 by passing in a different name.
801
802 @note: Apparently, we can't count on all rcp-compatible implementations
803 to return sensible errors for some error conditions. As an example, the
804 C{scp} command in Debian 'woody' returns a zero (normal) status even when
805 it can't find a host or if the login or path is invalid. Because of
806 this, the implementation of this method is rather convoluted.
807
808 @param collectIndicator: Name of the collect indicator file to check
809 @type collectIndicator: String representing name of a file in the collect directory
810
811 @return: Boolean true/false depending on whether the indicator exists.
812 @raise ValueError: If a path cannot be encoded properly.
813 """
814 try:
815 if collectIndicator is None:
816 sourceFile = os.path.join(self.collectDir, DEF_COLLECT_INDICATOR)
817 targetFile = os.path.join(self.workingDir, DEF_COLLECT_INDICATOR)
818 else:
819 collectIndicator = encodePath(collectIndicator)
820 sourceFile = os.path.join(self.collectDir, collectIndicator)
821 targetFile = os.path.join(self.workingDir, collectIndicator)
822 logger.debug("Fetch remote [%s] into [%s]." % (sourceFile, targetFile))
823 if os.path.exists(targetFile):
824 try:
825 os.remove(targetFile)
826 except:
827 raise Exception("Error: collect indicator [%s] already exists!" % targetFile)
828 try:
829 RemotePeer._copyRemoteFile(self.remoteUser, self.localUser, self.name,
830 self._rcpCommand, self._rcpCommandList,
831 sourceFile, targetFile,
832 overwrite=False)
833 if os.path.exists(targetFile):
834 return True
835 else:
836 return False
837 except Exception, e:
838 logger.info("Failed looking for collect indicator: %s" % e)
839 return False
840 finally:
841 if os.path.exists(targetFile):
842 try:
843 os.remove(targetFile)
844 except: pass
845
847 """
848 Writes the stage indicator in the peer's staging directory.
849
850 When the master has completed collecting its backup files, it will write
851 an empty indicator file into the peer's collect directory. The presence
852 of this file implies that the staging process is complete.
853
854 If you need to, you can override the name of the stage indicator file by
855 passing in a different name.
856
857 @note: If you have user/group as strings, call the L{util.getUidGid} function
858 to get the associated uid/gid as an ownership tuple.
859
860 @param stageIndicator: Name of the indicator file to write
861 @type stageIndicator: String representing name of a file in the collect directory
862
863 @raise ValueError: If a path cannot be encoded properly.
864 @raise IOError: If there is an IO error creating the file.
865 @raise OSError: If there is an OS error creating or changing permissions on the file
866 """
867 stageIndicator = encodePath(stageIndicator)
868 if stageIndicator is None:
869 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR)
870 targetFile = os.path.join(self.collectDir, DEF_STAGE_INDICATOR)
871 else:
872 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR)
873 targetFile = os.path.join(self.collectDir, stageIndicator)
874 try:
875 if not os.path.exists(sourceFile):
876 open(sourceFile, "w").write("")
877 RemotePeer._pushLocalFile(self.remoteUser, self.localUser, self.name,
878 self._rcpCommand, self._rcpCommandList,
879 sourceFile, targetFile)
880 finally:
881 if os.path.exists(sourceFile):
882 try:
883 os.remove(sourceFile)
884 except: pass
885
887 """
888 Executes a command on the peer via remote shell.
889
890 @param command: Command to execute
891 @type command: String command-line suitable for use with rsh.
892
893 @raise IOError: If there is an error executing the command on the remote peer.
894 """
895 RemotePeer._executeRemoteCommand(self.remoteUser, self.localUser,
896 self.name, self._rshCommand,
897 self._rshCommandList, command)
898
900 """
901 Executes a managed action on this peer.
902
903 @param action: Name of the action to execute.
904 @param fullBackup: Whether a full backup should be executed.
905
906 @raise IOError: If there is an error executing the action on the remote peer.
907 """
908 try:
909 command = RemotePeer._buildCbackCommand(self.cbackCommand, action, fullBackup)
910 self.executeRemoteCommand(command)
911 except IOError, e:
912 logger.info(e)
913 raise IOError("Failed to execute action [%s] on managed client [%s]." % (action, self.name))
914
915
916
917
918
919
920 - def _getDirContents(path):
921 """
922 Returns the contents of a directory in terms of a Set.
923
924 The directory's contents are read as a L{FilesystemList} containing only
925 files, and then the list is converted into a set object for later use.
926
927 @param path: Directory path to get contents for
928 @type path: String representing a path on disk
929
930 @return: Set of files in the directory
931 @raise ValueError: If path is not a directory or does not exist.
932 """
933 contents = FilesystemList()
934 contents.excludeDirs = True
935 contents.excludeLinks = True
936 contents.addDirContents(path)
937 try:
938 return set(contents)
939 except:
940 import sets
941 return sets.Set(contents)
942 _getDirContents = staticmethod(_getDirContents)
943
944 - def _copyRemoteDir(remoteUser, localUser, remoteHost, rcpCommand, rcpCommandList,
945 sourceDir, targetDir, ownership=None, permissions=None):
946 """
947 Copies files from the source directory to the target directory.
948
949 This function is not recursive. Only the files in the directory will be
950 copied. Ownership and permissions will be left at their default values
951 if new values are not specified. Behavior when copying soft links from
952 the collect directory is dependent on the behavior of the specified rcp
953 command.
954
955 @note: The returned count of copied files might be inaccurate if some of
956 the copied files already existed in the staging directory prior to the
957 copy taking place. We don't clear the staging directory first, because
958 some extension might also be using it.
959
960 @note: If you have user/group as strings, call the L{util.getUidGid} function
961 to get the associated uid/gid as an ownership tuple.
962
963 @note: We don't have a good way of knowing exactly what files we copied
964 down from the remote peer, unless we want to parse the output of the rcp
965 command (ugh). We could change permissions on everything in the target
966 directory, but that's kind of ugly too. Instead, we use Python's set
967 functionality to figure out what files were added while we executed the
968 rcp command. This isn't perfect - for instance, it's not correct if
969 someone else is messing with the directory at the same time we're doing
970 the remote copy - but it's about as good as we're going to get.
971
972 @note: Apparently, we can't count on all rcp-compatible implementations
973 to return sensible errors for some error conditions. As an example, the
974 C{scp} command in Debian 'woody' returns a zero (normal) status even
975 when it can't find a host or if the login or path is invalid. We try
976 to work around this by issuing C{IOError} if we don't copy any files from
977 the remote host.
978
979 @param remoteUser: Name of the Cedar Backup user on the remote peer
980 @type remoteUser: String representing a username, valid via the copy command
981
982 @param localUser: Name of the Cedar Backup user on the current host
983 @type localUser: String representing a username, valid on the current host
984
985 @param remoteHost: Hostname of the remote peer
986 @type remoteHost: String representing a hostname, accessible via the copy command
987
988 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
989 @type rcpCommand: String representing a system command including required arguments
990
991 @param rcpCommandList: An rcp-compatible copy command to use for copying files
992 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
993
994 @param sourceDir: Source directory
995 @type sourceDir: String representing a directory on disk
996
997 @param targetDir: Target directory
998 @type targetDir: String representing a directory on disk
999
1000 @param ownership: Owner and group that the copied files should have
1001 @type ownership: Tuple of numeric ids C{(uid, gid)}
1002
1003 @param permissions: Permissions that the staged files should have
1004 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
1005
1006 @return: Number of files copied from the source directory to the target directory.
1007
1008 @raise ValueError: If source or target is not a directory or does not exist.
1009 @raise IOError: If there is an IO error copying the files.
1010 """
1011 beforeSet = RemotePeer._getDirContents(targetDir)
1012 if localUser is not None:
1013 try:
1014 if os.getuid() != 0:
1015 raise IOError("Only root can remote copy as another user.")
1016 except AttributeError: pass
1017 actualCommand = "%s %s@%s:%s/* %s" % (rcpCommand, remoteUser, remoteHost, sourceDir, targetDir)
1018 command = resolveCommand(SU_COMMAND)
1019 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1020 if result != 0:
1021 raise IOError("Error (%d) copying files from remote host as local user [%s]." % (result, localUser))
1022 else:
1023 copySource = "%s@%s:%s/*" % (remoteUser, remoteHost, sourceDir)
1024 command = resolveCommand(rcpCommandList)
1025 result = executeCommand(command, [copySource, targetDir])[0]
1026 if result != 0:
1027 raise IOError("Error (%d) copying files from remote host." % result)
1028 afterSet = RemotePeer._getDirContents(targetDir)
1029 if len(afterSet) == 0:
1030 raise IOError("Did not copy any files from remote peer.")
1031 differenceSet = afterSet.difference(beforeSet)
1032 if len(differenceSet) == 0:
1033 raise IOError("Apparently did not copy any new files from remote peer.")
1034 for targetFile in differenceSet:
1035 if ownership is not None:
1036 os.chown(targetFile, ownership[0], ownership[1])
1037 if permissions is not None:
1038 os.chmod(targetFile, permissions)
1039 return len(differenceSet)
1040 _copyRemoteDir = staticmethod(_copyRemoteDir)
1041
1042 - def _copyRemoteFile(remoteUser, localUser, remoteHost,
1043 rcpCommand, rcpCommandList,
1044 sourceFile, targetFile, ownership=None,
1045 permissions=None, overwrite=True):
1046 """
1047 Copies a remote source file to a target file.
1048
1049 @note: Internally, we have to go through and escape any spaces in the
1050 source path with double-backslash, otherwise things get screwed up. It
1051 doesn't seem to be required in the target path. I hope this is portable
1052 to various different rcp methods, but I guess it might not be (all I have
1053 to test with is OpenSSH).
1054
1055 @note: If you have user/group as strings, call the L{util.getUidGid} function
1056 to get the associated uid/gid as an ownership tuple.
1057
1058 @note: We will not overwrite a target file that exists when this method
1059 is invoked. If the target already exists, we'll raise an exception.
1060
1061 @note: Apparently, we can't count on all rcp-compatible implementations
1062 to return sensible errors for some error conditions. As an example, the
1063 C{scp} command in Debian 'woody' returns a zero (normal) status even when
1064 it can't find a host or if the login or path is invalid. We try to work
1065 around this by issuing C{IOError} the target file does not exist when
1066 we're done.
1067
1068 @param remoteUser: Name of the Cedar Backup user on the remote peer
1069 @type remoteUser: String representing a username, valid via the copy command
1070
1071 @param remoteHost: Hostname of the remote peer
1072 @type remoteHost: String representing a hostname, accessible via the copy command
1073
1074 @param localUser: Name of the Cedar Backup user on the current host
1075 @type localUser: String representing a username, valid on the current host
1076
1077 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
1078 @type rcpCommand: String representing a system command including required arguments
1079
1080 @param rcpCommandList: An rcp-compatible copy command to use for copying files
1081 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
1082
1083 @param sourceFile: Source file to copy
1084 @type sourceFile: String representing a file on disk, as an absolute path
1085
1086 @param targetFile: Target file to create
1087 @type targetFile: String representing a file on disk, as an absolute path
1088
1089 @param ownership: Owner and group that the copied should have
1090 @type ownership: Tuple of numeric ids C{(uid, gid)}
1091
1092 @param permissions: Permissions that the staged files should have
1093 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
1094
1095 @param overwrite: Indicates whether it's OK to overwrite the target file.
1096 @type overwrite: Boolean true/false.
1097
1098 @raise IOError: If the target file already exists.
1099 @raise IOError: If there is an IO error copying the file
1100 @raise OSError: If there is an OS error changing permissions on the file
1101 """
1102 if not overwrite:
1103 if os.path.exists(targetFile):
1104 raise IOError("Target file [%s] already exists." % targetFile)
1105 if localUser is not None:
1106 try:
1107 if os.getuid() != 0:
1108 raise IOError("Only root can remote copy as another user.")
1109 except AttributeError: pass
1110 actualCommand = "%s %s@%s:%s %s" % (rcpCommand, remoteUser, remoteHost, sourceFile.replace(" ", "\\ "), targetFile)
1111 command = resolveCommand(SU_COMMAND)
1112 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1113 if result != 0:
1114 raise IOError("Error (%d) copying [%s] from remote host as local user [%s]." % (result, sourceFile, localUser))
1115 else:
1116 copySource = "%s@%s:%s" % (remoteUser, remoteHost, sourceFile.replace(" ", "\\ "))
1117 command = resolveCommand(rcpCommandList)
1118 result = executeCommand(command, [copySource, targetFile])[0]
1119 if result != 0:
1120 raise IOError("Error (%d) copying [%s] from remote host." % (result, sourceFile))
1121 if not os.path.exists(targetFile):
1122 raise IOError("Apparently unable to copy file from remote host.")
1123 if ownership is not None:
1124 os.chown(targetFile, ownership[0], ownership[1])
1125 if permissions is not None:
1126 os.chmod(targetFile, permissions)
1127 _copyRemoteFile = staticmethod(_copyRemoteFile)
1128
1129 - def _pushLocalFile(remoteUser, localUser, remoteHost,
1130 rcpCommand, rcpCommandList,
1131 sourceFile, targetFile, overwrite=True):
1132 """
1133 Copies a local source file to a remote host.
1134
1135 @note: We will not overwrite a target file that exists when this method
1136 is invoked. If the target already exists, we'll raise an exception.
1137
1138 @note: Internally, we have to go through and escape any spaces in the
1139 source and target paths with double-backslash, otherwise things get
1140 screwed up. I hope this is portable to various different rcp methods,
1141 but I guess it might not be (all I have to test with is OpenSSH).
1142
1143 @note: If you have user/group as strings, call the L{util.getUidGid} function
1144 to get the associated uid/gid as an ownership tuple.
1145
1146 @param remoteUser: Name of the Cedar Backup user on the remote peer
1147 @type remoteUser: String representing a username, valid via the copy command
1148
1149 @param localUser: Name of the Cedar Backup user on the current host
1150 @type localUser: String representing a username, valid on the current host
1151
1152 @param remoteHost: Hostname of the remote peer
1153 @type remoteHost: String representing a hostname, accessible via the copy command
1154
1155 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
1156 @type rcpCommand: String representing a system command including required arguments
1157
1158 @param rcpCommandList: An rcp-compatible copy command to use for copying files
1159 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
1160
1161 @param sourceFile: Source file to copy
1162 @type sourceFile: String representing a file on disk, as an absolute path
1163
1164 @param targetFile: Target file to create
1165 @type targetFile: String representing a file on disk, as an absolute path
1166
1167 @param overwrite: Indicates whether it's OK to overwrite the target file.
1168 @type overwrite: Boolean true/false.
1169
1170 @raise IOError: If there is an IO error copying the file
1171 @raise OSError: If there is an OS error changing permissions on the file
1172 """
1173 if not overwrite:
1174 if os.path.exists(targetFile):
1175 raise IOError("Target file [%s] already exists." % targetFile)
1176 if localUser is not None:
1177 try:
1178 if os.getuid() != 0:
1179 raise IOError("Only root can remote copy as another user.")
1180 except AttributeError: pass
1181 actualCommand = '%s "%s" "%s@%s:%s"' % (rcpCommand, sourceFile, remoteUser, remoteHost, targetFile)
1182 command = resolveCommand(SU_COMMAND)
1183 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1184 if result != 0:
1185 raise IOError("Error (%d) copying [%s] to remote host as local user [%s]." % (result, sourceFile, localUser))
1186 else:
1187 copyTarget = "%s@%s:%s" % (remoteUser, remoteHost, targetFile.replace(" ", "\\ "))
1188 command = resolveCommand(rcpCommandList)
1189 result = executeCommand(command, [sourceFile.replace(" ", "\\ "), copyTarget])[0]
1190 if result != 0:
1191 raise IOError("Error (%d) copying [%s] to remote host." % (result, sourceFile))
1192 _pushLocalFile = staticmethod(_pushLocalFile)
1193
1194 - def _executeRemoteCommand(remoteUser, localUser, remoteHost, rshCommand, rshCommandList, remoteCommand):
1195 """
1196 Executes a command on the peer via remote shell.
1197
1198 @param remoteUser: Name of the Cedar Backup user on the remote peer
1199 @type remoteUser: String representing a username, valid on the remote host
1200
1201 @param localUser: Name of the Cedar Backup user on the current host
1202 @type localUser: String representing a username, valid on the current host
1203
1204 @param remoteHost: Hostname of the remote peer
1205 @type remoteHost: String representing a hostname, accessible via the copy command
1206
1207 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer
1208 @type rshCommand: String representing a system command including required arguments
1209
1210 @param rshCommandList: An rsh-compatible copy command to use for remote shells to the peer
1211 @type rshCommandList: Command as a list to be passed to L{util.executeCommand}
1212
1213 @param remoteCommand: The command to be executed on the remote host
1214 @type remoteCommand: String command-line, with no special shell characters ($, <, etc.)
1215
1216 @raise IOError: If there is an error executing the remote command
1217 """
1218 actualCommand = "%s %s@%s '%s'" % (rshCommand, remoteUser, remoteHost, remoteCommand)
1219 if localUser is not None:
1220 try:
1221 if os.getuid() != 0:
1222 raise IOError("Only root can remote shell as another user.")
1223 except AttributeError: pass
1224 command = resolveCommand(SU_COMMAND)
1225 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1226 if result != 0:
1227 raise IOError("Command failed [su -c %s \"%s\"]" % (localUser, actualCommand))
1228 else:
1229 command = resolveCommand(rshCommandList)
1230 result = executeCommand(command, ["%s@%s" % (remoteUser, remoteHost), "%s" % remoteCommand])[0]
1231 if result != 0:
1232 raise IOError("Command failed [%s]" % (actualCommand))
1233 _executeRemoteCommand = staticmethod(_executeRemoteCommand)
1234
1236 """
1237 Builds a Cedar Backup command line for the named action.
1238
1239 @note: If the cback command is None, then DEF_CBACK_COMMAND is used.
1240
1241 @param cbackCommand: cback command to execute, including required options
1242 @param action: Name of the action to execute.
1243 @param fullBackup: Whether a full backup should be executed.
1244
1245 @return: String suitable for passing to L{_executeRemoteCommand} as remoteCommand.
1246 @raise ValueError: If action is None.
1247 """
1248 if action is None:
1249 raise ValueError("Action cannot be None.")
1250 if cbackCommand is None:
1251 cbackCommand = DEF_CBACK_COMMAND
1252 if fullBackup:
1253 return "%s --full %s" % (cbackCommand, action)
1254 else:
1255 return "%s %s" % (cbackCommand, action)
1256 _buildCbackCommand = staticmethod(_buildCbackCommand)
1257