1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39 """
40 Provides backup peer-related objects and utility functions.
41
42 @sort: LocalPeer, Remote Peer
43
44 @var DEF_COLLECT_INDICATOR: Name of the default collect indicator file.
45 @var DEF_STAGE_INDICATOR: Name of the default stage indicator file.
46
47 @author: Kenneth J. Pronovici <pronovic@ieee.org>
48 """
49
50
51
52
53
54
55
56 import os
57 import logging
58 import shutil
59 import sets
60 import re
61
62
63 from CedarBackup2.filesystem import FilesystemList
64 from CedarBackup2.util import resolveCommand, executeCommand
65 from CedarBackup2.util import splitCommandLine, encodePath
66
67
68
69
70
71
72 logger = logging.getLogger("CedarBackup2.log.peer")
73
74 DEF_RCP_COMMAND = [ "/usr/bin/scp", "-B", "-q", "-C" ]
75 DEF_COLLECT_INDICATOR = "cback.collect"
76 DEF_STAGE_INDICATOR = "cback.stage"
77
78 SU_COMMAND = [ "su" ]
79
80
81
82
83
84
86
87
88
89
90
91 """
92 Backup peer representing a local peer in a backup pool.
93
94 This is a class representing a local (non-network) peer in a backup pool.
95 Local peers are backed up by simple filesystem copy operations. A local
96 peer has associated with it a name (typically, but not necessarily, a
97 hostname) and a collect directory.
98
99 The public methods other than the constructor are part of a "backup peer"
100 interface shared with the C{RemotePeer} class.
101
102 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator,
103 _copyLocalDir, _copyLocalFile, name, collectDir
104 """
105
106
107
108
109
111 """
112 Initializes a local backup peer.
113
114 Note that the collect directory must be an absolute path, but does not
115 have to exist when the object is instantiated. We do a lazy validation
116 on this value since we could (potentially) be creating peer objects
117 before an ongoing backup completed.
118
119 @param name: Name of the backup peer
120 @type name: String, typically a hostname
121
122 @param collectDir: Path to the peer's collect directory
123 @type collectDir: String representing an absolute local path on disk
124
125 @raise ValueError: If the name is empty.
126 @raise ValueError: If collect directory is not an absolute path.
127 """
128 self._name = None
129 self._collectDir = None
130 self.name = name
131 self.collectDir = collectDir
132
133
134
135
136
137
139 """
140 Property target used to set the peer name.
141 The value must be a non-empty string and cannot be C{None}.
142 @raise ValueError: If the value is an empty string or C{None}.
143 """
144 if value is None or len(value) < 1:
145 raise ValueError("Peer name must be a non-empty string.")
146 self._name = value
147
149 """
150 Property target used to get the peer name.
151 """
152 return self._name
153
155 """
156 Property target used to set the collect directory.
157 The value must be an absolute path and cannot be C{None}.
158 It does not have to exist on disk at the time of assignment.
159 @raise ValueError: If the value is C{None} or is not an absolute path.
160 @raise ValueError: If a path cannot be encoded properly.
161 """
162 if value is None or not os.path.isabs(value):
163 raise ValueError("Collect directory must be an absolute path.")
164 self._collectDir = encodePath(value)
165
167 """
168 Property target used to get the collect directory.
169 """
170 return self._collectDir
171
172 name = property(_getName, _setName, None, "Name of the peer.")
173 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).")
174
175
176
177
178
179
180 - def stagePeer(self, targetDir, ownership=None, permissions=None):
181 """
182 Stages data from the peer into the indicated local target directory.
183
184 The collect and target directories must both already exist before this
185 method is called. If passed in, ownership and permissions will be
186 applied to the files that are copied.
187
188 @note: The caller is responsible for checking that the indicator exists,
189 if they care. This function only stages the files within the directory.
190
191 @note: If you have user/group as strings, call the L{util.getUidGid} function
192 to get the associated uid/gid as an ownership tuple.
193
194 @param targetDir: Target directory to write data into
195 @type targetDir: String representing a directory on disk
196
197 @param ownership: Owner and group that the staged files should have
198 @type ownership: Tuple of numeric ids C{(uid, gid)}
199
200 @param permissions: Permissions that the staged files should have
201 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
202
203 @return: Number of files copied from the source directory to the target directory.
204
205 @raise ValueError: If collect directory is not a directory or does not exist
206 @raise ValueError: If target directory is not a directory, does not exist or is not absolute.
207 @raise ValueError: If a path cannot be encoded properly.
208 @raise IOError: If there were no files to stage (i.e. the directory was empty)
209 @raise IOError: If there is an IO error copying a file.
210 @raise OSError: If there is an OS error copying or changing permissions on a file
211 """
212 targetDir = encodePath(targetDir)
213 if not os.path.isabs(targetDir):
214 logger.debug("Target directory [%s] not an absolute path." % targetDir)
215 raise ValueError("Target directory must be an absolute path.")
216 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir):
217 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir)
218 raise ValueError("Collect directory is not a directory or does not exist on disk.")
219 if not os.path.exists(targetDir) or not os.path.isdir(targetDir):
220 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir)
221 raise ValueError("Target directory is not a directory or does not exist on disk.")
222 count = LocalPeer._copyLocalDir(self.collectDir, targetDir, ownership, permissions)
223 if count == 0:
224 raise IOError("Did not copy any files from local peer.")
225 return count
226
228 """
229 Checks the collect indicator in the peer's staging directory.
230
231 When a peer has completed collecting its backup files, it will write an
232 empty indicator file into its collect directory. This method checks to
233 see whether that indicator has been written. We're "stupid" here - if
234 the collect directory doesn't exist, you'll naturally get back C{False}.
235
236 If you need to, you can override the name of the collect indicator file
237 by passing in a different name.
238
239 @param collectIndicator: Name of the collect indicator file to check
240 @type collectIndicator: String representing name of a file in the collect directory
241
242 @return: Boolean true/false depending on whether the indicator exists.
243 @raise ValueError: If a path cannot be encoded properly.
244 """
245 collectIndicator = encodePath(collectIndicator)
246 if collectIndicator is None:
247 return os.path.exists(os.path.join(self.collectDir, DEF_COLLECT_INDICATOR))
248 else:
249 return os.path.exists(os.path.join(self.collectDir, collectIndicator))
250
252 """
253 Writes the stage indicator in the peer's staging directory.
254
255 When the master has completed collecting its backup files, it will write
256 an empty indicator file into the peer's collect directory. The presence
257 of this file implies that the staging process is complete.
258
259 If you need to, you can override the name of the stage indicator file by
260 passing in a different name.
261
262 @note: If you have user/group as strings, call the L{util.getUidGid}
263 function to get the associated uid/gid as an ownership tuple.
264
265 @param stageIndicator: Name of the indicator file to write
266 @type stageIndicator: String representing name of a file in the collect directory
267
268 @param ownership: Owner and group that the indicator file should have
269 @type ownership: Tuple of numeric ids C{(uid, gid)}
270
271 @param permissions: Permissions that the indicator file should have
272 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
273
274 @raise ValueError: If collect directory is not a directory or does not exist
275 @raise ValueError: If a path cannot be encoded properly.
276 @raise IOError: If there is an IO error creating the file.
277 @raise OSError: If there is an OS error creating or changing permissions on the file
278 """
279 stageIndicator = encodePath(stageIndicator)
280 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir):
281 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir)
282 raise ValueError("Collect directory is not a directory or does not exist on disk.")
283 if stageIndicator is None:
284 fileName = os.path.join(self.collectDir, DEF_STAGE_INDICATOR)
285 else:
286 fileName = os.path.join(self.collectDir, stageIndicator)
287 LocalPeer._copyLocalFile(None, fileName, ownership, permissions)
288
289
290
291
292
293
294 - def _copyLocalDir(sourceDir, targetDir, ownership=None, permissions=None):
295 """
296 Copies files from the source directory to the target directory.
297
298 This function is not recursive. Only the files in the directory will be
299 copied. Ownership and permissions will be left at their default values
300 if new values are not specified. The source and target directories are
301 allowed to be soft links to a directory, but besides that soft links are
302 ignored.
303
304 @note: If you have user/group as strings, call the L{util.getUidGid}
305 function to get the associated uid/gid as an ownership tuple.
306
307 @param sourceDir: Source directory
308 @type sourceDir: String representing a directory on disk
309
310 @param targetDir: Target directory
311 @type targetDir: String representing a directory on disk
312
313 @param ownership: Owner and group that the copied files should have
314 @type ownership: Tuple of numeric ids C{(uid, gid)}
315
316 @param permissions: Permissions that the staged files should have
317 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
318
319 @return: Number of files copied from the source directory to the target directory.
320
321 @raise ValueError: If source or target is not a directory or does not exist.
322 @raise ValueError: If a path cannot be encoded properly.
323 @raise IOError: If there is an IO error copying the files.
324 @raise OSError: If there is an OS error copying or changing permissions on a files
325 """
326 filesCopied = 0
327 sourceDir = encodePath(sourceDir)
328 targetDir = encodePath(targetDir)
329 for fileName in os.listdir(sourceDir):
330 sourceFile = os.path.join(sourceDir, fileName)
331 targetFile = os.path.join(targetDir, fileName)
332 LocalPeer._copyLocalFile(sourceFile, targetFile, ownership, permissions)
333 filesCopied += 1
334 return filesCopied
335 _copyLocalDir = staticmethod(_copyLocalDir)
336
337 - def _copyLocalFile(sourceFile=None, targetFile=None, ownership=None, permissions=None, overwrite=True):
338 """
339 Copies a source file to a target file.
340
341 If the source file is C{None} then the target file will be created or
342 overwritten as an empty file. If the target file is C{None}, this method
343 is a no-op. Attempting to copy a soft link or a directory will result in
344 an exception.
345
346 @note: If you have user/group as strings, call the L{util.getUidGid}
347 function to get the associated uid/gid as an ownership tuple.
348
349 @note: We will not overwrite a target file that exists when this method
350 is invoked. If the target already exists, we'll raise an exception.
351
352 @param sourceFile: Source file to copy
353 @type sourceFile: String representing a file on disk, as an absolute path
354
355 @param targetFile: Target file to create
356 @type targetFile: String representing a file on disk, as an absolute path
357
358 @param ownership: Owner and group that the copied should have
359 @type ownership: Tuple of numeric ids C{(uid, gid)}
360
361 @param permissions: Permissions that the staged files should have
362 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
363
364 @param overwrite: Indicates whether it's OK to overwrite the target file.
365 @type overwrite: Boolean true/false.
366
367 @raise ValueError: If the passed-in source file is not a regular file.
368 @raise ValueError: If a path cannot be encoded properly.
369 @raise IOError: If the target file already exists.
370 @raise IOError: If there is an IO error copying the file
371 @raise OSError: If there is an OS error copying or changing permissions on a file
372 """
373 targetFile = encodePath(targetFile)
374 sourceFile = encodePath(sourceFile)
375 if targetFile is None:
376 return
377 if not overwrite:
378 if os.path.exists(targetFile):
379 raise IOError("Target file [%s] already exists." % targetFile)
380 if sourceFile is None:
381 open(targetFile, "w").write("")
382 else:
383 if os.path.isfile(sourceFile) and not os.path.islink(sourceFile):
384 shutil.copy(sourceFile, targetFile)
385 else:
386 logger.debug("Source [%s] is not a regular file." % sourceFile)
387 raise ValueError("Source is not a regular file.")
388 if ownership is not None:
389 os.chown(targetFile, ownership[0], ownership[1])
390 if permissions is not None:
391 os.chmod(targetFile, permissions)
392 _copyLocalFile = staticmethod(_copyLocalFile)
393
394
395
396
397
398
400
401
402
403
404
405 """
406 Backup peer representing a remote peer in a backup pool.
407
408 This is a class representing a remote (networked) peer in a backup pool.
409 Remote peers are backed up using an rcp-compatible copy command. A remote
410 peer has associated with it a name (which must be a valid hostname), a
411 collect directory, a working directory and a copy method (an rcp-compatible
412 command).
413
414 You can also set an optional local user value. This username will be used
415 as the local user for any remote copies that are required. It can only be
416 used if the root user is executing the backup. The root user will C{su} to
417 the local user and execute the remote copies as that user.
418
419 The copy method is associated with the peer and not with the actual request
420 to copy, because we can envision that each remote host might have a
421 different connect method.
422
423 The public methods other than the constructor are part of a "backup peer"
424 interface shared with the C{LocalPeer} class.
425
426 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator,
427 _getDirContents, _copyRemoteDir, _copyRemoteFile, _pushLocalFile,
428 name, collectDir, remoteUser, rcpCommand
429 """
430
431
432
433
434
435 - def __init__(self, name, collectDir, workingDir, remoteUser, rcpCommand=None, localUser=None):
436 """
437 Initializes a remote backup peer.
438
439 @note: If provided, the rcp command will eventually be parsed into a list
440 of strings suitable for passing to C{popen2.Popen4} in order to avoid
441 security holes related to shell interpolation. This parsing will be
442 done by the L{util.splitCommandLine} function. See the documentation for
443 that function for some important notes about its limitations.
444
445 @param name: Name of the backup peer
446 @type name: String, must be a valid DNS hostname
447
448 @param collectDir: Path to the peer's collect directory
449 @type collectDir: String representing an absolute path on the remote peer
450
451 @param workingDir: Working directory that can be used to create temporary files, etc.
452 @type workingDir: String representing an absolute path on the current host.
453
454 @param remoteUser: Name of the Cedar Backup user on the remote peer
455 @type remoteUser: String representing a username, valid via the copy command
456
457 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
458 @type rcpCommand: String representing a system command including required arguments
459
460 @param localUser: Name of the Cedar Backup user on the current host
461 @type localUser: String representing a username, valid on the current host
462
463 @raise ValueError: If collect directory is not an absolute path
464 """
465 self._name = None
466 self._collectDir = None
467 self._workingDir = None
468 self._remoteUser = None
469 self._localUser = None
470 self._rcpCommand = None
471 self._rcpCommandList = None
472 self.name = name
473 self.collectDir = collectDir
474 self.workingDir = workingDir
475 self.remoteUser = remoteUser
476 self.localUser = localUser
477 self.rcpCommand = rcpCommand
478
479
480
481
482
483
485 """
486 Property target used to set the peer name.
487 The value must be a non-empty string and cannot be C{None}.
488 @raise ValueError: If the value is an empty string or C{None}.
489 """
490 if value is None or len(value) < 1:
491 raise ValueError("Peer name must be a non-empty string.")
492 self._name = value
493
495 """
496 Property target used to get the peer name.
497 """
498 return self._name
499
501 """
502 Property target used to set the collect directory.
503 The value must be an absolute path and cannot be C{None}.
504 It does not have to exist on disk at the time of assignment.
505 @raise ValueError: If the value is C{None} or is not an absolute path.
506 @raise ValueError: If the value cannot be encoded properly.
507 """
508 if value is None or not os.path.isabs(value):
509 raise ValueError("Collect directory must be an absolute path.")
510 self._collectDir = encodePath(value)
511
513 """
514 Property target used to get the collect directory.
515 """
516 return self._collectDir
517
519 """
520 Property target used to set the working directory.
521 The value must be an absolute path and cannot be C{None}.
522 @raise ValueError: If the value is C{None} or is not an absolute path.
523 @raise ValueError: If the value cannot be encoded properly.
524 """
525 if value is None or not os.path.isabs(value):
526 raise ValueError("Working directory must be an absolute path.")
527 self._workingDir = encodePath(value)
528
530 """
531 Property target used to get the working directory.
532 """
533 return self._workingDir
534
536 """
537 Property target used to set the remote user.
538 The value must be a non-empty string and cannot be C{None}.
539 @raise ValueError: If the value is an empty string or C{None}.
540 """
541 if value is None or len(value) < 1:
542 raise ValueError("Peer remote user must be a non-empty string.")
543 self._remoteUser = value
544
546 """
547 Property target used to get the remote user.
548 """
549 return self._remoteUser
550
552 """
553 Property target used to set the local user.
554 The value must be a non-empty string if it is not C{None}.
555 @raise ValueError: If the value is an empty string.
556 """
557 if value is not None:
558 if len(value) < 1:
559 raise ValueError("Peer local user must be a non-empty string.")
560 self._localUser = value
561
563 """
564 Property target used to get the local user.
565 """
566 return self._localUser
567
569 """
570 Property target to set the rcp command.
571
572 The value must be a non-empty string or C{None}. Its value is stored in
573 the two forms: "raw" as provided by the client, and "parsed" into a list
574 suitable for being passed to L{util.executeCommand} via
575 L{util.splitCommandLine}.
576
577 However, all the caller will ever see via the property is the actual
578 value they set (which includes seeing C{None}, even if we translate that
579 internally to C{DEF_RCP_COMMAND}). Internally, we should always use
580 C{self._rcpCommandList} if we want the actual command list.
581
582 @raise ValueError: If the value is an empty string.
583 """
584 if value is None:
585 self._rcpCommand = None
586 self._rcpCommandList = DEF_RCP_COMMAND
587 else:
588 if len(value) >= 1:
589 self._rcpCommand = value
590 self._rcpCommandList = splitCommandLine(self._rcpCommand)
591 else:
592 raise ValueError("The rcp command must be a non-empty string.")
593
595 """
596 Property target used to get the rcp command.
597 """
598 return self._rcpCommand
599
600 name = property(_getName, _setName, None, "Name of the peer (a valid DNS hostname).")
601 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).")
602 workingDir = property(_getWorkingDir, _setWorkingDir, None, "Path to the peer's working directory (an absolute local path).")
603 remoteUser = property(_getRemoteUser, _setRemoteUser, None, "Name of the Cedar Backup user on the remote peer.")
604 localUser = property(_getLocalUser, _setLocalUser, None, "Name of the Cedar Backup user on the current host.")
605 rcpCommand = property(_getRcpCommand, _setRcpCommand, None, "An rcp-compatible copy command to use for copying files.")
606
607
608
609
610
611
612 - def stagePeer(self, targetDir, ownership=None, permissions=None):
613 """
614 Stages data from the peer into the indicated local target directory.
615
616 The target directory must already exist before this method is called. If
617 passed in, ownership and permissions will be applied to the files that
618 are copied.
619
620 @note: The returned count of copied files might be inaccurate if some of
621 the copied files already existed in the staging directory prior to the
622 copy taking place. We don't clear the staging directory first, because
623 some extension might also be using it.
624
625 @note: If you have user/group as strings, call the L{util.getUidGid} function
626 to get the associated uid/gid as an ownership tuple.
627
628 @note: Unlike the local peer version of this method, an I/O error might
629 or might not be raised if the directory is empty. Since we're using a
630 remote copy method, we just don't have the fine-grained control over our
631 exceptions that's available when we can look directly at the filesystem,
632 and we can't control whether the remote copy method thinks an empty
633 directory is an error.
634
635 @param targetDir: Target directory to write data into
636 @type targetDir: String representing a directory on disk
637
638 @param ownership: Owner and group that the staged files should have
639 @type ownership: Tuple of numeric ids C{(uid, gid)}
640
641 @param permissions: Permissions that the staged files should have
642 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
643
644 @return: Number of files copied from the source directory to the target directory.
645
646 @raise ValueError: If target directory is not a directory, does not exist or is not absolute.
647 @raise ValueError: If a path cannot be encoded properly.
648 @raise IOError: If there were no files to stage (i.e. the directory was empty)
649 @raise IOError: If there is an IO error copying a file.
650 @raise OSError: If there is an OS error copying or changing permissions on a file
651 """
652 targetDir = encodePath(targetDir)
653 if not os.path.isabs(targetDir):
654 logger.debug("Target directory [%s] not an absolute path." % targetDir)
655 raise ValueError("Target directory must be an absolute path.")
656 if not os.path.exists(targetDir) or not os.path.isdir(targetDir):
657 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir)
658 raise ValueError("Target directory is not a directory or does not exist on disk.")
659 count = RemotePeer._copyRemoteDir(self.remoteUser, self.localUser, self.name,
660 self._rcpCommand, self._rcpCommandList,
661 self.collectDir, targetDir,
662 ownership, permissions)
663 if count == 0:
664 raise IOError("Did not copy any files from local peer.")
665 return count
666
667
669 """
670 Checks the collect indicator in the peer's staging directory.
671
672 When a peer has completed collecting its backup files, it will write an
673 empty indicator file into its collect directory. This method checks to
674 see whether that indicator has been written. If the remote copy command
675 fails, we return C{False} as if the file weren't there.
676
677 If you need to, you can override the name of the collect indicator file
678 by passing in a different name.
679
680 @note: Apparently, we can't count on all rcp-compatible implementations
681 to return sensible errors for some error conditions. As an example, the
682 C{scp} command in Debian 'woody' returns a zero (normal) status even when
683 it can't find a host or if the login or path is invalid. Because of
684 this, the implementation of this method is rather convoluted.
685
686 @param collectIndicator: Name of the collect indicator file to check
687 @type collectIndicator: String representing name of a file in the collect directory
688
689 @return: Boolean true/false depending on whether the indicator exists.
690 @raise ValueError: If a path cannot be encoded properly.
691 """
692 try:
693 if collectIndicator is None:
694 sourceFile = os.path.join(self.collectDir, DEF_COLLECT_INDICATOR)
695 targetFile = os.path.join(self.workingDir, DEF_COLLECT_INDICATOR)
696 else:
697 collectIndicator = encodePath(collectIndicator)
698 sourceFile = os.path.join(self.collectDir, collectIndicator)
699 targetFile = os.path.join(self.workingDir, collectIndicator)
700 logger.debug("Fetch remote [%s] into [%s]." % (sourceFile, targetFile))
701 if os.path.exists(targetFile):
702 try:
703 os.remove(targetFile)
704 except:
705 raise Exception("Error: collect indicator [%s] already exists!" % targetFile)
706 try:
707 RemotePeer._copyRemoteFile(self.remoteUser, self.localUser, self.name,
708 self._rcpCommand, self._rcpCommandList,
709 sourceFile, targetFile,
710 overwrite=False)
711 if os.path.exists(targetFile):
712 return True
713 else:
714 return False
715 except:
716 return False
717 finally:
718 if os.path.exists(targetFile):
719 try:
720 os.remove(targetFile)
721 except: pass
722
724 """
725 Writes the stage indicator in the peer's staging directory.
726
727 When the master has completed collecting its backup files, it will write
728 an empty indicator file into the peer's collect directory. The presence
729 of this file implies that the staging process is complete.
730
731 If you need to, you can override the name of the stage indicator file by
732 passing in a different name.
733
734 @note: If you have user/group as strings, call the L{util.getUidGid} function
735 to get the associated uid/gid as an ownership tuple.
736
737 @param stageIndicator: Name of the indicator file to write
738 @type stageIndicator: String representing name of a file in the collect directory
739
740 @raise ValueError: If a path cannot be encoded properly.
741 @raise IOError: If there is an IO error creating the file.
742 @raise OSError: If there is an OS error creating or changing permissions on the file
743 """
744 stageIndicator = encodePath(stageIndicator)
745 if stageIndicator is None:
746 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR)
747 targetFile = os.path.join(self.collectDir, DEF_STAGE_INDICATOR)
748 else:
749 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR)
750 targetFile = os.path.join(self.collectDir, stageIndicator)
751 try:
752 if not os.path.exists(sourceFile):
753 open(sourceFile, "w").write("")
754 RemotePeer._pushLocalFile(self.remoteUser, self.localUser, self.name,
755 self._rcpCommand, self._rcpCommandList,
756 sourceFile, targetFile)
757 finally:
758 if os.path.exists(sourceFile):
759 try:
760 os.remove(sourceFile)
761 except: pass
762
763
764
765
766
767
768 - def _getDirContents(path):
769 """
770 Returns the contents of a directory in terms of a Set.
771
772 The directory's contents are read as a L{FilesystemList} containing only
773 files, and then the list is converted into a C{sets.Set} object for later
774 use.
775
776 @param path: Directory path to get contents for
777 @type path: String representing a path on disk
778
779 @return: Set of files in the directory
780 @raise ValueError: If path is not a directory or does not exist.
781 """
782 contents = FilesystemList()
783 contents.excludeDirs = True
784 contents.excludeLinks = True
785 contents.addDirContents(path)
786 return sets.Set(contents)
787 _getDirContents = staticmethod(_getDirContents)
788
789 - def _copyRemoteDir(remoteUser, localUser, remoteHost, rcpCommand, rcpCommandList,
790 sourceDir, targetDir, ownership=None, permissions=None):
791 """
792 Copies files from the source directory to the target directory.
793
794 This function is not recursive. Only the files in the directory will be
795 copied. Ownership and permissions will be left at their default values
796 if new values are not specified. Behavior when copying soft links from
797 the collect directory is dependent on the behavior of the specified rcp
798 command.
799
800 @note: The returned count of copied files might be inaccurate if some of
801 the copied files already existed in the staging directory prior to the
802 copy taking place. We don't clear the staging directory first, because
803 some extension might also be using it.
804
805 @note: If you have user/group as strings, call the L{util.getUidGid} function
806 to get the associated uid/gid as an ownership tuple.
807
808 @note: We don't have a good way of knowing exactly what files we copied
809 down from the remote peer, unless we want to parse the output of the rcp
810 command (ugh). We could change permissions on everything in the target
811 directory, but that's kind of ugly too. Instead, we use Python's set
812 functionality to figure out what files were added while we executed the
813 rcp command. This isn't perfect - for instance, it's not correct if
814 someone else is messing with the directory at the same time we're doing
815 the remote copy - but it's about as good as we're going to get.
816
817 @note: Apparently, we can't count on all rcp-compatible implementations
818 to return sensible errors for some error conditions. As an example, the
819 C{scp} command in Debian 'woody' returns a zero (normal) status even
820 when it can't find a host or if the login or path is invalid. We try
821 to work around this by issuing C{IOError} if we don't copy any files from
822 the remote host.
823
824 @param remoteUser: Name of the Cedar Backup user on the remote peer
825 @type remoteUser: String representing a username, valid via the copy command
826
827 @param localUser: Name of the Cedar Backup user on the current host
828 @type localUser: String representing a username, valid on the current host
829
830 @param remoteHost: Hostname of the remote peer
831 @type remoteHost: String representing a hostname, accessible via the copy command
832
833 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
834 @type rcpCommand: String representing a system command including required arguments
835
836 @param rcpCommandList: An rcp-compatible copy command to use for copying files
837 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
838
839 @param sourceDir: Source directory
840 @type sourceDir: String representing a directory on disk
841
842 @param targetDir: Target directory
843 @type targetDir: String representing a directory on disk
844
845 @param ownership: Owner and group that the copied files should have
846 @type ownership: Tuple of numeric ids C{(uid, gid)}
847
848 @param permissions: Permissions that the staged files should have
849 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
850
851 @return: Number of files copied from the source directory to the target directory.
852
853 @raise ValueError: If source or target is not a directory or does not exist.
854 @raise IOError: If there is an IO error copying the files.
855 """
856 beforeSet = RemotePeer._getDirContents(targetDir)
857 if localUser is not None:
858 try:
859 if os.getuid() != 0:
860 raise IOError("Only root can remote copy as another user.")
861 except AttributeError: pass
862 actualCommand = "%s %s@%s:%s/* %s" % (rcpCommand, remoteUser, remoteHost, sourceDir, targetDir)
863 command = resolveCommand(SU_COMMAND)
864 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
865 if result != 0:
866 raise IOError("Error (%d) copying files from remote host as local user [%s]." % (result, localUser))
867 else:
868 copySource = "%s@%s:%s/*" % (remoteUser, remoteHost, sourceDir)
869 command = resolveCommand(rcpCommandList)
870 result = executeCommand(command, [copySource, targetDir])[0]
871 if result != 0:
872 raise IOError("Error (%d) copying files from remote host (using no local user)." % result)
873 afterSet = RemotePeer._getDirContents(targetDir)
874 if len(afterSet) == 0:
875 raise IOError("Did not copy any files from remote peer.")
876 differenceSet = afterSet.difference(beforeSet)
877 if len(differenceSet) == 0:
878 raise IOError("Apparently did not copy any new files from remote peer.")
879 for targetFile in differenceSet:
880 if ownership is not None:
881 os.chown(targetFile, ownership[0], ownership[1])
882 if permissions is not None:
883 os.chmod(targetFile, permissions)
884 return len(differenceSet)
885 _copyRemoteDir = staticmethod(_copyRemoteDir)
886
887 - def _copyRemoteFile(remoteUser, localUser, remoteHost,
888 rcpCommand, rcpCommandList,
889 sourceFile, targetFile, ownership=None,
890 permissions=None, overwrite=True):
891 """
892 Copies a remote source file to a target file.
893
894 @note: Internally, we have to go through and escape any spaces in the
895 source path with double-backslash, otherwise things get screwed up. It
896 doesn't seem to be required in the target path. I hope this is portable
897 to various different rcp methods, but I guess it might not be (all I have
898 to test with is OpenSSH).
899
900 @note: If you have user/group as strings, call the L{util.getUidGid} function
901 to get the associated uid/gid as an ownership tuple.
902
903 @note: We will not overwrite a target file that exists when this method
904 is invoked. If the target already exists, we'll raise an exception.
905
906 @note: Apparently, we can't count on all rcp-compatible implementations
907 to return sensible errors for some error conditions. As an example, the
908 C{scp} command in Debian 'woody' returns a zero (normal) status even when
909 it can't find a host or if the login or path is invalid. We try to work
910 around this by issuing C{IOError} the target file does not exist when
911 we're done.
912
913 @param remoteUser: Name of the Cedar Backup user on the remote peer
914 @type remoteUser: String representing a username, valid via the copy command
915
916 @param remoteHost: Hostname of the remote peer
917 @type remoteHost: String representing a hostname, accessible via the copy command
918
919 @param localUser: Name of the Cedar Backup user on the current host
920 @type localUser: String representing a username, valid on the current host
921
922 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
923 @type rcpCommand: String representing a system command including required arguments
924
925 @param rcpCommandList: An rcp-compatible copy command to use for copying files
926 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
927
928 @param sourceFile: Source file to copy
929 @type sourceFile: String representing a file on disk, as an absolute path
930
931 @param targetFile: Target file to create
932 @type targetFile: String representing a file on disk, as an absolute path
933
934 @param ownership: Owner and group that the copied should have
935 @type ownership: Tuple of numeric ids C{(uid, gid)}
936
937 @param permissions: Permissions that the staged files should have
938 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
939
940 @param overwrite: Indicates whether it's OK to overwrite the target file.
941 @type overwrite: Boolean true/false.
942
943 @raise IOError: If the target file already exists.
944 @raise IOError: If there is an IO error copying the file
945 @raise OSError: If there is an OS error changing permissions on the file
946 """
947 if not overwrite:
948 if os.path.exists(targetFile):
949 raise IOError("Target file [%s] already exists." % targetFile)
950 if localUser is not None:
951 try:
952 if os.getuid() != 0:
953 raise IOError("Only root can remote copy as another user.")
954 except AttributeError: pass
955 actualCommand = "%s %s@%s:%s %s" % (rcpCommand, remoteUser, remoteHost, sourceFile.replace(" ", "\\ "), targetFile)
956 command = resolveCommand(SU_COMMAND)
957 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
958 if result != 0:
959 raise IOError("Error (%d) copying [%s] from remote host as local user [%s]." % (result, sourceFile, localUser))
960 else:
961 copySource = "%s@%s:%s" % (remoteUser, remoteHost, sourceFile.replace(" ", "\\ "))
962 command = resolveCommand(rcpCommandList)
963 result = executeCommand(command, [copySource, targetFile])[0]
964 if result != 0:
965 raise IOError("Error (%d) copying [%s] from remote host (using no local user)." % (result, sourceFile))
966 if not os.path.exists(targetFile):
967 raise IOError("Apparently unable to copy file from remote host.")
968 if ownership is not None:
969 os.chown(targetFile, ownership[0], ownership[1])
970 if permissions is not None:
971 os.chmod(targetFile, permissions)
972 _copyRemoteFile = staticmethod(_copyRemoteFile)
973
974 - def _pushLocalFile(remoteUser, localUser, remoteHost,
975 rcpCommand, rcpCommandList,
976 sourceFile, targetFile, overwrite=True):
977 """
978 Copies a local source file to a remote host.
979
980 @note: We will not overwrite a target file that exists when this method
981 is invoked. If the target already exists, we'll raise an exception.
982
983 @note: Internally, we have to go through and escape any spaces in the
984 source and target paths with double-backslash, otherwise things get
985 screwed up. I hope this is portable to various different rcp methods,
986 but I guess it might not be (all I have to test with is OpenSSH).
987
988 @note: If you have user/group as strings, call the L{util.getUidGid} function
989 to get the associated uid/gid as an ownership tuple.
990
991 @param remoteUser: Name of the Cedar Backup user on the remote peer
992 @type remoteUser: String representing a username, valid via the copy command
993
994 @param localUser: Name of the Cedar Backup user on the current host
995 @type localUser: String representing a username, valid on the current host
996
997 @param remoteHost: Hostname of the remote peer
998 @type remoteHost: String representing a hostname, accessible via the copy command
999
1000 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
1001 @type rcpCommand: String representing a system command including required arguments
1002
1003 @param rcpCommandList: An rcp-compatible copy command to use for copying files
1004 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
1005
1006 @param sourceFile: Source file to copy
1007 @type sourceFile: String representing a file on disk, as an absolute path
1008
1009 @param targetFile: Target file to create
1010 @type targetFile: String representing a file on disk, as an absolute path
1011
1012 @param overwrite: Indicates whether it's OK to overwrite the target file.
1013 @type overwrite: Boolean true/false.
1014
1015 @raise IOError: If there is an IO error copying the file
1016 @raise OSError: If there is an OS error changing permissions on the file
1017 """
1018 if not overwrite:
1019 if os.path.exists(targetFile):
1020 raise IOError("Target file [%s] already exists." % targetFile)
1021 if localUser is not None:
1022 try:
1023 if os.getuid() != 0:
1024 raise IOError("Only root can remote copy as another user.")
1025 except AttributeError: pass
1026 actualCommand = '%s "%s" "%s@%s:%s"' % (rcpCommand, sourceFile, remoteUser, remoteHost, targetFile)
1027 command = resolveCommand(SU_COMMAND)
1028 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1029 if result != 0:
1030 raise IOError("Error (%d) copying [%s] to remote host as local user [%s]." % (result, sourceFile, localUser))
1031 else:
1032 copyTarget = "%s@%s:%s" % (remoteUser, remoteHost, targetFile.replace(" ", "\\ "))
1033 command = resolveCommand(rcpCommandList)
1034 result = executeCommand(command, [sourceFile.replace(" ", "\\ "), copyTarget])[0]
1035 if result != 0:
1036 raise IOError("Error (%d) copying [%s] to remote host (using no local user)." % (result, sourceFile))
1037 _pushLocalFile = staticmethod(_pushLocalFile)
1038