Package CedarBackup2 :: Module peer
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup2.peer

   1  # -*- coding: iso-8859-1 -*- 
   2  # vim: set ft=python ts=3 sw=3 expandtab: 
   3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
   4  # 
   5  #              C E D A R 
   6  #          S O L U T I O N S       "Software done right." 
   7  #           S O F T W A R E 
   8  # 
   9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  10  # 
  11  # Copyright (c) 2004-2006 Kenneth J. Pronovici. 
  12  # All rights reserved. 
  13  # 
  14  # This program is free software; you can redistribute it and/or 
  15  # modify it under the terms of the GNU General Public License, 
  16  # Version 2, as published by the Free Software Foundation. 
  17  # 
  18  # This program is distributed in the hope that it will be useful, 
  19  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
  20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
  21  # 
  22  # Copies of the GNU General Public License are available from 
  23  # the Free Software Foundation website, http://www.gnu.org/. 
  24  # 
  25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  26  # 
  27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
  28  # Language : Python (>= 2.3) 
  29  # Project  : Cedar Backup, release 2 
  30  # Revision : $Id: peer.py 1212 2007-07-10 03:08:39Z pronovic $ 
  31  # Purpose  : Provides backup peer-related objects. 
  32  # 
  33  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  34   
  35  ######################################################################## 
  36  # Module documentation 
  37  ######################################################################## 
  38   
  39  """ 
  40  Provides backup peer-related objects and utility functions. 
  41   
  42  @sort: LocalPeer, Remote Peer 
  43   
  44  @var DEF_COLLECT_INDICATOR: Name of the default collect indicator file. 
  45  @var DEF_STAGE_INDICATOR: Name of the default stage indicator file. 
  46   
  47  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
  48  """ 
  49   
  50   
  51  ######################################################################## 
  52  # Imported modules 
  53  ######################################################################## 
  54   
  55  # System modules 
  56  import os 
  57  import logging 
  58  import shutil 
  59  import sets 
  60  import re 
  61   
  62  # Cedar Backup modules 
  63  from CedarBackup2.filesystem import FilesystemList 
  64  from CedarBackup2.util import resolveCommand, executeCommand 
  65  from CedarBackup2.util import splitCommandLine, encodePath 
  66   
  67   
  68  ######################################################################## 
  69  # Module-wide constants and variables 
  70  ######################################################################## 
  71   
  72  logger                  = logging.getLogger("CedarBackup2.log.peer") 
  73   
  74  DEF_RCP_COMMAND         = [ "/usr/bin/scp", "-B", "-q", "-C" ] 
  75  DEF_COLLECT_INDICATOR   = "cback.collect" 
  76  DEF_STAGE_INDICATOR     = "cback.stage" 
  77   
  78  SU_COMMAND              = [ "su" ] 
  79   
  80   
  81  ######################################################################## 
  82  # LocalPeer class definition 
  83  ######################################################################## 
  84   
85 -class LocalPeer(object):
86 87 ###################### 88 # Class documentation 89 ###################### 90 91 """ 92 Backup peer representing a local peer in a backup pool. 93 94 This is a class representing a local (non-network) peer in a backup pool. 95 Local peers are backed up by simple filesystem copy operations. A local 96 peer has associated with it a name (typically, but not necessarily, a 97 hostname) and a collect directory. 98 99 The public methods other than the constructor are part of a "backup peer" 100 interface shared with the C{RemotePeer} class. 101 102 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator, 103 _copyLocalDir, _copyLocalFile, name, collectDir 104 """ 105 106 ############## 107 # Constructor 108 ############## 109
110 - def __init__(self, name, collectDir):
111 """ 112 Initializes a local backup peer. 113 114 Note that the collect directory must be an absolute path, but does not 115 have to exist when the object is instantiated. We do a lazy validation 116 on this value since we could (potentially) be creating peer objects 117 before an ongoing backup completed. 118 119 @param name: Name of the backup peer 120 @type name: String, typically a hostname 121 122 @param collectDir: Path to the peer's collect directory 123 @type collectDir: String representing an absolute local path on disk 124 125 @raise ValueError: If the name is empty. 126 @raise ValueError: If collect directory is not an absolute path. 127 """ 128 self._name = None 129 self._collectDir = None 130 self.name = name 131 self.collectDir = collectDir
132 133 134 ############# 135 # Properties 136 ############# 137
138 - def _setName(self, value):
139 """ 140 Property target used to set the peer name. 141 The value must be a non-empty string and cannot be C{None}. 142 @raise ValueError: If the value is an empty string or C{None}. 143 """ 144 if value is None or len(value) < 1: 145 raise ValueError("Peer name must be a non-empty string.") 146 self._name = value
147
148 - def _getName(self):
149 """ 150 Property target used to get the peer name. 151 """ 152 return self._name
153
154 - def _setCollectDir(self, value):
155 """ 156 Property target used to set the collect directory. 157 The value must be an absolute path and cannot be C{None}. 158 It does not have to exist on disk at the time of assignment. 159 @raise ValueError: If the value is C{None} or is not an absolute path. 160 @raise ValueError: If a path cannot be encoded properly. 161 """ 162 if value is None or not os.path.isabs(value): 163 raise ValueError("Collect directory must be an absolute path.") 164 self._collectDir = encodePath(value)
165
166 - def _getCollectDir(self):
167 """ 168 Property target used to get the collect directory. 169 """ 170 return self._collectDir
171 172 name = property(_getName, _setName, None, "Name of the peer.") 173 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).") 174 175 176 ################# 177 # Public methods 178 ################# 179
180 - def stagePeer(self, targetDir, ownership=None, permissions=None):
181 """ 182 Stages data from the peer into the indicated local target directory. 183 184 The collect and target directories must both already exist before this 185 method is called. If passed in, ownership and permissions will be 186 applied to the files that are copied. 187 188 @note: The caller is responsible for checking that the indicator exists, 189 if they care. This function only stages the files within the directory. 190 191 @note: If you have user/group as strings, call the L{util.getUidGid} function 192 to get the associated uid/gid as an ownership tuple. 193 194 @param targetDir: Target directory to write data into 195 @type targetDir: String representing a directory on disk 196 197 @param ownership: Owner and group that the staged files should have 198 @type ownership: Tuple of numeric ids C{(uid, gid)} 199 200 @param permissions: Permissions that the staged files should have 201 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 202 203 @return: Number of files copied from the source directory to the target directory. 204 205 @raise ValueError: If collect directory is not a directory or does not exist 206 @raise ValueError: If target directory is not a directory, does not exist or is not absolute. 207 @raise ValueError: If a path cannot be encoded properly. 208 @raise IOError: If there were no files to stage (i.e. the directory was empty) 209 @raise IOError: If there is an IO error copying a file. 210 @raise OSError: If there is an OS error copying or changing permissions on a file 211 """ 212 targetDir = encodePath(targetDir) 213 if not os.path.isabs(targetDir): 214 logger.debug("Target directory [%s] not an absolute path." % targetDir) 215 raise ValueError("Target directory must be an absolute path.") 216 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir): 217 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir) 218 raise ValueError("Collect directory is not a directory or does not exist on disk.") 219 if not os.path.exists(targetDir) or not os.path.isdir(targetDir): 220 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir) 221 raise ValueError("Target directory is not a directory or does not exist on disk.") 222 count = LocalPeer._copyLocalDir(self.collectDir, targetDir, ownership, permissions) 223 if count == 0: 224 raise IOError("Did not copy any files from local peer.") 225 return count
226
227 - def checkCollectIndicator(self, collectIndicator=None):
228 """ 229 Checks the collect indicator in the peer's staging directory. 230 231 When a peer has completed collecting its backup files, it will write an 232 empty indicator file into its collect directory. This method checks to 233 see whether that indicator has been written. We're "stupid" here - if 234 the collect directory doesn't exist, you'll naturally get back C{False}. 235 236 If you need to, you can override the name of the collect indicator file 237 by passing in a different name. 238 239 @param collectIndicator: Name of the collect indicator file to check 240 @type collectIndicator: String representing name of a file in the collect directory 241 242 @return: Boolean true/false depending on whether the indicator exists. 243 @raise ValueError: If a path cannot be encoded properly. 244 """ 245 collectIndicator = encodePath(collectIndicator) 246 if collectIndicator is None: 247 return os.path.exists(os.path.join(self.collectDir, DEF_COLLECT_INDICATOR)) 248 else: 249 return os.path.exists(os.path.join(self.collectDir, collectIndicator))
250
251 - def writeStageIndicator(self, stageIndicator=None, ownership=None, permissions=None):
252 """ 253 Writes the stage indicator in the peer's staging directory. 254 255 When the master has completed collecting its backup files, it will write 256 an empty indicator file into the peer's collect directory. The presence 257 of this file implies that the staging process is complete. 258 259 If you need to, you can override the name of the stage indicator file by 260 passing in a different name. 261 262 @note: If you have user/group as strings, call the L{util.getUidGid} 263 function to get the associated uid/gid as an ownership tuple. 264 265 @param stageIndicator: Name of the indicator file to write 266 @type stageIndicator: String representing name of a file in the collect directory 267 268 @param ownership: Owner and group that the indicator file should have 269 @type ownership: Tuple of numeric ids C{(uid, gid)} 270 271 @param permissions: Permissions that the indicator file should have 272 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 273 274 @raise ValueError: If collect directory is not a directory or does not exist 275 @raise ValueError: If a path cannot be encoded properly. 276 @raise IOError: If there is an IO error creating the file. 277 @raise OSError: If there is an OS error creating or changing permissions on the file 278 """ 279 stageIndicator = encodePath(stageIndicator) 280 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir): 281 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir) 282 raise ValueError("Collect directory is not a directory or does not exist on disk.") 283 if stageIndicator is None: 284 fileName = os.path.join(self.collectDir, DEF_STAGE_INDICATOR) 285 else: 286 fileName = os.path.join(self.collectDir, stageIndicator) 287 LocalPeer._copyLocalFile(None, fileName, ownership, permissions) # None for sourceFile results in an empty target
288 289 290 ################## 291 # Private methods 292 ################## 293
294 - def _copyLocalDir(sourceDir, targetDir, ownership=None, permissions=None):
295 """ 296 Copies files from the source directory to the target directory. 297 298 This function is not recursive. Only the files in the directory will be 299 copied. Ownership and permissions will be left at their default values 300 if new values are not specified. The source and target directories are 301 allowed to be soft links to a directory, but besides that soft links are 302 ignored. 303 304 @note: If you have user/group as strings, call the L{util.getUidGid} 305 function to get the associated uid/gid as an ownership tuple. 306 307 @param sourceDir: Source directory 308 @type sourceDir: String representing a directory on disk 309 310 @param targetDir: Target directory 311 @type targetDir: String representing a directory on disk 312 313 @param ownership: Owner and group that the copied files should have 314 @type ownership: Tuple of numeric ids C{(uid, gid)} 315 316 @param permissions: Permissions that the staged files should have 317 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 318 319 @return: Number of files copied from the source directory to the target directory. 320 321 @raise ValueError: If source or target is not a directory or does not exist. 322 @raise ValueError: If a path cannot be encoded properly. 323 @raise IOError: If there is an IO error copying the files. 324 @raise OSError: If there is an OS error copying or changing permissions on a files 325 """ 326 filesCopied = 0 327 sourceDir = encodePath(sourceDir) 328 targetDir = encodePath(targetDir) 329 for fileName in os.listdir(sourceDir): 330 sourceFile = os.path.join(sourceDir, fileName) 331 targetFile = os.path.join(targetDir, fileName) 332 LocalPeer._copyLocalFile(sourceFile, targetFile, ownership, permissions) 333 filesCopied += 1 334 return filesCopied
335 _copyLocalDir = staticmethod(_copyLocalDir) 336
337 - def _copyLocalFile(sourceFile=None, targetFile=None, ownership=None, permissions=None, overwrite=True):
338 """ 339 Copies a source file to a target file. 340 341 If the source file is C{None} then the target file will be created or 342 overwritten as an empty file. If the target file is C{None}, this method 343 is a no-op. Attempting to copy a soft link or a directory will result in 344 an exception. 345 346 @note: If you have user/group as strings, call the L{util.getUidGid} 347 function to get the associated uid/gid as an ownership tuple. 348 349 @note: We will not overwrite a target file that exists when this method 350 is invoked. If the target already exists, we'll raise an exception. 351 352 @param sourceFile: Source file to copy 353 @type sourceFile: String representing a file on disk, as an absolute path 354 355 @param targetFile: Target file to create 356 @type targetFile: String representing a file on disk, as an absolute path 357 358 @param ownership: Owner and group that the copied should have 359 @type ownership: Tuple of numeric ids C{(uid, gid)} 360 361 @param permissions: Permissions that the staged files should have 362 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 363 364 @param overwrite: Indicates whether it's OK to overwrite the target file. 365 @type overwrite: Boolean true/false. 366 367 @raise ValueError: If the passed-in source file is not a regular file. 368 @raise ValueError: If a path cannot be encoded properly. 369 @raise IOError: If the target file already exists. 370 @raise IOError: If there is an IO error copying the file 371 @raise OSError: If there is an OS error copying or changing permissions on a file 372 """ 373 targetFile = encodePath(targetFile) 374 sourceFile = encodePath(sourceFile) 375 if targetFile is None: 376 return 377 if not overwrite: 378 if os.path.exists(targetFile): 379 raise IOError("Target file [%s] already exists." % targetFile) 380 if sourceFile is None: 381 open(targetFile, "w").write("") 382 else: 383 if os.path.isfile(sourceFile) and not os.path.islink(sourceFile): 384 shutil.copy(sourceFile, targetFile) 385 else: 386 logger.debug("Source [%s] is not a regular file." % sourceFile) 387 raise ValueError("Source is not a regular file.") 388 if ownership is not None: 389 os.chown(targetFile, ownership[0], ownership[1]) 390 if permissions is not None: 391 os.chmod(targetFile, permissions)
392 _copyLocalFile = staticmethod(_copyLocalFile)
393 394 395 ######################################################################## 396 # RemotePeer class definition 397 ######################################################################## 398
399 -class RemotePeer(object):
400 401 ###################### 402 # Class documentation 403 ###################### 404 405 """ 406 Backup peer representing a remote peer in a backup pool. 407 408 This is a class representing a remote (networked) peer in a backup pool. 409 Remote peers are backed up using an rcp-compatible copy command. A remote 410 peer has associated with it a name (which must be a valid hostname), a 411 collect directory, a working directory and a copy method (an rcp-compatible 412 command). 413 414 You can also set an optional local user value. This username will be used 415 as the local user for any remote copies that are required. It can only be 416 used if the root user is executing the backup. The root user will C{su} to 417 the local user and execute the remote copies as that user. 418 419 The copy method is associated with the peer and not with the actual request 420 to copy, because we can envision that each remote host might have a 421 different connect method. 422 423 The public methods other than the constructor are part of a "backup peer" 424 interface shared with the C{LocalPeer} class. 425 426 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator, 427 _getDirContents, _copyRemoteDir, _copyRemoteFile, _pushLocalFile, 428 name, collectDir, remoteUser, rcpCommand 429 """ 430 431 ############## 432 # Constructor 433 ############## 434
435 - def __init__(self, name, collectDir, workingDir, remoteUser, rcpCommand=None, localUser=None):
436 """ 437 Initializes a remote backup peer. 438 439 @note: If provided, the rcp command will eventually be parsed into a list 440 of strings suitable for passing to C{popen2.Popen4} in order to avoid 441 security holes related to shell interpolation. This parsing will be 442 done by the L{util.splitCommandLine} function. See the documentation for 443 that function for some important notes about its limitations. 444 445 @param name: Name of the backup peer 446 @type name: String, must be a valid DNS hostname 447 448 @param collectDir: Path to the peer's collect directory 449 @type collectDir: String representing an absolute path on the remote peer 450 451 @param workingDir: Working directory that can be used to create temporary files, etc. 452 @type workingDir: String representing an absolute path on the current host. 453 454 @param remoteUser: Name of the Cedar Backup user on the remote peer 455 @type remoteUser: String representing a username, valid via the copy command 456 457 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 458 @type rcpCommand: String representing a system command including required arguments 459 460 @param localUser: Name of the Cedar Backup user on the current host 461 @type localUser: String representing a username, valid on the current host 462 463 @raise ValueError: If collect directory is not an absolute path 464 """ 465 self._name = None 466 self._collectDir = None 467 self._workingDir = None 468 self._remoteUser = None 469 self._localUser = None 470 self._rcpCommand = None 471 self._rcpCommandList = None 472 self.name = name 473 self.collectDir = collectDir 474 self.workingDir = workingDir 475 self.remoteUser = remoteUser 476 self.localUser = localUser 477 self.rcpCommand = rcpCommand
478 479 480 ############# 481 # Properties 482 ############# 483
484 - def _setName(self, value):
485 """ 486 Property target used to set the peer name. 487 The value must be a non-empty string and cannot be C{None}. 488 @raise ValueError: If the value is an empty string or C{None}. 489 """ 490 if value is None or len(value) < 1: 491 raise ValueError("Peer name must be a non-empty string.") 492 self._name = value
493
494 - def _getName(self):
495 """ 496 Property target used to get the peer name. 497 """ 498 return self._name
499
500 - def _setCollectDir(self, value):
501 """ 502 Property target used to set the collect directory. 503 The value must be an absolute path and cannot be C{None}. 504 It does not have to exist on disk at the time of assignment. 505 @raise ValueError: If the value is C{None} or is not an absolute path. 506 @raise ValueError: If the value cannot be encoded properly. 507 """ 508 if value is None or not os.path.isabs(value): 509 raise ValueError("Collect directory must be an absolute path.") 510 self._collectDir = encodePath(value)
511
512 - def _getCollectDir(self):
513 """ 514 Property target used to get the collect directory. 515 """ 516 return self._collectDir
517
518 - def _setWorkingDir(self, value):
519 """ 520 Property target used to set the working directory. 521 The value must be an absolute path and cannot be C{None}. 522 @raise ValueError: If the value is C{None} or is not an absolute path. 523 @raise ValueError: If the value cannot be encoded properly. 524 """ 525 if value is None or not os.path.isabs(value): 526 raise ValueError("Working directory must be an absolute path.") 527 self._workingDir = encodePath(value)
528
529 - def _getWorkingDir(self):
530 """ 531 Property target used to get the working directory. 532 """ 533 return self._workingDir
534
535 - def _setRemoteUser(self, value):
536 """ 537 Property target used to set the remote user. 538 The value must be a non-empty string and cannot be C{None}. 539 @raise ValueError: If the value is an empty string or C{None}. 540 """ 541 if value is None or len(value) < 1: 542 raise ValueError("Peer remote user must be a non-empty string.") 543 self._remoteUser = value
544
545 - def _getRemoteUser(self):
546 """ 547 Property target used to get the remote user. 548 """ 549 return self._remoteUser
550
551 - def _setLocalUser(self, value):
552 """ 553 Property target used to set the local user. 554 The value must be a non-empty string if it is not C{None}. 555 @raise ValueError: If the value is an empty string. 556 """ 557 if value is not None: 558 if len(value) < 1: 559 raise ValueError("Peer local user must be a non-empty string.") 560 self._localUser = value
561
562 - def _getLocalUser(self):
563 """ 564 Property target used to get the local user. 565 """ 566 return self._localUser
567
568 - def _setRcpCommand(self, value):
569 """ 570 Property target to set the rcp command. 571 572 The value must be a non-empty string or C{None}. Its value is stored in 573 the two forms: "raw" as provided by the client, and "parsed" into a list 574 suitable for being passed to L{util.executeCommand} via 575 L{util.splitCommandLine}. 576 577 However, all the caller will ever see via the property is the actual 578 value they set (which includes seeing C{None}, even if we translate that 579 internally to C{DEF_RCP_COMMAND}). Internally, we should always use 580 C{self._rcpCommandList} if we want the actual command list. 581 582 @raise ValueError: If the value is an empty string. 583 """ 584 if value is None: 585 self._rcpCommand = None 586 self._rcpCommandList = DEF_RCP_COMMAND 587 else: 588 if len(value) >= 1: 589 self._rcpCommand = value 590 self._rcpCommandList = splitCommandLine(self._rcpCommand) 591 else: 592 raise ValueError("The rcp command must be a non-empty string.")
593
594 - def _getRcpCommand(self):
595 """ 596 Property target used to get the rcp command. 597 """ 598 return self._rcpCommand
599 600 name = property(_getName, _setName, None, "Name of the peer (a valid DNS hostname).") 601 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).") 602 workingDir = property(_getWorkingDir, _setWorkingDir, None, "Path to the peer's working directory (an absolute local path).") 603 remoteUser = property(_getRemoteUser, _setRemoteUser, None, "Name of the Cedar Backup user on the remote peer.") 604 localUser = property(_getLocalUser, _setLocalUser, None, "Name of the Cedar Backup user on the current host.") 605 rcpCommand = property(_getRcpCommand, _setRcpCommand, None, "An rcp-compatible copy command to use for copying files.") 606 607 608 ################# 609 # Public methods 610 ################# 611
612 - def stagePeer(self, targetDir, ownership=None, permissions=None):
613 """ 614 Stages data from the peer into the indicated local target directory. 615 616 The target directory must already exist before this method is called. If 617 passed in, ownership and permissions will be applied to the files that 618 are copied. 619 620 @note: The returned count of copied files might be inaccurate if some of 621 the copied files already existed in the staging directory prior to the 622 copy taking place. We don't clear the staging directory first, because 623 some extension might also be using it. 624 625 @note: If you have user/group as strings, call the L{util.getUidGid} function 626 to get the associated uid/gid as an ownership tuple. 627 628 @note: Unlike the local peer version of this method, an I/O error might 629 or might not be raised if the directory is empty. Since we're using a 630 remote copy method, we just don't have the fine-grained control over our 631 exceptions that's available when we can look directly at the filesystem, 632 and we can't control whether the remote copy method thinks an empty 633 directory is an error. 634 635 @param targetDir: Target directory to write data into 636 @type targetDir: String representing a directory on disk 637 638 @param ownership: Owner and group that the staged files should have 639 @type ownership: Tuple of numeric ids C{(uid, gid)} 640 641 @param permissions: Permissions that the staged files should have 642 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 643 644 @return: Number of files copied from the source directory to the target directory. 645 646 @raise ValueError: If target directory is not a directory, does not exist or is not absolute. 647 @raise ValueError: If a path cannot be encoded properly. 648 @raise IOError: If there were no files to stage (i.e. the directory was empty) 649 @raise IOError: If there is an IO error copying a file. 650 @raise OSError: If there is an OS error copying or changing permissions on a file 651 """ 652 targetDir = encodePath(targetDir) 653 if not os.path.isabs(targetDir): 654 logger.debug("Target directory [%s] not an absolute path." % targetDir) 655 raise ValueError("Target directory must be an absolute path.") 656 if not os.path.exists(targetDir) or not os.path.isdir(targetDir): 657 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir) 658 raise ValueError("Target directory is not a directory or does not exist on disk.") 659 count = RemotePeer._copyRemoteDir(self.remoteUser, self.localUser, self.name, 660 self._rcpCommand, self._rcpCommandList, 661 self.collectDir, targetDir, 662 ownership, permissions) 663 if count == 0: 664 raise IOError("Did not copy any files from local peer.") 665 return count
666 667
668 - def checkCollectIndicator(self, collectIndicator=None):
669 """ 670 Checks the collect indicator in the peer's staging directory. 671 672 When a peer has completed collecting its backup files, it will write an 673 empty indicator file into its collect directory. This method checks to 674 see whether that indicator has been written. If the remote copy command 675 fails, we return C{False} as if the file weren't there. 676 677 If you need to, you can override the name of the collect indicator file 678 by passing in a different name. 679 680 @note: Apparently, we can't count on all rcp-compatible implementations 681 to return sensible errors for some error conditions. As an example, the 682 C{scp} command in Debian 'woody' returns a zero (normal) status even when 683 it can't find a host or if the login or path is invalid. Because of 684 this, the implementation of this method is rather convoluted. 685 686 @param collectIndicator: Name of the collect indicator file to check 687 @type collectIndicator: String representing name of a file in the collect directory 688 689 @return: Boolean true/false depending on whether the indicator exists. 690 @raise ValueError: If a path cannot be encoded properly. 691 """ 692 try: 693 if collectIndicator is None: 694 sourceFile = os.path.join(self.collectDir, DEF_COLLECT_INDICATOR) 695 targetFile = os.path.join(self.workingDir, DEF_COLLECT_INDICATOR) 696 else: 697 collectIndicator = encodePath(collectIndicator) 698 sourceFile = os.path.join(self.collectDir, collectIndicator) 699 targetFile = os.path.join(self.workingDir, collectIndicator) 700 logger.debug("Fetch remote [%s] into [%s]." % (sourceFile, targetFile)) 701 if os.path.exists(targetFile): 702 try: 703 os.remove(targetFile) 704 except: 705 raise Exception("Error: collect indicator [%s] already exists!" % targetFile) 706 try: 707 RemotePeer._copyRemoteFile(self.remoteUser, self.localUser, self.name, 708 self._rcpCommand, self._rcpCommandList, 709 sourceFile, targetFile, 710 overwrite=False) 711 if os.path.exists(targetFile): 712 return True 713 else: 714 return False 715 except: 716 return False 717 finally: 718 if os.path.exists(targetFile): 719 try: 720 os.remove(targetFile) 721 except: pass
722
723 - def writeStageIndicator(self, stageIndicator=None):
724 """ 725 Writes the stage indicator in the peer's staging directory. 726 727 When the master has completed collecting its backup files, it will write 728 an empty indicator file into the peer's collect directory. The presence 729 of this file implies that the staging process is complete. 730 731 If you need to, you can override the name of the stage indicator file by 732 passing in a different name. 733 734 @note: If you have user/group as strings, call the L{util.getUidGid} function 735 to get the associated uid/gid as an ownership tuple. 736 737 @param stageIndicator: Name of the indicator file to write 738 @type stageIndicator: String representing name of a file in the collect directory 739 740 @raise ValueError: If a path cannot be encoded properly. 741 @raise IOError: If there is an IO error creating the file. 742 @raise OSError: If there is an OS error creating or changing permissions on the file 743 """ 744 stageIndicator = encodePath(stageIndicator) 745 if stageIndicator is None: 746 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR) 747 targetFile = os.path.join(self.collectDir, DEF_STAGE_INDICATOR) 748 else: 749 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR) 750 targetFile = os.path.join(self.collectDir, stageIndicator) 751 try: 752 if not os.path.exists(sourceFile): 753 open(sourceFile, "w").write("") 754 RemotePeer._pushLocalFile(self.remoteUser, self.localUser, self.name, 755 self._rcpCommand, self._rcpCommandList, 756 sourceFile, targetFile) 757 finally: 758 if os.path.exists(sourceFile): 759 try: 760 os.remove(sourceFile) 761 except: pass
762 763 764 ################## 765 # Private methods 766 ################## 767
768 - def _getDirContents(path):
769 """ 770 Returns the contents of a directory in terms of a Set. 771 772 The directory's contents are read as a L{FilesystemList} containing only 773 files, and then the list is converted into a C{sets.Set} object for later 774 use. 775 776 @param path: Directory path to get contents for 777 @type path: String representing a path on disk 778 779 @return: Set of files in the directory 780 @raise ValueError: If path is not a directory or does not exist. 781 """ 782 contents = FilesystemList() 783 contents.excludeDirs = True 784 contents.excludeLinks = True 785 contents.addDirContents(path) 786 return sets.Set(contents)
787 _getDirContents = staticmethod(_getDirContents) 788
789 - def _copyRemoteDir(remoteUser, localUser, remoteHost, rcpCommand, rcpCommandList, 790 sourceDir, targetDir, ownership=None, permissions=None):
791 """ 792 Copies files from the source directory to the target directory. 793 794 This function is not recursive. Only the files in the directory will be 795 copied. Ownership and permissions will be left at their default values 796 if new values are not specified. Behavior when copying soft links from 797 the collect directory is dependent on the behavior of the specified rcp 798 command. 799 800 @note: The returned count of copied files might be inaccurate if some of 801 the copied files already existed in the staging directory prior to the 802 copy taking place. We don't clear the staging directory first, because 803 some extension might also be using it. 804 805 @note: If you have user/group as strings, call the L{util.getUidGid} function 806 to get the associated uid/gid as an ownership tuple. 807 808 @note: We don't have a good way of knowing exactly what files we copied 809 down from the remote peer, unless we want to parse the output of the rcp 810 command (ugh). We could change permissions on everything in the target 811 directory, but that's kind of ugly too. Instead, we use Python's set 812 functionality to figure out what files were added while we executed the 813 rcp command. This isn't perfect - for instance, it's not correct if 814 someone else is messing with the directory at the same time we're doing 815 the remote copy - but it's about as good as we're going to get. 816 817 @note: Apparently, we can't count on all rcp-compatible implementations 818 to return sensible errors for some error conditions. As an example, the 819 C{scp} command in Debian 'woody' returns a zero (normal) status even 820 when it can't find a host or if the login or path is invalid. We try 821 to work around this by issuing C{IOError} if we don't copy any files from 822 the remote host. 823 824 @param remoteUser: Name of the Cedar Backup user on the remote peer 825 @type remoteUser: String representing a username, valid via the copy command 826 827 @param localUser: Name of the Cedar Backup user on the current host 828 @type localUser: String representing a username, valid on the current host 829 830 @param remoteHost: Hostname of the remote peer 831 @type remoteHost: String representing a hostname, accessible via the copy command 832 833 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 834 @type rcpCommand: String representing a system command including required arguments 835 836 @param rcpCommandList: An rcp-compatible copy command to use for copying files 837 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand} 838 839 @param sourceDir: Source directory 840 @type sourceDir: String representing a directory on disk 841 842 @param targetDir: Target directory 843 @type targetDir: String representing a directory on disk 844 845 @param ownership: Owner and group that the copied files should have 846 @type ownership: Tuple of numeric ids C{(uid, gid)} 847 848 @param permissions: Permissions that the staged files should have 849 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 850 851 @return: Number of files copied from the source directory to the target directory. 852 853 @raise ValueError: If source or target is not a directory or does not exist. 854 @raise IOError: If there is an IO error copying the files. 855 """ 856 beforeSet = RemotePeer._getDirContents(targetDir) 857 if localUser is not None: 858 try: 859 if os.getuid() != 0: 860 raise IOError("Only root can remote copy as another user.") 861 except AttributeError: pass 862 actualCommand = "%s %s@%s:%s/* %s" % (rcpCommand, remoteUser, remoteHost, sourceDir, targetDir) 863 command = resolveCommand(SU_COMMAND) 864 result = executeCommand(command, [localUser, "-c", actualCommand])[0] 865 if result != 0: 866 raise IOError("Error (%d) copying files from remote host as local user [%s]." % (result, localUser)) 867 else: 868 copySource = "%s@%s:%s/*" % (remoteUser, remoteHost, sourceDir) 869 command = resolveCommand(rcpCommandList) 870 result = executeCommand(command, [copySource, targetDir])[0] 871 if result != 0: 872 raise IOError("Error (%d) copying files from remote host (using no local user)." % result) 873 afterSet = RemotePeer._getDirContents(targetDir) 874 if len(afterSet) == 0: 875 raise IOError("Did not copy any files from remote peer.") 876 differenceSet = afterSet.difference(beforeSet) # files we added as part of copy 877 if len(differenceSet) == 0: 878 raise IOError("Apparently did not copy any new files from remote peer.") 879 for targetFile in differenceSet: 880 if ownership is not None: 881 os.chown(targetFile, ownership[0], ownership[1]) 882 if permissions is not None: 883 os.chmod(targetFile, permissions) 884 return len(differenceSet)
885 _copyRemoteDir = staticmethod(_copyRemoteDir) 886
887 - def _copyRemoteFile(remoteUser, localUser, remoteHost, 888 rcpCommand, rcpCommandList, 889 sourceFile, targetFile, ownership=None, 890 permissions=None, overwrite=True):
891 """ 892 Copies a remote source file to a target file. 893 894 @note: Internally, we have to go through and escape any spaces in the 895 source path with double-backslash, otherwise things get screwed up. It 896 doesn't seem to be required in the target path. I hope this is portable 897 to various different rcp methods, but I guess it might not be (all I have 898 to test with is OpenSSH). 899 900 @note: If you have user/group as strings, call the L{util.getUidGid} function 901 to get the associated uid/gid as an ownership tuple. 902 903 @note: We will not overwrite a target file that exists when this method 904 is invoked. If the target already exists, we'll raise an exception. 905 906 @note: Apparently, we can't count on all rcp-compatible implementations 907 to return sensible errors for some error conditions. As an example, the 908 C{scp} command in Debian 'woody' returns a zero (normal) status even when 909 it can't find a host or if the login or path is invalid. We try to work 910 around this by issuing C{IOError} the target file does not exist when 911 we're done. 912 913 @param remoteUser: Name of the Cedar Backup user on the remote peer 914 @type remoteUser: String representing a username, valid via the copy command 915 916 @param remoteHost: Hostname of the remote peer 917 @type remoteHost: String representing a hostname, accessible via the copy command 918 919 @param localUser: Name of the Cedar Backup user on the current host 920 @type localUser: String representing a username, valid on the current host 921 922 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 923 @type rcpCommand: String representing a system command including required arguments 924 925 @param rcpCommandList: An rcp-compatible copy command to use for copying files 926 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand} 927 928 @param sourceFile: Source file to copy 929 @type sourceFile: String representing a file on disk, as an absolute path 930 931 @param targetFile: Target file to create 932 @type targetFile: String representing a file on disk, as an absolute path 933 934 @param ownership: Owner and group that the copied should have 935 @type ownership: Tuple of numeric ids C{(uid, gid)} 936 937 @param permissions: Permissions that the staged files should have 938 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 939 940 @param overwrite: Indicates whether it's OK to overwrite the target file. 941 @type overwrite: Boolean true/false. 942 943 @raise IOError: If the target file already exists. 944 @raise IOError: If there is an IO error copying the file 945 @raise OSError: If there is an OS error changing permissions on the file 946 """ 947 if not overwrite: 948 if os.path.exists(targetFile): 949 raise IOError("Target file [%s] already exists." % targetFile) 950 if localUser is not None: 951 try: 952 if os.getuid() != 0: 953 raise IOError("Only root can remote copy as another user.") 954 except AttributeError: pass 955 actualCommand = "%s %s@%s:%s %s" % (rcpCommand, remoteUser, remoteHost, sourceFile.replace(" ", "\\ "), targetFile) 956 command = resolveCommand(SU_COMMAND) 957 result = executeCommand(command, [localUser, "-c", actualCommand])[0] 958 if result != 0: 959 raise IOError("Error (%d) copying [%s] from remote host as local user [%s]." % (result, sourceFile, localUser)) 960 else: 961 copySource = "%s@%s:%s" % (remoteUser, remoteHost, sourceFile.replace(" ", "\\ ")) 962 command = resolveCommand(rcpCommandList) 963 result = executeCommand(command, [copySource, targetFile])[0] 964 if result != 0: 965 raise IOError("Error (%d) copying [%s] from remote host (using no local user)." % (result, sourceFile)) 966 if not os.path.exists(targetFile): 967 raise IOError("Apparently unable to copy file from remote host.") 968 if ownership is not None: 969 os.chown(targetFile, ownership[0], ownership[1]) 970 if permissions is not None: 971 os.chmod(targetFile, permissions)
972 _copyRemoteFile = staticmethod(_copyRemoteFile) 973
974 - def _pushLocalFile(remoteUser, localUser, remoteHost, 975 rcpCommand, rcpCommandList, 976 sourceFile, targetFile, overwrite=True):
977 """ 978 Copies a local source file to a remote host. 979 980 @note: We will not overwrite a target file that exists when this method 981 is invoked. If the target already exists, we'll raise an exception. 982 983 @note: Internally, we have to go through and escape any spaces in the 984 source and target paths with double-backslash, otherwise things get 985 screwed up. I hope this is portable to various different rcp methods, 986 but I guess it might not be (all I have to test with is OpenSSH). 987 988 @note: If you have user/group as strings, call the L{util.getUidGid} function 989 to get the associated uid/gid as an ownership tuple. 990 991 @param remoteUser: Name of the Cedar Backup user on the remote peer 992 @type remoteUser: String representing a username, valid via the copy command 993 994 @param localUser: Name of the Cedar Backup user on the current host 995 @type localUser: String representing a username, valid on the current host 996 997 @param remoteHost: Hostname of the remote peer 998 @type remoteHost: String representing a hostname, accessible via the copy command 999 1000 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 1001 @type rcpCommand: String representing a system command including required arguments 1002 1003 @param rcpCommandList: An rcp-compatible copy command to use for copying files 1004 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand} 1005 1006 @param sourceFile: Source file to copy 1007 @type sourceFile: String representing a file on disk, as an absolute path 1008 1009 @param targetFile: Target file to create 1010 @type targetFile: String representing a file on disk, as an absolute path 1011 1012 @param overwrite: Indicates whether it's OK to overwrite the target file. 1013 @type overwrite: Boolean true/false. 1014 1015 @raise IOError: If there is an IO error copying the file 1016 @raise OSError: If there is an OS error changing permissions on the file 1017 """ 1018 if not overwrite: 1019 if os.path.exists(targetFile): 1020 raise IOError("Target file [%s] already exists." % targetFile) 1021 if localUser is not None: 1022 try: 1023 if os.getuid() != 0: 1024 raise IOError("Only root can remote copy as another user.") 1025 except AttributeError: pass 1026 actualCommand = '%s "%s" "%s@%s:%s"' % (rcpCommand, sourceFile, remoteUser, remoteHost, targetFile) 1027 command = resolveCommand(SU_COMMAND) 1028 result = executeCommand(command, [localUser, "-c", actualCommand])[0] 1029 if result != 0: 1030 raise IOError("Error (%d) copying [%s] to remote host as local user [%s]." % (result, sourceFile, localUser)) 1031 else: 1032 copyTarget = "%s@%s:%s" % (remoteUser, remoteHost, targetFile.replace(" ", "\\ ")) 1033 command = resolveCommand(rcpCommandList) 1034 result = executeCommand(command, [sourceFile.replace(" ", "\\ "), copyTarget])[0] 1035 if result != 0: 1036 raise IOError("Error (%d) copying [%s] to remote host (using no local user)." % (result, sourceFile))
1037 _pushLocalFile = staticmethod(_pushLocalFile)
1038