1 """
2 connection operations
3
4 Connection instances are used to communicate with the remote service at
5 the account level creating, listing and deleting Containers, and returning
6 Container instances.
7
8 See COPYING for license information.
9 """
10
11 import socket
12 import os
13 from urllib import quote
14 from httplib import HTTPSConnection, HTTPConnection, HTTPException
15 from container import Container, ContainerResults
16 from utils import unicode_quote, parse_url, THTTPConnection, THTTPSConnection
17 from errors import ResponseError, NoSuchContainer, ContainerNotEmpty, \
18 InvalidContainerName, CDNNotEnabled
19 from Queue import Queue, Empty, Full
20 from time import time
21 import consts
22 from authentication import Authentication
23 from fjson import json_loads
24 from sys import version_info
25
26
27
28
29
31 """
32 Manages the connection to the storage system and serves as a factory
33 for Container instances.
34
35 @undocumented: cdn_connect
36 @undocumented: http_connect
37 @undocumented: cdn_request
38 @undocumented: make_request
39 @undocumented: _check_container_name
40 """
41
42 - def __init__(self, username=None, api_key=None, timeout=5, **kwargs):
43 """
44 Accepts keyword arguments for Mosso username and api key.
45 Optionally, you can omit these keywords and supply an
46 Authentication object using the auth keyword. Setting the argument
47 servicenet to True will make use of Rackspace servicenet network.
48
49 @type username: str
50 @param username: a Mosso username
51 @type api_key: str
52 @param api_key: a Mosso API key
53 @type servicenet: bool
54 @param servicenet: Use Rackspace servicenet to access Cloud Files.
55 @type cdn_log_retention: bool
56 @param cdn_log_retention: set logs retention for this cdn enabled
57 container.
58 """
59 self.cdn_enabled = False
60 self.cdn_args = None
61 self.connection_args = None
62 self.cdn_connection = None
63 self.connection = None
64 self.token = None
65 self.debuglevel = int(kwargs.get('debuglevel', 0))
66 self.servicenet = kwargs.get('servicenet', False)
67 self.user_agent = kwargs.get('useragent', consts.user_agent)
68 self.timeout = timeout
69
70
71
72 if not 'servicenet' in kwargs \
73 and 'RACKSPACE_SERVICENET' in os.environ:
74 self.servicenet = True
75
76 self.auth = 'auth' in kwargs and kwargs['auth'] or None
77
78 if not self.auth:
79 authurl = kwargs.get('authurl', consts.us_authurl)
80 if username and api_key and authurl:
81 self.auth = Authentication(username, api_key, authurl=authurl,
82 useragent=self.user_agent)
83 else:
84 raise TypeError("Incorrect or invalid arguments supplied")
85
86 self._authenticate()
87
89 """
90 Authenticate and setup this instance with the values returned.
91 """
92 (url, self.cdn_url, self.token) = self.auth.authenticate()
93 url = self._set_storage_url(url)
94 self.connection_args = parse_url(url)
95
96 if version_info[0] <= 2 and version_info[1] < 6:
97 self.conn_class = self.connection_args[3] and THTTPSConnection or \
98 THTTPConnection
99 else:
100 self.conn_class = self.connection_args[3] and HTTPSConnection or \
101 HTTPConnection
102 self.http_connect()
103 if self.cdn_url:
104 self.cdn_connect()
105
107 if self.servicenet:
108 return "https://snet-%s" % url.replace("https://", "")
109 return url
110
112 """
113 Setup the http connection instance for the CDN service.
114 """
115 (host, port, cdn_uri, is_ssl) = parse_url(self.cdn_url)
116 self.cdn_connection = self.conn_class(host, port, timeout=self.timeout)
117 self.cdn_enabled = True
118
120 """
121 Setup the http connection instance.
122 """
123 (host, port, self.uri, is_ssl) = self.connection_args
124 self.connection = self.conn_class(host, port=port, \
125 timeout=self.timeout)
126 self.connection.set_debuglevel(self.debuglevel)
127
128 - def cdn_request(self, method, path=[], data='', hdrs=None):
129 """
130 Given a method (i.e. GET, PUT, POST, etc), a path, data, header and
131 metadata dicts, performs an http request against the CDN service.
132 """
133 if not self.cdn_enabled:
134 raise CDNNotEnabled()
135
136 path = '/%s/%s' % \
137 (self.uri.rstrip('/'), '/'.join([unicode_quote(i) for i in path]))
138 headers = {'Content-Length': str(len(data)),
139 'User-Agent': self.user_agent,
140 'X-Auth-Token': self.token}
141 if isinstance(hdrs, dict):
142 headers.update(hdrs)
143
144 def retry_request():
145 '''Re-connect and re-try a failed request once'''
146 self.cdn_connect()
147 self.cdn_connection.request(method, path, data, headers)
148 return self.cdn_connection.getresponse()
149
150 try:
151 self.cdn_connection.request(method, path, data, headers)
152 response = self.cdn_connection.getresponse()
153 except (socket.error, IOError, HTTPException):
154 response = retry_request()
155 if response.status == 401:
156 self._authenticate()
157 headers['X-Auth-Token'] = self.token
158 response = retry_request()
159
160 return response
161
162 - def make_request(self, method, path=[], data='', hdrs=None, parms=None):
163 """
164 Given a method (i.e. GET, PUT, POST, etc), a path, data, header and
165 metadata dicts, and an optional dictionary of query parameters,
166 performs an http request.
167 """
168 path = '/%s/%s' % \
169 (self.uri.rstrip('/'), '/'.join([unicode_quote(i) for i in path]))
170
171 if isinstance(parms, dict) and parms:
172 query_args = \
173 ['%s=%s' % (quote(x),
174 quote(str(y))) for (x, y) in parms.items()]
175 path = '%s?%s' % (path, '&'.join(query_args))
176
177 headers = {'Content-Length': str(len(data)),
178 'User-Agent': self.user_agent,
179 'X-Auth-Token': self.token}
180 isinstance(hdrs, dict) and headers.update(hdrs)
181
182 def retry_request():
183 '''Re-connect and re-try a failed request once'''
184 self.http_connect()
185 self.connection.request(method, path, data, headers)
186 return self.connection.getresponse()
187
188 try:
189 self.connection.request(method, path, data, headers)
190 response = self.connection.getresponse()
191 except (socket.error, IOError, HTTPException):
192 response = retry_request()
193 if response.status == 401:
194 self._authenticate()
195 headers['X-Auth-Token'] = self.token
196 response = retry_request()
197
198 return response
199
201 """
202 Return tuple for number of containers and total bytes in the account
203
204 >>> connection.get_info()
205 (5, 2309749)
206
207 @rtype: tuple
208 @return: a tuple containing the number of containers and total bytes
209 used by the account
210 """
211 response = self.make_request('HEAD')
212 count = size = None
213 for hdr in response.getheaders():
214 if hdr[0].lower() == 'x-account-container-count':
215 try:
216 count = int(hdr[1])
217 except ValueError:
218 count = 0
219 if hdr[0].lower() == 'x-account-bytes-used':
220 try:
221 size = int(hdr[1])
222 except ValueError:
223 size = 0
224 buff = response.read()
225 if (response.status < 200) or (response.status > 299):
226 raise ResponseError(response.status, response.reason)
227 return (count, size)
228
230 if not container_name or \
231 '/' in container_name or \
232 len(container_name) > consts.container_name_limit:
233 raise InvalidContainerName(container_name)
234
236 """
237 Given a container name, returns a L{Container} item, creating a new
238 Container if one does not already exist.
239
240 >>> connection.create_container('new_container')
241 <cloudfiles.container.Container object at 0xb77d628c>
242
243 @param container_name: name of the container to create
244 @type container_name: str
245 @rtype: L{Container}
246 @return: an object representing the newly created container
247 """
248 self._check_container_name(container_name)
249
250 response = self.make_request('PUT', [container_name])
251 buff = response.read()
252 if (response.status < 200) or (response.status > 299):
253 raise ResponseError(response.status, response.reason)
254 return Container(self, container_name)
255
257 """
258 Given a container name, delete it.
259
260 >>> connection.delete_container('old_container')
261
262 @param container_name: name of the container to delete
263 @type container_name: str
264 """
265 if isinstance(container_name, Container):
266 container_name = container_name.name
267 self._check_container_name(container_name)
268
269 response = self.make_request('DELETE', [container_name])
270
271 if (response.status == 409):
272 raise ContainerNotEmpty(container_name)
273 elif (response.status == 404):
274 raise NoSuchContainer
275 elif (response.status < 200) or (response.status > 299):
276 raise ResponseError(response.status, response.reason)
277
278 if self.cdn_enabled:
279 response = self.cdn_request('POST', [container_name],
280 hdrs={'X-CDN-Enabled': 'False'})
281
283 """
284 Returns a Container item result set.
285
286 >>> connection.get_all_containers()
287 ContainerResults: 4 containers
288 >>> print ', '.join([container.name for container in
289 connection.get_all_containers()])
290 new_container, old_container, pictures, music
291
292 @rtype: L{ContainerResults}
293 @return: an iterable set of objects representing all containers on the
294 account
295 @param limit: number of results to return, up to 10,000
296 @type limit: int
297 @param marker: return only results whose name is greater than "marker"
298 @type marker: str
299 """
300 if limit:
301 parms['limit'] = limit
302 if marker:
303 parms['marker'] = marker
304 return ContainerResults(self, self.list_containers_info(**parms))
305
307 """
308 Return a single Container item for the given Container.
309
310 >>> connection.get_container('old_container')
311 <cloudfiles.container.Container object at 0xb77d628c>
312 >>> container = connection.get_container('old_container')
313 >>> container.size_used
314 23074
315
316 @param container_name: name of the container to create
317 @type container_name: str
318 @rtype: L{Container}
319 @return: an object representing the container
320 """
321 self._check_container_name(container_name)
322
323 response = self.make_request('HEAD', [container_name])
324 count = size = None
325 for hdr in response.getheaders():
326 if hdr[0].lower() == 'x-container-object-count':
327 try:
328 count = int(hdr[1])
329 except ValueError:
330 count = 0
331 if hdr[0].lower() == 'x-container-bytes-used':
332 try:
333 size = int(hdr[1])
334 except ValueError:
335 size = 0
336 buff = response.read()
337 if response.status == 404:
338 raise NoSuchContainer(container_name)
339 if (response.status < 200) or (response.status > 299):
340 raise ResponseError(response.status, response.reason)
341 return Container(self, container_name, count, size)
342
344 """
345 Returns a list of containers that have been published to the CDN.
346
347 >>> connection.list_public_containers()
348 ['container1', 'container2', 'container3']
349
350 @rtype: list(str)
351 @return: a list of all CDN-enabled container names as strings
352 """
353 response = self.cdn_request('GET', [''])
354 if (response.status < 200) or (response.status > 299):
355 buff = response.read()
356 raise ResponseError(response.status, response.reason)
357 return response.read().splitlines()
358
360 """
361 Returns a list of Containers, including object count and size.
362
363 >>> connection.list_containers_info()
364 [{u'count': 510, u'bytes': 2081717, u'name': u'new_container'},
365 {u'count': 12, u'bytes': 23074, u'name': u'old_container'},
366 {u'count': 0, u'bytes': 0, u'name': u'container1'},
367 {u'count': 0, u'bytes': 0, u'name': u'container2'},
368 {u'count': 0, u'bytes': 0, u'name': u'container3'},
369 {u'count': 3, u'bytes': 2306, u'name': u'test'}]
370
371 @rtype: list({"name":"...", "count":..., "bytes":...})
372 @return: a list of all container info as dictionaries with the
373 keys "name", "count", and "bytes"
374 @param limit: number of results to return, up to 10,000
375 @type limit: int
376 @param marker: return only results whose name is greater than "marker"
377 @type marker: str
378 """
379 if limit:
380 parms['limit'] = limit
381 if marker:
382 parms['marker'] = marker
383 parms['format'] = 'json'
384 response = self.make_request('GET', [''], parms=parms)
385 if (response.status < 200) or (response.status > 299):
386 buff = response.read()
387 raise ResponseError(response.status, response.reason)
388 return json_loads(response.read())
389
391 """
392 Returns a list of Containers.
393
394 >>> connection.list_containers()
395 ['new_container',
396 'old_container',
397 'container1',
398 'container2',
399 'container3',
400 'test']
401
402 @rtype: list(str)
403 @return: a list of all containers names as strings
404 @param limit: number of results to return, up to 10,000
405 @type limit: int
406 @param marker: return only results whose name is greater than "marker"
407 @type marker: str
408 """
409 if limit:
410 parms['limit'] = limit
411 if marker:
412 parms['marker'] = marker
413 response = self.make_request('GET', [''], parms=parms)
414 if (response.status < 200) or (response.status > 299):
415 buff = response.read()
416 raise ResponseError(response.status, response.reason)
417 return response.read().splitlines()
418
420 """
421 Container objects can be grabbed from a connection using index
422 syntax.
423
424 >>> container = conn['old_container']
425 >>> container.size_used
426 23074
427
428 @rtype: L{Container}
429 @return: an object representing the container
430 """
431 return self.get_container(key)
432
433
435 """
436 A thread-safe connection pool object.
437
438 This component isn't required when using the cloudfiles library, but it may
439 be useful when building threaded applications.
440 """
441
442 - def __init__(self, username=None, api_key=None, **kwargs):
443 auth = kwargs.get('auth', None)
444 self.timeout = kwargs.get('timeout', 5)
445 self.connargs = {'username': username, 'api_key': api_key}
446 poolsize = kwargs.get('poolsize', 10)
447 Queue.__init__(self, poolsize)
448
450 """
451 Return a cloudfiles connection object.
452
453 @rtype: L{Connection}
454 @return: a cloudfiles connection object
455 """
456 try:
457 (create, connobj) = Queue.get(self, block=0)
458 except Empty:
459 connobj = Connection(**self.connargs)
460 return connobj
461
462 - def put(self, connobj):
463 """
464 Place a cloudfiles connection object back into the pool.
465
466 @param connobj: a cloudfiles connection object
467 @type connobj: L{Connection}
468 """
469 try:
470 Queue.put(self, (time(), connobj), block=0)
471 except Full:
472 del connobj
473
474