1# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ 2# 3# Permission is hereby granted, free of charge, to any person obtaining a 4# copy of this software and associated documentation files (the 5# "Software"), to deal in the Software without restriction, including 6# without limitation the rights to use, copy, modify, merge, publish, dis- 7# tribute, sublicense, and/or sell copies of the Software, and to permit 8# persons to whom the Software is furnished to do so, subject to the fol- 9# lowing conditions: 10# 11# The above copyright notice and this permission notice shall be included 12# in all copies or substantial portions of the Software. 13# 14# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 15# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- 16# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 17# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 18# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20# IN THE SOFTWARE. 21from __future__ import print_function 22 23""" 24Represents an SDB Domain 25""" 26 27from boto.sdb.queryresultset import SelectResultSet 28from boto.compat import six 29 30class Domain(object): 31 32 def __init__(self, connection=None, name=None): 33 self.connection = connection 34 self.name = name 35 self._metadata = None 36 37 def __repr__(self): 38 return 'Domain:%s' % self.name 39 40 def __iter__(self): 41 return iter(self.select("SELECT * FROM `%s`" % self.name)) 42 43 def startElement(self, name, attrs, connection): 44 return None 45 46 def endElement(self, name, value, connection): 47 if name == 'DomainName': 48 self.name = value 49 else: 50 setattr(self, name, value) 51 52 def get_metadata(self): 53 if not self._metadata: 54 self._metadata = self.connection.domain_metadata(self) 55 return self._metadata 56 57 def put_attributes(self, item_name, attributes, 58 replace=True, expected_value=None): 59 """ 60 Store attributes for a given item. 61 62 :type item_name: string 63 :param item_name: The name of the item whose attributes are being stored. 64 65 :type attribute_names: dict or dict-like object 66 :param attribute_names: The name/value pairs to store as attributes 67 68 :type expected_value: list 69 :param expected_value: If supplied, this is a list or tuple consisting 70 of a single attribute name and expected value. The list can be 71 of the form: 72 73 * ['name', 'value'] 74 75 In which case the call will first verify that the attribute 76 "name" of this item has a value of "value". If it does, the delete 77 will proceed, otherwise a ConditionalCheckFailed error will be 78 returned. The list can also be of the form: 79 80 * ['name', True|False] 81 82 which will simply check for the existence (True) or non-existence 83 (False) of the attribute. 84 85 :type replace: bool 86 :param replace: Whether the attribute values passed in will replace 87 existing values or will be added as addition values. 88 Defaults to True. 89 90 :rtype: bool 91 :return: True if successful 92 """ 93 return self.connection.put_attributes(self, item_name, attributes, 94 replace, expected_value) 95 96 def batch_put_attributes(self, items, replace=True): 97 """ 98 Store attributes for multiple items. 99 100 :type items: dict or dict-like object 101 :param items: A dictionary-like object. The keys of the dictionary are 102 the item names and the values are themselves dictionaries 103 of attribute names/values, exactly the same as the 104 attribute_names parameter of the scalar put_attributes 105 call. 106 107 :type replace: bool 108 :param replace: Whether the attribute values passed in will replace 109 existing values or will be added as addition values. 110 Defaults to True. 111 112 :rtype: bool 113 :return: True if successful 114 """ 115 return self.connection.batch_put_attributes(self, items, replace) 116 117 def get_attributes(self, item_name, attribute_name=None, 118 consistent_read=False, item=None): 119 """ 120 Retrieve attributes for a given item. 121 122 :type item_name: string 123 :param item_name: The name of the item whose attributes are being retrieved. 124 125 :type attribute_names: string or list of strings 126 :param attribute_names: An attribute name or list of attribute names. This 127 parameter is optional. If not supplied, all attributes 128 will be retrieved for the item. 129 130 :rtype: :class:`boto.sdb.item.Item` 131 :return: An Item mapping type containing the requested attribute name/values 132 """ 133 return self.connection.get_attributes(self, item_name, attribute_name, 134 consistent_read, item) 135 136 def delete_attributes(self, item_name, attributes=None, 137 expected_values=None): 138 """ 139 Delete attributes from a given item. 140 141 :type item_name: string 142 :param item_name: The name of the item whose attributes are being deleted. 143 144 :type attributes: dict, list or :class:`boto.sdb.item.Item` 145 :param attributes: Either a list containing attribute names which will cause 146 all values associated with that attribute name to be deleted or 147 a dict or Item containing the attribute names and keys and list 148 of values to delete as the value. If no value is supplied, 149 all attribute name/values for the item will be deleted. 150 151 :type expected_value: list 152 :param expected_value: If supplied, this is a list or tuple consisting 153 of a single attribute name and expected value. The list can be of 154 the form: 155 156 * ['name', 'value'] 157 158 In which case the call will first verify that the attribute "name" 159 of this item has a value of "value". If it does, the delete 160 will proceed, otherwise a ConditionalCheckFailed error will be 161 returned. The list can also be of the form: 162 163 * ['name', True|False] 164 165 which will simply check for the existence (True) or 166 non-existence (False) of the attribute. 167 168 :rtype: bool 169 :return: True if successful 170 """ 171 return self.connection.delete_attributes(self, item_name, attributes, 172 expected_values) 173 174 def batch_delete_attributes(self, items): 175 """ 176 Delete multiple items in this domain. 177 178 :type items: dict or dict-like object 179 :param items: A dictionary-like object. The keys of the dictionary are 180 the item names and the values are either: 181 182 * dictionaries of attribute names/values, exactly the 183 same as the attribute_names parameter of the scalar 184 put_attributes call. The attribute name/value pairs 185 will only be deleted if they match the name/value 186 pairs passed in. 187 * None which means that all attributes associated 188 with the item should be deleted. 189 190 :rtype: bool 191 :return: True if successful 192 """ 193 return self.connection.batch_delete_attributes(self, items) 194 195 def select(self, query='', next_token=None, consistent_read=False, max_items=None): 196 """ 197 Returns a set of Attributes for item names within domain_name that match the query. 198 The query must be expressed in using the SELECT style syntax rather than the 199 original SimpleDB query language. 200 201 :type query: string 202 :param query: The SimpleDB query to be performed. 203 204 :rtype: iter 205 :return: An iterator containing the results. This is actually a generator 206 function that will iterate across all search results, not just the 207 first page. 208 """ 209 return SelectResultSet(self, query, max_items=max_items, next_token=next_token, 210 consistent_read=consistent_read) 211 212 def get_item(self, item_name, consistent_read=False): 213 """ 214 Retrieves an item from the domain, along with all of its attributes. 215 216 :param string item_name: The name of the item to retrieve. 217 :rtype: :class:`boto.sdb.item.Item` or ``None`` 218 :keyword bool consistent_read: When set to true, ensures that the most 219 recent data is returned. 220 :return: The requested item, or ``None`` if there was no match found 221 """ 222 item = self.get_attributes(item_name, consistent_read=consistent_read) 223 if item: 224 item.domain = self 225 return item 226 else: 227 return None 228 229 def new_item(self, item_name): 230 return self.connection.item_cls(self, item_name) 231 232 def delete_item(self, item): 233 self.delete_attributes(item.name) 234 235 def to_xml(self, f=None): 236 """Get this domain as an XML DOM Document 237 :param f: Optional File to dump directly to 238 :type f: File or Stream 239 240 :return: File object where the XML has been dumped to 241 :rtype: file 242 """ 243 if not f: 244 from tempfile import TemporaryFile 245 f = TemporaryFile() 246 print('<?xml version="1.0" encoding="UTF-8"?>', file=f) 247 print('<Domain id="%s">' % self.name, file=f) 248 for item in self: 249 print('\t<Item id="%s">' % item.name, file=f) 250 for k in item: 251 print('\t\t<attribute id="%s">' % k, file=f) 252 values = item[k] 253 if not isinstance(values, list): 254 values = [values] 255 for value in values: 256 print('\t\t\t<value><![CDATA[', end=' ', file=f) 257 if isinstance(value, six.text_type): 258 value = value.encode('utf-8', 'replace') 259 else: 260 value = six.text_type(value, errors='replace').encode('utf-8', 'replace') 261 f.write(value) 262 print(']]></value>', file=f) 263 print('\t\t</attribute>', file=f) 264 print('\t</Item>', file=f) 265 print('</Domain>', file=f) 266 f.flush() 267 f.seek(0) 268 return f 269 270 271 def from_xml(self, doc): 272 """Load this domain based on an XML document""" 273 import xml.sax 274 handler = DomainDumpParser(self) 275 xml.sax.parse(doc, handler) 276 return handler 277 278 def delete(self): 279 """ 280 Delete this domain, and all items under it 281 """ 282 return self.connection.delete_domain(self) 283 284 285class DomainMetaData(object): 286 287 def __init__(self, domain=None): 288 self.domain = domain 289 self.item_count = None 290 self.item_names_size = None 291 self.attr_name_count = None 292 self.attr_names_size = None 293 self.attr_value_count = None 294 self.attr_values_size = None 295 296 def startElement(self, name, attrs, connection): 297 return None 298 299 def endElement(self, name, value, connection): 300 if name == 'ItemCount': 301 self.item_count = int(value) 302 elif name == 'ItemNamesSizeBytes': 303 self.item_names_size = int(value) 304 elif name == 'AttributeNameCount': 305 self.attr_name_count = int(value) 306 elif name == 'AttributeNamesSizeBytes': 307 self.attr_names_size = int(value) 308 elif name == 'AttributeValueCount': 309 self.attr_value_count = int(value) 310 elif name == 'AttributeValuesSizeBytes': 311 self.attr_values_size = int(value) 312 elif name == 'Timestamp': 313 self.timestamp = value 314 else: 315 setattr(self, name, value) 316 317import sys 318from xml.sax.handler import ContentHandler 319class DomainDumpParser(ContentHandler): 320 """ 321 SAX parser for a domain that has been dumped 322 """ 323 324 def __init__(self, domain): 325 self.uploader = UploaderThread(domain) 326 self.item_id = None 327 self.attrs = {} 328 self.attribute = None 329 self.value = "" 330 self.domain = domain 331 332 def startElement(self, name, attrs): 333 if name == "Item": 334 self.item_id = attrs['id'] 335 self.attrs = {} 336 elif name == "attribute": 337 self.attribute = attrs['id'] 338 elif name == "value": 339 self.value = "" 340 341 def characters(self, ch): 342 self.value += ch 343 344 def endElement(self, name): 345 if name == "value": 346 if self.value and self.attribute: 347 value = self.value.strip() 348 attr_name = self.attribute.strip() 349 if attr_name in self.attrs: 350 self.attrs[attr_name].append(value) 351 else: 352 self.attrs[attr_name] = [value] 353 elif name == "Item": 354 self.uploader.items[self.item_id] = self.attrs 355 # Every 20 items we spawn off the uploader 356 if len(self.uploader.items) >= 20: 357 self.uploader.start() 358 self.uploader = UploaderThread(self.domain) 359 elif name == "Domain": 360 # If we're done, spawn off our last Uploader Thread 361 self.uploader.start() 362 363from threading import Thread 364class UploaderThread(Thread): 365 """Uploader Thread""" 366 367 def __init__(self, domain): 368 self.db = domain 369 self.items = {} 370 super(UploaderThread, self).__init__() 371 372 def run(self): 373 try: 374 self.db.batch_put_attributes(self.items) 375 except: 376 print("Exception using batch put, trying regular put instead") 377 for item_name in self.items: 378 self.db.put_attributes(item_name, self.items[item_name]) 379 print(".", end=' ') 380 sys.stdout.flush() 381