Commit: daaf7ce6bbab25bf8718aee876ab288f4db0f549 - winazurestorage (git)

winazurestorageのフォーク

Commit MetaInfo

• Revision: daaf7ce6bbab25bf8718aee876ab288f4db0f549 (tree)
• Zeit: 2008-11-22 14:07:50
• Autor: Steve Marx <Steve.Marx@micr...>
• Commiter: Steve Marx

Log Message

Add early table support, support for non-path-style-URIs, and move to urllib2 for proxy support.

Ändern Zusammenfassung

• modified: winazurestorage.py

Diff

--- a/winazurestorage.py
+++ b/winazurestorage.py
@@ -7,22 +7,24 @@ Sriram Krishnan <[e-mail]>
 
 import base64
 import hmac
-import httplib
 import hashlib
 import time
-import urllib
-import urlparse
 import sys
 import os
 from xml.dom import minidom #TODO: Use a faster way of processing XML
+import re
+from urllib2 import Request, urlopen
+from urlparse import urlsplit
+from datetime import datetime, timedelta
 
 DEVSTORE_ACCOUNT = "devstoreaccount1"
 DEVSTORE_SECRET_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
 
+DEVSTORE_BLOB_HOST = "127.0.0.1:10000"
+DEVSTORE_TABLE_HOST = "127.0.0.1:10002"
 
-DEVSTORE_HOST="127.0.0.1:10000"
-CLOUD_HOST = "blob.core.windows.net"
-
+CLOUD_BLOB_HOST = "blob.core.windows.net"
+CLOUD_TABLE_HOST = "table.core.windows.net"
 
 PREFIX_PROPERTIES = "x-ms-prop-"
 PREFIX_METADATA = "x-ms-meta-"
@@ -34,36 +36,157 @@ DEBUG = False
 
 TIME_FORMAT ="%a, %d %b %Y %H:%M:%S %Z"
 
+def parse_edm_datetime(input):
+    d = datetime.strptime(input[:input.find('.')], "%Y-%m-%dT%H:%M:%S")
+    if input[:input.find('.')] != -1:
+        d += timedelta(0, 0, int(round(float(input[input.index('.'):-1])*1000000)))
+    return d
 
-class WAStorageConnection:
-    def __init__(self, server_url = DEVSTORE_HOST, account_name=DEVSTORE_ACCOUNT,secret_key = DEVSTORE_SECRET_KEY):
-        self.server_url = server_url
-        self.account_name = account_name
-        self.secret_key = base64.decodestring(secret_key)
-        
-       
-    def create_container(self,container_name, is_public):
-        headers ={}
-        if is_public:
-            headers[PREFIX_PROPERTIES+ "publicaccess"] = "true"
+def parse_edm_int32(input):
+    return int(input)
+
+class SharedKeyCredentials(object):
+    def __init__(self, account_name, account_key, use_path_style_uris = None):
+        self._account = account_name
+        self._key = base64.decodestring(account_key)
+
+    def _sign_request_impl(self, request, for_tables = False,  use_path_style_uris = None):
+        (scheme, host, path, query, fragment) = urlsplit(request.get_full_url())
+        if use_path_style_uris:
+            path = path[path.index('/'):]
+
+        canonicalized_resource = "/" + self._account + path
+        match = re.search(r'comp=[^&]*', query)
+        if match is not None:
+            canonicalized_resource += "?" + match.group(0)
             
-        return self._do_store_request(container_name, None, 'PUT', headers)
-  
-    def put_blob(self, container_name, key, data, content_type=None):
-        return self._do_store_request(container_name, key, 'PUT', {}, data, content_type )
-           
-    def get_blob(self, container_name, key):
-        response = self._do_store_request(container_name, key, 'GET' )
-        return response.read()
-                
-    def list_containers(self):
-        #TODO: Deal with nextmarker for large requests (only 5K containers returned by default)
-        #TODO: Use a different XML parsing scheme
-        
-        response = self._do_store_request(query_string = "?comp=list")
-        
+        if use_path_style_uris is None:
+            use_path_style_uris = re.match('^[\d.:]+$', host) is not None
+
+        request.add_header(PREFIX_STORAGE_HEADER + 'date', time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())) #RFC 1123
+        canonicalized_headers = NEW_LINE.join(('%s:%s' % (k.lower(), request.get_header(k).strip()) for k in sorted(request.headers.keys(), lambda x,y: cmp(x.lower(), y.lower())) if k.lower().startswith(PREFIX_STORAGE_HEADER)))
+
+        string_to_sign = request.get_method().upper() + NEW_LINE # verb
+        string_to_sign += NEW_LINE                               # MD5 not required
+        if request.get_header('Content-type') is not None:       # Content-Type
+            string_to_sign += request.get_header('Content-type')
+        string_to_sign += NEW_LINE
+        if for_tables: string_to_sign += request.get_header(PREFIX_STORAGE_HEADER.capitalize() + 'date') + NEW_LINE
+        else: string_to_sign += NEW_LINE                         # Date
+        if not for_tables:
+            string_to_sign += canonicalized_headers + NEW_LINE   # Canonicalized headers
+        string_to_sign += canonicalized_resource                 # Canonicalized resource
+
+        request.add_header('Authorization', 'SharedKey ' + self._account + ':' + base64.encodestring(hmac.new(self._key, unicode(string_to_sign).encode("utf-8"), hashlib.sha256).digest()).strip())
+        return request
+
+    def sign_request(self, request, use_path_style_uris = None):
+        return self._sign_request_impl(request, use_path_style_uris)
+
+    def sign_table_request(self, request, use_path_style_uris = None):
+        return self._sign_request_impl(request, for_tables = True, use_path_style_uris = use_path_style_uris)
+
+class RequestWithMethod(Request):
+    '''Subclass urllib2.Request to add the capability of using methods other than GET and POST.
+       Thanks to http://benjamin.smedbergs.us/blog/2008-10-21/putting-and-deleteing-in-python-urllib2/'''
+    def __init__(self, method, *args, **kwargs):
+        self._method = method
+        Request.__init__(self, *args, **kwargs)
+
+    def get_method(self):
+        return self._method
+
+class Table(object):
+    def __init__(self, url, name):
+        self.url = url
+        self.name = name
+
+class Storage(object):
+    def __init__(self, host, account_name, secret_key, use_path_style_uris):
+        self._host = host
+        self._account = account_name
+        self._key = secret_key
+        if use_path_style_uris is None:
+            use_path_style_uris = re.match(r'^[^:]*[\d:]+$', self._host)
+        self._use_path_style_uris = use_path_style_uris
+        self._credentials = SharedKeyCredentials(self._account, self._key)
+
+    def get_base_url(self):
+        if self._use_path_style_uris:
+            return "http://%s/%s" % (self._host, self._account)
+        else:
+            return "http://%s.%s" % (self._account, self._host)
+
+class TableEntity(object): pass
+
+class TableStorage(Storage):
+    '''Due to local development storage not supporting SharedKeyLite authentication, this class
+       will only work against cloud storage.'''
+    def __init__(self, host, account_name, secret_key, use_path_style_uris = None):
+        super(TableStorage, self).__init__(host, account_name, secret_key, use_path_style_uris)
+
+    def list_tables(self):
+        req = Request("%s/Tables" % self.get_base_url())
+        self._credentials.sign_table_request(req)
+        response = urlopen(req)
+
         dom = minidom.parseString(response.read())
         
+        entries = dom.getElementsByTagName("entry")
+        for entry in entries:
+            table_url = entry.getElementsByTagName("id")[0].firstChild.data
+            table_name = entry.getElementsByTagName("content")[0].getElementsByTagName("m:properties")[0].getElementsByTagName("d:TableName")[0].firstChild.data
+            yield Table(table_url, table_name)
+        dom.unlink()
+
+    def get_entity(self, table_name, partition_key, row_key):
+        dom = minidom.parseString(urlopen(self._credentials.sign_table_request(Request("%s/%s(PartitionKey='%s',RowKey='%s')" % (self.get_base_url(), table_name, partition_key, row_key)))).read())
+        entity = self._parse_entity(dom.getElementsByTagName("entry")[0])
+        dom.unlink()
+        return entity
+
+    def _parse_entity(self, entry):
+        entity = TableEntity()
+        for property in (p for p in entry.getElementsByTagName("m:properties")[0].childNodes if p.nodeType == minidom.Node.ELEMENT_NODE):
+            key = property.tagName[2:]
+            if property.hasAttribute('m:type'):
+                t = property.getAttribute('m:type')
+                if t.lower() == 'edm.datetime': value = parse_edm_datetime(property.firstChild.data)
+                elif t.lower() == 'edm.int32': value = parse_edm_int32(property.firstChild.data)
+                else: raise Exception(t.lower())
+            else: value = property.firstChild.data
+            setattr(entity, key, value)
+        return entity
+
+    def get_all(self, table_name):
+        dom = minidom.parseString(urlopen(self._credentials.sign_table_request(Request("%s/%s" % (self.get_base_url(), table_name)))).read())
+        entries = dom.getElementsByTagName("entry")
+        entities = []
+        for entry in entries:
+            entities.append(self._parse_entity(entry))
+        dom.unlink()
+        return entities
+
+class BlobStorage(Storage):
+    def __init__(self, host = DEVSTORE_BLOB_HOST, account_name = DEVSTORE_ACCOUNT, secret_key = DEVSTORE_SECRET_KEY, use_path_style_uris = None):
+        super(BlobStorage, self).__init__(host, account_name, secret_key, use_path_style_uris)
+
+    def create_container(self, container_name, is_public = False):
+        req = RequestWithMethod("PUT", "%s/%s" % (self.get_base_url(), container_name))
+        req.add_header("Content-Length", "0")
+        self._credentials.sign_request(req)
+        if is_public: req.add_header(PREFIX_PROPERTIES + "publicaccess", "true")
+        return urlopen(req)
+
+    def delete_container(self, container_name):
+        req = RequestWithMethod("DELETE", "%s/%s" % (self.get_base_url(), container_name))
+        self._credentials.sign_request(req)
+        return urlopen(req)
+
+    def list_containers(self):
+        req = Request("%s/?comp=list" % self.get_base_url())
+        self._credentials.sign_request(req)
+        dom = minidom.parseString(urlopen(req).read())
         containers = dom.getElementsByTagName("Container")
         for container in containers:
             container_name = container.getElementsByTagName("Name")[0].firstChild.data
@@ -72,85 +195,21 @@ class WAStorageConnection:
             yield (container_name, etag, last_modified)
         
         dom.unlink() #Docs say to do this to force GC. Ugh.
-  
-    def _get_auth_header(self, http_method, path, data, headers):
-        string_to_sign =""
-        
-        #First element is the method
-        string_to_sign += http_method + NEW_LINE
-        
-        #Second is the optional content MD5
-        string_to_sign += NEW_LINE
-        
-        #content type - this should have been initialized atleast to a blank value
-        if headers.has_key("content-type"):
-            string_to_sign += headers["content-type"] 
-        string_to_sign += NEW_LINE
-        
-        # date - we don't need to add header here since the special date storage header
-        # always exists in our implementation
-        string_to_sign +=  NEW_LINE
-        
-        # Construct canonicalized headers. 
-        # TODO: Note that this doesn't implement parts of the spec - combining header fields with same name,
-        # unfolding long lines and trimming white spaces around the colon
-        
-        ms_headers =[header_key for header_key in headers.keys() if header_key.startswith(PREFIX_STORAGE_HEADER)]
-        ms_headers.sort()
-        for header_key in ms_headers:
-            string_to_sign += "%s:%s%s" % (header_key, headers[header_key], NEW_LINE) 
-        
-        # Add canonicalized resource
-        string_to_sign += "/" + self.account_name + path
-        utf8_string_to_sign = unicode(string_to_sign).encode("utf-8")
-        hmac_digest = hmac.new(self.secret_key, utf8_string_to_sign, hashlib.sha256).digest()
-        return base64.encodestring(hmac_digest).strip()
-        
-        
-        
-    def _do_store_request(self, container=None, blob_name=None, http_method="GET", headers = {},data = "", 
-                    content_type=None, query_string = None, signed=True):
-        connection = httplib.HTTPConnection(self.server_url)
-        
-        # Construct right path based on account name , container name and blob name if any
-        path = "/" + self.account_name + "/"
-        if container!= None:
-            path = path  + container +"/"
-            if blob_name != None:
-                path = path + blob_name
-        
-        if query_string != None:
-            path = path + query_string
-      
-        
-        headers[PREFIX_STORAGE_HEADER + "date"] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) #RFC 1123
-        if content_type != None:
-            headers["content-type"] = content_type  
-               
-        if signed:
-            auth_header = self._get_auth_header(http_method,path, data, headers)
-            headers["Authorization"] = "SharedKey " + self.account_name + ":" + auth_header
-            headers["content-length"] = len(data)
-         
-        connection.request(http_method, path, data, headers)
-        response = connection.getresponse()
-        if DEBUG:
-            print response.status
-        return response
-
-    
-def main():
-    conn = WAStorageConnection()
-    for (container_name,etag, last_modified ) in  conn.list_containers():
-        print container_name
-        print etag
-        print last_modified
-        
-    conn.create_container("testcontainer", False)
-    conn.put_blob("testcontainer","test","Hello World!" )
-    print conn.get_blob("testcontainer", "test")
 
+    def put_blob(self, container_name, blob_name, data, content_type = None):
+        req = RequestWithMethod("PUT", "%s/%s/%s" % (self.get_base_url(), container_name, blob_name), data=data)
+        req.add_header("Content-Length", "%d" % len(data))
+        if content_type is not None: req.add_header("Content-Type", content_type)
+        self._credentials.sign_request(req)
+        return urlopen(req)
+
+    def get_blob(self, container_name, blob_name):
+        req = Request("%s/%s/%s" % (self.get_base_url(), container_name, blob_name))
+        self._credentials.sign_request(req)
+        return urlopen(req).read()
+
+def main():
+    pass
 
 if __name__ == '__main__':
-    main()      
-      
+    main()
\ No newline at end of file