opener.py (3225B)
1 from contextlib import closing 2 import logging 3 import sys 4 5 if sys.version_info >= (3, 0): 6 _is_py3 = True 7 from urllib.request import build_opener, ContentTooShortError 8 else: 9 _is_py3 = False 10 from urllib2 import build_opener 11 from urllib import FancyURLopener 12 13 14 logger = logging.getLogger('glad.opener') 15 16 17 def build_urllib_opener(user_agent, *args, **kwargs): 18 if _is_py3: 19 return None 20 21 class UrllibURLOpener(FancyURLopener): 22 version = user_agent 23 return UrllibURLOpener(*args, **kwargs) 24 25 26 def _urlretrieve_with_opener(opener, url, filename, data=None): 27 if not _is_py3: 28 raise SyntaxError('Only call this in Python 3 code.') 29 30 # borrowed from the original implementation at urllib.request.urlretrieve. 31 with closing(opener.open(url, data=data)) as src: 32 headers = src.info() 33 34 with open(filename, 'wb') as dest: 35 result = filename, headers 36 bs = 1024*8 37 size = -1 38 read = 0 39 blocknum = 0 40 if "content-length" in headers: 41 size = int(headers["Content-Length"]) 42 43 while True: 44 block = src.read(bs) 45 if not block: 46 break 47 read += len(block) 48 dest.write(block) 49 blocknum += 1 50 51 if size >= 0 and read < size: 52 raise ContentTooShortError( 53 'retrieval incomplete: got only %i out of %i bytes' 54 % (read, size), result) 55 56 return result 57 58 59 class URLOpener(object): 60 """ 61 Class to download/find Khronos related files, like 62 the official specs and khrplatform.h. 63 64 Can also be used to download files, exists mainly because of 65 Python 2 and Python 3 incompatibilities. 66 """ 67 def __init__(self, user_agent='Mozilla/5.0'): 68 # the urllib2/urllib.request opener 69 self.opener = build_opener() 70 self.opener.addheaders = [('User-agent', user_agent)] 71 72 # the urllib opener (Python 2 only) 73 self.opener2 = build_urllib_opener(user_agent) 74 75 def urlopen(self, url, data=None, *args, **kwargs): 76 """ 77 Same as urllib2.urlopen or urllib.request.urlopen, 78 the only difference is that it links to the internal opener. 79 """ 80 logger.info('opening: \'%s\'', url) 81 82 if data is None: 83 return self.opener.open(url) 84 85 return self.opener.open(url, data) 86 87 def urlretrieve(self, url, filename, data=None): 88 """ 89 Similar to urllib.urlretrieve or urllib.request.urlretrieve 90 only that *filname* is required. 91 92 :param url: URL to download. 93 :param filename: Filename to save the content to. 94 :param data: Valid URL-encoded data. 95 :return: Tuple containing path and headers. 96 """ 97 logger.info('saving: \'%s\' to \'%s\'', url, filename) 98 99 if _is_py3: 100 return _urlretrieve_with_opener(self.opener, url, filename, data=data) 101 102 return self.opener2.retrieve(url, filename, data=data) 103 104 # just a singleton helper: 105 _default = None 106 107 @classmethod 108 def default(cls): 109 if cls._default is None: 110 cls._default = cls() 111 112 return cls._default