restkit.client

43 44 """ A client handle a connection at a time. A client is threadsafe, 45 but an handled shouldn't be shared between threads. All connections 46 are shared between threads via a pool. 47 48 >>> from restkit import * 49 >>> c = Client() 50 >>> r = c.request("http://google.com") 51 r>>> r.status 52 '301 Moved Permanently' 53 >>> r.body_string() 54 '<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has moved\n<A HREF="http://www.google.com/">here</A>.\r\n</BODY></HTML>\r\n' 55 >>> c.follow_redirect = True 56 >>> r = c.request("http://google.com") 57 >>> r.status 58 '200 OK' 59 60 """ 61 62 version = (1, 1) 63 response_class=Response 64

65 - def __init__(self, 66 follow_redirect=False, 67 force_follow_redirect=False, 68 max_follow_redirect=MAX_FOLLOW_REDIRECTS, 69 filters=None, 70 decompress=True, 71 max_status_line_garbage=None, 72 max_header_count=0, 73 pool=None, 74 response_class=None, 75 timeout=None, 76 use_proxy=False, 77 max_tries=3, 78 wait_tries=0.3, 79 backend="thread", 80 **ssl_args):

81 """ 82 Client parameters 83 ~~~~~~~~~~~~~~~~~ 84 85 :param follow_redirect: follow redirection, by default False 86 :param max_ollow_redirect: number of redirections available 87 :filters: http filters to pass 88 :param decompress: allows the client to decompress the response 89 body 90 :param max_status_line_garbage: defines the maximum number of ignorable 91 lines before we expect a HTTP response's status line. With 92 HTTP/1.1 persistent connections, the problem arises that broken 93 scripts could return a wrong Content-Length (there are more 94 bytes sent than specified). Unfortunately, in some cases, this 95 cannot be detected after the bad response, but only before the 96 next one. So the client is abble to skip bad lines using this 97 limit. 0 disable garbage collection, None means unlimited number 98 of tries. 99 :param max_header_count: determines the maximum HTTP header count 100 allowed. by default no limit. 101 :param manager: the manager to use. By default we use the global 102 one. 103 :parama response_class: the response class to use 104 :param timeout: the default timeout of the connection 105 (SO_TIMEOUT) 106 107 :param max_tries: the number of tries before we give up a 108 connection 109 :param wait_tries: number of time we wait between each tries. 110 :param ssl_args: named argument, see ssl module for more 111 informations 112 """ 113 self.follow_redirect = follow_redirect 114 self.force_follow_redirect = force_follow_redirect 115 self.max_follow_redirect = max_follow_redirect 116 self.decompress = decompress 117 self.filters = filters or [] 118 self.max_status_line_garbage = max_status_line_garbage 119 self.max_header_count = max_header_count 120 self.use_proxy = use_proxy 121 122 self.request_filters = [] 123 self.response_filters = [] 124 self.load_filters() 125 126 127 # set manager 128 129 session_options = dict( 130 retry_delay=wait_tries, 131 retry_max = max_tries, 132 timeout = timeout) 133 134 135 if pool is None: 136 pool = get_session(backend, **session_options) 137 self._pool = pool 138 self.backend = backend 139 140 # change default response class 141 if response_class is not None: 142 self.response_class = response_class 143 144 self.max_tries = max_tries 145 self.wait_tries = wait_tries 146 self.timeout = timeout 147 148 self._nb_redirections = self.max_follow_redirect 149 self._url = None 150 self._initial_url = None 151 self._write_cb = None 152 self._headers = None 153 self._sock_key = None 154 self._sock = None 155 self._original = None 156 157 self.method = 'GET' 158 self.body = None 159 self.ssl_args = ssl_args or {}

160

161 - def load_filters(self):

162 """ Populate filters from self.filters. 163 Must be called each time self.filters is updated. 164 """ 165 for f in self.filters: 166 if hasattr(f, "on_request"): 167 self.request_filters.append(f) 168 if hasattr(f, "on_response"): 169 self.response_filters.append(f)

170 171 172

173 - def get_connection(self, request):

174 """ get a connection from the pool or create new one. """ 175 176 addr = parse_netloc(request.parsed_url) 177 is_ssl = request.is_ssl() 178 179 extra_headers = [] 180 conn = None 181 if self.use_proxy: 182 conn = self.proxy_connection(request, 183 addr, is_ssl) 184 if not conn: 185 conn = self._pool.get(host=addr[0], port=addr[1], 186 pool=self._pool, is_ssl=is_ssl, 187 extra_headers=extra_headers, **self.ssl_args) 188 189 190 return conn

191

192 - def proxy_connection(self, request, req_addr, is_ssl):

193 """ do the proxy connection """ 194 proxy_settings = os.environ.get('%s_proxy' % 195 request.parsed_url.scheme) 196 197 if proxy_settings and proxy_settings is not None: 198 request.is_proxied = True 199 200 proxy_settings, proxy_auth = _get_proxy_auth(proxy_settings) 201 addr = parse_netloc(urlparse.urlparse(proxy_settings)) 202 203 if is_ssl: 204 if proxy_auth: 205 proxy_auth = 'Proxy-authorization: %s' % proxy_auth 206 proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % req_addr 207 208 user_agent = request.headers.iget('user_agent') 209 if not user_agent: 210 user_agent = "User-Agent: restkit/%s\r\n" % __version__ 211 212 proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, 213 user_agent) 214 215 216 conn = self._pool.get(host=addr[0], port=addr[1], 217 pool=self._pool, is_ssl=is_ssl, 218 extra_headers=[], **self.ssl_args) 219 220 221 conn.send(proxy_pieces) 222 p = HttpStream(SocketReader(conn.socket()), kind=1, 223 decompress=True) 224 225 if p.status_code != 200: 226 raise ProxyError("Tunnel connection failed: %d %s" % 227 (resp.status_int, body)) 228 229 _ = p.body_string() 230 231 else: 232 headers = [] 233 if proxy_auth: 234 headers = [('Proxy-authorization', proxy_auth)] 235 236 conn = self._pool.get(host=addr[0], port=addr[1], 237 pool=self._pool, is_ssl=False, 238 extra_headers=[], **self.ssl_args) 239 return conn 240 241 return

242

243 - def make_headers_string(self, request, extra_headers=None):

244 """ create final header string """ 245 headers = request.headers.copy() 246 if extra_headers is not None: 247 for k, v in extra_headers: 248 headers[k] = v 249 250 if not request.body and request.method in ('POST', 'PUT',): 251 headers['Content-Length'] = 0 252 253 if self.version == (1,1): 254 httpver = "HTTP/1.1" 255 else: 256 httpver = "HTTP/1.0" 257 258 ua = headers.iget('user_agent') 259 if not ua: 260 ua = USER_AGENT 261 host = request.host 262 263 accept_encoding = headers.iget('accept-encoding') 264 if not accept_encoding: 265 accept_encoding = 'identity' 266 267 if request.is_proxied: 268 full_path = ("https://" if request.is_ssl() else "http://") + request.host + request.path 269 else: 270 full_path = request.path 271 272 lheaders = [ 273 "%s %s %s\r\n" % (request.method, full_path, httpver), 274 "Host: %s\r\n" % host, 275 "User-Agent: %s\r\n" % ua, 276 "Accept-Encoding: %s\r\n" % accept_encoding 277 ] 278 279 lheaders.extend(["%s: %s\r\n" % (k, str(v)) for k, v in \ 280 headers.items() if k.lower() not in \ 281 ('user-agent', 'host', 'accept-encoding',)]) 282 if log.isEnabledFor(logging.DEBUG): 283 log.debug("Send headers: %s" % lheaders) 284 return "%s\r\n" % "".join(lheaders)

285

286 - def perform(self, request):

287 """ perform the request. If an error happen it will first try to 288 restart it """ 289 290 if log.isEnabledFor(logging.DEBUG): 291 log.debug("Start to perform request: %s %s %s" % 292 (request.host, request.method, request.path)) 293 tries = 0 294 while True: 295 conn = None 296 try: 297 # get or create a connection to the remote host 298 conn = self.get_connection(request) 299 300 # send headers 301 msg = self.make_headers_string(request, 302 conn.extra_headers) 303 304 # send body 305 if request.body is not None: 306 chunked = request.is_chunked() 307 if request.headers.iget('content-length') is None and \ 308 not chunked: 309 raise RequestError( 310 "Can't determine content length and " + 311 "Transfer-Encoding header is not chunked") 312 313 314 # handle 100-Continue status 315 # http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html#sec8.2.3 316 hdr_expect = request.headers.iget("expect") 317 if hdr_expect is not None and \ 318 hdr_expect.lower() == "100-continue": 319 conn.send(msg) 320 msg = None 321 p = HttpStream(SocketReader(conn.socket()), kind=1, 322 decompress=True) 323 324 325 if p.status_code != 100: 326 self.reset_request() 327 if log.isEnabledFor(logging.DEBUG): 328 log.debug("return response class") 329 return self.response_class(conn, request, p) 330 331 chunked = request.is_chunked() 332 if log.isEnabledFor(logging.DEBUG): 333 log.debug("send body (chunked: %s)" % chunked) 334 335 336 if isinstance(request.body, types.StringTypes): 337 if msg is not None: 338 conn.send(msg + request.body, chunked) 339 else: 340 conn.send(request.body, chunked) 341 else: 342 if msg is not None: 343 conn.send(msg) 344 345 if hasattr(request.body, 'read'): 346 if hasattr(request.body, 'seek'): 347 request.body.seek(0) 348 conn.sendfile(request.body, chunked) 349 else: 350 conn.sendlines(request.body, chunked) 351 if chunked: 352 conn.send_chunk("") 353 else: 354 conn.send(msg) 355 356 return self.get_response(request, conn) 357 except socket.gaierror, e: 358 if conn is not None: 359 conn.close() 360 raise RequestError(str(e)) 361 except socket.timeout, e: 362 if conn is not None: 363 conn.close() 364 raise RequestTimeout(str(e)) 365 except socket.error, e: 366 if log.isEnabledFor(logging.DEBUG): 367 log.debug("socket error: %s" % str(e)) 368 if conn is not None: 369 conn.close() 370 371 if e[0] not in (errno.EAGAIN, errno.EPIPE, errno.EBADF) or \ 372 tries >= self.max_tries: 373 raise RequestError("socket.error: %s" % str(e)) 374 375 # should raised an exception in other cases 376 request.maybe_rewind(msg=str(e)) 377 378 except BadStatusLine: 379 if conn is not None: 380 conn.close() 381 382 # should raised an exception in other cases 383 request.maybe_rewind(msg="bad status line") 384 385 if tries >= self.max_tries: 386 raise 387 except Exception: 388 # unkown error 389 log.debug("unhandled exception %s" % 390 traceback.format_exc()) 391 raise 392 tries += 1 393 self._pool.backend_mod.sleep(self.wait_tries)

394

395 - def request(self, url, method='GET', body=None, headers=None):

396 """ perform immediatly a new request """ 397 398 request = Request(url, method=method, body=body, 399 headers=headers) 400 401 # apply request filters 402 # They are applied only once time. 403 for f in self.request_filters: 404 ret = f.on_request(request) 405 if isinstance(ret, Response): 406 # a response instance has been provided. 407 # just return it. Useful for cache filters 408 return ret 409 410 # no response has been provided, do the request 411 self._nb_redirections = self.max_follow_redirect 412 return self.perform(request)

413

414 - def redirect(self, location, request):

415 """ reset request, set new url of request and perform it """ 416 if self._nb_redirections <= 0: 417 raise RedirectLimit("Redirection limit is reached") 418 419 if request.initial_url is None: 420 request.initial_url = self.url 421 422 # make sure location follow rfc2616 423 location = rewrite_location(request.url, location) 424 425 if log.isEnabledFor(logging.DEBUG): 426 log.debug("Redirect to %s" % location) 427 428 # change request url and method if needed 429 request.url = location 430 431 self._nb_redirections -= 1 432 433 #perform a new request 434 return self.perform(request)

435

436 - def get_response(self, request, connection):

437 """ return final respons, it is only accessible via peform 438 method """ 439 if log.isEnabledFor(logging.DEBUG): 440 log.debug("Start to parse response") 441 442 p = HttpStream(SocketReader(connection.socket()), kind=1, 443 decompress=self.decompress) 444 445 if log.isEnabledFor(logging.DEBUG): 446 log.debug("Got response: %s %s" % (p.version(), p.status())) 447 log.debug("headers: [%s]" % p.headers()) 448 449 location = p.headers().get('location') 450 451 if self.follow_redirect: 452 if p.status_code() in (301, 302, 307,): 453 connection.close() 454 if request.method in ('GET', 'HEAD',) or \ 455 self.force_follow_redirect: 456 if hasattr(self.body, 'read'): 457 try: 458 self.body.seek(0) 459 except AttributeError: 460 raise RequestError("Can't redirect %s to %s " 461 "because body has already been read" 462 % (self.url, location)) 463 return self.redirect(location, request) 464 465 elif p.status_code() == 303 and self.method == "POST": 466 connection.close() 467 request.method = "GET" 468 request.body = None 469 return self.redirect(location, request) 470 471 # create response object 472 resp = self.response_class(connection, request, p) 473 474 # apply response filters 475 for f in self.response_filters: 476 f.on_response(resp, request) 477 478 if log.isEnabledFor(logging.DEBUG): 479 log.debug("return response class") 480 481 # return final response 482 return resp

Source Code for Module restkit.client