这是关于chunk encoding传输以前相关传输编码的处理。没有做压缩解码的处理。
import tornado.ioloop import tornado.iostream import socket class WebRequest(object): LINE_END = b' ' def __init__(self, host, port=80, callback=None): self.host = host self.port =port self.headers = {} self.data = '' self.callback = callback self.transfer = 'stream' self.http_ver_with_status = '' s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) self.stream = tornado.iostream.IOStream(s) self.stream.connect((self.host, self.port), self.send_request) def send_request(self): self.stream.write(b"GET / HTTP/1.0 Host: %s " %self.host) self.stream.read_until(self.LINE_END*2, self.on_headers) def on_headers(self, data): lines = data.split(self.LINE_END) self.http_ver_with_status = lines.pop(0) for line in lines: parts = line.split(b":", 1) if len(parts) == 2: self.headers[parts[0].strip()] = parts[1].strip() if 'Content-Length' in self.headers: if int(self.headers[b"Content-Length"]) > 0: self.transfer = 'content_length' self.stream.read_bytes(int(self.headers[b"Content-Length"]), self.on_body) else: self.callback(self) self.stream.close() elif self.headers.get(b'Transfer-Encoding') == 'chunked': self.transfer = 'chunked' self.on_chunk() else: self.transfer = 'stream' callback = lambda data: self.on_stream(data, True) streaming_callback = lambda data: self.on_stream(data, False) self.stream.read_until_close(callback, streaming_callback) def on_body(self, data): self.data += data self.callback(self) self.stream.close() def on_chunk(self): self.stream.read_until_regex(self.LINE_END, self.on_chunk_header) def on_chunk_header(self, data): data = data.strip(self.LINE_END) length = int(data, 16) if length > 0: self.stream.read_bytes(length+len(self.LINE_END), self.on_chunk_data) else: self.stream.read_bytes(length+len(self.LINE_END), self.on_chunk_end) def on_chunk_end(self, data): self.callback(self) self.stream.close() def on_chunk_data(self, data): self.data += data self.on_chunk() def on_stream(self, data, finish=False): # do some thing if finish: self.data += data self.callback(self) self.stream.close() else: self.data += data def callback(req): print req.http_ver_with_status print "Transfer :" , req.transfer print "Headers: ", req.headers #print "Data: ", req.data req = WebRequest('cn.bing.com', 80, on_bing) def on_bing(req): print req.http_ver_with_status print "Transfer :" , req.transfer print "Headers: ", req.headers #print "Data: ", req.data tornado.ioloop.IOLoop.instance().stop() if __name__ == '__main__': req = WebRequest('valpha.gameloft.com', 20000, callback) tornado.ioloop.IOLoop.instance().start()