I'm trying to serve large files in response to an HTTP GET request with a Python web server. I've overridden do_GET(self) successfully and everything works great for sending small zip files but with larger files I only write around 137MB into the wfile output stream before I get a broken pipe error as BaseHTTPServer is calling finish before I'm done writing.
self.send_header("Content-Type", "application/octet-stream")
self.end_headers()
f = open(filename, 'rb')
fullyWritten = False
count = 0
while not fullyWritten:
tmp = f.read(32768)
count = count + len(tmp)
print count # always a different number before crash
if not tmp:
fullyWritten = True
else:
self.wfile.write(tmp) # never finishes writing large files
print "wrote to http body"
...
...
wrote to http body
137134080
wrote to http body
137166848
wrote to http body
137199616
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 43644)
Traceback (most recent call last):
File "/usr/lib/python2.7/SocketServer.py", line 295, in _handle_request_noblock
self.process_request(request, client_address)
File "/usr/lib/python2.7/SocketServer.py", line 321, in process_request
self.finish_request(request, client_address)
File "/usr/lib/python2.7/SocketServer.py", line 334, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/lib/python2.7/SocketServer.py", line 651, in __init__
self.finish()
File "/usr/lib/python2.7/SocketServer.py", line 710, in finish
self.wfile.close()
File "/usr/lib/python2.7/socket.py", line 279, in close
self.flush()
File "/usr/lib/python2.7/socket.py", line 303, in flush
self._sock.sendall(view[write_offset:write_offset+buffer_size])
error: [Errno 32] Broken pipe
----------------------------------------
GO code that's calling the end point:
res, err := http.Get(remoteUri)
if err != nil {
return err
}
defer res.Body.Close()
bodyBytesBuffer := new(bytes.Buffer)
_, err = io.Copy(bodyBytesBuffer, res.Body)
if err != nil {
return err
}
urlReader := bytes.NewReader(bodyBytesBuffer.Bytes())
var zipReader *zip.Reader
zipReader, err = zip.NewReader(urlReader, int64(urlReader.Len()))
