golang/python下载大文件时怎样避免oom

    问题场景&＃xff1a;高频系统中&＃xff0c;agent 会向ATS 服务器发出刷新和预缓存的请求&＃xff0c;这里的请求head 里面有GET &＃xff0c;PURGE等&＃xff0c;因为一般的预缓存都是小文件&＃xff0c;但是某天&＃xff0c;突然服务器oom。。。罪魁祸首发现是并发GET 大文件将服务器打死了。第一个版本是python 的&＃xff0c;第二个版本是golang 实现的&＃xff0c; 这里记录下两种语言的 下载大文件的实现方式。

该文章后续仍在不断的更新修改中&＃xff0c; 请移步到原文地址http://dmwan.cc

    第一种是python&＃xff0c;使用的是request 库, 使用流式读取的方式&＃xff0c;写到空设备中去。

res &＃61; self.session.request(method, url, data&＃61;body, headers&＃61;header, timeout&＃61;timeout, proxies&＃61;proxies, stream&＃61;True)
with open("/dev/null", &＃39;wb&＃39;) as f:for chunk in res.iter_content(chunk_size&＃61;1024):if chunk: # filter out keep-alive new chunksf.write(chunk)f.flush()


    第二种方式&＃xff0c;对于golang &＃xff0c;使用io.Copy(), 将response copy 到空设备中。

func downLoadFile(url string)(len int, err error){//err write /dev/null: bad file descriptor#out, err :&＃61; os.OpenFile("/dev/null", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)defer out.Close()resp, err :&＃61; http.Get(url)defer resp.Body.Close()n, err :&＃61; io.Copy(out, resp.Body)return n, err
}

    使用这种方式为什么不会出现oom 的情况&＃xff1f;因为两个原因&＃xff0c;第一个&＃xff0c; resp.Body 只是个reader 并没有发生真实的读取操作&＃xff0c;第二个是io.copy 这个函数设置了缓冲区大小限制为3m&＃xff0c;不会一次全部读取到内存中&＃xff0c;下面是标准库的源码&＃xff1a;

func Copy(dst Writer, src Reader) (written int64, err error) {return copyBuffer(dst, src, nil)
}// copyBuffer is the actual implementation of Copy and CopyBuffer.
// if buf is nil, one is allocated.
func copyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) {// If the reader has a WriteTo method, use it to do the copy.// Avoids an allocation and a copy.if wt, ok :&＃61; src.(WriterTo); ok {return wt.WriteTo(dst)}// Similarly, if the writer has a ReadFrom method, use it to do the copy.if rt, ok :&＃61; dst.(ReaderFrom); ok {return rt.ReadFrom(src)}if buf &＃61;&＃61; nil {buf &＃61; make([]byte, 32*1024) //这一步可以控制每次缓冲区迭代的大小&＃xff0c;默认大小是3m}for {nr, er :&＃61; src.Read(buf)if nr > 0 {nw, ew :&＃61; dst.Write(buf[0:nr])if nw > 0 {written &＃43;&＃61; int64(nw)}if ew !&＃61; nil {err &＃61; ewbreak}if nr !&＃61; nw {err &＃61; ErrShortWritebreak}}if er !&＃61; nil {if er !&＃61; EOF {err &＃61; er}break}}return written, err
}