Go并发读写文件、分片写、分片下载文件的实现示例
作者：ProblemTerminator
读写文件在很多项目中都可以用到,本文主要介绍了Go并发读写文件、分片写、分片下载文件的实现示例,具有一定的参考价值,感兴趣的可以了解一下
简单读取
func ReadFile(filePath string) (chunks []byte, err error) {
f, err := os.Open(filePath)
if err != nil {
return
}
defer f.Close()
reader := bufio.NewReader(f)
for {
dataByte := make([]byte, 5*1024)
var n int
n, err = reader.Read(dataByte)
if err != nil || 0 == n {
break
}
chunks = append(chunks, dataByte[:n]...)
fmt.Printf("file: %s, len(chunks):%v", filePath, len(chunks))
}
isEOF := strings.Compare(err.Error(), "EOF")
if isEOF == 0 {
err = nil
fmt.Printf("read %s success: \n, len=%v", filePath, len(chunks))
return
}
fmt.Printf("readFile over")
return
}可以看到如文件较大，chunks会变得很大，此法只适用特定条件下的一般做法。
读取&分片写
读取文件流+分片写-1
var bufLen = 2 * 1024 * 1024
func DownLoadFileShardByFilePath1(writerFilePath string, body io.Reader) (err error) {
f, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModeAppend)
defer f.Close()
if err != nil {
fmt.Println("open err:" + err.Error())
return
}
writer := bufio.NewWriter(f)
bs := make([]byte, bufLen)
for {
var read int
read, err = body.Read(bs)
if err != nil || 0 == read {
break
}
_, err = writer.Write(bs[:read])
if err != nil {
fmt.Println("write err:" + err.Error())
break
}
}
if err == io.EOF {
err = nil
}
if err != nil {
return
}
if err = writer.Flush(); err != nil {
fmt.Println("writer flush err: ", err.Error())
return
}
fmt.Printf("downLoad over")
return
}读取文件流+分片写-2
var bufLen = 2 * 1024 * 1024
func DownLoadFileShard(writerFilePath string, body io.Reader) (err error) {
f, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModeAppend)
if err != nil {
fmt.Println("open err:" + err.Error())
return
}
defer f.Close()
bs := make([]byte, bufLen)
writer := bufio.NewWriter(f)
for {
var read int
switch read, err = body.Read(bs[:]); true {
case read < 0:
fmt.Println("read err: ", err.Error())
return
case read == 0, err == io.EOF:
fmt.Printf("downLoad over")
return writer.Flush()
case read > 0:
_, err = writer.Write(bs[:read])
if err != nil {
fmt.Println("write err:" + err.Error())
return
}
}
}
return
}读取文件流+并发分片写
type FileShard struct {
Data []byte
Err error
Code int // 0-正常 -1=失败
}
var bufLen = 2 * 1024 * 1024
func DownLoadFileShardCon(writerFilePath string, body io.Reader) (err error) {
writerFile, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModeAppend)
if err != nil {
fmt.Println("open err:" + err.Error())
return
}
defer writerFile.Close()
ch, complete := make(chan *FileShard), make(chan struct{})
go func() {
writer := bufio.NewWriter(writerFile)
youKnow:
for {
select {
case data := <-ch:
if data == nil {
err = writer.Flush()
break youKnow
}
if data.Code != 0 {
err = data.Err
break youKnow
}
if _, err = writer.Write(data.Data); err != nil {
fmt.Println("write err:", err.Error())
}
}
}
close(complete)
}()
go func() {
bs := make([]byte, bufLen)
for {
switch read, readErr := body.Read(bs[:]); true {
case read < 0:
ch <- &FileShard{Code: -1, Err: readErr}
close(ch)
return
case read == 0, err == io.EOF:
close(ch)
return
case read > 0:
ch <- &FileShard{Data: bs[:read], Code: 0}
}
}
}()
select {
case <-complete:
break
}
fmt.Printf("downLoad over")
return
}并发思路有很多种，看你代码怎么写哦，条条大路通罗马！
更好用的Copy方法
要提醒的是，还有一个很不错的方法在io包里，就是io.Copy()，可防止大文件处理时的内存溢出，也可以替换上述主要流程，比如：
func IOCopyExample(writerFilePath string, body io.Reader) (err error) {
f, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_TRUNC, 0666)
if err != nil {
return
}
defer f.Close()
writer := bufio.NewWriter(f)
_, err = io.Copy(writer, body)
_ = writer.Flush()
return
}http并发、分片下载
基于http的Range来完成：
func DownloadFileRange(url, writeFile string) error {
f, err := os.OpenFile(writeFile , os.O_CREATE|os.O_TRUNC, 0666)
if err != nil {
return err
}
defer f.Close()
resp, err := http.Head(url)
if err != nil {
return err
}
size, err := strconv.Atoi(resp.Header.Get("Content-Length"))
if err != nil {
return err
}
con := getSize(size) // getSize函数用来计算每次的并发数，可按自己方式自行指定
var start, end int64
for i := 0; i < con; i++ {
start = int64(i) * int64(size/con)
end = start + int64(size/con) - 1
go func(n int, offset, end int64) {
req := &http.Request{}
req, err = http.NewRequest(http.MethodGet, url, nil)
req.Header.Set("Range", fmt.Sprintf("bytes=%v-%v", offset, end))
client := &http.Client{}
resp, err = client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
f.Seek(offset, 0)
_, err = io.Copy(f, resp.Body)
if err != nil {
// log
}
}(i, start, end)
}
return nil
}到此这篇关于Go并发读写文件、分片写、分片下载文件的实现示例的文章就介绍到这了,更多相关Go并发读写、分片写、分片下载内容请搜索脚本之家以前的文章或继续浏览下面的相关文章希望大家以后多多支持脚本之家！