storage/minio: add workaround for unspecified size in Save when multipart is disabled

This commit is contained in:
Steve Fan
2022-11-11 16:04:34 +00:00
parent c71fa9b8db
commit 866f567a3d

View File

@@ -5,6 +5,7 @@
package storage
import (
"bytes"
"context"
"io"
"net/url"
@@ -141,6 +142,22 @@ func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error)
disableSignature, disableMultipart = m.config.DisableSignature, m.config.DisableMultipart
}
if disableMultipart && size < 0 {
// Attempts to read everything from the source into memory. This can take a big toll on memory, and it can become a potential DoS source
// but since we have disabled multipart upload this mean we can't really stream write anymore...
// well, unless we have a better way to estimate the stream size, this would be a workaround
buf := &bytes.Buffer{}
if n, err := io.Copy(buf, r); err != nil {
// I guess this would likely be EOF or OOM...?
return -1, err
} else {
// Since we read all the data from the source, it might not be usable again,
// so we should swap the reader location to our memory buffer
r, size = buf, n
}
}
uploadInfo, err := m.client.PutObject(
m.ctx,
m.bucket,