Add resumable downloads, download paths, queue controls and history
This commit is contained in:
SirBlob committed
1 parent
434f6441da
commit
6027cfabb9
43 files changed
+2138
-291
No files matched your search
@@ -3,25 +3,15 @@ package fetch
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"ddc/internal/diskspace"
|
||||
"ddc/internal/files"
|
||||
"ddc/internal/guard"
|
||||
)
|
||||
|
||||
const (
|
||||
userAgent = "ddc/1.0"
|
||||
bufferSize = 1 << 20
|
||||
stallTimeout = 2 * time.Minute
|
||||
spaceCheckInterval = int64(256) << 20
|
||||
)
|
||||
|
||||
var (
|
||||
ErrIncomplete = errors.New("download ended before the full file was received")
|
||||
ErrStalled = errors.New("download stalled, no data received for too long")
|
||||
@@ -29,8 +19,21 @@ var (
|
||||
|
||||
type Progress interface {
|
||||
SetTotal(total int64)
|
||||
SetDone(done int64)
|
||||
Add(delta int64)
|
||||
SetFileName(name string)
|
||||
SetNotice(text string)
|
||||
}
|
||||
|
||||
type Request struct {
|
||||
URL string
|
||||
Directory string
|
||||
FileName string
|
||||
ReservedBytes int64
|
||||
Connections int
|
||||
State *State
|
||||
Progress Progress
|
||||
Checkpoint func(State)
|
||||
}
|
||||
|
||||
type Downloader struct {
|
||||
@@ -42,111 +45,99 @@ func NewDownloader(policy *guard.Policy) *Downloader {
|
||||
return &Downloader{policy: policy, client: policy.NewHTTPClient()}
|
||||
}
|
||||
|
||||
func (downloader *Downloader) Download(ctx context.Context, rawURL, directory string, reservedBytes int64, progress Progress) (string, error) {
|
||||
target, err := downloader.policy.ValidateURL(rawURL)
|
||||
func (downloader *Downloader) Download(ctx context.Context, request Request) (string, error) {
|
||||
target, err := downloader.policy.ValidateURL(request.URL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
defer cancel(nil)
|
||||
|
||||
response, err := downloader.open(ctx, target.String())
|
||||
source := &remoteFile{url: target.String()}
|
||||
partPath := filepath.Join(request.Directory, files.PartFileName)
|
||||
state, probe, err := downloader.prepare(ctx, source, request, partPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
fileName := FileNameFromResponse(response)
|
||||
progress.SetFileName(fileName)
|
||||
if err := diskspace.EnsureFree(directory, max(response.ContentLength, 0), reservedBytes); err != nil {
|
||||
progress := request.Progress
|
||||
progress.SetFileName(state.FileName)
|
||||
progress.SetTotal(state.TotalBytes)
|
||||
progress.SetDone(state.DoneBytes())
|
||||
remainingBytes := max(state.TotalBytes-state.DoneBytes(), 0)
|
||||
if err := diskspace.EnsureFree(request.Directory, remainingBytes, request.ReservedBytes); err != nil {
|
||||
closeBody(probe)
|
||||
return "", err
|
||||
}
|
||||
progress.SetTotal(max(response.ContentLength, 0))
|
||||
|
||||
partPath := filepath.Join(directory, files.PartFileName)
|
||||
written, err := writePart(cancel, partPath, response.Body, progress, directory, reservedBytes)
|
||||
file, err := openPartFile(partPath, probe != nil)
|
||||
if err != nil {
|
||||
os.Remove(partPath)
|
||||
if errors.Is(context.Cause(ctx), ErrStalled) {
|
||||
return "", ErrStalled
|
||||
}
|
||||
closeBody(probe)
|
||||
return "", err
|
||||
}
|
||||
if response.ContentLength > 0 && written != response.ContentLength {
|
||||
os.Remove(partPath)
|
||||
return "", ErrIncomplete
|
||||
runErr := newSession(downloader, source, file, request, state).run(ctx, probe)
|
||||
closeErr := file.Close()
|
||||
if runErr != nil {
|
||||
return "", runErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return "", closeErr
|
||||
}
|
||||
|
||||
finalPath := files.UniquePath(filepath.Join(directory, fileName))
|
||||
finalPath := files.UniquePath(filepath.Join(request.Directory, state.FileName))
|
||||
if err := os.Rename(partPath, finalPath); err != nil {
|
||||
os.Remove(partPath)
|
||||
return "", err
|
||||
}
|
||||
return finalPath, nil
|
||||
}
|
||||
|
||||
func (downloader *Downloader) open(ctx context.Context, target string) (*http.Response, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
func (downloader *Downloader) prepare(ctx context.Context, source *remoteFile, request Request, partPath string) (State, *http.Response, error) {
|
||||
if request.State != nil && request.State.valid() && fileExists(partPath) {
|
||||
state := *request.State
|
||||
source.validator = state.Validator
|
||||
source.totalBytes = state.TotalBytes
|
||||
return state, nil, nil
|
||||
}
|
||||
request.Header.Set("User-Agent", userAgent)
|
||||
request.Header.Set("Accept-Encoding", "identity")
|
||||
|
||||
response, err := downloader.client.Do(request)
|
||||
probe, err := downloader.open(ctx, source, 0, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return State{}, nil, err
|
||||
}
|
||||
if response.StatusCode != http.StatusOK {
|
||||
source.describe(probe)
|
||||
|
||||
connections := 1
|
||||
if probe.Header.Get("Accept-Ranges") == "bytes" {
|
||||
connections = min(max(request.Connections, 1), MaxConnections)
|
||||
}
|
||||
state := State{
|
||||
FileName: chooseFileName(request.FileName, probe),
|
||||
Validator: source.validator,
|
||||
TotalBytes: source.totalBytes,
|
||||
Segments: planSegments(source.totalBytes, connections),
|
||||
}
|
||||
return state, probe, nil
|
||||
}
|
||||
|
||||
func chooseFileName(requested string, response *http.Response) string {
|
||||
if name := SanitizeFileName(requested); name != "" {
|
||||
return name
|
||||
}
|
||||
return FileNameFromResponse(response)
|
||||
}
|
||||
|
||||
func openPartFile(partPath string, fresh bool) (*os.File, error) {
|
||||
flags := os.O_WRONLY | os.O_CREATE
|
||||
if fresh {
|
||||
flags |= os.O_TRUNC
|
||||
}
|
||||
return os.OpenFile(partPath, flags, 0o644)
|
||||
}
|
||||
|
||||
func fileExists(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
return err == nil && info.Mode().IsRegular()
|
||||
}
|
||||
|
||||
func closeBody(response *http.Response) {
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
return nil, fmt.Errorf("server responded with %s", response.Status)
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func writePart(cancel context.CancelCauseFunc, partPath string, body io.Reader, progress Progress, directory string, reservedBytes int64) (int64, error) {
|
||||
file, err := os.OpenFile(partPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
written, copyErr := copyBody(cancel, file, body, progress, directory, reservedBytes)
|
||||
closeErr := file.Close()
|
||||
if copyErr != nil {
|
||||
return written, copyErr
|
||||
}
|
||||
return written, closeErr
|
||||
}
|
||||
|
||||
func copyBody(cancel context.CancelCauseFunc, destination io.Writer, body io.Reader, progress Progress, directory string, reservedBytes int64) (int64, error) {
|
||||
stallTimer := time.AfterFunc(stallTimeout, func() { cancel(ErrStalled) })
|
||||
defer stallTimer.Stop()
|
||||
|
||||
buffer := make([]byte, bufferSize)
|
||||
var written, sinceSpaceCheck int64
|
||||
for {
|
||||
count, readErr := body.Read(buffer)
|
||||
if count > 0 {
|
||||
stallTimer.Reset(stallTimeout)
|
||||
if _, err := destination.Write(buffer[:count]); err != nil {
|
||||
return written, err
|
||||
}
|
||||
written += int64(count)
|
||||
sinceSpaceCheck += int64(count)
|
||||
progress.Add(int64(count))
|
||||
|
||||
if sinceSpaceCheck >= spaceCheckInterval {
|
||||
sinceSpaceCheck = 0
|
||||
if err := diskspace.EnsureFree(directory, 0, reservedBytes); err != nil {
|
||||
return written, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if errors.Is(readErr, io.EOF) {
|
||||
return written, nil
|
||||
}
|
||||
if readErr != nil {
|
||||
return written, readErr
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user