Initial DDC release

internal/jobs/job.go

@@ -0,0 +1,126 @@
+package jobs
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type Status string
+
+const (
+	StatusQueued      Status = "queued"
+	StatusDownloading Status = "downloading"
+	StatusExtracting  Status = "extracting"
+	StatusReady       Status = "ready"
+	StatusMoving      Status = "moving"
+	StatusCompleted   Status = "completed"
+	StatusFailed      Status = "failed"
+	StatusCancelled   Status = "cancelled"
+)
+
+type Snapshot struct {
+	ID         string    `json:"id"`
+	SourceURL  string    `json:"sourceUrl"`
+	FileName   string    `json:"fileName"`
+	Status     Status    `json:"status"`
+	Message    string    `json:"message"`
+	BytesDone  int64     `json:"bytesDone"`
+	BytesTotal int64     `json:"bytesTotal"`
+	CreatedAt  time.Time `json:"createdAt"`
+}
+
+type Job struct {
+	id          string
+	sourceURL   string
+	displayURL  string
+	directory   string
+	autoExtract bool
+	createdAt   time.Time
+	jobContext  context.Context
+	cancel      context.CancelFunc
+	workers     sync.WaitGroup
+
+	mutex    sync.Mutex
+	status   Status
+	message  string
+	fileName string
+	deleted  bool
+
+	bytesDone  atomic.Int64
+	bytesTotal atomic.Int64
+}
+
+func (job *Job) SetTotal(total int64) {
+	job.bytesTotal.Store(total)
+}
+
+func (job *Job) Add(delta int64) {
+	job.bytesDone.Add(delta)
+}
+
+func (job *Job) SetFileName(name string) {
+	job.mutex.Lock()
+	defer job.mutex.Unlock()
+	job.fileName = name
+}
+
+func (job *Job) resetProgress() {
+	job.bytesDone.Store(0)
+	job.bytesTotal.Store(0)
+}
+
+func (job *Job) setStatus(status Status, message string) {
+	job.mutex.Lock()
+	defer job.mutex.Unlock()
+	if job.deleted {
+		return
+	}
+	job.status = status
+	job.message = message
+}
+
+func (job *Job) currentStatus() Status {
+	job.mutex.Lock()
+	defer job.mutex.Unlock()
+	return job.status
+}
+
+func (job *Job) begin(from, to Status) bool {
+	job.mutex.Lock()
+	defer job.mutex.Unlock()
+	if job.deleted || job.status != from {
+		return false
+	}
+	job.status = to
+	job.message = ""
+	job.workers.Add(1)
+	return true
+}
+
+func (job *Job) markDeleted() bool {
+	job.mutex.Lock()
+	defer job.mutex.Unlock()
+	if job.status == StatusMoving {
+		return false
+	}
+	job.deleted = true
+	job.status = StatusCancelled
+	return true
+}
+
+func (job *Job) snapshot() Snapshot {
+	job.mutex.Lock()
+	defer job.mutex.Unlock()
+	return Snapshot{
+		ID:         job.id,
+		SourceURL:  job.displayURL,
+		FileName:   job.fileName,
+		Status:     job.status,
+		Message:    job.message,
+		BytesDone:  job.bytesDone.Load(),
+		BytesTotal: job.bytesTotal.Load(),
+		CreatedAt:  job.createdAt,
+	}
+}

internal/jobs/manager.go

@@ -0,0 +1,208 @@
+package jobs
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"sync"
+	"time"
+
+	"ddc/internal/config"
+	"ddc/internal/diskspace"
+	"ddc/internal/fetch"
+	"ddc/internal/files"
+	"ddc/internal/guard"
+	"ddc/internal/libraries"
+	"ddc/internal/preferences"
+)
+
+var (
+	ErrJobNotFound    = errors.New("download not found")
+	ErrJobBusy        = errors.New("download is busy, try again when it is ready")
+	ErrInvalidRequest = errors.New("invalid request")
+)
+
+type Manager struct {
+	settings    config.Settings
+	preferences *preferences.Store
+	policy      *guard.Policy
+	downloader  *fetch.Downloader
+	rootContext context.Context
+	cancelAll   context.CancelFunc
+	slots       slotLimiter
+	mutex       sync.RWMutex
+	jobs        map[string]*Job
+}
+
+func NewManager(settings config.Settings, store *preferences.Store) *Manager {
+	policy := guard.NewPolicy(store.Get().AllowedPorts)
+	rootContext, cancelAll := context.WithCancel(context.Background())
+	return &Manager{
+		settings:    settings,
+		preferences: store,
+		policy:      policy,
+		downloader:  fetch.NewDownloader(policy),
+		rootContext: rootContext,
+		cancelAll:   cancelAll,
+		jobs:        make(map[string]*Job),
+	}
+}
+
+func (manager *Manager) Destinations() []libraries.Destination {
+	return libraries.Discover(manager.settings.LibraryRoots)
+}
+
+func (manager *Manager) Preferences() preferences.Preferences {
+	return manager.preferences.Get()
+}
+
+func (manager *Manager) UpdatePreferences(next preferences.Preferences) error {
+	if err := manager.preferences.Update(next); err != nil {
+		return err
+	}
+	manager.policy.SetAllowedPorts(next.AllowedPorts)
+	return nil
+}
+
+func (manager *Manager) concurrencyLimit() int {
+	return manager.preferences.Get().MaxConcurrentDownloads
+}
+
+func (manager *Manager) reservedBytes() int64 {
+	return manager.preferences.Get().MinFreeBytes()
+}
+
+func (manager *Manager) AvailableBytes() (int64, error) {
+	return diskspace.Available(manager.settings.DownloadDir)
+}
+
+func (manager *Manager) Create(rawURL string, autoExtract bool) (Snapshot, error) {
+	target, err := manager.policy.ValidateURL(rawURL)
+	if err != nil {
+		return Snapshot{}, invalid(err)
+	}
+	if err := diskspace.EnsureFree(manager.settings.DownloadDir, 0, manager.reservedBytes()); err != nil {
+		return Snapshot{}, err
+	}
+
+	id, err := newJobID()
+	if err != nil {
+		return Snapshot{}, err
+	}
+	directory := filepath.Join(manager.settings.DownloadDir, id)
+	if err := os.Mkdir(directory, 0o755); err != nil {
+		return Snapshot{}, err
+	}
+
+	job := manager.newJob(id, directory, time.Now())
+	job.sourceURL = target.String()
+	job.displayURL = target.Redacted()
+	job.autoExtract = autoExtract
+	job.status = StatusQueued
+	job.fileName = initialFileName(target.Path, target.Host)
+	job.workers.Add(1)
+
+	manager.store(job)
+	go manager.download(job)
+	return job.snapshot(), nil
+}
+
+func (manager *Manager) List() []Snapshot {
+	manager.mutex.RLock()
+	snapshots := make([]Snapshot, 0, len(manager.jobs))
+	for _, job := range manager.jobs {
+		snapshots = append(snapshots, job.snapshot())
+	}
+	manager.mutex.RUnlock()
+
+	sort.Slice(snapshots, func(first, second int) bool {
+		return snapshots[first].CreatedAt.After(snapshots[second].CreatedAt)
+	})
+	return snapshots
+}
+
+func (manager *Manager) Files(id string) ([]files.Node, error) {
+	job, err := manager.lookup(id)
+	if err != nil {
+		return nil, err
+	}
+	if job.currentStatus() != StatusReady {
+		return nil, ErrJobBusy
+	}
+	return files.BuildTree(job.directory)
+}
+
+func (manager *Manager) Delete(id string) error {
+	job, err := manager.lookup(id)
+	if err != nil {
+		return err
+	}
+	if !job.markDeleted() {
+		return ErrJobBusy
+	}
+
+	job.cancel()
+	job.workers.Wait()
+
+	manager.mutex.Lock()
+	delete(manager.jobs, id)
+	manager.mutex.Unlock()
+
+	return os.RemoveAll(job.directory)
+}
+
+func (manager *Manager) CancelAll() {
+	manager.cancelAll()
+}
+
+func (manager *Manager) newJob(id, directory string, createdAt time.Time) *Job {
+	jobContext, cancel := context.WithCancel(manager.rootContext)
+	return &Job{
+		id:         id,
+		directory:  directory,
+		createdAt:  createdAt,
+		jobContext: jobContext,
+		cancel:     cancel,
+	}
+}
+
+func (manager *Manager) store(job *Job) {
+	manager.mutex.Lock()
+	defer manager.mutex.Unlock()
+	manager.jobs[job.id] = job
+}
+
+func (manager *Manager) lookup(id string) (*Job, error) {
+	manager.mutex.RLock()
+	defer manager.mutex.RUnlock()
+	job, found := manager.jobs[id]
+	if !found {
+		return nil, ErrJobNotFound
+	}
+	return job, nil
+}
+
+func newJobID() (string, error) {
+	buffer := make([]byte, 16)
+	if _, err := rand.Read(buffer); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(buffer), nil
+}
+
+func initialFileName(urlPath, host string) string {
+	if name := fetch.SanitizeFileName(path.Base(urlPath)); name != "" {
+		return name
+	}
+	return host
+}
+
+func invalid(err error) error {
+	return fmt.Errorf("%w: %w", ErrInvalidRequest, err)
+}

internal/jobs/move.go

@@ -0,0 +1,125 @@
+package jobs
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"ddc/internal/files"
+	"ddc/internal/libraries"
+)
+
+type MoveRequest struct {
+	Paths       []string `json:"paths"`
+	Destination string   `json:"destination"`
+	Subfolder   string   `json:"subfolder"`
+	Cleanup     bool     `json:"cleanup"`
+}
+
+func (manager *Manager) Move(id string, request MoveRequest) error {
+	job, err := manager.lookup(id)
+	if err != nil {
+		return err
+	}
+	if job.currentStatus() != StatusReady {
+		return ErrJobBusy
+	}
+	if len(request.Paths) == 0 {
+		return invalid(errors.New("select at least one file or folder"))
+	}
+
+	library, err := libraries.Resolve(manager.settings.LibraryRoots, request.Destination)
+	if err != nil {
+		return invalid(err)
+	}
+	subfolder, err := files.SanitizeSubfolder(request.Subfolder)
+	if err != nil {
+		return invalid(err)
+	}
+	sources, err := resolveSources(job.directory, request.Paths)
+	if err != nil {
+		return err
+	}
+	destination, err := files.EnsureSubdirectory(library.Path, subfolder)
+	if err != nil {
+		return invalid(err)
+	}
+	if err := checkTargets(sources, destination); err != nil {
+		return err
+	}
+
+	if !job.begin(StatusReady, StatusMoving) {
+		return ErrJobBusy
+	}
+	go func() {
+		defer job.workers.Done()
+		manager.moveFiles(job, library, sources, destination, request.Cleanup)
+	}()
+	return nil
+}
+
+func resolveSources(root string, relativePaths []string) ([]string, error) {
+	sources := make([]string, 0, len(relativePaths))
+	seenNames := make(map[string]bool)
+
+	for _, relative := range relativePaths {
+		source, err := files.ResolveInside(root, relative)
+		if err != nil {
+			return nil, invalid(err)
+		}
+		name := filepath.Base(source)
+		if seenNames[name] {
+			return nil, invalid(fmt.Errorf("more than one selected item is named %q", name))
+		}
+		seenNames[name] = true
+		sources = append(sources, source)
+	}
+
+	for _, outer := range sources {
+		for _, inner := range sources {
+			if files.IsWithin(outer, inner) {
+				return nil, invalid(errors.New("selection contains a folder and items inside it"))
+			}
+		}
+	}
+	return sources, nil
+}
+
+func checkTargets(sources []string, destination string) error {
+	for _, source := range sources {
+		name := filepath.Base(source)
+		if _, err := os.Lstat(filepath.Join(destination, name)); err == nil {
+			return fmt.Errorf("%w: %s", files.ErrDestinationExists, name)
+		}
+	}
+	return nil
+}
+
+func (manager *Manager) moveFiles(job *Job, library libraries.Destination, sources []string, destination string, cleanup bool) {
+	job.resetProgress()
+	var totalBytes int64
+	for _, source := range sources {
+		if size, err := files.TreeSize(source); err == nil {
+			totalBytes += size
+		}
+	}
+	job.SetTotal(totalBytes)
+
+	for _, source := range sources {
+		if err := files.MoveInto(source, destination, manager.reservedBytes(), job.Add); err != nil {
+			job.setStatus(StatusReady, "Move failed: "+err.Error())
+			return
+		}
+	}
+
+	message := fmt.Sprintf("Moved %d item(s) to %s", len(sources), library.Path)
+	if !cleanup {
+		job.setStatus(StatusReady, message)
+		return
+	}
+	if err := os.RemoveAll(job.directory); err != nil {
+		message += ", cleanup failed: " + err.Error()
+	}
+	job.setStatus(StatusCompleted, message)
+}

internal/jobs/restore.go

@@ -0,0 +1,51 @@
+package jobs
+
+import (
+	"os"
+	"path/filepath"
+	"regexp"
+	"time"
+
+	"ddc/internal/files"
+)
+
+var jobIDPattern = regexp.MustCompile(`^[0-9a-f]{32}$`)
+
+func (manager *Manager) RestoreExisting() error {
+	entries, err := os.ReadDir(manager.settings.DownloadDir)
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range entries {
+		if !entry.IsDir() || !jobIDPattern.MatchString(entry.Name()) {
+			continue
+		}
+		directory := filepath.Join(manager.settings.DownloadDir, entry.Name())
+		os.Remove(filepath.Join(directory, files.PartFileName))
+
+		createdAt := time.Now()
+		if info, err := entry.Info(); err == nil {
+			createdAt = info.ModTime()
+		}
+
+		job := manager.newJob(entry.Name(), directory, createdAt)
+		job.status = StatusReady
+		job.fileName = firstEntryName(directory, entry.Name())
+		manager.store(job)
+	}
+	return nil
+}
+
+func firstEntryName(directory, fallback string) string {
+	entries, err := os.ReadDir(directory)
+	if err != nil {
+		return fallback
+	}
+	for _, entry := range entries {
+		if entry.Name() != files.PartFileName {
+			return entry.Name()
+		}
+	}
+	return fallback
+}

internal/jobs/slots.go

@@ -0,0 +1,45 @@
+package jobs
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+const slotWaitInterval = 500 * time.Millisecond
+
+type slotLimiter struct {
+	mutex  sync.Mutex
+	active int
+}
+
+func (limiter *slotLimiter) acquire(ctx context.Context, limit func() int) bool {
+	ticker := time.NewTicker(slotWaitInterval)
+	defer ticker.Stop()
+	for {
+		if limiter.tryAcquire(limit()) {
+			return true
+		}
+		select {
+		case <-ctx.Done():
+			return false
+		case <-ticker.C:
+		}
+	}
+}
+
+func (limiter *slotLimiter) tryAcquire(limit int) bool {
+	limiter.mutex.Lock()
+	defer limiter.mutex.Unlock()
+	if limiter.active >= limit {
+		return false
+	}
+	limiter.active++
+	return true
+}
+
+func (limiter *slotLimiter) release() {
+	limiter.mutex.Lock()
+	defer limiter.mutex.Unlock()
+	limiter.active--
+}

internal/jobs/workflows.go

@@ -0,0 +1,102 @@
+package jobs
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+
+	"ddc/internal/diskspace"
+	"ddc/internal/extract"
+	"ddc/internal/files"
+)
+
+const maxArchiveEntries = 100000
+
+var errNotArchive = errors.New("selected file is not a supported archive")
+
+func (manager *Manager) download(job *Job) {
+	defer job.workers.Done()
+
+	if !manager.slots.acquire(job.jobContext, manager.concurrencyLimit) {
+		return
+	}
+	defer manager.slots.release()
+
+	job.setStatus(StatusDownloading, "")
+	filePath, err := manager.downloader.Download(job.jobContext, job.sourceURL, job.directory, manager.reservedBytes(), job)
+	if err != nil {
+		manager.finishWithError(job, StatusFailed, "", err)
+		return
+	}
+	job.SetFileName(filepath.Base(filePath))
+
+	if job.autoExtract && extract.IsArchive(filePath) {
+		job.setStatus(StatusExtracting, "")
+		destination, err := manager.extractArchive(job, filePath)
+		if err != nil {
+			manager.finishWithError(job, StatusReady, "Extraction failed: ", err)
+			return
+		}
+		job.SetFileName(filepath.Base(destination))
+	}
+
+	job.setStatus(StatusReady, "")
+}
+
+func (manager *Manager) Extract(id, relativePath string) error {
+	job, err := manager.lookup(id)
+	if err != nil {
+		return err
+	}
+
+	archivePath, err := files.ResolveInside(job.directory, relativePath)
+	if err != nil {
+		return invalid(err)
+	}
+	info, err := os.Stat(archivePath)
+	if err != nil || !info.Mode().IsRegular() || !extract.IsArchive(archivePath) {
+		return invalid(errNotArchive)
+	}
+
+	if !job.begin(StatusReady, StatusExtracting) {
+		return ErrJobBusy
+	}
+	go func() {
+		defer job.workers.Done()
+		if _, err := manager.extractArchive(job, archivePath); err != nil {
+			manager.finishWithError(job, StatusReady, "Extraction failed: ", err)
+			return
+		}
+		job.setStatus(StatusReady, "")
+	}()
+	return nil
+}
+
+func (manager *Manager) extractArchive(job *Job, archivePath string) (string, error) {
+	available, err := diskspace.Available(job.directory)
+	if err != nil {
+		return "", err
+	}
+	limit := available - manager.reservedBytes()
+	if limit <= 0 {
+		return "", diskspace.ErrInsufficientSpace
+	}
+
+	directoryName := extract.DirectoryName(filepath.Base(archivePath))
+	destination := files.UniquePath(filepath.Join(filepath.Dir(archivePath), directoryName))
+	limits := extract.Limits{MaxBytes: limit, MaxEntries: maxArchiveEntries}
+
+	job.resetProgress()
+	if err := extract.Archive(job.jobContext, archivePath, destination, limits, job); err != nil {
+		return "", err
+	}
+	return destination, os.Remove(archivePath)
+}
+
+func (manager *Manager) finishWithError(job *Job, status Status, prefix string, err error) {
+	if job.jobContext.Err() != nil {
+		job.setStatus(StatusCancelled, "")
+		return
+	}
+	job.setStatus(status, prefix+err.Error())
+}