Initial DDC release

This commit is contained in:
SirBlob committed 2026-10-25 20:24:50 -04:00
commit 434f6441da
57 files changed
+4606

No files matched your search

+18
View File
@@ -0,0 +1,18 @@
package auth
import (
"crypto/sha256"
"crypto/subtle"
)
func CredentialsMatch(expectedUsername, expectedPassword, username, password string) bool {
usernameMatches := constantTimeEqual(expectedUsername, username)
passwordMatches := constantTimeEqual(expectedPassword, password)
return usernameMatches && passwordMatches
}
func constantTimeEqual(expected, actual string) bool {
expectedHash := sha256.Sum256([]byte(expected))
actualHash := sha256.Sum256([]byte(actual))
return subtle.ConstantTimeCompare(expectedHash[:], actualHash[:]) == 1
}
+76
View File
@@ -0,0 +1,76 @@
package auth
import (
"sync"
"time"
)
const maxTrackedClients = 10000
type attemptWindow struct {
failures int
startedAt time.Time
}
type LoginLimiter struct {
mutex sync.Mutex
maxFailures int
window time.Duration
attempts map[string]*attemptWindow
}
func NewLoginLimiter(maxFailures int, window time.Duration) *LoginLimiter {
return &LoginLimiter{
maxFailures: maxFailures,
window: window,
attempts: make(map[string]*attemptWindow),
}
}
func (limiter *LoginLimiter) Allowed(client string) bool {
limiter.mutex.Lock()
defer limiter.mutex.Unlock()
attempt := limiter.current(client)
return attempt == nil || attempt.failures < limiter.maxFailures
}
func (limiter *LoginLimiter) RecordFailure(client string) {
limiter.mutex.Lock()
defer limiter.mutex.Unlock()
attempt := limiter.current(client)
if attempt == nil {
if len(limiter.attempts) >= maxTrackedClients {
limiter.removeExpired()
}
attempt = &attemptWindow{startedAt: time.Now()}
limiter.attempts[client] = attempt
}
attempt.failures++
}
func (limiter *LoginLimiter) Reset(client string) {
limiter.mutex.Lock()
defer limiter.mutex.Unlock()
delete(limiter.attempts, client)
}
func (limiter *LoginLimiter) current(client string) *attemptWindow {
attempt, found := limiter.attempts[client]
if !found {
return nil
}
if time.Since(attempt.startedAt) > limiter.window {
delete(limiter.attempts, client)
return nil
}
return attempt
}
func (limiter *LoginLimiter) removeExpired() {
for client, attempt := range limiter.attempts {
if time.Since(attempt.startedAt) > limiter.window {
delete(limiter.attempts, client)
}
}
}
+75
View File
@@ -0,0 +1,75 @@
package auth
import (
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"sync"
"time"
)
const tokenBytes = 32
type SessionStore struct {
mutex sync.Mutex
ttl time.Duration
expirations map[string]time.Time
}
func NewSessionStore(ttl time.Duration) *SessionStore {
return &SessionStore{ttl: ttl, expirations: make(map[string]time.Time)}
}
func (store *SessionStore) Create() (string, error) {
buffer := make([]byte, tokenBytes)
if _, err := rand.Read(buffer); err != nil {
return "", err
}
token := base64.RawURLEncoding.EncodeToString(buffer)
store.mutex.Lock()
defer store.mutex.Unlock()
now := time.Now()
store.removeExpired(now)
store.expirations[hashToken(token)] = now.Add(store.ttl)
return token, nil
}
func (store *SessionStore) Valid(token string) bool {
if token == "" {
return false
}
key := hashToken(token)
store.mutex.Lock()
defer store.mutex.Unlock()
expiresAt, found := store.expirations[key]
if !found {
return false
}
if time.Now().After(expiresAt) {
delete(store.expirations, key)
return false
}
return true
}
func (store *SessionStore) Delete(token string) {
store.mutex.Lock()
defer store.mutex.Unlock()
delete(store.expirations, hashToken(token))
}
func (store *SessionStore) removeExpired(now time.Time) {
for key, expiresAt := range store.expirations {
if now.After(expiresAt) {
delete(store.expirations, key)
}
}
}
func hashToken(token string) string {
sum := sha256.Sum256([]byte(token))
return hex.EncodeToString(sum[:])
}
+73
View File
@@ -0,0 +1,73 @@
package config
import (
"errors"
"fmt"
"path/filepath"
"time"
"ddc/internal/preferences"
)
const (
minPasswordLength = 12
defaultLibraryRoots = "/media/blob/TeaNet/Plex;/media/blob/ChaiNet/Plex"
)
type Settings struct {
ListenAddr string
Username string
Password string
CookieSecure bool
SessionTTL time.Duration
TrustProxyHeaders bool
DownloadDir string
LibraryRoots []string
Defaults preferences.Preferences
}
func Load() (Settings, error) {
reader := &envReader{}
settings := Settings{
ListenAddr: reader.text("LISTEN_ADDR", ":8080"),
Username: reader.text("AUTH_USERNAME", ""),
Password: reader.text("AUTH_PASSWORD", ""),
CookieSecure: reader.boolean("COOKIE_SECURE", true),
SessionTTL: time.Duration(reader.integer("SESSION_TTL_HOURS", 12)) * time.Hour,
TrustProxyHeaders: reader.boolean("TRUST_PROXY_HEADERS", false),
DownloadDir: filepath.Clean(reader.text("DOWNLOAD_DIR", "/data/downloads")),
Defaults: preferences.Preferences{
MaxConcurrentDownloads: reader.integer("MAX_CONCURRENT_DOWNLOADS", 2),
MinFreeGB: reader.integer("MIN_FREE_GB", 10),
AllowedPorts: reader.ports("ALLOWED_PORTS", "80,443"),
AutoExtract: true,
CleanupAfterMove: true,
},
}
rawRoots := reader.text("LIBRARY_ROOTS", defaultLibraryRoots)
if reader.err != nil {
return Settings{}, reader.err
}
roots, err := parseLibraryRoots(rawRoots)
if err != nil {
return Settings{}, err
}
settings.LibraryRoots = roots
return settings, settings.validate()
}
func (settings Settings) validate() error {
switch {
case settings.Username == "":
return errors.New("AUTH_USERNAME is required")
case len(settings.Password) < minPasswordLength:
return fmt.Errorf("AUTH_PASSWORD must be at least %d characters", minPasswordLength)
case !filepath.IsAbs(settings.DownloadDir):
return errors.New("DOWNLOAD_DIR must be an absolute path")
case settings.SessionTTL <= 0:
return errors.New("SESSION_TTL_HOURS must be greater than zero")
}
return settings.Defaults.Validate()
}
+78
View File
@@ -0,0 +1,78 @@
package config
import (
"fmt"
"os"
"strconv"
"strings"
)
type envReader struct {
err error
}
func (reader *envReader) fail(name string, err error) {
if reader.err == nil {
reader.err = fmt.Errorf("%s: %w", name, err)
}
}
func (reader *envReader) text(name, fallback string) string {
value := strings.TrimSpace(os.Getenv(name))
if value == "" {
return fallback
}
return value
}
func (reader *envReader) boolean(name string, fallback bool) bool {
raw := reader.text(name, "")
if raw == "" {
return fallback
}
value, err := strconv.ParseBool(raw)
if err != nil {
reader.fail(name, err)
return fallback
}
return value
}
func (reader *envReader) integer(name string, fallback int) int {
raw := reader.text(name, "")
if raw == "" {
return fallback
}
value, err := strconv.Atoi(raw)
if err != nil {
reader.fail(name, err)
return fallback
}
return value
}
func (reader *envReader) ports(name, fallback string) []uint16 {
raw := reader.text(name, fallback)
if raw == "*" {
return nil
}
var ports []uint16
for _, part := range strings.Split(raw, ",") {
part = strings.TrimSpace(part)
if part == "" {
continue
}
value, err := strconv.ParseUint(part, 10, 16)
if err != nil || value == 0 {
reader.fail(name, fmt.Errorf("invalid port %q", part))
return nil
}
ports = append(ports, uint16(value))
}
if len(ports) == 0 {
reader.fail(name, fmt.Errorf("no ports listed, use * to allow any port"))
}
return ports
}
+56
View File
@@ -0,0 +1,56 @@
package config
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
)
func parseLibraryRoots(raw string) ([]string, error) {
var roots []string
seenRoots := make(map[string]bool)
for _, entry := range strings.Split(raw, ";") {
entry = strings.TrimSpace(entry)
if entry == "" {
continue
}
root, err := resolveRoot(entry)
if err != nil {
return nil, err
}
if seenRoots[root] {
continue
}
seenRoots[root] = true
roots = append(roots, root)
}
if len(roots) == 0 {
return nil, errors.New("LIBRARY_ROOTS must list at least one folder")
}
return roots, nil
}
func resolveRoot(entry string) (string, error) {
cleaned := filepath.Clean(entry)
if !filepath.IsAbs(cleaned) || cleaned == "/" {
return "", fmt.Errorf("LIBRARY_ROOTS: invalid path %q", entry)
}
resolved, err := filepath.EvalSymlinks(cleaned)
if err != nil {
return "", fmt.Errorf("LIBRARY_ROOTS: %w", err)
}
info, err := os.Stat(resolved)
if err != nil {
return "", fmt.Errorf("LIBRARY_ROOTS: %w", err)
}
if !info.IsDir() {
return "", fmt.Errorf("LIBRARY_ROOTS: %q is not a directory", entry)
}
return resolved, nil
}
+38
View File
@@ -0,0 +1,38 @@
package diskspace
import (
"errors"
"fmt"
"math"
"syscall"
)
var ErrInsufficientSpace = errors.New("not enough free disk space")
func Available(path string) (int64, error) {
var stats syscall.Statfs_t
if err := syscall.Statfs(path, &stats); err != nil {
return 0, err
}
available := stats.Bavail * uint64(stats.Bsize)
if available > math.MaxInt64 {
return math.MaxInt64, nil
}
return int64(available), nil
}
func EnsureFree(path string, requiredBytes, reservedBytes int64) error {
available, err := Available(path)
if err != nil {
return err
}
if available-reservedBytes < requiredBytes {
return fmt.Errorf("%w: %.1f GiB needed, %.1f GiB usable after keeping %.1f GiB free",
ErrInsufficientSpace, toGiB(requiredBytes), toGiB(available-reservedBytes), toGiB(reservedBytes))
}
return nil
}
func toGiB(bytes int64) float64 {
return float64(bytes) / float64(1<<30)
}
+48
View File
@@ -0,0 +1,48 @@
package extract
import (
"context"
"errors"
"os"
"path/filepath"
)
var (
ErrUnsupportedFormat = errors.New("unsupported archive format")
ErrUnsafeEntry = errors.New("archive contains an unsafe path")
ErrTooLarge = errors.New("archive exceeds the extraction size limit")
ErrTooManyEntries = errors.New("archive contains too many entries")
)
type Limits struct {
MaxBytes int64
MaxEntries int
}
type Progress interface {
SetTotal(total int64)
Add(delta int64)
}
func Archive(ctx context.Context, archivePath, destination string, limits Limits, progress Progress) error {
archiveFormat, _ := detect(filepath.Base(archivePath))
if archiveFormat == formatNone {
return ErrUnsupportedFormat
}
if err := os.Mkdir(destination, 0o755); err != nil {
return err
}
writer := &entryWriter{ctx: ctx, root: destination, limits: limits, progress: progress}
var err error
if archiveFormat == formatZip {
err = extractZip(archivePath, writer)
} else {
err = extractTar(archivePath, archiveFormat, writer)
}
if err != nil {
os.RemoveAll(destination)
}
return err
}
+50
View File
@@ -0,0 +1,50 @@
package extract
import "strings"
type format int
const (
formatNone format = iota
formatZip
formatTar
formatTarGzip
formatTarBzip2
)
var knownSuffixes = []struct {
suffix string
format format
}{
{".tar.gz", formatTarGzip},
{".tgz", formatTarGzip},
{".tar.bz2", formatTarBzip2},
{".tbz2", formatTarBzip2},
{".tbz", formatTarBzip2},
{".tar", formatTar},
{".zip", formatZip},
}
func detect(name string) (format, string) {
lowerName := strings.ToLower(name)
for _, known := range knownSuffixes {
if strings.HasSuffix(lowerName, known.suffix) {
return known.format, known.suffix
}
}
return formatNone, ""
}
func IsArchive(name string) bool {
archiveFormat, _ := detect(name)
return archiveFormat != formatNone
}
func DirectoryName(archiveName string) string {
_, suffix := detect(archiveName)
base := strings.Trim(archiveName[:len(archiveName)-len(suffix)], ". ")
if base == "" {
return "extracted"
}
return base
}
+61
View File
@@ -0,0 +1,61 @@
package extract
import (
"archive/tar"
"compress/bzip2"
"compress/gzip"
"errors"
"io"
"os"
)
func extractTar(archivePath string, archiveFormat format, writer *entryWriter) error {
file, err := os.Open(archivePath)
if err != nil {
return err
}
defer file.Close()
if archiveFormat == formatTar {
if info, err := file.Stat(); err == nil {
writer.progress.SetTotal(info.Size())
}
}
source, err := decompress(file, archiveFormat)
if err != nil {
return err
}
reader := tar.NewReader(source)
for {
header, err := reader.Next()
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return err
}
switch header.Typeflag {
case tar.TypeDir:
err = writer.writeDirectory(header.Name)
case tar.TypeReg:
err = writer.writeFile(header.Name, reader)
}
if err != nil {
return err
}
}
}
func decompress(file *os.File, archiveFormat format) (io.Reader, error) {
switch archiveFormat {
case formatTarGzip:
return gzip.NewReader(file)
case formatTarBzip2:
return bzip2.NewReader(file), nil
default:
return file, nil
}
}
+106
View File
@@ -0,0 +1,106 @@
package extract
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
)
type entryWriter struct {
ctx context.Context
root string
limits Limits
progress Progress
written int64
entries int
}
func (writer *entryWriter) writeDirectory(name string) error {
target, err := writer.prepare(name)
if err != nil || target == "" {
return err
}
return os.MkdirAll(target, 0o755)
}
func (writer *entryWriter) writeFile(name string, content io.Reader) error {
target, err := writer.prepare(name)
if err != nil || target == "" {
return err
}
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
return err
}
file, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
if err != nil {
return err
}
remaining := writer.limits.MaxBytes - writer.written
source := &progressReader{ctx: writer.ctx, reader: io.LimitReader(content, remaining+1), progress: writer.progress}
copied, copyErr := io.Copy(file, source)
closeErr := file.Close()
writer.written += copied
if copyErr != nil {
return copyErr
}
if writer.written > writer.limits.MaxBytes {
return ErrTooLarge
}
return closeErr
}
func (writer *entryWriter) prepare(name string) (string, error) {
writer.entries++
if writer.entries > writer.limits.MaxEntries {
return "", ErrTooManyEntries
}
if err := writer.ctx.Err(); err != nil {
return "", err
}
return writer.resolve(name)
}
func (writer *entryWriter) resolve(name string) (string, error) {
normalized := strings.ReplaceAll(name, "\\", "/")
if strings.HasPrefix(normalized, "/") {
return "", fmt.Errorf("%w: %s", ErrUnsafeEntry, name)
}
cleaned := filepath.Clean(filepath.FromSlash(normalized))
if cleaned == "." {
return "", nil
}
if !filepath.IsLocal(cleaned) {
return "", fmt.Errorf("%w: %s", ErrUnsafeEntry, name)
}
target := filepath.Join(writer.root, cleaned)
relative, err := filepath.Rel(writer.root, target)
if err != nil || !filepath.IsLocal(relative) {
return "", fmt.Errorf("%w: %s", ErrUnsafeEntry, name)
}
return target, nil
}
type progressReader struct {
ctx context.Context
reader io.Reader
progress Progress
}
func (reader *progressReader) Read(buffer []byte) (int, error) {
if err := reader.ctx.Err(); err != nil {
return 0, err
}
count, err := reader.reader.Read(buffer)
if count > 0 {
reader.progress.Add(int64(count))
}
return count, err
}
+44
View File
@@ -0,0 +1,44 @@
package extract
import "archive/zip"
func extractZip(archivePath string, writer *entryWriter) error {
reader, err := zip.OpenReader(archivePath)
if err != nil {
return err
}
defer reader.Close()
var declaredBytes uint64
for _, file := range reader.File {
declaredBytes += file.UncompressedSize64
}
if declaredBytes > uint64(writer.limits.MaxBytes) {
return ErrTooLarge
}
writer.progress.SetTotal(int64(declaredBytes))
for _, file := range reader.File {
if err := extractZipEntry(file, writer); err != nil {
return err
}
}
return nil
}
func extractZipEntry(file *zip.File, writer *entryWriter) error {
mode := file.Mode()
switch {
case mode.IsDir():
return writer.writeDirectory(file.Name)
case mode.IsRegular():
content, err := file.Open()
if err != nil {
return err
}
defer content.Close()
return writer.writeFile(file.Name, content)
default:
return nil
}
}
+152
View File
@@ -0,0 +1,152 @@
package fetch
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"time"
"ddc/internal/diskspace"
"ddc/internal/files"
"ddc/internal/guard"
)
const (
userAgent = "ddc/1.0"
bufferSize = 1 << 20
stallTimeout = 2 * time.Minute
spaceCheckInterval = int64(256) << 20
)
var (
ErrIncomplete = errors.New("download ended before the full file was received")
ErrStalled = errors.New("download stalled, no data received for too long")
)
type Progress interface {
SetTotal(total int64)
Add(delta int64)
SetFileName(name string)
}
type Downloader struct {
policy *guard.Policy
client *http.Client
}
func NewDownloader(policy *guard.Policy) *Downloader {
return &Downloader{policy: policy, client: policy.NewHTTPClient()}
}
func (downloader *Downloader) Download(ctx context.Context, rawURL, directory string, reservedBytes int64, progress Progress) (string, error) {
target, err := downloader.policy.ValidateURL(rawURL)
if err != nil {
return "", err
}
ctx, cancel := context.WithCancelCause(ctx)
defer cancel(nil)
response, err := downloader.open(ctx, target.String())
if err != nil {
return "", err
}
defer response.Body.Close()
fileName := FileNameFromResponse(response)
progress.SetFileName(fileName)
if err := diskspace.EnsureFree(directory, max(response.ContentLength, 0), reservedBytes); err != nil {
return "", err
}
progress.SetTotal(max(response.ContentLength, 0))
partPath := filepath.Join(directory, files.PartFileName)
written, err := writePart(cancel, partPath, response.Body, progress, directory, reservedBytes)
if err != nil {
os.Remove(partPath)
if errors.Is(context.Cause(ctx), ErrStalled) {
return "", ErrStalled
}
return "", err
}
if response.ContentLength > 0 && written != response.ContentLength {
os.Remove(partPath)
return "", ErrIncomplete
}
finalPath := files.UniquePath(filepath.Join(directory, fileName))
if err := os.Rename(partPath, finalPath); err != nil {
os.Remove(partPath)
return "", err
}
return finalPath, nil
}
func (downloader *Downloader) open(ctx context.Context, target string) (*http.Response, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
if err != nil {
return nil, err
}
request.Header.Set("User-Agent", userAgent)
request.Header.Set("Accept-Encoding", "identity")
response, err := downloader.client.Do(request)
if err != nil {
return nil, err
}
if response.StatusCode != http.StatusOK {
response.Body.Close()
return nil, fmt.Errorf("server responded with %s", response.Status)
}
return response, nil
}
func writePart(cancel context.CancelCauseFunc, partPath string, body io.Reader, progress Progress, directory string, reservedBytes int64) (int64, error) {
file, err := os.OpenFile(partPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
if err != nil {
return 0, err
}
written, copyErr := copyBody(cancel, file, body, progress, directory, reservedBytes)
closeErr := file.Close()
if copyErr != nil {
return written, copyErr
}
return written, closeErr
}
func copyBody(cancel context.CancelCauseFunc, destination io.Writer, body io.Reader, progress Progress, directory string, reservedBytes int64) (int64, error) {
stallTimer := time.AfterFunc(stallTimeout, func() { cancel(ErrStalled) })
defer stallTimer.Stop()
buffer := make([]byte, bufferSize)
var written, sinceSpaceCheck int64
for {
count, readErr := body.Read(buffer)
if count > 0 {
stallTimer.Reset(stallTimeout)
if _, err := destination.Write(buffer[:count]); err != nil {
return written, err
}
written += int64(count)
sinceSpaceCheck += int64(count)
progress.Add(int64(count))
if sinceSpaceCheck >= spaceCheckInterval {
sinceSpaceCheck = 0
if err := diskspace.EnsureFree(directory, 0, reservedBytes); err != nil {
return written, err
}
}
}
if errors.Is(readErr, io.EOF) {
return written, nil
}
if readErr != nil {
return written, readErr
}
}
}
+65
View File
@@ -0,0 +1,65 @@
package fetch
import (
"mime"
"net/http"
"path"
"path/filepath"
"strings"
"unicode/utf8"
)
const (
maxFileNameBytes = 200
maxExtensionBytes = 16
fallbackFileName = "download"
)
func FileNameFromResponse(response *http.Response) string {
if disposition := response.Header.Get("Content-Disposition"); disposition != "" {
if _, parameters, err := mime.ParseMediaType(disposition); err == nil {
if name := SanitizeFileName(parameters["filename"]); name != "" {
return name
}
}
}
if response.Request != nil && response.Request.URL != nil {
if name := SanitizeFileName(path.Base(response.Request.URL.Path)); name != "" {
return name
}
}
return fallbackFileName
}
func SanitizeFileName(raw string) string {
name := path.Base(strings.ReplaceAll(raw, "\\", "/"))
name = strings.Map(dropUnsafeRune, name)
name = strings.TrimSpace(strings.TrimLeft(name, ". "))
if name == "" {
return ""
}
return truncate(name)
}
func dropUnsafeRune(character rune) rune {
if character < 0x20 || character == 0x7f || character == '/' || character == utf8.RuneError {
return -1
}
return character
}
func truncate(name string) string {
if len(name) <= maxFileNameBytes {
return name
}
extension := filepath.Ext(name)
if len(extension) > maxExtensionBytes {
extension = ""
}
base := strings.TrimSuffix(name, extension)
for len(base)+len(extension) > maxFileNameBytes {
_, size := utf8.DecodeLastRuneInString(base)
base = base[:len(base)-size]
}
return base + extension
}
+101
View File
@@ -0,0 +1,101 @@
package files
import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"syscall"
"ddc/internal/diskspace"
)
var ErrDestinationExists = errors.New("an item with this name already exists in the destination")
func MoveInto(source, destinationDir string, reservedBytes int64, progress func(int64)) error {
name := filepath.Base(source)
target := filepath.Join(destinationDir, name)
if exists(target) {
return fmt.Errorf("%w: %s", ErrDestinationExists, name)
}
size, err := TreeSize(source)
if err != nil {
return err
}
renameErr := os.Rename(source, target)
if renameErr == nil {
progress(size)
return nil
}
if !errors.Is(renameErr, syscall.EXDEV) {
return renameErr
}
if err := diskspace.EnsureFree(destinationDir, size, reservedBytes); err != nil {
return err
}
if err := copyTree(source, target, progress); err != nil {
os.RemoveAll(target)
return err
}
return os.RemoveAll(source)
}
func copyTree(source, target string, progress func(int64)) error {
return filepath.WalkDir(source, func(current string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
relative, err := filepath.Rel(source, current)
if err != nil {
return err
}
destination := filepath.Join(target, relative)
switch {
case entry.IsDir():
return os.MkdirAll(destination, 0o755)
case entry.Type().IsRegular():
return copyFile(current, destination, progress)
default:
return nil
}
})
}
func copyFile(source, destination string, progress func(int64)) error {
input, err := os.Open(source)
if err != nil {
return err
}
defer input.Close()
output, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
if err != nil {
return err
}
_, copyErr := io.Copy(output, &progressReader{reader: input, progress: progress})
closeErr := output.Close()
if copyErr != nil {
return copyErr
}
return closeErr
}
type progressReader struct {
reader io.Reader
progress func(int64)
}
func (reader *progressReader) Read(buffer []byte) (int, error) {
count, err := reader.reader.Read(buffer)
if count > 0 {
reader.progress(int64(count))
}
return count, err
}
+99
View File
@@ -0,0 +1,99 @@
package files
import (
"errors"
"io/fs"
"os"
"path/filepath"
"strings"
)
const (
PartFileName = ".download.part"
maxSubfolderDepth = 4
maxNameBytes = 255
)
var ErrUnsafePath = errors.New("path is not allowed")
func ResolveInside(root, relative string) (string, error) {
cleaned := filepath.Clean(filepath.FromSlash(relative))
if relative == "" || cleaned == "." || !filepath.IsLocal(cleaned) {
return "", ErrUnsafePath
}
resolvedRoot, err := filepath.EvalSymlinks(root)
if err != nil {
return "", err
}
candidate := filepath.Join(resolvedRoot, cleaned)
info, err := os.Lstat(candidate)
if err != nil {
return "", err
}
if info.Mode()&fs.ModeSymlink != 0 {
return "", ErrUnsafePath
}
resolved, err := filepath.EvalSymlinks(candidate)
if err != nil {
return "", err
}
if !IsWithin(resolvedRoot, resolved) {
return "", ErrUnsafePath
}
return resolved, nil
}
func IsWithin(root, target string) bool {
relative, err := filepath.Rel(root, target)
return err == nil && relative != "." && filepath.IsLocal(relative)
}
func SanitizeSubfolder(raw string) (string, error) {
trimmed := strings.Trim(strings.TrimSpace(raw), "/")
if trimmed == "" {
return "", nil
}
parts := strings.Split(trimmed, "/")
if len(parts) > maxSubfolderDepth {
return "", ErrUnsafePath
}
for index, part := range parts {
part = strings.TrimSpace(part)
if part == "" || part == "." || part == ".." || len(part) > maxNameBytes || strings.ContainsFunc(part, isUnsafeRune) {
return "", ErrUnsafePath
}
parts[index] = part
}
return filepath.Join(parts...), nil
}
func EnsureSubdirectory(root, subfolder string) (string, error) {
current := root
if subfolder == "" {
return current, nil
}
for _, part := range strings.Split(subfolder, string(filepath.Separator)) {
current = filepath.Join(current, part)
info, err := os.Lstat(current)
switch {
case errors.Is(err, fs.ErrNotExist):
if err := os.Mkdir(current, 0o755); err != nil && !errors.Is(err, fs.ErrExist) {
return "", err
}
case err != nil:
return "", err
case !info.IsDir():
return "", ErrUnsafePath
}
}
return current, nil
}
func isUnsafeRune(character rune) bool {
return character < 0x20 || character == 0x7f || character == '\\'
}
+52
View File
@@ -0,0 +1,52 @@
package files
import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
)
func TreeSize(root string) (int64, error) {
var total int64
err := filepath.WalkDir(root, func(_ string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if !entry.Type().IsRegular() {
return nil
}
info, err := entry.Info()
if err != nil {
return err
}
total += info.Size()
return nil
})
return total, err
}
func UniquePath(candidate string) string {
if !exists(candidate) {
return candidate
}
directory, name := filepath.Split(candidate)
extension := filepath.Ext(name)
base := strings.TrimSuffix(name, extension)
for index := 1; index < 1000; index++ {
next := filepath.Join(directory, fmt.Sprintf("%s (%d)%s", base, index, extension))
if !exists(next) {
return next
}
}
return filepath.Join(directory, fmt.Sprintf("%s-%d%s", base, time.Now().UnixNano(), extension))
}
func exists(path string) bool {
_, err := os.Lstat(path)
return !errors.Is(err, fs.ErrNotExist)
}
+85
View File
@@ -0,0 +1,85 @@
package files
import (
"os"
"path"
"path/filepath"
"sort"
"strings"
"ddc/internal/extract"
)
const (
maxTreeEntries = 20000
maxTreeDepth = 32
)
type Node struct {
Name string `json:"name"`
Path string `json:"path"`
IsDir bool `json:"isDir"`
IsArchive bool `json:"isArchive"`
Size int64 `json:"size"`
Children []Node `json:"children,omitempty"`
}
func BuildTree(root string) ([]Node, error) {
entryCount := 0
return readLevel(root, "", 0, &entryCount)
}
func readLevel(root, relative string, depth int, entryCount *int) ([]Node, error) {
entries, err := os.ReadDir(filepath.Join(root, relative))
if err != nil {
return nil, err
}
nodes := make([]Node, 0, len(entries))
for _, entry := range entries {
if *entryCount >= maxTreeEntries {
break
}
entryType := entry.Type()
if entry.Name() == PartFileName || (!entryType.IsDir() && !entryType.IsRegular()) {
continue
}
*entryCount++
node := Node{
Name: entry.Name(),
Path: path.Join(filepath.ToSlash(relative), entry.Name()),
IsDir: entry.IsDir(),
}
if node.IsDir {
if depth < maxTreeDepth {
children, err := readLevel(root, filepath.Join(relative, entry.Name()), depth+1, entryCount)
if err != nil {
return nil, err
}
node.Children = children
for _, child := range children {
node.Size += child.Size
}
}
} else {
info, err := entry.Info()
if err != nil {
continue
}
node.Size = info.Size()
node.IsArchive = extract.IsArchive(entry.Name())
}
nodes = append(nodes, node)
}
sort.SliceStable(nodes, func(first, second int) bool {
if nodes[first].IsDir != nodes[second].IsDir {
return nodes[first].IsDir
}
return strings.ToLower(nodes[first].Name) < strings.ToLower(nodes[second].Name)
})
return nodes, nil
}
+54
View File
@@ -0,0 +1,54 @@
package guard
import "net/netip"
var blockedPrefixes = []netip.Prefix{
netip.MustParsePrefix("0.0.0.0/8"),
netip.MustParsePrefix("10.0.0.0/8"),
netip.MustParsePrefix("100.64.0.0/10"),
netip.MustParsePrefix("127.0.0.0/8"),
netip.MustParsePrefix("169.254.0.0/16"),
netip.MustParsePrefix("172.16.0.0/12"),
netip.MustParsePrefix("192.0.0.0/24"),
netip.MustParsePrefix("192.0.2.0/24"),
netip.MustParsePrefix("192.88.99.0/24"),
netip.MustParsePrefix("192.168.0.0/16"),
netip.MustParsePrefix("198.18.0.0/15"),
netip.MustParsePrefix("198.51.100.0/24"),
netip.MustParsePrefix("203.0.113.0/24"),
netip.MustParsePrefix("224.0.0.0/4"),
netip.MustParsePrefix("240.0.0.0/4"),
netip.MustParsePrefix("::/128"),
netip.MustParsePrefix("::1/128"),
netip.MustParsePrefix("::ffff:0:0/96"),
netip.MustParsePrefix("64:ff9b::/96"),
netip.MustParsePrefix("64:ff9b:1::/48"),
netip.MustParsePrefix("100::/64"),
netip.MustParsePrefix("2001::/32"),
netip.MustParsePrefix("2001:db8::/32"),
netip.MustParsePrefix("2002::/16"),
netip.MustParsePrefix("fc00::/7"),
netip.MustParsePrefix("fe80::/10"),
netip.MustParsePrefix("fec0::/10"),
netip.MustParsePrefix("ff00::/8"),
}
func IsPublicAddress(address netip.Addr) bool {
address = address.Unmap().WithZone("")
if !address.IsValid() ||
!address.IsGlobalUnicast() ||
address.IsPrivate() ||
address.IsLoopback() ||
address.IsLinkLocalUnicast() ||
address.IsMulticast() ||
address.IsUnspecified() {
return false
}
for _, prefix := range blockedPrefixes {
if prefix.Contains(address) {
return false
}
}
return true
}
+55
View File
@@ -0,0 +1,55 @@
package guard
import (
"net"
"net/http"
"net/netip"
"syscall"
"time"
)
func (policy *Policy) NewHTTPClient() *http.Client {
dialer := &net.Dialer{
Timeout: 15 * time.Second,
KeepAlive: 30 * time.Second,
Control: policy.controlConnection,
}
transport := &http.Transport{
Proxy: nil,
DialContext: dialer.DialContext,
ForceAttemptHTTP2: true,
TLSHandshakeTimeout: 15 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
ExpectContinueTimeout: time.Second,
MaxIdleConns: 10,
IdleConnTimeout: 90 * time.Second,
}
return &http.Client{
Transport: transport,
CheckRedirect: policy.checkRedirect,
}
}
func (policy *Policy) controlConnection(_ string, address string, _ syscall.RawConn) error {
addressPort, err := netip.ParseAddrPort(address)
if err != nil {
return ErrBlockedAddress
}
if !IsPublicAddress(addressPort.Addr()) {
return ErrBlockedAddress
}
if !policy.portAllowed(addressPort.Port()) {
return ErrBlockedPort
}
return nil
}
func (policy *Policy) checkRedirect(request *http.Request, previous []*http.Request) error {
if len(previous) >= maxRedirects {
return ErrTooManyRedirects
}
_, err := policy.ValidateURL(request.URL.String())
return err
}
+98
View File
@@ -0,0 +1,98 @@
package guard
import (
"errors"
"fmt"
"net/netip"
"net/url"
"strconv"
"strings"
"sync"
)
const (
maxURLLength = 4096
maxRedirects = 5
)
var (
ErrURLTooLong = errors.New("URL is too long")
ErrUnsupportedScheme = errors.New("only http and https URLs are allowed")
ErrMissingHost = errors.New("URL must include a host")
ErrBlockedAddress = errors.New("destination address is not allowed")
ErrBlockedPort = errors.New("destination port is not allowed")
ErrTooManyRedirects = errors.New("too many redirects")
)
type Policy struct {
mutex sync.RWMutex
allowedPorts map[uint16]bool
}
func NewPolicy(allowedPorts []uint16) *Policy {
policy := &Policy{}
policy.SetAllowedPorts(allowedPorts)
return policy
}
func (policy *Policy) SetAllowedPorts(allowedPorts []uint16) {
ports := make(map[uint16]bool, len(allowedPorts))
for _, port := range allowedPorts {
ports[port] = true
}
policy.mutex.Lock()
defer policy.mutex.Unlock()
policy.allowedPorts = ports
}
func (policy *Policy) ValidateURL(raw string) (*url.URL, error) {
raw = strings.TrimSpace(raw)
if len(raw) > maxURLLength {
return nil, ErrURLTooLong
}
parsed, err := url.Parse(raw)
if err != nil {
return nil, fmt.Errorf("invalid URL: %w", err)
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return nil, ErrUnsupportedScheme
}
host := parsed.Hostname()
if host == "" {
return nil, ErrMissingHost
}
if address, err := netip.ParseAddr(host); err == nil && !IsPublicAddress(address) {
return nil, ErrBlockedAddress
}
port, err := portOf(parsed)
if err != nil {
return nil, err
}
if !policy.portAllowed(port) {
return nil, ErrBlockedPort
}
return parsed, nil
}
func (policy *Policy) portAllowed(port uint16) bool {
policy.mutex.RLock()
defer policy.mutex.RUnlock()
return len(policy.allowedPorts) == 0 || policy.allowedPorts[port]
}
func portOf(parsed *url.URL) (uint16, error) {
if raw := parsed.Port(); raw != "" {
value, err := strconv.ParseUint(raw, 10, 16)
if err != nil {
return 0, fmt.Errorf("invalid port: %w", err)
}
return uint16(value), nil
}
if parsed.Scheme == "https" {
return 443, nil
}
return 80, nil
}
+126
View File
@@ -0,0 +1,126 @@
package jobs
import (
"context"
"sync"
"sync/atomic"
"time"
)
type Status string
const (
StatusQueued Status = "queued"
StatusDownloading Status = "downloading"
StatusExtracting Status = "extracting"
StatusReady Status = "ready"
StatusMoving Status = "moving"
StatusCompleted Status = "completed"
StatusFailed Status = "failed"
StatusCancelled Status = "cancelled"
)
type Snapshot struct {
ID string `json:"id"`
SourceURL string `json:"sourceUrl"`
FileName string `json:"fileName"`
Status Status `json:"status"`
Message string `json:"message"`
BytesDone int64 `json:"bytesDone"`
BytesTotal int64 `json:"bytesTotal"`
CreatedAt time.Time `json:"createdAt"`
}
type Job struct {
id string
sourceURL string
displayURL string
directory string
autoExtract bool
createdAt time.Time
jobContext context.Context
cancel context.CancelFunc
workers sync.WaitGroup
mutex sync.Mutex
status Status
message string
fileName string
deleted bool
bytesDone atomic.Int64
bytesTotal atomic.Int64
}
func (job *Job) SetTotal(total int64) {
job.bytesTotal.Store(total)
}
func (job *Job) Add(delta int64) {
job.bytesDone.Add(delta)
}
func (job *Job) SetFileName(name string) {
job.mutex.Lock()
defer job.mutex.Unlock()
job.fileName = name
}
func (job *Job) resetProgress() {
job.bytesDone.Store(0)
job.bytesTotal.Store(0)
}
func (job *Job) setStatus(status Status, message string) {
job.mutex.Lock()
defer job.mutex.Unlock()
if job.deleted {
return
}
job.status = status
job.message = message
}
func (job *Job) currentStatus() Status {
job.mutex.Lock()
defer job.mutex.Unlock()
return job.status
}
func (job *Job) begin(from, to Status) bool {
job.mutex.Lock()
defer job.mutex.Unlock()
if job.deleted || job.status != from {
return false
}
job.status = to
job.message = ""
job.workers.Add(1)
return true
}
func (job *Job) markDeleted() bool {
job.mutex.Lock()
defer job.mutex.Unlock()
if job.status == StatusMoving {
return false
}
job.deleted = true
job.status = StatusCancelled
return true
}
func (job *Job) snapshot() Snapshot {
job.mutex.Lock()
defer job.mutex.Unlock()
return Snapshot{
ID: job.id,
SourceURL: job.displayURL,
FileName: job.fileName,
Status: job.status,
Message: job.message,
BytesDone: job.bytesDone.Load(),
BytesTotal: job.bytesTotal.Load(),
CreatedAt: job.createdAt,
}
}
+208
View File
@@ -0,0 +1,208 @@
package jobs
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"os"
"path"
"path/filepath"
"sort"
"sync"
"time"
"ddc/internal/config"
"ddc/internal/diskspace"
"ddc/internal/fetch"
"ddc/internal/files"
"ddc/internal/guard"
"ddc/internal/libraries"
"ddc/internal/preferences"
)
var (
ErrJobNotFound = errors.New("download not found")
ErrJobBusy = errors.New("download is busy, try again when it is ready")
ErrInvalidRequest = errors.New("invalid request")
)
type Manager struct {
settings config.Settings
preferences *preferences.Store
policy *guard.Policy
downloader *fetch.Downloader
rootContext context.Context
cancelAll context.CancelFunc
slots slotLimiter
mutex sync.RWMutex
jobs map[string]*Job
}
func NewManager(settings config.Settings, store *preferences.Store) *Manager {
policy := guard.NewPolicy(store.Get().AllowedPorts)
rootContext, cancelAll := context.WithCancel(context.Background())
return &Manager{
settings: settings,
preferences: store,
policy: policy,
downloader: fetch.NewDownloader(policy),
rootContext: rootContext,
cancelAll: cancelAll,
jobs: make(map[string]*Job),
}
}
func (manager *Manager) Destinations() []libraries.Destination {
return libraries.Discover(manager.settings.LibraryRoots)
}
func (manager *Manager) Preferences() preferences.Preferences {
return manager.preferences.Get()
}
func (manager *Manager) UpdatePreferences(next preferences.Preferences) error {
if err := manager.preferences.Update(next); err != nil {
return err
}
manager.policy.SetAllowedPorts(next.AllowedPorts)
return nil
}
func (manager *Manager) concurrencyLimit() int {
return manager.preferences.Get().MaxConcurrentDownloads
}
func (manager *Manager) reservedBytes() int64 {
return manager.preferences.Get().MinFreeBytes()
}
func (manager *Manager) AvailableBytes() (int64, error) {
return diskspace.Available(manager.settings.DownloadDir)
}
func (manager *Manager) Create(rawURL string, autoExtract bool) (Snapshot, error) {
target, err := manager.policy.ValidateURL(rawURL)
if err != nil {
return Snapshot{}, invalid(err)
}
if err := diskspace.EnsureFree(manager.settings.DownloadDir, 0, manager.reservedBytes()); err != nil {
return Snapshot{}, err
}
id, err := newJobID()
if err != nil {
return Snapshot{}, err
}
directory := filepath.Join(manager.settings.DownloadDir, id)
if err := os.Mkdir(directory, 0o755); err != nil {
return Snapshot{}, err
}
job := manager.newJob(id, directory, time.Now())
job.sourceURL = target.String()
job.displayURL = target.Redacted()
job.autoExtract = autoExtract
job.status = StatusQueued
job.fileName = initialFileName(target.Path, target.Host)
job.workers.Add(1)
manager.store(job)
go manager.download(job)
return job.snapshot(), nil
}
func (manager *Manager) List() []Snapshot {
manager.mutex.RLock()
snapshots := make([]Snapshot, 0, len(manager.jobs))
for _, job := range manager.jobs {
snapshots = append(snapshots, job.snapshot())
}
manager.mutex.RUnlock()
sort.Slice(snapshots, func(first, second int) bool {
return snapshots[first].CreatedAt.After(snapshots[second].CreatedAt)
})
return snapshots
}
func (manager *Manager) Files(id string) ([]files.Node, error) {
job, err := manager.lookup(id)
if err != nil {
return nil, err
}
if job.currentStatus() != StatusReady {
return nil, ErrJobBusy
}
return files.BuildTree(job.directory)
}
func (manager *Manager) Delete(id string) error {
job, err := manager.lookup(id)
if err != nil {
return err
}
if !job.markDeleted() {
return ErrJobBusy
}
job.cancel()
job.workers.Wait()
manager.mutex.Lock()
delete(manager.jobs, id)
manager.mutex.Unlock()
return os.RemoveAll(job.directory)
}
func (manager *Manager) CancelAll() {
manager.cancelAll()
}
func (manager *Manager) newJob(id, directory string, createdAt time.Time) *Job {
jobContext, cancel := context.WithCancel(manager.rootContext)
return &Job{
id: id,
directory: directory,
createdAt: createdAt,
jobContext: jobContext,
cancel: cancel,
}
}
func (manager *Manager) store(job *Job) {
manager.mutex.Lock()
defer manager.mutex.Unlock()
manager.jobs[job.id] = job
}
func (manager *Manager) lookup(id string) (*Job, error) {
manager.mutex.RLock()
defer manager.mutex.RUnlock()
job, found := manager.jobs[id]
if !found {
return nil, ErrJobNotFound
}
return job, nil
}
func newJobID() (string, error) {
buffer := make([]byte, 16)
if _, err := rand.Read(buffer); err != nil {
return "", err
}
return hex.EncodeToString(buffer), nil
}
func initialFileName(urlPath, host string) string {
if name := fetch.SanitizeFileName(path.Base(urlPath)); name != "" {
return name
}
return host
}
func invalid(err error) error {
return fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
+125
View File
@@ -0,0 +1,125 @@
package jobs
import (
"errors"
"fmt"
"os"
"path/filepath"
"ddc/internal/files"
"ddc/internal/libraries"
)
type MoveRequest struct {
Paths []string `json:"paths"`
Destination string `json:"destination"`
Subfolder string `json:"subfolder"`
Cleanup bool `json:"cleanup"`
}
func (manager *Manager) Move(id string, request MoveRequest) error {
job, err := manager.lookup(id)
if err != nil {
return err
}
if job.currentStatus() != StatusReady {
return ErrJobBusy
}
if len(request.Paths) == 0 {
return invalid(errors.New("select at least one file or folder"))
}
library, err := libraries.Resolve(manager.settings.LibraryRoots, request.Destination)
if err != nil {
return invalid(err)
}
subfolder, err := files.SanitizeSubfolder(request.Subfolder)
if err != nil {
return invalid(err)
}
sources, err := resolveSources(job.directory, request.Paths)
if err != nil {
return err
}
destination, err := files.EnsureSubdirectory(library.Path, subfolder)
if err != nil {
return invalid(err)
}
if err := checkTargets(sources, destination); err != nil {
return err
}
if !job.begin(StatusReady, StatusMoving) {
return ErrJobBusy
}
go func() {
defer job.workers.Done()
manager.moveFiles(job, library, sources, destination, request.Cleanup)
}()
return nil
}
func resolveSources(root string, relativePaths []string) ([]string, error) {
sources := make([]string, 0, len(relativePaths))
seenNames := make(map[string]bool)
for _, relative := range relativePaths {
source, err := files.ResolveInside(root, relative)
if err != nil {
return nil, invalid(err)
}
name := filepath.Base(source)
if seenNames[name] {
return nil, invalid(fmt.Errorf("more than one selected item is named %q", name))
}
seenNames[name] = true
sources = append(sources, source)
}
for _, outer := range sources {
for _, inner := range sources {
if files.IsWithin(outer, inner) {
return nil, invalid(errors.New("selection contains a folder and items inside it"))
}
}
}
return sources, nil
}
func checkTargets(sources []string, destination string) error {
for _, source := range sources {
name := filepath.Base(source)
if _, err := os.Lstat(filepath.Join(destination, name)); err == nil {
return fmt.Errorf("%w: %s", files.ErrDestinationExists, name)
}
}
return nil
}
func (manager *Manager) moveFiles(job *Job, library libraries.Destination, sources []string, destination string, cleanup bool) {
job.resetProgress()
var totalBytes int64
for _, source := range sources {
if size, err := files.TreeSize(source); err == nil {
totalBytes += size
}
}
job.SetTotal(totalBytes)
for _, source := range sources {
if err := files.MoveInto(source, destination, manager.reservedBytes(), job.Add); err != nil {
job.setStatus(StatusReady, "Move failed: "+err.Error())
return
}
}
message := fmt.Sprintf("Moved %d item(s) to %s", len(sources), library.Path)
if !cleanup {
job.setStatus(StatusReady, message)
return
}
if err := os.RemoveAll(job.directory); err != nil {
message += ", cleanup failed: " + err.Error()
}
job.setStatus(StatusCompleted, message)
}
+51
View File
@@ -0,0 +1,51 @@
package jobs
import (
"os"
"path/filepath"
"regexp"
"time"
"ddc/internal/files"
)
var jobIDPattern = regexp.MustCompile(`^[0-9a-f]{32}$`)
func (manager *Manager) RestoreExisting() error {
entries, err := os.ReadDir(manager.settings.DownloadDir)
if err != nil {
return err
}
for _, entry := range entries {
if !entry.IsDir() || !jobIDPattern.MatchString(entry.Name()) {
continue
}
directory := filepath.Join(manager.settings.DownloadDir, entry.Name())
os.Remove(filepath.Join(directory, files.PartFileName))
createdAt := time.Now()
if info, err := entry.Info(); err == nil {
createdAt = info.ModTime()
}
job := manager.newJob(entry.Name(), directory, createdAt)
job.status = StatusReady
job.fileName = firstEntryName(directory, entry.Name())
manager.store(job)
}
return nil
}
func firstEntryName(directory, fallback string) string {
entries, err := os.ReadDir(directory)
if err != nil {
return fallback
}
for _, entry := range entries {
if entry.Name() != files.PartFileName {
return entry.Name()
}
}
return fallback
}
+45
View File
@@ -0,0 +1,45 @@
package jobs
import (
"context"
"sync"
"time"
)
const slotWaitInterval = 500 * time.Millisecond
type slotLimiter struct {
mutex sync.Mutex
active int
}
func (limiter *slotLimiter) acquire(ctx context.Context, limit func() int) bool {
ticker := time.NewTicker(slotWaitInterval)
defer ticker.Stop()
for {
if limiter.tryAcquire(limit()) {
return true
}
select {
case <-ctx.Done():
return false
case <-ticker.C:
}
}
}
func (limiter *slotLimiter) tryAcquire(limit int) bool {
limiter.mutex.Lock()
defer limiter.mutex.Unlock()
if limiter.active >= limit {
return false
}
limiter.active++
return true
}
func (limiter *slotLimiter) release() {
limiter.mutex.Lock()
defer limiter.mutex.Unlock()
limiter.active--
}
+102
View File
@@ -0,0 +1,102 @@
package jobs
import (
"errors"
"os"
"path/filepath"
"ddc/internal/diskspace"
"ddc/internal/extract"
"ddc/internal/files"
)
const maxArchiveEntries = 100000
var errNotArchive = errors.New("selected file is not a supported archive")
func (manager *Manager) download(job *Job) {
defer job.workers.Done()
if !manager.slots.acquire(job.jobContext, manager.concurrencyLimit) {
return
}
defer manager.slots.release()
job.setStatus(StatusDownloading, "")
filePath, err := manager.downloader.Download(job.jobContext, job.sourceURL, job.directory, manager.reservedBytes(), job)
if err != nil {
manager.finishWithError(job, StatusFailed, "", err)
return
}
job.SetFileName(filepath.Base(filePath))
if job.autoExtract && extract.IsArchive(filePath) {
job.setStatus(StatusExtracting, "")
destination, err := manager.extractArchive(job, filePath)
if err != nil {
manager.finishWithError(job, StatusReady, "Extraction failed: ", err)
return
}
job.SetFileName(filepath.Base(destination))
}
job.setStatus(StatusReady, "")
}
func (manager *Manager) Extract(id, relativePath string) error {
job, err := manager.lookup(id)
if err != nil {
return err
}
archivePath, err := files.ResolveInside(job.directory, relativePath)
if err != nil {
return invalid(err)
}
info, err := os.Stat(archivePath)
if err != nil || !info.Mode().IsRegular() || !extract.IsArchive(archivePath) {
return invalid(errNotArchive)
}
if !job.begin(StatusReady, StatusExtracting) {
return ErrJobBusy
}
go func() {
defer job.workers.Done()
if _, err := manager.extractArchive(job, archivePath); err != nil {
manager.finishWithError(job, StatusReady, "Extraction failed: ", err)
return
}
job.setStatus(StatusReady, "")
}()
return nil
}
func (manager *Manager) extractArchive(job *Job, archivePath string) (string, error) {
available, err := diskspace.Available(job.directory)
if err != nil {
return "", err
}
limit := available - manager.reservedBytes()
if limit <= 0 {
return "", diskspace.ErrInsufficientSpace
}
directoryName := extract.DirectoryName(filepath.Base(archivePath))
destination := files.UniquePath(filepath.Join(filepath.Dir(archivePath), directoryName))
limits := extract.Limits{MaxBytes: limit, MaxEntries: maxArchiveEntries}
job.resetProgress()
if err := extract.Archive(job.jobContext, archivePath, destination, limits, job); err != nil {
return "", err
}
return destination, os.Remove(archivePath)
}
func (manager *Manager) finishWithError(job *Job, status Status, prefix string, err error) {
if job.jobContext.Err() != nil {
job.setStatus(StatusCancelled, "")
return
}
job.setStatus(status, prefix+err.Error())
}
+57
View File
@@ -0,0 +1,57 @@
package libraries
import (
"errors"
"os"
"path/filepath"
"strings"
)
var ErrUnknownDestination = errors.New("destination is not a first level folder of a library root")
type Destination struct {
Name string `json:"name"`
Root string `json:"root"`
Path string `json:"path"`
}
func Discover(roots []string) []Destination {
destinations := []Destination{}
for _, root := range roots {
entries, err := os.ReadDir(root)
if err != nil {
continue
}
for _, entry := range entries {
if !entry.IsDir() || strings.HasPrefix(entry.Name(), ".") {
continue
}
destinations = append(destinations, Destination{
Name: entry.Name(),
Root: root,
Path: filepath.Join(root, entry.Name()),
})
}
}
return destinations
}
func Resolve(roots []string, requested string) (Destination, error) {
cleaned := filepath.Clean(requested)
name := filepath.Base(cleaned)
if strings.HasPrefix(name, ".") {
return Destination{}, ErrUnknownDestination
}
for _, root := range roots {
if filepath.Dir(cleaned) != root {
continue
}
info, err := os.Lstat(cleaned)
if err != nil || !info.IsDir() {
return Destination{}, ErrUnknownDestination
}
return Destination{Name: name, Root: root, Path: cleaned}, nil
}
return Destination{}, ErrUnknownDestination
}
+41
View File
@@ -0,0 +1,41 @@
package preferences
import (
"errors"
"fmt"
)
const (
gigabyte = int64(1) << 30
maxConcurrentLimit = 10
maxReservedGigabytes = 100000
)
var ErrInvalid = errors.New("invalid settings")
type Preferences struct {
MaxConcurrentDownloads int `json:"maxConcurrentDownloads"`
MinFreeGB int `json:"minFreeGB"`
AllowedPorts []uint16 `json:"allowedPorts"`
AutoExtract bool `json:"autoExtract"`
CleanupAfterMove bool `json:"cleanupAfterMove"`
}
func (preferences Preferences) MinFreeBytes() int64 {
return int64(preferences.MinFreeGB) * gigabyte
}
func (preferences Preferences) Validate() error {
if preferences.MaxConcurrentDownloads < 1 || preferences.MaxConcurrentDownloads > maxConcurrentLimit {
return fmt.Errorf("%w: parallel downloads must be between 1 and %d", ErrInvalid, maxConcurrentLimit)
}
if preferences.MinFreeGB < 0 || preferences.MinFreeGB > maxReservedGigabytes {
return fmt.Errorf("%w: free space to keep must be between 0 and %d GB", ErrInvalid, maxReservedGigabytes)
}
for _, port := range preferences.AllowedPorts {
if port == 0 {
return fmt.Errorf("%w: port 0 is not allowed", ErrInvalid)
}
}
return nil
}
+87
View File
@@ -0,0 +1,87 @@
package preferences
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"slices"
"sync"
)
type Store struct {
mutex sync.RWMutex
path string
current Preferences
}
func Open(path string, defaults Preferences) (*Store, error) {
store := &Store{path: path, current: defaults}
data, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return store, nil
}
if err != nil {
return nil, err
}
loaded := defaults
if err := json.Unmarshal(data, &loaded); err != nil {
return nil, fmt.Errorf("settings file %s: %w", path, err)
}
if err := loaded.Validate(); err != nil {
return nil, fmt.Errorf("settings file %s: %w", path, err)
}
store.current = loaded
return store, nil
}
func (store *Store) Get() Preferences {
store.mutex.RLock()
defer store.mutex.RUnlock()
current := store.current
current.AllowedPorts = slices.Clone(current.AllowedPorts)
if current.AllowedPorts == nil {
current.AllowedPorts = []uint16{}
}
return current
}
func (store *Store) Update(next Preferences) error {
if err := next.Validate(); err != nil {
return err
}
data, err := json.MarshalIndent(next, "", " ")
if err != nil {
return err
}
store.mutex.Lock()
defer store.mutex.Unlock()
if err := writeAtomically(store.path, data); err != nil {
return err
}
store.current = next
return nil
}
func writeAtomically(path string, data []byte) error {
temporary, err := os.CreateTemp(filepath.Dir(path), ".settings-*.tmp")
if err != nil {
return err
}
_, writeErr := temporary.Write(data)
closeErr := temporary.Close()
if writeErr != nil || closeErr != nil {
os.Remove(temporary.Name())
return errors.Join(writeErr, closeErr)
}
if err := os.Rename(temporary.Name(), path); err != nil {
os.Remove(temporary.Name())
return err
}
return nil
}
+83
View File
@@ -0,0 +1,83 @@
package server
import (
"net"
"net/http"
"strings"
"ddc/internal/auth"
)
type loginRequest struct {
Username string `json:"username"`
Password string `json:"password"`
}
type sessionResponse struct {
Authenticated bool `json:"authenticated"`
}
func (server *Server) handleLogin(writer http.ResponseWriter, request *http.Request) {
client := server.clientAddress(request)
if !server.limiter.Allowed(client) {
writeError(writer, http.StatusTooManyRequests, "too many failed attempts, try again later")
return
}
var body loginRequest
if err := decodeJSON(request, &body); err != nil {
writeError(writer, http.StatusBadRequest, "invalid request body")
return
}
if !auth.CredentialsMatch(server.settings.Username, server.settings.Password, body.Username, body.Password) {
server.limiter.RecordFailure(client)
writeError(writer, http.StatusUnauthorized, "invalid username or password")
return
}
server.limiter.Reset(client)
token, err := server.sessions.Create()
if err != nil {
writeError(writer, http.StatusInternalServerError, "could not create session")
return
}
http.SetCookie(writer, server.sessionCookie(token, int(server.settings.SessionTTL.Seconds())))
writeJSON(writer, http.StatusOK, sessionResponse{Authenticated: true})
}
func (server *Server) handleLogout(writer http.ResponseWriter, request *http.Request) {
if cookie, err := request.Cookie(sessionCookieName); err == nil {
server.sessions.Delete(cookie.Value)
}
http.SetCookie(writer, server.sessionCookie("", -1))
writeJSON(writer, http.StatusOK, sessionResponse{Authenticated: false})
}
func (server *Server) handleSession(writer http.ResponseWriter, request *http.Request) {
writeJSON(writer, http.StatusOK, sessionResponse{Authenticated: server.authenticated(request)})
}
func (server *Server) sessionCookie(value string, maxAge int) *http.Cookie {
return &http.Cookie{
Name: sessionCookieName,
Value: value,
Path: "/",
MaxAge: maxAge,
HttpOnly: true,
Secure: server.settings.CookieSecure,
SameSite: http.SameSiteStrictMode,
}
}
func (server *Server) clientAddress(request *http.Request) string {
if server.settings.TrustProxyHeaders {
if realIP := strings.TrimSpace(request.Header.Get("X-Real-IP")); realIP != "" {
return realIP
}
}
host, _, err := net.SplitHostPort(request.RemoteAddr)
if err != nil {
return request.RemoteAddr
}
return host
}
+99
View File
@@ -0,0 +1,99 @@
package server
import (
"net/http"
"ddc/internal/jobs"
)
type createJobRequest struct {
URL string `json:"url"`
AutoExtract bool `json:"autoExtract"`
}
type extractRequest struct {
Path string `json:"path"`
}
type diskResponse struct {
AvailableBytes int64 `json:"availableBytes"`
ReservedBytes int64 `json:"reservedBytes"`
}
type startedResponse struct {
Started bool `json:"started"`
}
func (server *Server) handleLibraries(writer http.ResponseWriter, _ *http.Request) {
writeJSON(writer, http.StatusOK, server.manager.Destinations())
}
func (server *Server) handleDisk(writer http.ResponseWriter, _ *http.Request) {
available, err := server.manager.AvailableBytes()
if err != nil {
writeFailure(writer, err)
return
}
writeJSON(writer, http.StatusOK, diskResponse{AvailableBytes: available, ReservedBytes: server.manager.Preferences().MinFreeBytes()})
}
func (server *Server) handleListJobs(writer http.ResponseWriter, _ *http.Request) {
writeJSON(writer, http.StatusOK, server.manager.List())
}
func (server *Server) handleCreateJob(writer http.ResponseWriter, request *http.Request) {
var body createJobRequest
if err := decodeJSON(request, &body); err != nil {
writeError(writer, http.StatusBadRequest, "invalid request body")
return
}
snapshot, err := server.manager.Create(body.URL, body.AutoExtract)
if err != nil {
writeFailure(writer, err)
return
}
writeJSON(writer, http.StatusCreated, snapshot)
}
func (server *Server) handleJobFiles(writer http.ResponseWriter, request *http.Request) {
tree, err := server.manager.Files(request.PathValue("id"))
if err != nil {
writeFailure(writer, err)
return
}
writeJSON(writer, http.StatusOK, tree)
}
func (server *Server) handleExtract(writer http.ResponseWriter, request *http.Request) {
var body extractRequest
if err := decodeJSON(request, &body); err != nil {
writeError(writer, http.StatusBadRequest, "invalid request body")
return
}
if err := server.manager.Extract(request.PathValue("id"), body.Path); err != nil {
writeFailure(writer, err)
return
}
writeJSON(writer, http.StatusAccepted, startedResponse{Started: true})
}
func (server *Server) handleMove(writer http.ResponseWriter, request *http.Request) {
var body jobs.MoveRequest
if err := decodeJSON(request, &body); err != nil {
writeError(writer, http.StatusBadRequest, "invalid request body")
return
}
if err := server.manager.Move(request.PathValue("id"), body); err != nil {
writeFailure(writer, err)
return
}
writeJSON(writer, http.StatusAccepted, startedResponse{Started: true})
}
func (server *Server) handleDeleteJob(writer http.ResponseWriter, request *http.Request) {
if err := server.manager.Delete(request.PathValue("id")); err != nil {
writeFailure(writer, err)
return
}
writer.WriteHeader(http.StatusNoContent)
}
+73
View File
@@ -0,0 +1,73 @@
package server
import (
"net/http"
"net/url"
"strings"
)
const (
sessionCookieName = "ddc_session"
maxRequestBodyBytes = 64 << 10
contentSecurityPolicy = "default-src 'self'; img-src 'self' data:; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'"
)
func securityHeaders(next http.Handler) http.Handler {
return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
header := writer.Header()
header.Set("Content-Security-Policy", contentSecurityPolicy)
header.Set("X-Content-Type-Options", "nosniff")
header.Set("X-Frame-Options", "DENY")
header.Set("Referrer-Policy", "no-referrer")
header.Set("Cross-Origin-Opener-Policy", "same-origin")
header.Set("Permissions-Policy", "camera=(), microphone=(), geolocation=()")
if strings.HasPrefix(request.URL.Path, "/api/") {
header.Set("Cache-Control", "no-store")
}
next.ServeHTTP(writer, request)
})
}
func protectStateChanges(next http.Handler) http.Handler {
return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.Method == http.MethodGet || request.Method == http.MethodHead {
next.ServeHTTP(writer, request)
return
}
if !isSameOrigin(request) {
writeError(writer, http.StatusForbidden, "cross-origin request rejected")
return
}
if request.Method == http.MethodPost && !strings.HasPrefix(request.Header.Get("Content-Type"), "application/json") {
writeError(writer, http.StatusUnsupportedMediaType, "content type must be application/json")
return
}
request.Body = http.MaxBytesReader(writer, request.Body, maxRequestBodyBytes)
next.ServeHTTP(writer, request)
})
}
func isSameOrigin(request *http.Request) bool {
origin := request.Header.Get("Origin")
if origin == "" {
fetchSite := request.Header.Get("Sec-Fetch-Site")
return fetchSite == "" || fetchSite == "same-origin" || fetchSite == "none"
}
parsed, err := url.Parse(origin)
return err == nil && strings.EqualFold(parsed.Host, request.Host)
}
func (server *Server) requireSession(next http.Handler) http.Handler {
return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if !server.authenticated(request) {
writeError(writer, http.StatusUnauthorized, "authentication required")
return
}
next.ServeHTTP(writer, request)
})
}
func (server *Server) authenticated(request *http.Request) bool {
cookie, err := request.Cookie(sessionCookieName)
return err == nil && server.sessions.Valid(cookie.Value)
}
+47
View File
@@ -0,0 +1,47 @@
package server
import (
"encoding/json"
"errors"
"net/http"
"ddc/internal/diskspace"
"ddc/internal/files"
"ddc/internal/jobs"
"ddc/internal/preferences"
)
func writeJSON(writer http.ResponseWriter, status int, payload any) {
writer.Header().Set("Content-Type", "application/json; charset=utf-8")
writer.WriteHeader(status)
json.NewEncoder(writer).Encode(payload)
}
func writeError(writer http.ResponseWriter, status int, message string) {
writeJSON(writer, status, map[string]string{"error": message})
}
func writeFailure(writer http.ResponseWriter, err error) {
writeError(writer, statusFor(err), err.Error())
}
func decodeJSON(request *http.Request, target any) error {
decoder := json.NewDecoder(request.Body)
decoder.DisallowUnknownFields()
return decoder.Decode(target)
}
func statusFor(err error) int {
switch {
case errors.Is(err, jobs.ErrJobNotFound):
return http.StatusNotFound
case errors.Is(err, jobs.ErrJobBusy), errors.Is(err, files.ErrDestinationExists):
return http.StatusConflict
case errors.Is(err, jobs.ErrInvalidRequest), errors.Is(err, preferences.ErrInvalid):
return http.StatusBadRequest
case errors.Is(err, diskspace.ErrInsufficientSpace):
return http.StatusInsufficientStorage
default:
return http.StatusInternalServerError
}
}
+73
View File
@@ -0,0 +1,73 @@
package server
import (
"net/http"
"strings"
"time"
"ddc/internal/auth"
"ddc/internal/config"
"ddc/internal/jobs"
"ddc/web"
)
const (
maxLoginFailures = 5
loginLockout = 15 * time.Minute
)
type Server struct {
settings config.Settings
sessions *auth.SessionStore
limiter *auth.LoginLimiter
manager *jobs.Manager
}
func New(settings config.Settings, sessions *auth.SessionStore, manager *jobs.Manager) http.Handler {
server := &Server{
settings: settings,
sessions: sessions,
limiter: auth.NewLoginLimiter(maxLoginFailures, loginLockout),
manager: manager,
}
return server.routes()
}
func (server *Server) routes() http.Handler {
api := http.NewServeMux()
api.HandleFunc("POST /api/logout", server.handleLogout)
api.HandleFunc("GET /api/libraries", server.handleLibraries)
api.HandleFunc("GET /api/disk", server.handleDisk)
api.HandleFunc("GET /api/settings", server.handleGetSettings)
api.HandleFunc("POST /api/settings", server.handleUpdateSettings)
api.HandleFunc("GET /api/jobs", server.handleListJobs)
api.HandleFunc("POST /api/jobs", server.handleCreateJob)
api.HandleFunc("GET /api/jobs/{id}/files", server.handleJobFiles)
api.HandleFunc("POST /api/jobs/{id}/extract", server.handleExtract)
api.HandleFunc("POST /api/jobs/{id}/move", server.handleMove)
api.HandleFunc("DELETE /api/jobs/{id}", server.handleDeleteJob)
root := http.NewServeMux()
root.HandleFunc("POST /api/login", server.handleLogin)
root.HandleFunc("GET /api/session", server.handleSession)
root.HandleFunc("GET /healthz", handleHealth)
root.Handle("/api/", server.requireSession(api))
root.Handle("/", staticHandler())
return securityHeaders(protectStateChanges(root))
}
func staticHandler() http.Handler {
fileServer := http.FileServerFS(web.Static())
return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.URL.Path != "/" && strings.HasSuffix(request.URL.Path, "/") {
http.NotFound(writer, request)
return
}
fileServer.ServeHTTP(writer, request)
})
}
func handleHealth(writer http.ResponseWriter, _ *http.Request) {
writer.WriteHeader(http.StatusNoContent)
}
+38
View File
@@ -0,0 +1,38 @@
package server
import (
"net/http"
"ddc/internal/preferences"
)
type settingsResponse struct {
Settings preferences.Preferences `json:"settings"`
DownloadDir string `json:"downloadDir"`
LibraryRoots []string `json:"libraryRoots"`
}
func (server *Server) handleGetSettings(writer http.ResponseWriter, _ *http.Request) {
writeJSON(writer, http.StatusOK, server.settingsSnapshot())
}
func (server *Server) handleUpdateSettings(writer http.ResponseWriter, request *http.Request) {
var body preferences.Preferences
if err := decodeJSON(request, &body); err != nil {
writeError(writer, http.StatusBadRequest, "invalid request body")
return
}
if err := server.manager.UpdatePreferences(body); err != nil {
writeFailure(writer, err)
return
}
writeJSON(writer, http.StatusOK, server.settingsSnapshot())
}
func (server *Server) settingsSnapshot() settingsResponse {
return settingsResponse{
Settings: server.manager.Preferences(),
DownloadDir: server.settings.DownloadDir,
LibraryRoots: server.settings.LibraryRoots,
}
}