sonda/internal/duplicates/hashjob.go
tgiachi a880b87a9e chore: move the module path to git.kmalloc.org/squid/sonda
Migrating hosting from Codeberg to git.kmalloc.org. Renames the
module path in go.mod, every internal import, and the OCI image
source label in the Dockerfile. Build and full test suite verified
green after the rename.
2026-11-15 17:30:29 +01:00

120 lines
3.1 KiB
Go

package duplicates
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"slices"
"sort"
"sync"
"time"
"git.kmalloc.org/squid/sonda/internal/jobs"
)
// OutcomeHashed is what the check counts for a file it read and fingerprinted.
const OutcomeHashed = "hashed"
// NewHashJob builds the job that reads the files of the groups of copies and keeps their SHA-256, so
// that a group of files that are byte for byte the same can be said to be identical. It is started by
// hand like the other bulk jobs: it reads whole files, which on a NAS takes a while.
func NewHashJob(s *Service) *jobs.Job {
t := &hashTask{s: s}
return jobs.New(jobs.Task{Noun: "files", Count: t.count, List: t.list, Do: t.do}, s.now)
}
// hashTask keeps, between Count and the Lists that follow, the files to read: those of the groups
// that have no fingerprint yet (all of them to do everything again).
type hashTask struct {
s *Service
mu sync.Mutex
ids []int64
}
func (t *hashTask) count(ctx context.Context, refresh bool, _ time.Time) (int, error) {
groups, err := t.s.groups(ctx)
if err != nil {
return 0, err
}
var ids []int64
for _, g := range groups {
for _, f := range g.Files {
if refresh || f.Hash == "" {
ids = append(ids, f.FileID)
}
}
}
slices.Sort(ids)
t.mu.Lock()
t.ids = ids
t.mu.Unlock()
return len(ids), nil
}
func (t *hashTask) list(_ context.Context, after int64, limit int, _ bool, _ time.Time) ([]int64, error) {
t.mu.Lock()
defer t.mu.Unlock()
start := sort.Search(len(t.ids), func(i int) bool { return t.ids[i] > after })
return slices.Clone(t.ids[start:min(start+limit, len(t.ids))]), nil
}
// do reads one file and keeps its fingerprint, if the file is still the one the scan saw.
func (t *hashTask) do(ctx context.Context, fileID int64, _ bool) (string, error) {
rec, err := t.s.store.FileByID(ctx, fileID)
if errors.Is(err, ErrTrackNotFound) || err == nil && rec.Missing {
return "", jobs.ErrSkip
}
if err != nil {
return "", err
}
root, err := openRoot(rec.Root)
if err != nil {
return "", err // a disk that is not mounted: every file fails the same way, and the job gives up
}
defer root.Close()
rel, err := relPath(rec)
if err != nil {
return "", jobs.ErrSkip
}
if err := sameAsScanned(root, rel, rec); errors.Is(err, ErrChanged) {
return "", jobs.ErrSkip
} else if err != nil {
return "", err
}
f, err := root.Open(rel)
if err != nil {
return "", fmt.Errorf("open the file: %w", err)
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, &contextReader{ctx: ctx, r: f}); err != nil {
return "", fmt.Errorf("read the file: %w", err)
}
saved, err := t.s.store.SaveHash(ctx, fileID, rec.SizeBytes, rec.MTimeUnix, hex.EncodeToString(h.Sum(nil)))
if err != nil {
return "", err
}
if !saved {
return "", jobs.ErrSkip // it changed while it was read
}
return OutcomeHashed, nil
}
// contextReader stops reading when the job is stopped, instead of finishing a long file first.
type contextReader struct {
ctx context.Context
r *os.File
}
func (c *contextReader) Read(p []byte) (int, error) {
if err := c.ctx.Err(); err != nil {
return 0, err
}
return c.r.Read(p)
}