Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
| dae5118354 | |||
| 3490849837 | |||
|
|
9905c4788f | ||
|
|
a478c95536 | ||
| 985bd3ed13 | |||
|
|
89f1b5275a |
12 changed files with 734 additions and 6 deletions
88
.forgejo/workflows/release.yml
Normal file
88
.forgejo/workflows/release.yml
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
# What happens at every merge into main:
|
||||
# 1. semantic-release reads the commits since the last release (Conventional Commits), works out the next version,
|
||||
# writes CHANGELOG.md, makes the release on the forge and tags it. A merge with nothing to release (docs, tests, ci)
|
||||
# makes none.
|
||||
# 2. When there is a release, the image is built from the tag and pushed to git.kmalloc.org/squid/sonda as
|
||||
# X.Y.Z, X.Y and latest.
|
||||
# The runner has no Docker, so the image is built with kaniko, which needs no daemon.
|
||||
# Secret needed: FORGEJO_TOKEN, a token of the owner with the scopes write:repository and write:package.
|
||||
name: release
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
# One release at a time: two merges close together must not make two versions at once.
|
||||
concurrency:
|
||||
group: release
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: docker
|
||||
# The commit that the release makes (CHANGELOG.md) says [skip ci]; this is the belt for those braces.
|
||||
if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # the whole history: the next version depends on every commit since the last tag
|
||||
token: ${{ secrets.FORGEJO_TOKEN }}
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
- name: Release
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.FORGEJO_TOKEN }}
|
||||
GITEA_URL: https://git.kmalloc.org
|
||||
GIT_AUTHOR_NAME: sonda-release
|
||||
GIT_AUTHOR_EMAIL: squid@kmalloc.org
|
||||
GIT_COMMITTER_NAME: sonda-release
|
||||
GIT_COMMITTER_EMAIL: squid@kmalloc.org
|
||||
run: >
|
||||
npx --yes
|
||||
-p semantic-release@25
|
||||
-p @semantic-release/commit-analyzer@13.0.1
|
||||
-p @semantic-release/release-notes-generator@14.1.1
|
||||
-p @semantic-release/changelog@7.0.0
|
||||
-p @semantic-release/git@11.0.1
|
||||
-p conventional-changelog-conventionalcommits@8.0.0
|
||||
-p @saithodev/semantic-release-gitea@2.1.0
|
||||
semantic-release
|
||||
- name: Which version was made, if any
|
||||
id: version
|
||||
run: |
|
||||
# The tag of a release points at the commit the release made, which is where the checkout now is.
|
||||
tag="$(git tag --points-at HEAD | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -n 1 || true)"
|
||||
echo "version=${tag#v}" >> "$GITHUB_OUTPUT"
|
||||
echo "released: ${tag:-nothing}"
|
||||
|
||||
image:
|
||||
needs: release
|
||||
if: ${{ needs.release.outputs.version != '' }}
|
||||
runs-on: docker
|
||||
container:
|
||||
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||
steps:
|
||||
- name: Build the image from the tag and push it
|
||||
env:
|
||||
VERSION: ${{ needs.release.outputs.version }}
|
||||
REGISTRY_USER: squid
|
||||
REGISTRY_TOKEN: ${{ secrets.FORGEJO_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
mkdir -p /kaniko/.docker
|
||||
auth="$(printf '%s:%s' "$REGISTRY_USER" "$REGISTRY_TOKEN" | base64 | tr -d '\n')"
|
||||
printf '{"auths":{"git.kmalloc.org":{"auth":"%s"}}}' "$auth" > /kaniko/.docker/config.json
|
||||
image=git.kmalloc.org/squid/sonda
|
||||
minor="${VERSION%.*}"
|
||||
/kaniko/executor \
|
||||
--dockerfile Dockerfile \
|
||||
--context "git://git.kmalloc.org/squid/sonda.git#refs/tags/v${VERSION}" \
|
||||
--build-arg "VERSION=${VERSION}" \
|
||||
--label "org.opencontainers.image.version=${VERSION}" \
|
||||
--destination "${image}:${VERSION}" \
|
||||
--destination "${image}:${minor}" \
|
||||
--destination "${image}:latest" \
|
||||
--verbosity info
|
||||
35
.releaserc.json
Normal file
35
.releaserc.json
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
{
|
||||
"branches": ["main"],
|
||||
"tagFormat": "v${version}",
|
||||
"plugins": [
|
||||
["@semantic-release/commit-analyzer", { "preset": "conventionalcommits" }],
|
||||
[
|
||||
"@semantic-release/release-notes-generator",
|
||||
{
|
||||
"preset": "conventionalcommits",
|
||||
"presetConfig": {
|
||||
"types": [
|
||||
{ "type": "feat", "section": "Features" },
|
||||
{ "type": "fix", "section": "Bug fixes" },
|
||||
{ "type": "perf", "section": "Performance" },
|
||||
{ "type": "refactor", "section": "Refactoring" },
|
||||
{ "type": "docs", "section": "Documentation" },
|
||||
{ "type": "style", "hidden": true },
|
||||
{ "type": "test", "hidden": true },
|
||||
{ "type": "chore", "hidden": true },
|
||||
{ "type": "ci", "hidden": true }
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
["@semantic-release/changelog", { "changelogFile": "CHANGELOG.md", "changelogTitle": "# Changelog" }],
|
||||
[
|
||||
"@semantic-release/git",
|
||||
{
|
||||
"assets": ["CHANGELOG.md"],
|
||||
"message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
|
||||
}
|
||||
],
|
||||
["@saithodev/semantic-release-gitea", { "giteaUrl": "https://git.kmalloc.org" }]
|
||||
]
|
||||
}
|
||||
7
CHANGELOG.md
Normal file
7
CHANGELOG.md
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# Changelog
|
||||
|
||||
## [0.2.0](https://git.kmalloc.org/squid/sonda/compare/v0.1.0...v0.2.0) (2026-11-16)
|
||||
|
||||
### Features
|
||||
|
||||
* scan the libraries by themselves when their music changed ([89f1b52](https://git.kmalloc.org/squid/sonda/commit/89f1b5275a3f1d0115f1cdd22512daebebf964d2))
|
||||
|
|
@ -20,7 +20,9 @@ COPY cmd ./cmd
|
|||
COPY internal ./internal
|
||||
COPY migrations ./migrations
|
||||
COPY --from=web /src/internal/webui/dist ./internal/webui/dist
|
||||
RUN CGO_ENABLED=0 go build -tags webui -trimpath -ldflags="-s -w" -o /out/sondad ./cmd/sondad
|
||||
# The version of the release (see .releaserc.json), as GET /version and the User-Agent of the providers say it.
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=0 go build -tags webui -trimpath -ldflags="-s -w -X main.version=${VERSION}" -o /out/sondad ./cmd/sondad
|
||||
|
||||
FROM alpine:3.22
|
||||
LABEL org.opencontainers.image.title="Sonda" \
|
||||
|
|
|
|||
5
Makefile
5
Makefile
|
|
@ -8,8 +8,11 @@
|
|||
|
||||
.PHONY: build web test
|
||||
|
||||
# The version is the last release tag, and how far this checkout is from it (v1.2.3-4-gabcdef), or "dev" outside git.
|
||||
VERSION ?= $(shell git describe --tags --always 2>/dev/null | sed 's/^v//' || echo dev)
|
||||
|
||||
build: web
|
||||
CGO_ENABLED=0 go build -tags webui -trimpath -ldflags="-s -w" -o bin/sondad ./cmd/sondad
|
||||
CGO_ENABLED=0 go build -tags webui -trimpath -ldflags="-s -w -X main.version=$(VERSION)" -o bin/sondad ./cmd/sondad
|
||||
|
||||
web:
|
||||
cd web && npm ci && npm run build
|
||||
|
|
|
|||
16
README.md
16
README.md
|
|
@ -10,7 +10,7 @@ It is a single executable, one database (SQLite by default, PostgreSQL if you pr
|
|||
|
||||
- **Accounts.** An account is an email address and a password. The first user to register becomes the admin and registration then closes; admins create the other users. The address only identifies the account: Sonda sends no mail, does not check that the address exists, and keeps it on your server. It is trimmed and lowercased, and unique. Accounts made before addresses were asked for keep the plain name they had as their login. Short-lived access tokens (JWT) plus rotating refresh tokens, so a stolen refresh token is detected.
|
||||
- **Libraries.** A library is a name and a folder, managed through the API. Sonda only reads your music folders: it never writes or renames a file, and it deletes one only when an admin asks to remove the extra copy of a song and you turned on `library.allow_delete` (see **Duplicates**), which is off by default.
|
||||
- **Scanner.** Reads MP3 and FLAC tags and audio properties with several workers in parallel. Rescans are incremental, renamed files keep their identity, vanished files are flagged (never deleted), and a library whose folder looks empty or unreadable (an unmounted NAS) is left alone instead of being wiped. Audio in other formats (m4a, ogg, opus, wav, wma and similar) is not read yet, but it is counted as `unsupported` in the scan result instead of disappearing silently. NAS housekeeping folders (`@Recycle`, `#recycle`, `@eaDir`, `.@__thumb`, `$RECYCLE.BIN`) are skipped.
|
||||
- **Scanner.** Reads MP3 and FLAC tags and audio properties with several workers in parallel. Rescans are incremental, renamed files keep their identity, vanished files are flagged (never deleted), and a library whose folder looks empty or unreadable (an unmounted NAS) is left alone instead of being wiped. Audio in other formats (m4a, ogg, opus, wav, wma and similar) is not read yet, but it is counted as `unsupported` in the scan result instead of disappearing silently. NAS housekeeping folders (`@Recycle`, `#recycle`, `@eaDir`, `.@__thumb`, `$RECYCLE.BIN`) are skipped. **Scans by itself.** Every 15 minutes (`library.auto_scan_minutes`) Sonda looks at the folders of each library, which costs only the names of the files, and scans a library whose music changed: a song or a folder came, went or was renamed. A library is also scanned at least once a day, to find a song that was tagged again where it stands. It never scans while another scan runs, and a folder it cannot read (a NAS that is not mounted) is left alone.
|
||||
- **Catalog.** Browse and search artists, albums and songs, with paging, by library, artist or album. Only songs you can actually play are listed.
|
||||
- **Duplicates.** `GET /library/duplicates` (admin) lists the songs that are in more than one file: the same set of main artists, the same title and durations within 3 seconds, possibly on different albums. It is worked out from the index, touches no file, and returns no file path. Each group proposes the copy to keep (lossless first, then the higher bitrate, then the bigger file, then the oldest). A background job (`POST /library/duplicates/check`, like the other bulk jobs) reads the files of the groups and keeps their SHA-256, so a group can be called `identical` (byte for byte). `POST /library/duplicates/clean` **deletes the extra copies of one song from the disk**, and it is the one thing Sonda does that writes in your music folders, so it is off until you set `library.allow_delete` (or `SONDA_ALLOW_DELETE=true`) and mount the folders writable. It deletes only when an admin asks, never by itself, and the server checks again at that moment that no scan is running, that the songs are copies of one song, and that every file is still the one the last scan saw (same size and date, a regular file, not a link, inside its library): so the last copy of a song is never deleted, a file that changed is not deleted, and every file is checked before the first is deleted. The listens and the playlist items of a deleted copy move to the one that stays, so your taste and your playlists are not lost. Folders are never removed, and nothing is sent anywhere. The web app has a Duplicates section in Admin: a table of the copies of each song with the copy to keep already chosen (you can pick another), the check with its progress and a Stop button, **Delete the others** for one song and **Clean all suggestions** for all of them, each after a question that lists the files and the space it frees; with deleting off everything is visible and the delete buttons are disabled.
|
||||
- **Streaming.** `GET` and `HEAD` with `Range` support, so players can seek. Opening or prefetching a stream does not record a listen. `GET /tracks/{id}/peaks` gives the shape of a song for the seek bar: 120 levels from 0 to 255, how loud it is along its length. The server finds it by decoding the file (MP3 and FLAC, in pure Go: nothing else to install), which takes a few seconds, so the first request starts the work in the background and answers `202` with `Retry-After`; it is done once for each file, worked out again if the size or the date of the file change, and one song at a time, so a NAS is not kept busy. A file that cannot be decoded answers `waveform_unavailable`, and that is remembered too. A friend's song has no shape yet.
|
||||
|
|
@ -104,6 +104,7 @@ Precedence is: built-in defaults, then the file, then environment variables. An
|
|||
|---|---|---|---|
|
||||
| `server.listen` | `SONDA_LISTEN` | `127.0.0.1:8080` | Address to listen on. Localhost by default. |
|
||||
| `library.scan_workers` | | `4` | Files whose tags are read in parallel during a scan. |
|
||||
| `library.auto_scan_minutes` | `SONDA_AUTO_SCAN_MINUTES` | `15` | How often, in minutes, Sonda looks at the folders of the libraries and scans the ones whose music changed (a song or a folder came, went or was renamed); `0` turns it off. Looking reads the names of the files and nothing else, so it is cheap for a NAS. A library is scanned at least once a day anyway, which finds a song that was tagged again where it stands. A folder that cannot be read (a NAS that is not mounted) is left alone. |
|
||||
| `library.allow_delete` | `SONDA_ALLOW_DELETE` | `false` | Lets an admin delete the extra copy of a song from the music folders (see **Duplicates**). It is the one thing that makes Sonda write there, so it needs the folders mounted writable. A value that is not `true` or `false` is an error. |
|
||||
| `database.driver` | `SONDA_DB_DRIVER` | `sqlite` | `sqlite` or `postgres`. |
|
||||
| `database.path` | `SONDA_DB_PATH` | `sonda.db` | SQLite file (driver `sqlite`). |
|
||||
|
|
@ -403,6 +404,19 @@ What it does today: sign in (and create the first account of a new server), brow
|
|||
|
||||
More commands, all run in `web/`: `npm test`, `npm run build`, and `npm run gen:api`, which rebuilds `src/api/schema.d.ts` from the OpenAPI spec (it fetches the generator with `npx`, so it needs the network once). `npm run build` writes to `internal/webui/dist` (git-ignored), not to `web/dist`.
|
||||
|
||||
## Releases
|
||||
|
||||
Every merge into `main` can make a release, by itself (`.forgejo/workflows/release.yml`, [semantic-release](https://semantic-release.gitbook.io)). The next version comes from the messages of the commits since the last one, written as [Conventional Commits](https://www.conventionalcommits.org):
|
||||
|
||||
| Commit | Release |
|
||||
|---|---|
|
||||
| `fix: ...`, `perf: ...` | patch (0.2.0 to 0.2.1) |
|
||||
| `feat: ...` | minor (0.2.0 to 0.3.0) |
|
||||
| `feat!: ...`, or `BREAKING CHANGE:` in the body | major (0.2.0 to 1.0.0) |
|
||||
| `docs:`, `test:`, `chore:`, `ci:`, `style:` | none by themselves |
|
||||
|
||||
A release adds its entry to [CHANGELOG.md](CHANGELOG.md), tags the commit (`v0.3.0`), makes the release on the forge, and the image is built from the tag and pushed to `git.kmalloc.org/squid/sonda` as `0.3.0`, `0.3` and `latest`. The version is inside the binary: `GET /version` and the User-Agent of the providers say it (a build that no release made says `dev`). The commit that writes `CHANGELOG.md` says `[skip ci]`. The runner has no Docker, so the image is built with [kaniko](https://github.com/GoogleContainerTools/kaniko). The workflow needs the secret `FORGEJO_TOKEN`: a token of the owner with the scopes `write:repository` and `write:package`.
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -60,8 +60,10 @@ import (
|
|||
|
||||
const defaultConfigFile = "sonda.toml"
|
||||
|
||||
// version goes into the User-Agent sent to providers.
|
||||
const version = "0.1.0"
|
||||
// version goes into the User-Agent sent to providers, and is what GET /version says. The release sets it
|
||||
// (-ldflags "-X main.version=1.2.3": see the Dockerfile and the Makefile); a build that was not made by a release says
|
||||
// "dev".
|
||||
var version = "dev"
|
||||
|
||||
// lastfmMinInterval spaces the requests to Last.fm well below its limit.
|
||||
const lastfmMinInterval = 250 * time.Millisecond
|
||||
|
|
@ -153,6 +155,17 @@ func run() error {
|
|||
scanner := library.NewScanner(libraryRepo, library.TaglibReader{}, library.NewHub(), cfg.Library.ScanWorkers)
|
||||
scans := library.NewManager(scanner, libraryRepo)
|
||||
logScanResults(logger, scans)
|
||||
// The libraries are scanned by themselves when their music changed: looking costs only the names of the files.
|
||||
if cfg.Library.AutoScanMinutes > 0 {
|
||||
auto := library.NewAutoScanner(scans, libraryRepo, time.Duration(cfg.Library.AutoScanMinutes)*time.Minute)
|
||||
auto.OnScan = func(lib library.Library, signatureErr, _ error) {
|
||||
if signatureErr != nil { // a NAS that is not mounted says this all the time: not news
|
||||
logger.Debug("auto scan: the folder of a library cannot be read", zap.Int64("library", lib.ID), zap.Error(signatureErr))
|
||||
}
|
||||
}
|
||||
go auto.Run(ctx)
|
||||
logger.Info("libraries are scanned by themselves when their music changed", zap.Int("every_minutes", cfg.Library.AutoScanMinutes))
|
||||
}
|
||||
|
||||
libraries := library.NewLibraries(libraryRepo)
|
||||
libraries.SetBusy(func() bool { return scans.Status().Running })
|
||||
|
|
|
|||
|
|
@ -42,6 +42,9 @@ type Server struct {
|
|||
// in the database and are managed through the API.
|
||||
type Library struct {
|
||||
ScanWorkers int `toml:"scan_workers"` // parallel tag readers per scan
|
||||
// AutoScanMinutes is how often Sonda looks at the folders of the libraries and scans the ones whose music changed;
|
||||
// 0 turns it off. Looking reads the names of the files and nothing else.
|
||||
AutoScanMinutes int `toml:"auto_scan_minutes"`
|
||||
// AllowDelete lets an admin delete the extra copies of a song from the music folders. It is the
|
||||
// one thing that makes Sonda write there, so it is off until the operator turns it on (and it
|
||||
// needs the folders mounted writable).
|
||||
|
|
@ -142,7 +145,7 @@ type Storage struct {
|
|||
func Default() Config {
|
||||
return Config{
|
||||
Server: Server{Listen: "127.0.0.1:8080"},
|
||||
Library: Library{ScanWorkers: 4},
|
||||
Library: Library{ScanWorkers: 4, AutoScanMinutes: 15},
|
||||
Database: Database{Driver: "sqlite", Path: "sonda.db"},
|
||||
Log: Log{Env: "production"},
|
||||
Auth: Auth{AccessTTLMinutes: 15, RefreshTTLHours: 720}, // 15 minutes, 30 days
|
||||
|
|
@ -185,6 +188,13 @@ func Load(path string) (Config, error) {
|
|||
}
|
||||
cfg.Library.AllowDelete = allow
|
||||
}
|
||||
if v := os.Getenv("SONDA_AUTO_SCAN_MINUTES"); v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("SONDA_AUTO_SCAN_MINUTES must be a whole number of minutes, got %q", v)
|
||||
}
|
||||
cfg.Library.AutoScanMinutes = n
|
||||
}
|
||||
if cfg.Storage.DataDir == "" {
|
||||
cfg.Storage.DataDir = defaultDataDir(cfg.Database)
|
||||
}
|
||||
|
|
@ -285,6 +295,9 @@ func (c Config) Validate() error {
|
|||
if c.Storage.DataDir == "" {
|
||||
errs = append(errs, errors.New("storage.data_dir must not be empty"))
|
||||
}
|
||||
if c.Library.AutoScanMinutes < 0 || c.Library.AutoScanMinutes > 10080 {
|
||||
errs = append(errs, errors.New("library.auto_scan_minutes must be between 0 (off) and 10080 (a week)"))
|
||||
}
|
||||
if c.Library.ScanWorkers < 1 {
|
||||
errs = append(errs, errors.New("library.scan_workers must be at least 1"))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,6 +37,9 @@ func TestLoadEmptyPathReturnsDefaults(t *testing.T) {
|
|||
if cfg.Library.ScanWorkers != 4 {
|
||||
t.Errorf("scan workers = %d", cfg.Library.ScanWorkers)
|
||||
}
|
||||
if cfg.Library.AutoScanMinutes != 15 {
|
||||
t.Errorf("auto scan = %d minutes, want 15", cfg.Library.AutoScanMinutes)
|
||||
}
|
||||
if cfg.Listening.MinListen() != 20*time.Second {
|
||||
t.Errorf("min listen = %v, want 20s", cfg.Listening.MinListen())
|
||||
}
|
||||
|
|
@ -146,6 +149,9 @@ func TestValidate(t *testing.T) {
|
|||
{"two wikipedia languages", func(c *Config) { c.Wikipedia.Languages = []string{"it", "en"} }, ""},
|
||||
{"empty data dir", func(c *Config) { c.Storage.DataDir = "" }, "storage.data_dir"},
|
||||
{"bad workers", func(c *Config) { c.Library.ScanWorkers = 0 }, "library.scan_workers"},
|
||||
{"a negative auto scan", func(c *Config) { c.Library.AutoScanMinutes = -1 }, "library.auto_scan_minutes"},
|
||||
{"an auto scan of a year", func(c *Config) { c.Library.AutoScanMinutes = 525600 }, "library.auto_scan_minutes"},
|
||||
{"auto scan off", func(c *Config) { c.Library.AutoScanMinutes = 0 }, ""},
|
||||
{"bad access ttl", func(c *Config) { c.Auth.AccessTTLMinutes = 0 }, "auth.access_ttl_minutes"},
|
||||
{"bad refresh ttl", func(c *Config) { c.Auth.RefreshTTLHours = 0 }, "auth.refresh_ttl_hours"},
|
||||
}
|
||||
|
|
@ -341,3 +347,20 @@ func TestAllowDeleteEnvironmentVariableMustBeABoolean(t *testing.T) {
|
|||
t.Errorf("got %v, want an error naming SONDA_ALLOW_DELETE", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAutoScanComesFromTheFileOrTheEnvironment(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := writeFile(t, dir, "[library]\nauto_scan_minutes = 5\n")
|
||||
cfg, err := Load(p)
|
||||
if err != nil || cfg.Library.AutoScanMinutes != 5 {
|
||||
t.Fatalf("file: %d %v", cfg.Library.AutoScanMinutes, err)
|
||||
}
|
||||
t.Setenv("SONDA_AUTO_SCAN_MINUTES", "0")
|
||||
if cfg, err = Load(p); err != nil || cfg.Library.AutoScanMinutes != 0 {
|
||||
t.Fatalf("environment over file: %d %v", cfg.Library.AutoScanMinutes, err)
|
||||
}
|
||||
t.Setenv("SONDA_AUTO_SCAN_MINUTES", "often")
|
||||
if _, err = Load(""); err == nil {
|
||||
t.Fatal("a value that is not a number was taken")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
199
internal/library/autoscan.go
Normal file
199
internal/library/autoscan.go
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
package library
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// forceEvery is how long a library may go without a scan even when its folders look the same: a song that is tagged
|
||||
// again keeps its place and its name, so only a scan of the file finds it.
|
||||
const forceEvery = 24 * time.Hour
|
||||
|
||||
// Signature says how a folder looks: a hash of the names of everything in it, folder by folder, and of the time of each
|
||||
// folder. It reads the names and nothing else (no file is opened or measured), which is why it is cheap enough to
|
||||
// ask often of a NAS, and it changes when a song or a folder comes, goes or is renamed. It does not change when a
|
||||
// file is changed where it stands. The housekeeping folders of a NAS (see isSystemDir) and the links inside the folder
|
||||
// are left out, as the scanner leaves them out; a link as the folder itself is followed. A folder that cannot be read is
|
||||
// an error.
|
||||
func Signature(ctx context.Context, root string) (string, error) {
|
||||
top, err := filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
info, err := os.Stat(top)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", fmt.Errorf("%s is not a directory", root)
|
||||
}
|
||||
h := sha256.New()
|
||||
var walk func(dir, rel string) error
|
||||
walk = func(dir, rel string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
if rel == "" {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(h, "! %s\n", rel) // a folder that cannot be read is a state of its own
|
||||
return nil
|
||||
}
|
||||
var mtime int64
|
||||
if st, err := os.Stat(dir); err == nil {
|
||||
mtime = st.ModTime().UnixNano()
|
||||
}
|
||||
fmt.Fprintf(h, "d %s %d\n", rel, mtime)
|
||||
for _, e := range entries { // os.ReadDir sorts them by name
|
||||
if e.IsDir() && isSystemDir(e.Name()) {
|
||||
continue
|
||||
}
|
||||
kind := 'f'
|
||||
switch {
|
||||
case e.Type()&os.ModeSymlink != 0:
|
||||
kind = 'l'
|
||||
case e.IsDir():
|
||||
kind = 'd'
|
||||
}
|
||||
fmt.Fprintf(h, "%c %s\n", kind, e.Name())
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir() && e.Type()&os.ModeSymlink == 0 && !isSystemDir(e.Name()) {
|
||||
if err := walk(filepath.Join(dir, e.Name()), rel+e.Name()+"/"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := walk(top, ""); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// scans is what the watcher asks of the manager of the scans: *Manager.
|
||||
type scans interface {
|
||||
StartLibrary(ctx context.Context, id int64, opts ...ScanOption) error
|
||||
Status() Status
|
||||
}
|
||||
|
||||
// AutoScanner scans the libraries by itself when their music changed. Every `interval` it takes the Signature of each
|
||||
// library and scans the ones that do not look as they did the last time (the first time, all of them), and every library
|
||||
// at least once a day. A scan is the usual one: it reads only the files that are new or changed. Nothing is scanned
|
||||
// while another scan runs, and a scan that failed is tried again at the next look.
|
||||
type AutoScanner struct {
|
||||
scans scans
|
||||
libs LibrarySource
|
||||
interval time.Duration
|
||||
// first is how long after the start the first look comes: the server is not busy with it while it starts.
|
||||
first time.Duration
|
||||
now func() time.Time
|
||||
poll time.Duration // how often it asks whether a scan it started is over
|
||||
// OnScan, when set, is told about each library it scans and about each one it could not look at.
|
||||
OnScan func(lib Library, signatureErr, scanErr error)
|
||||
|
||||
seen map[int64]seenLibrary
|
||||
}
|
||||
|
||||
type seenLibrary struct {
|
||||
sig string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// NewAutoScanner builds an AutoScanner that looks every interval.
|
||||
func NewAutoScanner(s scans, libs LibrarySource, interval time.Duration) *AutoScanner {
|
||||
return &AutoScanner{
|
||||
scans: s, libs: libs, interval: interval, first: 30 * time.Second,
|
||||
now: time.Now, poll: time.Second, seen: map[int64]seenLibrary{},
|
||||
}
|
||||
}
|
||||
|
||||
// Run looks at the libraries until ctx ends.
|
||||
func (a *AutoScanner) Run(ctx context.Context) {
|
||||
timer := time.NewTimer(a.first)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-timer.C:
|
||||
a.tick(ctx)
|
||||
timer.Reset(a.interval)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tick looks at every library once.
|
||||
func (a *AutoScanner) tick(ctx context.Context) {
|
||||
libs, err := a.libs.ListLibraries(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
present := map[int64]bool{}
|
||||
for _, l := range libs {
|
||||
present[l.ID] = true
|
||||
}
|
||||
for id := range a.seen {
|
||||
if !present[id] {
|
||||
delete(a.seen, id) // a library that is gone: a new one with its number starts from nothing
|
||||
}
|
||||
}
|
||||
for _, lib := range libs {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
sig, err := Signature(ctx, lib.Path)
|
||||
if err != nil {
|
||||
// Not there (a NAS that is not mounted): nothing is known about it, and the scan would only say so.
|
||||
a.told(lib, err, nil)
|
||||
continue
|
||||
}
|
||||
prev, ok := a.seen[lib.ID]
|
||||
if ok && prev.sig == sig && a.now().Sub(prev.at) < forceEvery {
|
||||
continue
|
||||
}
|
||||
switch err := a.scans.StartLibrary(ctx, lib.ID); {
|
||||
case errors.Is(err, ErrScanRunning):
|
||||
return // another scan has the turn: the rest wait for the next look
|
||||
case err != nil:
|
||||
a.told(lib, nil, err)
|
||||
continue
|
||||
}
|
||||
scanErr := a.wait(ctx)
|
||||
a.told(lib, nil, scanErr)
|
||||
if scanErr == nil && ctx.Err() == nil {
|
||||
// The signature is the one taken before the scan: what changed while it ran is seen at the next look.
|
||||
a.seen[lib.ID] = seenLibrary{sig: sig, at: a.now()}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// wait holds until the scan that was started is over, and says how it ended.
|
||||
func (a *AutoScanner) wait(ctx context.Context) error {
|
||||
for a.scans.Status().Running {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(a.poll):
|
||||
}
|
||||
}
|
||||
if msg := a.scans.Status().LastError; msg != "" {
|
||||
return errors.New(msg)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *AutoScanner) told(lib Library, sigErr, scanErr error) {
|
||||
if a.OnScan != nil {
|
||||
a.OnScan(lib, sigErr, scanErr)
|
||||
}
|
||||
}
|
||||
328
internal/library/autoscan_test.go
Normal file
328
internal/library/autoscan_test.go
Normal file
|
|
@ -0,0 +1,328 @@
|
|||
package library
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ---- the signature of a folder ----
|
||||
|
||||
func tree(t *testing.T) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for _, f := range []string{"a/one.mp3", "a/two.flac", "b/c/three.mp3"} {
|
||||
touch(t, root, f)
|
||||
}
|
||||
settle(t, root)
|
||||
return root
|
||||
}
|
||||
|
||||
func touch(t *testing.T, root, rel string) {
|
||||
t.Helper()
|
||||
p := filepath.Join(root, filepath.FromSlash(rel))
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// settle gives every folder of the tree a moment far in the past, so that a change made after it moves the time of the
|
||||
// folder it touches, whatever the resolution of the file system.
|
||||
func settle(t *testing.T, root string) {
|
||||
t.Helper()
|
||||
old := time.Date(2020, 1, 1, 12, 0, 0, 0, time.UTC) // always the same, so that two calls leave the same time
|
||||
var dirs []string
|
||||
_ = filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error {
|
||||
if err == nil && d.IsDir() {
|
||||
dirs = append(dirs, p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
for i := len(dirs) - 1; i >= 0; i-- {
|
||||
if err := os.Chtimes(dirs[i], old, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sig(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
s, err := Signature(context.Background(), root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestTheSignatureOfAFolderChangesWhenMusicComesOrGoes(t *testing.T) {
|
||||
root := tree(t)
|
||||
base := sig(t, root)
|
||||
if sig(t, root) != base {
|
||||
t.Fatal("the same folder, two signatures")
|
||||
}
|
||||
|
||||
for name, change := range map[string]func(){
|
||||
"a song comes": func() { touch(t, root, "a/new.mp3") },
|
||||
"a song goes": func() { _ = os.Remove(filepath.Join(root, "a/one.mp3")) },
|
||||
"a song is renamed": func() { _ = os.Rename(filepath.Join(root, "a/two.flac"), filepath.Join(root, "a/deux.flac")) },
|
||||
"a folder comes": func() { touch(t, root, "d/new.mp3") },
|
||||
"a folder goes": func() { _ = os.RemoveAll(filepath.Join(root, "b")) },
|
||||
"something is dropped": func() { touch(t, root, "b/c/cover.jpg") },
|
||||
"a deep folder changes": func() { touch(t, root, "b/c/more.flac") },
|
||||
} {
|
||||
fresh := tree(t)
|
||||
before := sig(t, fresh)
|
||||
root = fresh
|
||||
change()
|
||||
if sig(t, fresh) == before {
|
||||
t.Errorf("%s: the signature did not change", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSignatureLeavesOutTheHousekeepingFoldersOfANAS(t *testing.T) {
|
||||
root := tree(t)
|
||||
base := sig(t, root)
|
||||
// A NAS writes thumbnails in @eaDir all the time: what happens inside it is not a change of the music.
|
||||
if err := os.MkdirAll(filepath.Join(root, "a", "@eaDir"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settle(t, root)
|
||||
withDir := sig(t, root)
|
||||
touch(t, root, "a/@eaDir/thumb.jpg")
|
||||
touch(t, root, "a/@eaDir/deeper/thumb2.jpg")
|
||||
settle(t, root)
|
||||
if got := sig(t, root); got != withDir {
|
||||
t.Errorf("a change inside @eaDir moved the signature")
|
||||
}
|
||||
_ = base
|
||||
}
|
||||
|
||||
func TestTheSignatureOfAFolderThatCannotBeReadIsAnError(t *testing.T) {
|
||||
if _, err := Signature(context.Background(), filepath.Join(t.TempDir(), "nothing")); err == nil {
|
||||
t.Fatal("no error for a folder that is not there")
|
||||
}
|
||||
file := filepath.Join(t.TempDir(), "f")
|
||||
_ = os.WriteFile(file, nil, 0o644)
|
||||
if _, err := Signature(context.Background(), file); err == nil {
|
||||
t.Fatal("no error for a file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSignatureFollowsTheLinkOfTheRootButNotOthers(t *testing.T) {
|
||||
root := tree(t)
|
||||
link := filepath.Join(t.TempDir(), "music")
|
||||
if err := os.Symlink(root, link); err != nil {
|
||||
t.Skip("no symlinks here")
|
||||
}
|
||||
if sig(t, link) != sig(t, root) {
|
||||
t.Error("a library that is a link has another signature than its target")
|
||||
}
|
||||
outside := tree(t)
|
||||
if err := os.Symlink(outside, filepath.Join(root, "outside")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settle(t, root)
|
||||
before := sig(t, root)
|
||||
touch(t, outside, "a/changed.mp3")
|
||||
if sig(t, root) != before {
|
||||
t.Error("a link inside the folder was followed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSignatureStopsWithTheContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := Signature(ctx, tree(t)); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the watcher ----
|
||||
|
||||
type fakeScans struct {
|
||||
mu sync.Mutex
|
||||
started []int64
|
||||
running bool
|
||||
busy bool // StartLibrary answers ErrScanRunning
|
||||
lastErr string // what the scan that finished says
|
||||
finishes int // how many Status calls say "running" before it is over
|
||||
left int
|
||||
}
|
||||
|
||||
func (f *fakeScans) StartLibrary(_ context.Context, id int64, _ ...ScanOption) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.busy {
|
||||
return ErrScanRunning
|
||||
}
|
||||
f.started = append(f.started, id)
|
||||
f.left = f.finishes
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeScans) Status() Status {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.left > 0 {
|
||||
f.left--
|
||||
return Status{Running: true}
|
||||
}
|
||||
return Status{LastError: f.lastErr}
|
||||
}
|
||||
|
||||
func (f *fakeScans) count() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.started)
|
||||
}
|
||||
|
||||
type fakeLibs struct{ libs []Library }
|
||||
|
||||
func (f *fakeLibs) ListLibraries(context.Context) ([]Library, error) { return f.libs, nil }
|
||||
func (f *fakeLibs) GetLibrary(_ context.Context, id int64) (Library, error) {
|
||||
for _, l := range f.libs {
|
||||
if l.ID == id {
|
||||
return l, nil
|
||||
}
|
||||
}
|
||||
return Library{}, ErrLibraryNotFound
|
||||
}
|
||||
|
||||
func newWatcher(scans *fakeScans, libs ...Library) (*AutoScanner, *time.Time) {
|
||||
now := time.Unix(1_700_000_000, 0)
|
||||
a := NewAutoScanner(scans, &fakeLibs{libs: libs}, time.Minute)
|
||||
a.now = func() time.Time { return now }
|
||||
a.poll = time.Millisecond
|
||||
return a, &now
|
||||
}
|
||||
|
||||
func TestTheWatcherScansALibraryThatChangedAndLeavesOneThatDidNot(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
rock, jazz := tree(t), tree(t)
|
||||
scans := &fakeScans{}
|
||||
a, _ := newWatcher(scans, Library{ID: 1, Path: rock}, Library{ID: 2, Path: jazz})
|
||||
|
||||
a.tick(ctx) // the first time nothing is known: each library is looked at
|
||||
if scans.count() != 2 {
|
||||
t.Fatalf("first tick: %v", scans.started)
|
||||
}
|
||||
a.tick(ctx)
|
||||
if scans.count() != 2 {
|
||||
t.Fatalf("nothing changed, yet %v", scans.started)
|
||||
}
|
||||
touch(t, jazz, "a/new.flac")
|
||||
a.tick(ctx)
|
||||
if got := scans.started; len(got) != 3 || got[2] != 2 {
|
||||
t.Fatalf("only jazz changed: %v", got)
|
||||
}
|
||||
a.tick(ctx)
|
||||
if scans.count() != 3 {
|
||||
t.Fatalf("after the scan nothing changed: %v", scans.started)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAScanThatCouldNotStartOrFailedIsTriedAgain(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := tree(t)
|
||||
scans := &fakeScans{busy: true}
|
||||
a, _ := newWatcher(scans, Library{ID: 1, Path: root})
|
||||
|
||||
a.tick(ctx) // somebody else is scanning
|
||||
scans.busy = false
|
||||
a.tick(ctx)
|
||||
if scans.count() != 1 {
|
||||
t.Fatalf("a scan that did not start was forgotten: %v", scans.started)
|
||||
}
|
||||
|
||||
scans.lastErr = "directory is empty but 10 indexed files exist"
|
||||
touch(t, root, "a/new.mp3")
|
||||
a.tick(ctx) // starts, and fails
|
||||
a.tick(ctx) // the same change is still not scanned: try again
|
||||
if scans.count() != 3 {
|
||||
t.Fatalf("a failed scan was not tried again: %v", scans.started)
|
||||
}
|
||||
scans.lastErr = ""
|
||||
a.tick(ctx)
|
||||
a.tick(ctx)
|
||||
if scans.count() != 4 {
|
||||
t.Fatalf("after it worked it was scanned again: %v", scans.started)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreadableLibraryIsLeftAlone(t *testing.T) {
|
||||
scans := &fakeScans{}
|
||||
a, _ := newWatcher(scans, Library{ID: 1, Path: filepath.Join(t.TempDir(), "unmounted")})
|
||||
a.tick(context.Background())
|
||||
if scans.count() != 0 {
|
||||
t.Fatalf("a library that cannot be read was scanned: %v", scans.started)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryLibraryIsScannedOnceADayEvenIfNothingMovedItsFolders(t *testing.T) {
|
||||
// A song that is tagged again keeps its folder as it was: only the file knows. A scan once a day finds it.
|
||||
ctx := context.Background()
|
||||
scans := &fakeScans{}
|
||||
a, now := newWatcher(scans, Library{ID: 1, Path: tree(t)})
|
||||
a.tick(ctx)
|
||||
*now = now.Add(23 * time.Hour)
|
||||
a.tick(ctx)
|
||||
if scans.count() != 1 {
|
||||
t.Fatalf("after 23 hours: %v", scans.started)
|
||||
}
|
||||
*now = now.Add(2 * time.Hour)
|
||||
a.tick(ctx)
|
||||
if scans.count() != 2 {
|
||||
t.Fatalf("after 25 hours: %v", scans.started)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheWatcherWaitsForTheScanToEndBeforeItGoesOn(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
scans := &fakeScans{finishes: 3}
|
||||
a, _ := newWatcher(scans, Library{ID: 1, Path: tree(t)}, Library{ID: 2, Path: tree(t)})
|
||||
a.tick(ctx)
|
||||
if scans.count() != 2 || scans.left != 0 {
|
||||
t.Fatalf("%v, %d left", scans.started, scans.left)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADeletedLibraryIsForgottenAndANewOneIsLookedAt(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
scans := &fakeScans{}
|
||||
libs := &fakeLibs{libs: []Library{{ID: 1, Path: tree(t)}}}
|
||||
a := NewAutoScanner(scans, libs, time.Minute)
|
||||
a.poll = time.Millisecond
|
||||
a.tick(ctx)
|
||||
libs.libs = nil
|
||||
a.tick(ctx)
|
||||
if len(a.seen) != 0 {
|
||||
t.Fatalf("a library that is gone is still remembered: %v", a.seen)
|
||||
}
|
||||
libs.libs = []Library{{ID: 1, Path: tree(t)}} // another one with the same id, later
|
||||
a.tick(ctx)
|
||||
if scans.count() != 2 {
|
||||
t.Fatalf("a new library with an old id: %v", scans.started)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStopsWithItsContext(t *testing.T) {
|
||||
scans := &fakeScans{}
|
||||
a := NewAutoScanner(scans, &fakeLibs{}, time.Hour)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { a.Run(ctx); close(done) }()
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Run did not stop")
|
||||
}
|
||||
}
|
||||
|
|
@ -9,6 +9,9 @@ listen = "127.0.0.1:8080" # localhost by default
|
|||
# Libraries (name + directory) are managed through the API: POST /api/v1/libraries.
|
||||
[library]
|
||||
scan_workers = 4 # parallel tag readers during a scan
|
||||
auto_scan_minutes = 15 # how often to look at the folders and scan the libraries whose music changed
|
||||
# (0 turns it off). Looking reads the names of the files, nothing else, so it is
|
||||
# cheap for a NAS; a library is scanned at least once a day. Or SONDA_AUTO_SCAN_MINUTES.
|
||||
# allow_delete = true # let an admin delete the extra copy of a song from the music folders.
|
||||
# Off by default: it is the one thing that makes Sonda write there, and it
|
||||
# needs the folders mounted writable. Or set SONDA_ALLOW_DELETE=true.
|
||||
|
|
|
|||
Loading…
Reference in a new issue