diff --git a/api/openapi.yaml b/api/openapi.yaml index a46b6b4..5c971a3 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -2105,6 +2105,39 @@ paths: default: $ref: "#/components/responses/Error" + /rooms/{roomId}/queue/{itemId}/beatmap: + parameters: + - $ref: "#/components/parameters/RoomId" + - name: itemId + in: path + required: true + schema: + type: string + get: + operationId: getQueueItemBeatMap + summary: A queued song's beat map + description: | + The song's tempo, beats, bars and sections, and its spectrum from + moment to moment, for visuals that move with the music on every + screen in the room. Worked out from the audio once, usually soon + after the song is queued; if it hasn't been yet, it's worked out + now, which can take a few seconds. 404 `no_beat_map` if the song + can't have one (its service has no audio, or it couldn't be + decoded). + tags: [queue] + security: + - session: [] + - display: [] + responses: + "200": + description: The beat map + content: + application/json: + schema: + $ref: "#/components/schemas/BeatMap" + default: + $ref: "#/components/responses/Error" + /rooms/{roomId}/queue/{itemId}/lyrics: parameters: - $ref: "#/components/parameters/RoomId" @@ -2937,8 +2970,13 @@ paths: else sends `vote_skip` (and `unvote_skip` to take it back); the song is skipped once `skipVotes.needed` have voted. `play` needs a speaker (`no_player`); pausing, skipping, seeking or - voting with nothing playing is `nothing_playing`. + voting with nothing playing is `nothing_playing`. A display with + `audio` on may `play`, `pause` and `skip` in its own room, as + whoever paired it. tags: [playback] + security: + - session: [] + - display: [] requestBody: required: true content: @@ -3321,6 +3359,33 @@ components: The guest expired or was removed. They're kept only so the room's history and recaps can show their name. + RoomScreens: + type: object + description: | + How the room's big screens show the music (MAD-779). Set from any + phone by whoever manages the room; every screen follows. + required: [look, scene, intensity] + properties: + look: + type: string + enum: [stage, visualizer, auto] + default: auto + description: | + `stage`: now playing, lyrics and what's next. `visualizer`: + full-screen visuals with now playing small. `auto`: the stage, + with visuals taking over for songs with no words to sing. + scene: + type: string + maxLength: 32 + default: "" + description: The visualizer's scene, or "" to pick one that suits each song. + intensity: + type: number + minimum: 0.5 + maximum: 2 + default: 1.5 + description: How strongly the screens move with the music. + RoomGuests: type: object description: | @@ -5013,6 +5078,82 @@ components: light: $ref: "#/components/schemas/OklchColor" + BeatMap: + type: object + description: | + A song's beat map (MAD-773). Times are in ms from the start of the + song, on the same clock as the playback position. + required: [durationMs, bpm, confidence, beats, downbeat, sections, frameRate, bandCount, bands, loudness, features] + properties: + durationMs: + type: integer + format: int64 + bpm: + type: number + description: The tempo; 0 if the song has no steady beat, and then `beats` is empty. + confidence: + type: number + description: How clear the beat is, 0–1. + beats: + type: array + description: The beats. + items: + type: integer + format: int64 + downbeat: + type: integer + description: The index in `beats` of the first bar's first beat; every fourth beat after it starts a bar. + sections: + type: array + description: The song's parts (intro, verse, drop), in order; the first starts at 0. + items: + $ref: "#/components/schemas/BeatMapSection" + frameRate: + type: number + description: Frames a second in `bands` and `loudness`. + bandCount: + type: integer + description: Spectrum bands per frame in `bands`. + bands: + type: string + format: byte + description: | + Base64: each frame's level in each band, low to high, a byte + (0–255) each, `bandCount` bytes a frame. Each band is scaled to + its own range in the song. + loudness: + type: string + format: byte + description: "Base64: each frame's overall level, a byte (0–255) each, scaled to the song." + features: + $ref: "#/components/schemas/BeatMapFeatures" + + BeatMapSection: + type: object + required: [startMs, energy] + properties: + startMs: + type: integer + format: int64 + energy: + type: number + description: How loud it is next to the rest of the song, 0–1. + + BeatMapFeatures: + type: object + description: The song as a whole, 0–1 each, for choosing how it looks. + required: [energy, brightness, dynamics] + properties: + energy: + type: number + description: How loud it's mastered, from a quiet acoustic recording to a club track. + brightness: + type: number + description: How much treble there is. + dynamics: + type: number + description: How much its loudness changes. + Accent: type: object required: [h, c] @@ -5244,6 +5385,34 @@ components: $ref: "#/components/schemas/QueueItem" skipVotes: $ref: "#/components/schemas/SkipVotes" + playNow: + $ref: "#/components/schemas/PlayNowVote" + + PlayNowVote: + type: object + description: | + Someone's request to play a queued song now, in a room whose skip + permission is `vote`. It passes on the same share of the room as a + skip, and lapses at `expires`, when the song leaves the queue, or + when whoever asked takes it back. + required: [itemId, by, voters, needed, expires] + properties: + itemId: + type: string + by: + type: string + description: Who asked. Their vote is the first. + voters: + type: array + description: IDs of the users who agreed, in order. + items: + type: string + needed: + type: integer + description: How many agreeing play the song. + expires: + type: string + format: date-time SkipVotes: type: object @@ -5290,7 +5459,13 @@ components: properties: action: type: string - enum: [play, pause, skip, seek, vote_skip, unvote_skip] + enum: [play, pause, skip, seek, vote_skip, unvote_skip, play_now, vote_play_now, unvote_play_now] + description: | + `play_now` plays the queued song `itemId` straight away, skipping + the one playing. It takes the skip permission; in a room that + votes on skips, it asks the room instead (`playNow`), and + `vote_play_now` agrees. `unvote_play_now` takes a vote back, or + withdraws the request from whoever asked. positionMs: type: integer format: int64 @@ -5300,7 +5475,8 @@ components: type: string description: | Skip (or vote) only if this is still the current song, so two - people tapping skip skip one song. + people tapping skip skip one song. For `play_now` and its votes, + the queued song to play. ClaimPlayerRequest: type: object @@ -5315,6 +5491,13 @@ components: type: string maxLength: 64 example: Collin's phone + playItemId: + type: string + description: | + A queued song to play now, as `play_now` would, instead of + starting the next in fair order: pressing play on a song with no + speaker. Ignored if the caller would have to ask the room, or + the song isn't waiting. PlayerReport: type: object @@ -5601,7 +5784,7 @@ components: Room: type: object - required: [id, name, ownerId, visibility, approveJoins, fairnessMode, fairness, matching, autopilot, guests, permissions, skipVotePercent, createdAt] + required: [id, name, ownerId, visibility, approveJoins, fairnessMode, fairness, matching, autopilot, guests, screens, permissions, skipVotePercent, createdAt] properties: id: type: string @@ -5624,6 +5807,8 @@ components: $ref: "#/components/schemas/RoomAutopilot" guests: $ref: "#/components/schemas/RoomGuests" + screens: + $ref: "#/components/schemas/RoomScreens" permissions: $ref: "#/components/schemas/RoomPermissions" skipVotePercent: @@ -5654,6 +5839,8 @@ components: $ref: "#/components/schemas/RoomAutopilot" guests: $ref: "#/components/schemas/RoomGuests" + screens: + $ref: "#/components/schemas/RoomScreens" permissions: $ref: "#/components/schemas/RoomPermissionsChange" skipVotePercent: @@ -5684,6 +5871,8 @@ components: $ref: "#/components/schemas/RoomAutopilot" guests: $ref: "#/components/schemas/RoomGuests" + screens: + $ref: "#/components/schemas/RoomScreens" permissions: $ref: "#/components/schemas/RoomPermissionsChange" skipVotePercent: diff --git a/docs/adr/0003-playback-engine.md b/docs/adr/0003-playback-engine.md index 4a16693..dae14a0 100644 --- a/docs/adr/0003-playback-engine.md +++ b/docs/adr/0003-playback-engine.md @@ -43,9 +43,10 @@ The song's service decides the driver, by whether its session implements `provid | --- | --- | | The song can't start (link removed, service error, speaker reports `error`) | Skip it and push a `playback.notice` | | A streamed song doesn't start within 20s | Skip it with a notice | +| …and the speaker said nothing about it the whole time (its tab is asleep, or gone) | Pause on it with a notice instead. The song isn't to blame, and skipping would burn the queue. The speaker acknowledges each song it starts loading (a `progress` report), so a speaker that's there but can't play the song still skips it. Play loads it again | | The speaker never reports `ended` | Move on 15s after the song's length | | The speaker goes quiet for 2 minutes while playing | Pause with a notice. The timeout is generous because mobile browsers throttle background tabs | -| 3 songs in a row fail | Stop, with a notice, instead of skipping through the whole queue. Pressing play resumes | +| 3 songs in a row fail | Stop, with a notice, instead of skipping through the whole queue. Pressing play resumes; the room shows a play button while it's stopped | ### Permissions @@ -57,6 +58,14 @@ When `skip` is `vote`, the owner and the song's requester can still skip outrigh The vote passes once more than `skipVotePercent` (default 50, a majority) of the room has voted. "The room" is everyone connected to it, plus anyone who voted and has since left, minus whoever queued the song, since they'd just skip it. The engine re-counts whenever someone joins or leaves and whenever the owner changes the settings, so a vote can pass because a member walked away. A passed vote skips with a `playback.notice`. +#### Play now + +`play_now` with a queued `itemId` makes that song current straight away, skipping the one playing. The fair order isn't changed: the song simply jumps it, and the fair order then sees its requester just played. It takes the `skip` permission. When `skip` is `vote`, the owner still plays it outright, and anyone else asks the room: `nowplaying.updated` carries the request (`playNow`), everyone gets a notice, and the room agrees with `vote_play_now` on the same share as a skip vote. One request is open at a time. It lapses after 2 minutes, when its song leaves the queue, or when whoever asked withdraws it (`unvote_play_now`). + +#### One tap to play + +With no speaker, a member who may be the speaker pressing play becomes the speaker and starts the room in the same tap. "Play on this phone" does the same for a paused room. + ### Restarts Playback state lives in memory. After a restart, a song that was playing comes back **paused at its start**: nobody knows how far the old speaker got. The stored player device is cleared, and the speaker claims the room again when it reconnects. diff --git a/server/cmd/syncphony/main.go b/server/cmd/syncphony/main.go index 546166e..83d772f 100644 --- a/server/cmd/syncphony/main.go +++ b/server/cmd/syncphony/main.go @@ -17,6 +17,7 @@ import ( "time" "github.com/madeofpendletonwool/syncphony/server/internal/admin" + "github.com/madeofpendletonwool/syncphony/server/internal/analysis" "github.com/madeofpendletonwool/syncphony/server/internal/artwork" "github.com/madeofpendletonwool/syncphony/server/internal/auth" "github.com/madeofpendletonwool/syncphony/server/internal/autopilot" @@ -195,22 +196,32 @@ func run() error { go repairMusicGraph(ctx, graph) } roomSvc := rooms.New(db, a.bus) - go sweep(ctx, db, accounts, roomSvc, lyricsSvc, mb, notes, graph) queueSvc := queue.New(db, roomSvc, a.links) art := artwork.New(a.links, mb) palettes := palette.New(db, art) go palettes.Run(ctx) + // Beat maps need ffmpeg to decode the songs. + var beatMaps *analysis.Service + if ff := (analysis.FFmpeg{}); ffmpegAvailable() { + beatMaps = analysis.New(db, a.links, ff) + go beatMaps.Run(ctx) + } else { + slog.Warn("ffmpeg not found: songs get no beat maps, so the app can't move with the music") + } queueSvc.OnAdd = func(ts []provider.Track) { if mb != nil { mb.Enqueue(ts...) } palettes.Enqueue(ts...) + if beatMaps != nil { + beatMaps.Enqueue(ts...) + } if len(graph.Sources()) > 0 { graph.Warm(ts...) } } var transcoder transcode.Transcoder - if ff := (transcode.FFmpeg{}); ff.Available() { + if ff := (transcode.FFmpeg{}); ffmpegAvailable() { transcoder = ff } else { slog.Warn("ffmpeg not found: songs in formats the player can't decode won't play") @@ -245,7 +256,7 @@ func run() error { } nightSvc := nights.New(db, a.bus) api := &httpapi.Server{ - Version: version, StartedAt: time.Now().UTC(), Admin: admin.New(db, filepath.Join(cfg.DataDir, "backups")), Auth: accounts, Links: a.links, Lyrics: lyricsSvc, LinerNotes: notes, Artwork: art, Palettes: palettes, + Version: version, StartedAt: time.Now().UTC(), Admin: admin.New(db, filepath.Join(cfg.DataDir, "backups")), Auth: accounts, Links: a.links, Lyrics: lyricsSvc, LinerNotes: notes, Artwork: art, Palettes: palettes, BeatMaps: beatMaps, Rooms: roomSvc, Queue: queueSvc, Playback: player, Nights: nightSvc, Suggest: suggestions, Autopilot: pilot, Bus: a.bus, Presence: presence, BaseURL: cfg.BaseURL, TrustedProxies: cfg.TrustedProxies, @@ -253,6 +264,7 @@ func run() error { if len(graph.Sources()) > 0 { api.Graph = graph } + go sweep(ctx, db, accounts, roomSvc, lyricsSvc, mb, notes, graph, beatMaps) go tick(ctx, api, nightSvc) mux := http.NewServeMux() mux.Handle("/api/", api.Handler()) @@ -289,6 +301,9 @@ func run() error { return nil } +// ffmpegAvailable reports whether ffmpeg is on $PATH, for transcoding and beat maps. +func ffmpegAvailable() bool { return (transcode.FFmpeg{}).Available() } + // tick ends guests whose time is up, and the night in rooms that went // quiet, every minute until ctx is done. func tick(ctx context.Context, api *httpapi.Server, ns *nights.Service) { @@ -356,7 +371,7 @@ func repairMusicGraph(ctx context.Context, graph *musicgraph.Service) { // sweep deletes expired sessions, displays and room invites, cached lyrics, // MusicBrainz matches, liner notes and music knowledge every hour until ctx // is done. -func sweep(ctx context.Context, db *store.Store, accounts *auth.Service, rs *rooms.Service, ly *lyrics.Service, mb *musicbrainz.Service, notes *linernotes.Service, graph *musicgraph.Service) { +func sweep(ctx context.Context, db *store.Store, accounts *auth.Service, rs *rooms.Service, ly *lyrics.Service, mb *musicbrainz.Service, notes *linernotes.Service, graph *musicgraph.Service, beatMaps *analysis.Service) { t := time.NewTicker(time.Hour) defer t.Stop() for { @@ -388,6 +403,11 @@ func sweep(ctx context.Context, db *store.Store, accounts *auth.Service, rs *roo if err := graph.Sweep(ctx); err != nil { slog.Warn("sweeping expired music knowledge", "err", err) } + if beatMaps != nil { + if err := beatMaps.Sweep(ctx); err != nil { + slog.Warn("sweeping unused beat maps", "err", err) + } + } select { case <-ctx.Done(): return diff --git a/server/internal/analysis/analysis.go b/server/internal/analysis/analysis.go new file mode 100644 index 0000000..b24b19a --- /dev/null +++ b/server/internal/analysis/analysis.go @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package analysis works out a song's beat map (MAD-773): its tempo, where +// the beats and bars fall, how loud each part of the spectrum is from +// moment to moment, and where its sections change. Every screen in a room +// uses the map to move with the music in step, without hearing it. +// +// The song is decoded once, to mono at 22.05 kHz, and analysed in one pass +// of short overlapping windows (46 ms, every 23 ms). Onsets come from +// spectral flux; tempo from the onset curve's autocorrelation with a +// prior around 120 BPM; beats from dynamic programming (Ellis, 2007); +// downbeats from where the kicks land; sections from the novelty of a +// bar-by-bar self-similarity matrix (Foote, 2000). +package analysis + +import ( + "errors" + "io" + "math" + "slices" +) + +const ( + // SampleRate is the rate songs are decoded to for analysis. + SampleRate = 22050 + // Version changes whenever an older map should be worked out again. + Version = 1 + // Bands is how many spectrum bands the map's envelope has. + Bands = 12 + + window = 1024 + hop = 512 + minHz = 40.0 + // lowHz bounds the kick drum's part of the spectrum, for downbeats. + lowHz = 200.0 + // Below these a song has no beat: its spectrum barely changes (music + // is in the tens, a steady tone under one), or its changes don't repeat. + minFlux = 2.0 + minConfidence = 0.1 + // noiseFloor is about −68 dB below a full-scale sine's peak bin + // (window/4): under anything audible in a mix, over 8-bit noise. + noiseFloor = 0.1 +) + +// frameRate is how many analysis windows there are a second. +const frameRate = float64(SampleRate) / hop + +// Map is a song's beat map. +type Map struct { + Version int `json:"v"` + DurationMs int64 `json:"durationMs"` + // BPM is the tempo, 0 if the song has no steady beat (or no sound). + BPM float64 `json:"bpm"` + // Confidence is how clear the beat is, 0–1. + Confidence float64 `json:"confidence"` + // BeatsMs are the beats, in ms from the start. + BeatsMs []int64 `json:"beats"` + // Downbeat is the index in BeatsMs of the first bar's first beat; + // every fourth beat after it starts a bar. + Downbeat int `json:"downbeat"` + Sections []Section `json:"sections"` + // FrameRate is how many envelope frames there are a second. + FrameRate float64 `json:"frameRate"` + // BandsEnv is each frame's level in each band, low to high, 0–255: + // Bands bytes a frame. Each band is scaled to its own range in the song. + BandsEnv []byte `json:"bands"` + // Loudness is each frame's overall level, 0–255, scaled to the song. + Loudness []byte `json:"loudness"` + Features Features `json:"features"` +} + +// Section is a part of the song: an intro, a verse, a drop. +type Section struct { + StartMs int64 `json:"startMs"` + // Energy is how loud it is next to the rest of the song, 0–1. + Energy float64 `json:"energy"` +} + +// Features describe the whole song, for choosing how it looks. +type Features struct { + // Energy is how loud it's mastered, 0 (quiet acoustic) – 1 (a club track). + Energy float64 `json:"energy"` + // Brightness is how much treble there is, 0–1. + Brightness float64 `json:"brightness"` + // Dynamics is how much its loudness changes, 0–1. + Dynamics float64 `json:"dynamics"` +} + +// Analyze reads a song as mono 32-bit float samples at SampleRate and +// works out its beat map. +func Analyze(pcm io.Reader) (Map, error) { + a := newAnalyzer() + buf := make([]byte, 64<<10) + var carry int + for { + n, err := pcm.Read(buf[carry:]) + n += carry + whole := n - n%4 + a.write(buf[:whole]) + carry = copy(buf, buf[whole:n]) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return Map{}, err + } + } + return a.finish(), nil +} + +// analyzer takes samples as they're decoded and keeps a few numbers per window. +type analyzer struct { + fft *fft + hann []float64 + pending []float64 + samples int64 + + spec []complex128 + prevLog []float64 + bandOf []int + lowBins int + + bandDB [][Bands]float64 + flux []float64 + lowFlux []float64 + rmsDB []float64 + centroid []float64 +} + +func newAnalyzer() *analyzer { + a := &analyzer{ + fft: newFFT(window), + hann: make([]float64, window), + spec: make([]complex128, window), + prevLog: make([]float64, window/2+1), + bandOf: make([]int, window/2+1), + // Half a window of silence first, so window t is centred on t·hop. + pending: make([]float64, window/2, 8*window), + } + for i := range a.hann { + a.hann[i] = 0.5 - 0.5*math.Cos(2*math.Pi*float64(i)/float64(window)) + } + nyquist := float64(SampleRate) / 2 + for k := range a.bandOf { + hz := binHz(k) + a.bandOf[k] = -1 + if hz >= minHz { + b := int(float64(Bands) * math.Log(hz/minHz) / math.Log(nyquist/minHz)) + a.bandOf[k] = min(b, Bands-1) + } + if hz < lowHz { + a.lowBins = k + 1 + } + } + return a +} + +func binHz(k int) float64 { return float64(k) * SampleRate / window } + +func (a *analyzer) write(b []byte) { + for i := 0; i+4 <= len(b); i += 4 { + bits := uint32(b[i]) | uint32(b[i+1])<<8 | uint32(b[i+2])<<16 | uint32(b[i+3])<<24 + a.pending = append(a.pending, float64(math.Float32frombits(bits))) + } + a.samples += int64(len(b) / 4) + start := 0 + for len(a.pending)-start >= window { + a.frame(a.pending[start : start+window]) + start += hop + } + a.pending = append(a.pending[:0], a.pending[start:]...) +} + +func (a *analyzer) frame(x []float64) { + var sq float64 + for i, v := range x { + sq += v * v + a.spec[i] = complex(v*a.hann[i], 0) + } + a.fft.transform(a.spec) + + var bands [Bands]float64 + var counts [Bands]int + var flux, low, power, weighted float64 + for k := range a.prevLog { + re, im := real(a.spec[k]), imag(a.spec[k]) + p := re*re + im*im + m := math.Sqrt(p) + // Log-compressed magnitude, so flux hears quiet notes too, but + // nothing under the noise floor: hiss and quantisation noise + // change every window, and would look like a beat. + lm := math.Log1p(10 * math.Max(0, m-noiseFloor)) + if k > 0 { + if d := lm - a.prevLog[k]; d > 0 { + flux += d + if k < a.lowBins { + low += d + } + } + } + a.prevLog[k] = lm + if b := a.bandOf[k]; b >= 0 { + bands[b] += p + counts[b]++ + } + power += p + weighted += p * binHz(k) + } + var db [Bands]float64 + for b := range bands { + db[b] = 10 * math.Log10(bands[b]/float64(max(counts[b], 1))+1e-12) + } + a.bandDB = append(a.bandDB, db) + a.flux = append(a.flux, flux) + a.lowFlux = append(a.lowFlux, low) + a.rmsDB = append(a.rmsDB, 20*math.Log10(math.Sqrt(sq/window)+1e-9)) + c := 0.0 + if power > 1e-9 { + c = weighted / power + } + a.centroid = append(a.centroid, c) +} + +func (a *analyzer) finish() Map { + m := Map{ + Version: Version, + DurationMs: a.samples * 1000 / SampleRate, + BeatsMs: []int64{}, + Sections: []Section{}, + FrameRate: frameRate / envStep, + } + n := len(a.flux) + if n == 0 { + return m + } + loud := percentile(a.rmsDB, 0.9) + if loud < -60 { + // Silence: nothing to follow. + return m + } + + period, conf := tempo(a.flux) + var beats []float64 + if steady(a.flux, conf) { + beats = trimQuiet(trackBeats(onsetStrength(a.flux), period), a.rmsDB, loud-30) + } + if len(beats) >= 8 { + m.BeatsMs = make([]int64, len(beats)) + for i, t := range beats { + m.BeatsMs[i] = frameMs(t) + } + m.BPM = bpmOf(m.BeatsMs) + m.Confidence = conf + m.Downbeat = downbeat(beats, a.lowFlux, a.rmsDB) + m.Sections = sections(beats, m.Downbeat, a.bandDB, a.rmsDB) + } + if len(m.Sections) == 0 { + m.Sections = []Section{{StartMs: 0, Energy: 0.5}} + } + + m.BandsEnv, m.Loudness = envelopes(a.bandDB, a.rmsDB) + m.Features = features(a.rmsDB, a.centroid, loud) + return m +} + +// steady reports whether the song has a beat to follow: something starts +// often enough (a held tone or a drone barely changes), and the starts +// repeat. +func steady(flux []float64, confidence float64) bool { + var sum float64 + for _, f := range flux { + sum += f + } + return sum/float64(len(flux)) >= minFlux && confidence >= minConfidence +} + +// frameMs is the time of window t, which is centred on t·hop. +func frameMs(t float64) int64 { return int64(math.Round(t * hop * 1000 / SampleRate)) } + +// bpmOf is the tempo the beats keep: the slope of a line through them, +// which a few stray beats barely move, and which isn't limited to the +// analysis windows' resolution as single gaps are. +func bpmOf(beats []int64) float64 { + n := float64(len(beats)) + mi := (n - 1) / 2 + var mt float64 + for _, b := range beats { + mt += float64(b) + } + mt /= n + var num, den float64 + for i, b := range beats { + num += (float64(i) - mi) * (float64(b) - mt) + den += (float64(i) - mi) * (float64(i) - mi) + } + if den == 0 || num <= 0 { + return 0 + } + return math.Round(60000/(num/den)*10) / 10 +} + +// trimQuiet drops beats in the silence before a song starts and after it ends. +func trimQuiet(beats []float64, rmsDB []float64, floor float64) []float64 { + quiet := func(t float64) bool { + i := min(int(math.Round(t)), len(rmsDB)-1) + return rmsDB[i] < floor + } + for len(beats) > 0 && quiet(beats[0]) { + beats = beats[1:] + } + for len(beats) > 0 && quiet(beats[len(beats)-1]) { + beats = beats[:len(beats)-1] + } + return beats +} + +// percentile is the value p (0–1) of the way up xs, sorted. xs is unchanged. +func percentile(xs []float64, p float64) float64 { + if len(xs) == 0 { + return 0 + } + s := slices.Clone(xs) + slices.Sort(s) + return s[min(int(p*float64(len(s)-1)+0.5), len(s)-1)] +} + +func clamp01(x float64) float64 { return math.Max(0, math.Min(1, x)) } diff --git a/server/internal/analysis/analysis_test.go b/server/internal/analysis/analysis_test.go new file mode 100644 index 0000000..4cff440 --- /dev/null +++ b/server/internal/analysis/analysis_test.go @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package analysis + +import ( + "bytes" + "encoding/binary" + "io" + "math" + "math/rand/v2" + "testing" +) + +// song synthesises a drum pattern: a kick on 1 (heavier) and 3, a snare on 2 and 4, +// hi-hats on the eighths, starting lead seconds in. From bar loudFrom on, +// a bass line and louder drums come in (a "drop"). Returns the PCM and the +// true beat times in ms. +func song(bpm float64, bars, loudFrom int, lead float64) ([]byte, []float64) { + period := 60 / bpm + total := lead + float64(bars*4)*period + 1 + pcm := make([]float32, int(total*SampleRate)) + rng := rand.New(rand.NewPCG(1, 2)) //nolint:gosec // repeatable noise for a test signal + add := func(at float64, dur float64, f func(t float64) float64) { + i0 := int(at * SampleRate) + for i := 0; i < int(dur*SampleRate) && i0+i < len(pcm); i++ { + pcm[i0+i] += float32(f(float64(i) / SampleRate)) + } + } + kick := func(gain float64) func(float64) float64 { + return func(t float64) float64 { + f := 50 + 100*math.Exp(-t*30) + return gain * math.Sin(2*math.Pi*f*t) * math.Exp(-t*12) + } + } + noise := func(gain, decay float64) func(float64) float64 { + return func(t float64) float64 { return gain * (rng.Float64()*2 - 1) * math.Exp(-t*decay) } + } + var beats []float64 + for b := range bars * 4 { + at := lead + float64(b)*period + beats = append(beats, at*1000) + loud := b/4 >= loudFrom + g := 0.35 + if loud { + g = 0.6 + } + switch b % 4 { + case 0: + // Like most songs, the one is a little heavier than the three. + add(at, 0.25, kick(g*1.6)) + case 2: + add(at, 0.25, kick(g*1.3)) + case 1, 3: + add(at, 0.15, noise(g*0.6, 25)) + } + add(at, 0.05, noise(g*0.15, 80)) + add(at+period/2, 0.05, noise(g*0.12, 80)) + if loud { + add(at, period*0.9, func(t float64) float64 { return 0.25 * math.Sin(2*math.Pi*55*t) }) + } + } + return f32le(pcm), beats +} + +func f32le(pcm []float32) []byte { + var buf bytes.Buffer + _ = binary.Write(&buf, binary.LittleEndian, pcm) + return buf.Bytes() +} + +func analyze(t *testing.T, pcm []byte) Map { + t.Helper() + m, err := Analyze(bytes.NewReader(pcm)) + if err != nil { + t.Fatal(err) + } + return m +} + +func TestTempoAndBeats(t *testing.T) { + for _, bpm := range []float64{84, 96, 110, 120, 128, 140, 150, 160} { + pcm, truth := song(bpm, 32, 99, 0.37) + m := analyze(t, pcm) + if math.Abs(m.BPM-bpm) > 1 { + t.Errorf("%v BPM: found %v", bpm, m.BPM) + continue + } + // Nearly every beat within 25 ms of a true one. + hits := 0 + for _, b := range m.BeatsMs { + for _, tr := range truth { + if math.Abs(float64(b)-tr) <= 25 { + hits++ + break + } + } + } + if len(m.BeatsMs) < len(truth)*9/10 || hits < len(m.BeatsMs)*95/100 { + t.Errorf("%v BPM: %d beats found, %d of them on time, of %d", bpm, len(m.BeatsMs), hits, len(truth)) + } + // The first downbeat lands on a true "one". + if len(m.BeatsMs) > m.Downbeat { + d := float64(m.BeatsMs[m.Downbeat]) + n := int(math.Round((d - truth[0]) / (60000 / bpm))) + if n%4 != 0 { + t.Errorf("%v BPM: downbeat on beat %d of the bar", bpm, n%4+1) + } + } + } +} + +func TestBeatOffset(t *testing.T) { + // Beats aren't consistently early or late. + pcm, truth := song(120, 32, 99, 0.5) + m := analyze(t, pcm) + var sum float64 + var n int + for _, b := range m.BeatsMs { + for _, tr := range truth { + if d := float64(b) - tr; math.Abs(d) <= 40 { + sum += d + n++ + break + } + } + } + if n == 0 || math.Abs(sum/float64(n)) > 8 { + t.Errorf("mean offset %.1f ms over %d beats", sum/float64(max(n, 1)), n) + } +} + +func TestSections(t *testing.T) { + const bpm = 124.0 + pcm, truth := song(bpm, 48, 24, 0.2) + m := analyze(t, pcm) + if len(m.Sections) < 2 { + t.Fatalf("sections: %+v", m.Sections) + } + drop := truth[24*4] + found := false + for i, s := range m.Sections[1:] { + if math.Abs(float64(s.StartMs)-drop) < 60000/bpm*4+50 { + found = true + if s.Energy <= m.Sections[i].Energy { + t.Errorf("the drop is no louder: %+v", m.Sections) + } + } + } + if !found { + t.Errorf("no section starts at the drop (%.0f ms): %+v", drop, m.Sections) + } + if m.Sections[0].StartMs != 0 { + t.Errorf("first section starts at %d", m.Sections[0].StartMs) + } +} + +func TestEnvelopes(t *testing.T) { + pcm, _ := song(120, 16, 8, 0) + m := analyze(t, pcm) + frames := len(m.Loudness) + want := float64(m.DurationMs) / 1000 * m.FrameRate + if math.Abs(float64(frames)-want) > 3 { + t.Errorf("%d frames for %d ms at %.2f/s", frames, m.DurationMs, m.FrameRate) + } + if len(m.BandsEnv) != frames*Bands { + t.Errorf("%d band bytes for %d frames", len(m.BandsEnv), frames) + } + // The bass comes in with the drop: the lowest bands get louder. + mean := func(band, from, to int) float64 { + var s float64 + for f := from; f < to; f++ { + s += float64(m.BandsEnv[f*Bands+band]) + } + return s / float64(to-from) + } + half := frames / 2 + if before, after := mean(1, 0, half-20), mean(1, half+20, frames-20); after <= before+20 { + t.Errorf("band 1 before the bass %.0f, after %.0f", before, after) + } + if m.Features.Energy <= 0 || m.Features.Energy > 1 || m.Features.Brightness < 0 || m.Features.Brightness > 1 { + t.Errorf("features out of range: %+v", m.Features) + } +} + +func TestNoBeat(t *testing.T) { + // A steady tone, like the dev library's songs, has no beat to follow. + pcm := make([]float32, 20*SampleRate) + for i := range pcm { + pcm[i] = float32(0.2 * math.Sin(2*math.Pi*440*float64(i)/SampleRate)) + } + m := analyze(t, f32le(pcm)) + if m.BPM != 0 || len(m.BeatsMs) != 0 { + t.Errorf("a steady tone: %v BPM, %d beats", m.BPM, len(m.BeatsMs)) + } + if len(m.Loudness) == 0 || len(m.Sections) != 1 { + t.Errorf("a steady tone still has envelopes and one section: %d frames, %+v", len(m.Loudness), m.Sections) + } +} + +func TestSilence(t *testing.T) { + m := analyze(t, make([]byte, 4*5*SampleRate)) + if m.BPM != 0 || len(m.BeatsMs) != 0 { + t.Errorf("silence: %+v", m) + } + if m.DurationMs != 5000 { + t.Errorf("duration %d", m.DurationMs) + } +} + +func TestOddReads(t *testing.T) { + // Reads that split a sample in two still decode. + pcm, _ := song(120, 8, 99, 0) + whole := analyze(t, pcm) + m, err := Analyze(&trickle{b: pcm, n: 7}) + if err != nil { + t.Fatal(err) + } + if m.BPM != whole.BPM || len(m.BeatsMs) != len(whole.BeatsMs) { + t.Errorf("trickled %v BPM / %d beats, whole %v / %d", m.BPM, len(m.BeatsMs), whole.BPM, len(whole.BeatsMs)) + } +} + +type trickle struct { + b []byte + n int +} + +func (r *trickle) Read(p []byte) (int, error) { + if len(r.b) == 0 { + return 0, io.EOF + } + n := copy(p[:min(len(p), r.n)], r.b) + r.b = r.b[n:] + return n, nil +} diff --git a/server/internal/analysis/fft.go b/server/internal/analysis/fft.go new file mode 100644 index 0000000..f0e0ca0 --- /dev/null +++ b/server/internal/analysis/fft.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package analysis + +import ( + "math" + "math/cmplx" +) + +// fft is an in-place radix-2 FFT for one window size, with its twiddles +// and bit-reversal table worked out once. +type fft struct { + n int + rev []int + twiddle []complex128 +} + +func newFFT(n int) *fft { + if n&(n-1) != 0 { + panic("analysis: FFT size must be a power of two") + } + bits := 0 + for 1<= n || b >= n { + continue + } + // Gaussian taper; + within a side, − across the boundary. + g := math.Exp(-0.5 * (float64(i*i+j*j) / (kernelBars * kernelBars / 4))) + sign := 1.0 + if (i < 0) != (j < 0) { + sign = -1 + } + s += sign * g * sim(a, b) + w += g + } + } + novelty[c] = s / w + } + threshold := percentile(novelty, 0.5) + 0.5*spread(novelty) + + bounds := []int{0} + for c := 1; c < n; c++ { + if novelty[c] < threshold || c-bounds[len(bounds)-1] < minSectionBars || n-c < minSectionBars { + continue + } + peak := true + for d := -minSectionBars / 2; d <= minSectionBars/2; d++ { + if j := c + d; j != c && j > 0 && j < n && novelty[j] > novelty[c] { + peak = false + } + } + if peak { + bounds = append(bounds, c) + } + } + + lo, hi := percentile(loud, 0.05), percentile(loud, 0.95) + out := make([]Section, len(bounds)) + for i, b := range bounds { + end := n + if i+1 < len(bounds) { + end = bounds[i+1] + } + var l float64 + for j := b; j < end; j++ { + l += loud[j] + } + l /= float64(end - b) + e := 0.5 + if hi-lo > 0.5 { + e = clamp01((l - lo) / (hi - lo)) + } + start := frameMs(starts[b]) + if i == 0 { + start = 0 + } + out[i] = Section{StartMs: start, Energy: math.Round(e*100) / 100} + } + return out +} + +func spread(xs []float64) float64 { + var mean, sq float64 + for _, x := range xs { + mean += x + } + mean /= float64(len(xs)) + for _, x := range xs { + sq += (x - mean) * (x - mean) + } + return math.Sqrt(sq / float64(len(xs))) +} + +// envelopes packs the band levels and loudness into bytes, envStep windows +// to a frame (keeping the loudest, so hits aren't averaged away). Each band +// is scaled to its own loud level in the song, so the treble moves as +// visibly as the bass. +func envelopes(bandDB [][Bands]float64, rmsDB []float64) (bands, loudness []byte) { + frames := (len(rmsDB) + envStep - 1) / envStep + bands = make([]byte, frames*Bands) + loudness = make([]byte, frames) + var top [Bands]float64 + col := make([]float64, len(bandDB)) + for k := range Bands { + for t := range bandDB { + col[t] = bandDB[t][k] + } + top[k] = percentile(col, 0.98) + } + loudTop := percentile(rmsDB, 0.98) + level := func(db, top float64) byte { + return byte(math.Round(255 * clamp01((db-(top-envRangeDB))/envRangeDB))) + } + for f := range frames { + for t := f * envStep; t < min((f+1)*envStep, len(rmsDB)); t++ { + for k := range Bands { + bands[f*Bands+k] = max(bands[f*Bands+k], level(bandDB[t][k], top[k])) + } + loudness[f] = max(loudness[f], level(rmsDB[t], loudTop)) + } + } + return bands, loudness +} + +// features describe the song as a whole. loud is its 90th percentile level. +func features(rmsDB, centroid []float64, loud float64) Features { + var c []float64 + for t, v := range centroid { + if rmsDB[t] > loud-30 { + c = append(c, v) + } + } + bright := 0.0 + if len(c) > 0 { + // 500 Hz is dull, 4 kHz bright. + bright = clamp01(math.Log2(math.Max(percentile(c, 0.5), 1)/500) / 3) + } + quiet := percentile(rmsDB, 0.1) + return Features{ + // RMS around −30 dBFS is a quiet recording, −8 a loud master. + Energy: round2(clamp01((percentile(rmsDB, 0.5) + 30) / 22)), + Brightness: round2(bright), + Dynamics: round2(clamp01((loud - math.Max(quiet, loud-40)) / 30)), + } +} + +func round2(x float64) float64 { return math.Round(x*100) / 100 } diff --git a/server/internal/analysis/service.go b/server/internal/analysis/service.go new file mode 100644 index 0000000..3e15f23 --- /dev/null +++ b/server/internal/analysis/service.go @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package analysis + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "os" + "os/exec" + "strings" + "sync" + "time" + + "golang.org/x/sync/singleflight" + + "github.com/madeofpendletonwool/syncphony/server/internal/provider" + "github.com/madeofpendletonwool/syncphony/server/internal/store" +) + +const ( + // analyzeTimeout bounds downloading, decoding and analysing one song. + analyzeTimeout = 3 * time.Minute + // maxDuration skips DJ mixes and audiobooks: hours of beats nobody needs. + maxDuration = 30 * time.Minute + queueSize = 256 + // retryMisses is how long a song that couldn't be analysed is left alone. + retryMisses = 7 * 24 * time.Hour + // keepUnused is how long a map nobody plays is kept. + keepUnused = 180 * 24 * time.Hour +) + +// ErrUnavailable means a song has no beat map and won't get one: its +// service has no audio to analyse, or it can't be decoded. +var ErrUnavailable = errors.New("analysis: no beat map for this song") + +// Sessions opens a service link, for its audio. +type Sessions interface { + Open(ctx context.Context, linkID string) (provider.Session, error) +} + +// Decoder turns a song's file into mono 32-bit float samples at SampleRate. +type Decoder interface { + Decode(ctx context.Context, path string) (io.ReadCloser, error) +} + +// Service works out songs' beat maps and keeps them. Call Run to analyse +// songs in the background as they're queued, so the map is ready when the +// song starts. +type Service struct { + db *store.Store + sessions Sessions + decoder Decoder + + jobs chan source + mu sync.Mutex + pending map[string]bool + group singleflight.Group +} + +// source is where a song's audio comes from. +type source struct { + provider, linkID, trackID string + duration time.Duration +} + +func (s source) key() string { return s.provider + "\x00" + s.trackID } + +// New returns a Service. +func New(db *store.Store, sessions Sessions, decoder Decoder) *Service { + return &Service{db: db, sessions: sessions, decoder: decoder, jobs: make(chan source, queueSize), pending: map[string]bool{}} +} + +// Enqueue asks for newly queued songs to be analysed. +func (s *Service) Enqueue(ts ...provider.Track) { + s.mu.Lock() + defer s.mu.Unlock() + for _, t := range ts { + src := source{provider: t.Ref.Provider, linkID: t.Ref.LinkID, trackID: t.Ref.ID, duration: t.Duration} + k := src.key() + if src.trackID == "" || src.linkID == "" || s.pending[k] { + continue + } + select { + case s.jobs <- src: + s.pending[k] = true + default: + slog.Debug("analysis: queue full, dropping a track", "provider", src.provider, "track", src.trackID) + } + } +} + +// Run analyses queued songs, one at a time, until ctx is done. +func (s *Service) Run(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case src := <-s.jobs: + if _, err := s.get(ctx, src); err != nil && !errors.Is(err, ErrUnavailable) && ctx.Err() == nil { + slog.Debug("analysis", "provider", src.provider, "track", src.trackID, "err", err) + } + s.mu.Lock() + delete(s.pending, src.key()) + s.mu.Unlock() + } + } +} + +// ForItem returns the beat map of the song a queue item plays (from the +// service standing in for its own, if one is), analysing it now if it +// hasn't been. It's ErrUnavailable if the song can't have one. +func (s *Service) ForItem(ctx context.Context, it store.QueueItem, duration time.Duration) (Map, error) { + src := source{provider: it.Provider, linkID: it.LinkID.String, trackID: it.TrackID, duration: duration} + if it.ViaLinkID.Valid { + src = source{provider: it.ViaProvider.String, linkID: it.ViaLinkID.String, trackID: it.ViaTrackID.String, duration: duration} + } + return s.get(ctx, src) +} + +// Sweep forgets maps nobody has played in a long while. +func (s *Service) Sweep(ctx context.Context) error { + now := store.Now() + _, err := s.db.SweepBeatMaps(ctx, store.SweepBeatMapsParams{UnusedSince: now.Add(-keepUnused), MissedBefore: now.Add(-retryMisses)}) + return err +} + +// get returns src's map from the cache, or analyses it. +func (s *Service) get(ctx context.Context, src source) (Map, error) { + row, err := s.db.GetBeatMap(ctx, store.GetBeatMapParams{Provider: src.provider, TrackID: src.trackID}) + switch { + case err == nil && row.Version == Version: + now := store.Now() + _ = s.db.TouchBeatMap(ctx, store.TouchBeatMapParams{Now: now, Provider: src.provider, TrackID: src.trackID, Since: now.Add(-24 * time.Hour)}) + if !row.Found { + if now.Sub(row.AnalyzedAt) < retryMisses { + return Map{}, ErrUnavailable + } + break + } + var m Map + if err := json.Unmarshal([]byte(row.Map), &m); err == nil { + return m, nil + } + case err != nil && !store.IsNotFound(err): + return Map{}, err + } + if src.linkID == "" { + return Map{}, ErrUnavailable + } + v, err, _ := s.group.Do(src.key(), func() (any, error) { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), analyzeTimeout) + defer cancel() + start := time.Now() + m, err := s.analyze(ctx, src) + if errors.Is(err, ErrUnavailable) { + return Map{}, s.save(ctx, src, nil) + } + if err != nil { + return Map{}, err + } + slog.Debug("analysed a song", "provider", src.provider, "track", src.trackID, "bpm", m.BPM, "took", time.Since(start).Round(time.Millisecond)) + return m, s.save(ctx, src, &m) + }) + if err != nil { + return Map{}, err + } + m := v.(Map) + if m.Version == 0 { + return Map{}, ErrUnavailable + } + return m, nil +} + +// save keeps src's map, or that it has none. +func (s *Service) save(ctx context.Context, src source, m *Map) error { + body := "{}" + if m != nil { + b, err := json.Marshal(m) + if err != nil { + return err + } + body = string(b) + } + now := store.Now() + if err := s.db.PutBeatMap(ctx, store.PutBeatMapParams{ + Provider: src.provider, TrackID: src.trackID, Version: Version, Found: m != nil, Map: body, AnalyzedAt: now, UsedAt: now, + }); err != nil { + return err + } + if m == nil { + return ErrUnavailable + } + return nil +} + +// analyze downloads src's audio to a temporary file (decoders can't read +// some files front to back, an MP4 with its index at the end), decodes +// and analyses it. +func (s *Service) analyze(ctx context.Context, src source) (Map, error) { + if s.decoder == nil || src.duration > maxDuration { + return Map{}, ErrUnavailable + } + sess, err := s.sessions.Open(ctx, src.linkID) + if err != nil { + return Map{}, err + } + defer sess.Close() + st, ok := sess.(provider.Streamer) + if !ok { + return Map{}, ErrUnavailable + } + a, err := st.Stream(ctx, src.trackID, provider.StreamOpts{}) + if errors.Is(err, provider.ErrNotFound) || errors.Is(err, provider.ErrNotPlayable) { + return Map{}, ErrUnavailable + } else if err != nil { + return Map{}, err + } + f, err := os.CreateTemp("", "syncphony-analysis-*") + if err != nil { + a.Body.Close() + return Map{}, err + } + defer os.Remove(f.Name()) + _, err = io.Copy(f, a.Body) + a.Body.Close() + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + return Map{}, fmt.Errorf("analysis: downloading: %w", err) + } + pcm, err := s.decoder.Decode(ctx, f.Name()) + if err != nil { + return Map{}, err + } + defer pcm.Close() + m, err := Analyze(pcm) + if err != nil { + if ctx.Err() != nil { + return Map{}, ctx.Err() + } + // ffmpeg couldn't read it: it never will. + slog.Debug("analysis: decoding", "provider", src.provider, "track", src.trackID, "err", err) + return Map{}, ErrUnavailable + } + return m, nil +} + +// FFmpeg decodes with an ffmpeg binary. +type FFmpeg struct { + // Path is the ffmpeg binary. Default "ffmpeg" on $PATH. + Path string +} + +// Decode implements Decoder. ffmpeg is stopped when the reader is closed +// or ctx is done; reading to the end reports how it exited. +func (f FFmpeg) Decode(ctx context.Context, path string) (io.ReadCloser, error) { + bin := f.Path + if bin == "" { + bin = "ffmpeg" + } + ctx, cancel := context.WithCancel(ctx) + // The binary comes from server config, the path from os.CreateTemp. + cmd := exec.CommandContext(ctx, bin, //nolint:gosec // see above + "-hide_banner", "-loglevel", "error", "-nostdin", "-i", path, + "-vn", "-ac", "1", "-ar", fmt.Sprint(SampleRate), "-f", "f32le", "pipe:1") + var stderr strings.Builder + cmd.Stderr = &limited{w: &stderr, n: 2 << 10} + out, err := cmd.StdoutPipe() + if err != nil { + cancel() + return nil, err + } + if err := cmd.Start(); err != nil { + cancel() + return nil, fmt.Errorf("analysis: starting ffmpeg: %w", err) + } + return &decoding{out: out, cmd: cmd, cancel: cancel, stderr: &stderr}, nil +} + +type decoding struct { + out io.ReadCloser + cmd *exec.Cmd + cancel context.CancelFunc + stderr *strings.Builder + once sync.Once + err error +} + +func (d *decoding) Read(p []byte) (int, error) { + n, err := d.out.Read(p) + if errors.Is(err, io.EOF) { + if werr := d.wait(); werr != nil { + return n, werr + } + } + return n, err +} + +func (d *decoding) Close() error { + d.cancel() + _ = d.wait() + return nil +} + +func (d *decoding) wait() error { + d.once.Do(func() { + if err := d.cmd.Wait(); err != nil { + d.err = fmt.Errorf("analysis: ffmpeg: %w: %s", err, strings.TrimSpace(d.stderr.String())) + } + d.cancel() + }) + return d.err +} + +// limited writes the first n bytes to w and drops the rest. +type limited struct { + w io.Writer + n int +} + +func (l *limited) Write(p []byte) (int, error) { + if l.n > 0 { + k := min(len(p), l.n) + _, _ = l.w.Write(p[:k]) + l.n -= k + } + return len(p), nil +} diff --git a/server/internal/analysis/service_test.go b/server/internal/analysis/service_test.go new file mode 100644 index 0000000..c5d6173 --- /dev/null +++ b/server/internal/analysis/service_test.go @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package analysis + +import ( + "bytes" + "context" + "database/sql" + "encoding/binary" + "errors" + "io" + "math" + "os" + "os/exec" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/madeofpendletonwool/syncphony/server/internal/provider" + "github.com/madeofpendletonwool/syncphony/server/internal/store" +) + +// links opens sessions that stream a 120 BPM drum pattern, or, for the +// link "nostream", sessions with no audio at all. +type links struct { + mu sync.Mutex + opened []string + streams []string +} + +func (l *links) Open(_ context.Context, linkID string) (provider.Session, error) { + l.mu.Lock() + defer l.mu.Unlock() + l.opened = append(l.opened, linkID) + if linkID == "nostream" { + return session{}, nil + } + return streamer{session{}, l}, nil +} + +type session struct{} + +func (session) Search(context.Context, provider.SearchQuery) (provider.SearchPage, error) { + return provider.SearchPage{}, nil +} + +func (session) Track(context.Context, string) (provider.Track, error) { return provider.Track{}, nil } + +func (session) Album(context.Context, string) (provider.Album, []provider.Track, error) { + return provider.Album{}, nil, nil +} + +func (session) Artist(context.Context, string) (provider.Artist, []provider.Album, error) { + return provider.Artist{}, nil, nil +} + +func (session) Artwork(context.Context, provider.ArtworkRef, int) (io.ReadCloser, string, error) { + return nil, "", provider.ErrNotFound +} +func (session) Close() error { return nil } + +type streamer struct { + session + l *links +} + +func (s streamer) Stream(_ context.Context, trackID string, _ provider.StreamOpts) (*provider.AudioStream, error) { + s.l.mu.Lock() + s.l.streams = append(s.l.streams, trackID) + s.l.mu.Unlock() + if trackID == "gone" { + return nil, provider.ErrNotFound + } + pcm, _ := song(120, 24, 99, 0.25) + return &provider.AudioStream{Body: io.NopCloser(bytes.NewReader(pcm)), ContentType: "audio/x-f32le", Length: -1, Size: -1}, nil +} + +// raw "decodes" files that already hold the samples. +type raw struct{} + +func (raw) Decode(_ context.Context, path string) (io.ReadCloser, error) { + return os.Open(path) //nolint:gosec // the service's own temporary file +} + +func newService(t *testing.T, dec Decoder) (*Service, *links, *store.Store) { + t.Helper() + db, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + l := &links{} + return New(db, l, dec), l, db +} + +func item(link, track string) store.QueueItem { + return store.QueueItem{ID: store.NewID(), Provider: "fake", LinkID: sql.NullString{String: link, Valid: true}, TrackID: track} +} + +func TestForItemAnalysesOnce(t *testing.T) { + svc, l, db := newService(t, raw{}) + m, err := svc.ForItem(t.Context(), item("link1", "t1"), 3*time.Minute) + if err != nil { + t.Fatal(err) + } + if math.Abs(m.BPM-120) > 1 || len(m.BeatsMs) < 80 || m.Version != Version { + t.Fatalf("map: %v BPM, %d beats, v%d", m.BPM, len(m.BeatsMs), m.Version) + } + // Again, from another item and a fresh service: it's kept. + again, err := New(db, l, raw{}).ForItem(t.Context(), item("link1", "t1"), 3*time.Minute) + if err != nil { + t.Fatal(err) + } + if len(l.streams) != 1 || again.BPM != m.BPM || len(again.BandsEnv) != len(m.BandsEnv) { + t.Errorf("streamed %v; kept map %v BPM, %d band bytes", l.streams, again.BPM, len(again.BandsEnv)) + } +} + +func TestForItemUnavailable(t *testing.T) { + svc, l, _ := newService(t, raw{}) + for _, it := range []store.QueueItem{item("nostream", "t1"), item("link1", "gone")} { + for range 2 { + if _, err := svc.ForItem(t.Context(), it, time.Minute); !errors.Is(err, ErrUnavailable) { + t.Errorf("%s/%s: %v", it.LinkID.String, it.TrackID, err) + } + } + } + // Misses are kept: each song was tried once. + if len(l.opened) != 2 { + t.Errorf("opened %v", l.opened) + } + // A DJ mix is too long to bother with, and no link means no audio. + if _, err := svc.ForItem(t.Context(), item("link1", "mix"), 2*time.Hour); !errors.Is(err, ErrUnavailable) { + t.Errorf("a two-hour mix: %v", err) + } + if _, err := svc.ForItem(t.Context(), store.QueueItem{Provider: "fake", TrackID: "t9"}, time.Minute); !errors.Is(err, ErrUnavailable) { + t.Errorf("no link: %v", err) + } +} + +func TestForItemVia(t *testing.T) { + svc, l, _ := newService(t, raw{}) + it := item("nostream", "t1") + it.ViaProvider = sql.NullString{String: "other", Valid: true} + it.ViaLinkID = sql.NullString{String: "link2", Valid: true} + it.ViaTrackID = sql.NullString{String: "t2", Valid: true} + if _, err := svc.ForItem(t.Context(), it, time.Minute); err != nil { + t.Fatal(err) + } + if len(l.opened) != 1 || l.opened[0] != "link2" || l.streams[0] != "t2" { + t.Errorf("opened %v, streamed %v", l.opened, l.streams) + } +} + +func TestOldVersionIsRedone(t *testing.T) { + svc, l, db := newService(t, raw{}) + if err := db.PutBeatMap(t.Context(), store.PutBeatMapParams{ + Provider: "fake", TrackID: "t1", Version: Version - 1, Found: true, Map: `{"v":0,"bpm":99}`, AnalyzedAt: store.Now(), UsedAt: store.Now(), + }); err != nil { + t.Fatal(err) + } + m, err := svc.ForItem(t.Context(), item("link1", "t1"), time.Minute) + if err != nil || math.Abs(m.BPM-120) > 1 || len(l.streams) != 1 { + t.Errorf("%v BPM, %v, streamed %v", m.BPM, err, l.streams) + } +} + +func TestEnqueueAnalysesInBackground(t *testing.T) { + svc, _, db := newService(t, raw{}) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + go svc.Run(ctx) + svc.Enqueue(provider.Track{Ref: provider.TrackRef{Provider: "fake", LinkID: "link1", ID: "t1"}, Duration: time.Minute}) + deadline := time.Now().Add(10 * time.Second) + for { + row, err := db.GetBeatMap(t.Context(), store.GetBeatMapParams{Provider: "fake", TrackID: "t1"}) + if err == nil && row.Found { + return + } + if time.Now().After(deadline) { + t.Fatalf("not analysed: %v", err) + } + time.Sleep(20 * time.Millisecond) + } +} + +func TestSweep(t *testing.T) { + svc, _, db := newService(t, raw{}) + old := store.Now().Add(-200 * 24 * time.Hour) + for _, r := range []store.PutBeatMapParams{ + {TrackID: "unused", Found: true, AnalyzedAt: old, UsedAt: old}, + {TrackID: "used", Found: true, AnalyzedAt: old, UsedAt: store.Now()}, + {TrackID: "oldmiss", Found: false, AnalyzedAt: store.Now().Add(-8 * 24 * time.Hour), UsedAt: store.Now()}, + {TrackID: "newmiss", Found: false, AnalyzedAt: store.Now(), UsedAt: store.Now()}, + } { + r.Provider, r.Version, r.Map = "fake", Version, "{}" + if err := db.PutBeatMap(t.Context(), r); err != nil { + t.Fatal(err) + } + } + if err := svc.Sweep(t.Context()); err != nil { + t.Fatal(err) + } + for track, want := range map[string]bool{"unused": false, "used": true, "oldmiss": false, "newmiss": true} { + _, err := db.GetBeatMap(t.Context(), store.GetBeatMapParams{Provider: "fake", TrackID: track}) + if got := err == nil; got != want { + t.Errorf("%s kept: %v", track, got) + } + } +} + +func TestFFmpegDecode(t *testing.T) { + if _, err := exec.LookPath("ffmpeg"); err != nil { + t.Skip("ffmpeg not installed") + } + // A 44.1 kHz stereo WAV, as a provider might send. + pcm, _ := song(128, 16, 99, 0.3) + var mono []float32 + for i := 0; i+4 <= len(pcm); i += 4 { + mono = append(mono, math.Float32frombits(binary.LittleEndian.Uint32(pcm[i:]))) + } + path := filepath.Join(t.TempDir(), "song.wav") + if err := os.WriteFile(path, wav44(mono), 0o600); err != nil { + t.Fatal(err) + } + r, err := FFmpeg{}.Decode(t.Context(), path) + if err != nil { + t.Fatal(err) + } + defer r.Close() + m, err := Analyze(r) + if err != nil { + t.Fatal(err) + } + if math.Abs(m.BPM-128) > 1 { + t.Errorf("decoded with ffmpeg: %v BPM", m.BPM) + } + // A file ffmpeg can't read fails when read. + bad := filepath.Join(t.TempDir(), "bad.mp3") + _ = os.WriteFile(bad, []byte("not audio"), 0o600) + r, err = FFmpeg{}.Decode(t.Context(), bad) + if err == nil { + _, err = Analyze(r) + r.Close() + } + if err == nil { + t.Error("decoding garbage didn't fail") + } +} + +// wav44 resamples 22.05 kHz mono to 44.1 kHz stereo 16-bit PCM in a WAV. +func wav44(mono []float32) []byte { + var data bytes.Buffer + for _, v := range mono { + s := int16(max(-1, min(1, v)) * 32767) + for range 4 { // two samples × two channels + _ = binary.Write(&data, binary.LittleEndian, s) + } + } + var b bytes.Buffer + w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) } + b.WriteString("RIFF") + w(uint32(36 + data.Len())) //nolint:gosec // a test file, far under 4 GB + b.WriteString("WAVEfmt ") + w(uint32(16)) + w(uint16(1)) + w(uint16(2)) + w(uint32(44100)) + w(uint32(44100 * 4)) + w(uint16(4)) + w(uint16(16)) + b.WriteString("data") + w(uint32(data.Len())) //nolint:gosec // as above + b.Write(data.Bytes()) + return b.Bytes() +} diff --git a/server/internal/analysis/tempo.go b/server/internal/analysis/tempo.go new file mode 100644 index 0000000..368bfa8 --- /dev/null +++ b/server/internal/analysis/tempo.go @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package analysis + +import "math" + +const ( + minBPM = 55.0 + maxBPM = 215.0 + // The tempo prior: songs cluster around priorBPM. Tuned so 84–160 BPM + // drum patterns come out right; outside that, a song can read at half or + // double its tempo (a 72 BPM ballad with busy hi-hats as 144), the + // ambiguity every beat tracker has. + priorBPM = 128.0 + priorOctave = 0.6 + // tightness is how much the beat tracker resists changing tempo. + tightness = 100.0 +) + +// onsetStrength is the flux with its slow trend taken out, scaled to unit +// spread: how sharply something starts in each window. +func onsetStrength(flux []float64) []float64 { + const span = 16 // about 370 ms + o := make([]float64, len(flux)) + var sum float64 + for i, v := range flux { + sum += v + if i >= span { + sum -= flux[i-span] + } + mean := sum / float64(min(i+1, span)) + o[i] = math.Max(0, v-mean) + } + var sq float64 + for _, v := range o { + sq += v * v + } + if sd := math.Sqrt(sq / float64(len(o))); sd > 0 { + for i := range o { + o[i] /= sd + } + } + return o +} + +// tempo is the beat period in windows (fractional), and how clear it is +// (0–1): the onset strength's correlation with itself a period later. +func tempo(flux []float64) (period, confidence float64) { + // Blurred a little, so a period that falls between whole windows + // (120 BPM is 21.53) scores as well as one that doesn't. + o := smooth(onsetStrength(flux), 1.5) + lo := int(math.Floor(frameRate * 60 / maxBPM)) + hi := int(math.Ceil(frameRate * 60 / minBPM)) + if len(o) < 2*hi { + hi = len(o) / 2 + } + if hi <= lo+2 { + return frameRate * 60 / priorBPM, 0 + } + var mean float64 + for _, v := range o { + mean += v + } + mean /= float64(len(o)) + // ac picks the period; cov, centred, says how clear it is. + ac := make([]float64, hi+2) + cov := func(lag int) float64 { + var s float64 + for t := 0; t+lag < len(o); t++ { + s += (o[t] - mean) * (o[t+lag] - mean) + } + return s / float64(len(o)-lag) + } + for lag := lo - 1; lag <= hi+1 && lag < len(o); lag++ { + var s float64 + for t := 0; t+lag < len(o); t++ { + s += o[t] * o[t+lag] + } + ac[lag] = s / float64(len(o)-lag) + } + weight := func(lag float64) float64 { + octaves := math.Log2(frameRate * 60 / lag / priorBPM) + return math.Exp(-0.5 * (octaves / priorOctave) * (octaves / priorOctave)) + } + best, bestScore := lo, math.Inf(-1) + for lag := lo; lag <= hi; lag++ { + s := ac[lag] * weight(float64(lag)) + if s > bestScore { + best, bestScore = lag, s + } + } + // A parabola through the peak finds it between whole windows. + period = float64(best) + if a, b, c := ac[best-1], ac[best], ac[best+1]; a-2*b+c < 0 { + period += 0.5 * (a - c) / (a - 2*b + c) + } + if v := cov(0); v > 0 { + confidence = clamp01(cov(best) / v) + } + return period, confidence +} + +// trackBeats places beats on the onset strength o, keeping close to period +// (in windows): Ellis's dynamic programming beat tracker. Each window's +// score is its own onset plus the best score of a beat about a period +// before it, less a penalty for straying from the period, if that's +// worth having. Returns the +// beats' times in windows, refined to fractions of a window. +func trackBeats(o []float64, period float64) []float64 { + n := len(o) + if n == 0 || period <= 0 { + return nil + } + // Smooth the onsets a little, so a beat may land just off a peak. + local := smooth(o, period/32) + score := make([]float64, n) + back := make([]int, n) + from, to := int(math.Round(period/2)), int(math.Round(2*period)) + for t := range n { + back[t] = -1 + best := 0.0 + for p := t - to; p <= t-from; p++ { + if p < 0 { + continue + } + d := math.Log(float64(t-p) / period) + if s := score[p] - tightness*d*d; back[t] < 0 || s > best { + best, back[t] = s, p + } + } + score[t] = local[t] + if back[t] >= 0 && best > 0 { + score[t] += best + } else { + // Nothing worth following before this (the silence before + // the song): start afresh, so the silence can't set the phase. + back[t] = -1 + } + } + // The last beat: the best score in the final stretch. + end := n - 1 + for t := max(0, n-int(2*period)); t < n; t++ { + if score[t] > score[end] { + end = t + } + } + var beats []float64 + for t := end; t >= 0; t = back[t] { + beats = append(beats, refine(o, t)) + } + for i, j := 0, len(beats)-1; i < j; i, j = i+1, j-1 { + beats[i], beats[j] = beats[j], beats[i] + } + return beats +} + +// refine moves a beat at window t to the onset peak's centre, between windows. +func refine(o []float64, t int) float64 { + if t <= 0 || t >= len(o)-1 { + return float64(t) + } + a, b, c := o[t-1], o[t], o[t+1] + if b < a || b < c || a-2*b+c >= 0 { + return float64(t) + } + return float64(t) + 0.5*(a-c)/(a-2*b+c) +} + +// smooth convolves x with a Gaussian of the given width, in samples. +func smooth(x []float64, sigma float64) []float64 { + if sigma < 0.5 { + return x + } + r := int(math.Ceil(3 * sigma)) + k := make([]float64, 2*r+1) + for i := range k { + d := float64(i-r) / sigma + k[i] = math.Exp(-0.5 * d * d) + } + out := make([]float64, len(x)) + for t := range x { + var s, w float64 + for i, kv := range k { + if j := t + i - r; j >= 0 && j < len(x) { + s += x[j] * kv + w += kv + } + } + out[t] = s / w + } + return out +} + +// downbeat picks which of the first four beats starts a bar: the one whose +// place in the bar gets the most kick drum, and the most energy. +func downbeat(beats []float64, lowFlux, rmsDB []float64) int { + at := func(xs []float64, t float64) float64 { + i := int(math.Round(t)) + v := math.Inf(-1) + for j := max(0, i-1); j <= min(len(xs)-1, i+1); j++ { + v = math.Max(v, xs[j]) + } + return v + } + var kick, loud [4]float64 + var count [4]int + for i, t := range beats { + kick[i%4] += at(lowFlux, t) + loud[i%4] += at(rmsDB, t) + count[i%4]++ + } + var kickSum float64 + for p := range 4 { + kick[p] /= float64(max(count[p], 1)) + loud[p] /= float64(max(count[p], 1)) + kickSum += kick[p] + } + best, bestScore := 0, math.Inf(-1) + for p := range 4 { + s := loud[p] / 6 // a dB or so counts for about as much as… + if kickSum > 0 { + s += 4 * kick[p] / kickSum // …a share of the kicks. + } + if s > bestScore { + best, bestScore = p, s + } + } + return best +} diff --git a/server/internal/httpapi/accounts_test.go b/server/internal/httpapi/accounts_test.go index 4e7e4cb..8028aca 100644 --- a/server/internal/httpapi/accounts_test.go +++ b/server/internal/httpapi/accounts_test.go @@ -12,6 +12,7 @@ import ( "net/http/cookiejar" "net/http/httptest" "net/url" + "os/exec" "path/filepath" "slices" "strings" @@ -20,6 +21,7 @@ import ( "time" "github.com/madeofpendletonwool/syncphony/server/internal/admin" + "github.com/madeofpendletonwool/syncphony/server/internal/analysis" "github.com/madeofpendletonwool/syncphony/server/internal/artwork" "github.com/madeofpendletonwool/syncphony/server/internal/auth" "github.com/madeofpendletonwool/syncphony/server/internal/autopilot" @@ -110,6 +112,10 @@ func newEnv(t *testing.T) *env { e.mb = musicbrainz.New(db, musicbrainz.Options{BaseURL: mbURL, CoverArtURL: mbURL, Interval: -1, Now: e.clock}) art := artwork.New(e.links, e.mb) e.palettes = palette.New(db, art) + var beatMaps *analysis.Service + if _, err := exec.LookPath("ffmpeg"); err == nil { + beatMaps = analysis.New(db, e.links, analysis.FFmpeg{}) + } qs.OnAdd = func(ts []provider.Track) { e.mb.Enqueue(ts...) e.palettes.Enqueue(ts...) @@ -127,7 +133,7 @@ func newEnv(t *testing.T) *env { pilot.Now, pilot.Memory = e.clock, &dj.Memory{DB: db, Now: e.clock} t.Cleanup(pilot.Close) api := &httpapi.Server{ - Version: "test", StartedAt: e.clock(), Admin: e.server, Auth: e.svc, Links: e.links, Lyrics: ly, LinerNotes: notes, Artwork: art, Palettes: e.palettes, Rooms: e.rooms, Queue: qs, Playback: e.playback, Nights: e.nights, Bus: e.bus, Presence: presence, + Version: "test", StartedAt: e.clock(), Admin: e.server, Auth: e.svc, Links: e.links, Lyrics: ly, LinerNotes: notes, Artwork: art, Palettes: e.palettes, BeatMaps: beatMaps, Rooms: e.rooms, Queue: qs, Playback: e.playback, Nights: e.nights, Bus: e.bus, Presence: presence, Suggest: suggest.New(db, e.rooms, e.links), Autopilot: pilot, BaseURL: e.base, PingEvery: 50 * time.Millisecond, } diff --git a/server/internal/httpapi/api.gen.go b/server/internal/httpapi/api.gen.go index a73ae46..431a034 100644 --- a/server/internal/httpapi/api.gen.go +++ b/server/internal/httpapi/api.gen.go @@ -242,12 +242,15 @@ func (e PermissionLevel) Valid() bool { // Defines values for PlaybackCommandAction. const ( - Pause PlaybackCommandAction = "pause" - Play PlaybackCommandAction = "play" - Seek PlaybackCommandAction = "seek" - Skip PlaybackCommandAction = "skip" - UnvoteSkip PlaybackCommandAction = "unvote_skip" - VoteSkip PlaybackCommandAction = "vote_skip" + Pause PlaybackCommandAction = "pause" + Play PlaybackCommandAction = "play" + PlayNow PlaybackCommandAction = "play_now" + Seek PlaybackCommandAction = "seek" + Skip PlaybackCommandAction = "skip" + UnvotePlayNow PlaybackCommandAction = "unvote_play_now" + UnvoteSkip PlaybackCommandAction = "unvote_skip" + VotePlayNow PlaybackCommandAction = "vote_play_now" + VoteSkip PlaybackCommandAction = "vote_skip" ) // Valid indicates whether the value is a known member of the PlaybackCommandAction enum. @@ -257,12 +260,18 @@ func (e PlaybackCommandAction) Valid() bool { return true case Play: return true + case PlayNow: + return true case Seek: return true case Skip: return true + case UnvotePlayNow: + return true case UnvoteSkip: return true + case VotePlayNow: + return true case VoteSkip: return true default: @@ -639,6 +648,27 @@ func (e RoomMembersChangedChange) Valid() bool { } } +// Defines values for RoomScreensLook. +const ( + Auto RoomScreensLook = "auto" + Stage RoomScreensLook = "stage" + Visualizer RoomScreensLook = "visualizer" +) + +// Valid indicates whether the value is a known member of the RoomScreensLook enum. +func (e RoomScreensLook) Valid() bool { + switch e { + case Auto: + return true + case Stage: + return true + case Visualizer: + return true + default: + return false + } +} + // Defines values for RoomVisibility. const ( Open RoomVisibility = "open" @@ -995,6 +1025,62 @@ type Backup struct { Name string `json:"name"` } +// BeatMap A song's beat map (MAD-773). Times are in ms from the start of the +// song, on the same clock as the playback position. +type BeatMap struct { + // BandCount Spectrum bands per frame in `bands`. + BandCount int `json:"bandCount"` + + // Bands Base64: each frame's level in each band, low to high, a byte + // (0–255) each, `bandCount` bytes a frame. Each band is scaled to + // its own range in the song. + Bands []byte `json:"bands"` + + // Beats The beats. + Beats []int64 `json:"beats"` + + // Bpm The tempo; 0 if the song has no steady beat, and then `beats` is empty. + Bpm float32 `json:"bpm"` + + // Confidence How clear the beat is, 0–1. + Confidence float32 `json:"confidence"` + + // Downbeat The index in `beats` of the first bar's first beat; every fourth beat after it starts a bar. + Downbeat int `json:"downbeat"` + DurationMs int64 `json:"durationMs"` + + // Features The song as a whole, 0–1 each, for choosing how it looks. + Features BeatMapFeatures `json:"features"` + + // FrameRate Frames a second in `bands` and `loudness`. + FrameRate float32 `json:"frameRate"` + + // Loudness Base64: each frame's overall level, a byte (0–255) each, scaled to the song. + Loudness []byte `json:"loudness"` + + // Sections The song's parts (intro, verse, drop), in order; the first starts at 0. + Sections []BeatMapSection `json:"sections"` +} + +// BeatMapFeatures The song as a whole, 0–1 each, for choosing how it looks. +type BeatMapFeatures struct { + // Brightness How much treble there is. + Brightness float32 `json:"brightness"` + + // Dynamics How much its loudness changes. + Dynamics float32 `json:"dynamics"` + + // Energy How loud it's mastered, from a quiet acoustic recording to a club track. + Energy float32 `json:"energy"` +} + +// BeatMapSection defines model for BeatMapSection. +type BeatMapSection struct { + // Energy How loud it is next to the rest of the song, 0–1. + Energy float32 `json:"energy"` + StartMs int64 `json:"startMs"` +} + // BeginOAuthLinkRequest Set `provider` to link a new account, or `linkId` to re-link one. For // providers that pair first, set only `pairingId`, the approved pairing. type BeginOAuthLinkRequest struct { @@ -1027,6 +1113,12 @@ type ClaimPlayerRequest struct { // Name Example: Collin's phone Name string `json:"name"` + + // PlayItemId A queued song to play now, as `play_now` would, instead of + // starting the next in fair order: pressing play on a song with no + // speaker. Ignored if the caller would have to ask the room, or + // the song isn't waiting. + PlayItemId *string `json:"playItemId,omitempty"` } // ClearedLane defines model for ClearedLane. @@ -1098,6 +1190,10 @@ type CreateRoomRequest struct { // Permissions Permissions to change. Missing ones stay as they are (`everyone` for a new room). Permissions *RoomPermissionsChange `json:"permissions,omitempty"` + // Screens How the room's big screens show the music (MAD-779). Set from any + // phone by whoever manages the room; every screen follows. + Screens *RoomScreens `json:"screens,omitempty"` + // SkipVotePercent A skip vote passes once more than this percent of the room has // voted (not counting whoever queued the song): 50 is a majority. SkipVotePercent *SkipVotePercent `json:"skipVotePercent,omitempty"` @@ -1487,6 +1583,12 @@ type NowPlaying struct { Item *QueueItem `json:"item,omitempty"` Next *QueueItem `json:"next,omitempty"` + // PlayNow Someone's request to play a queued song now, in a room whose skip + // permission is `vote`. It passes on the same share of the room as a + // skip, and lapses at `expires`, when the song leaves the queue, or + // when whoever asked takes it back. + PlayNow *PlayNowVote `json:"playNow,omitempty"` + // Player The device a room plays through. Player *Player `json:"player,omitempty"` PositionMs int64 `json:"positionMs"` @@ -1621,19 +1723,46 @@ type PersonStats struct { UserId string `json:"userId"` } +// PlayNowVote Someone's request to play a queued song now, in a room whose skip +// permission is `vote`. It passes on the same share of the room as a +// skip, and lapses at `expires`, when the song leaves the queue, or +// when whoever asked takes it back. +type PlayNowVote struct { + // By Who asked. Their vote is the first. + By string `json:"by"` + Expires time.Time `json:"expires"` + ItemId string `json:"itemId"` + + // Needed How many agreeing play the song. + Needed int `json:"needed"` + + // Voters IDs of the users who agreed, in order. + Voters []string `json:"voters"` +} + // PlaybackCommand defines model for PlaybackCommand. type PlaybackCommand struct { + // Action `play_now` plays the queued song `itemId` straight away, skipping + // the one playing. It takes the skip permission; in a room that + // votes on skips, it asks the room instead (`playNow`), and + // `vote_play_now` agrees. `unvote_play_now` takes a vote back, or + // withdraws the request from whoever asked. Action PlaybackCommandAction `json:"action"` // ItemId Skip (or vote) only if this is still the current song, so two - // people tapping skip skip one song. + // people tapping skip skip one song. For `play_now` and its votes, + // the queued song to play. ItemId *string `json:"itemId,omitempty"` // PositionMs Where to seek to. PositionMs *int64 `json:"positionMs,omitempty"` } -// PlaybackCommandAction defines model for PlaybackCommand.Action. +// PlaybackCommandAction `play_now` plays the queued song `itemId` straight away, skipping +// the one playing. It takes the skip permission; in a room that +// votes on skips, it asks the room instead (`playNow`), and +// `vote_play_now` agrees. `unvote_play_now` takes a vote back, or +// withdraws the request from whoever asked. type PlaybackCommandAction string // PlaybackNotice Something members should hear about, like a song skipped because its service failed. @@ -1958,6 +2087,10 @@ type Room struct { OwnerId string `json:"ownerId"` Permissions RoomPermissions `json:"permissions"` + // Screens How the room's big screens show the music (MAD-779). Set from any + // phone by whoever manages the room; every screen follows. + Screens RoomScreens `json:"screens"` + // SkipVotePercent A skip vote passes once more than this percent of the room has // voted (not counting whoever queued the song): 50 is a majority. SkipVotePercent SkipVotePercent `json:"skipVotePercent"` @@ -2240,6 +2373,26 @@ type RoomPermissionsChange struct { Speaker *PermissionLevel `json:"speaker,omitempty"` } +// RoomScreens How the room's big screens show the music (MAD-779). Set from any +// phone by whoever manages the room; every screen follows. +type RoomScreens struct { + // Intensity How strongly the screens move with the music. + Intensity float32 `json:"intensity"` + + // Look `stage`: now playing, lyrics and what's next. `visualizer`: + // full-screen visuals with now playing small. `auto`: the stage, + // with visuals taking over for songs with no words to sing. + Look RoomScreensLook `json:"look"` + + // Scene The visualizer's scene, or "" to pick one that suits each song. + Scene string `json:"scene"` +} + +// RoomScreensLook `stage`: now playing, lyrics and what's next. `visualizer`: +// full-screen visuals with now playing small. `auto`: the stage, +// with visuals taking over for songs with no words to sing. +type RoomScreensLook string + // RoomStats defines model for RoomStats. type RoomStats struct { First *PlayedItem `json:"first,omitempty"` @@ -2548,6 +2701,10 @@ type UpdateRoomRequest struct { // Permissions Permissions to change. Missing ones stay as they are (`everyone` for a new room). Permissions *RoomPermissionsChange `json:"permissions,omitempty"` + // Screens How the room's big screens show the music (MAD-779). Set from any + // phone by whoever manages the room; every screen follows. + Screens *RoomScreens `json:"screens,omitempty"` + // SkipVotePercent A skip vote passes once more than this percent of the room has // voted (not counting whoever queued the song): 50 is a majority. SkipVotePercent *SkipVotePercent `json:"skipVotePercent,omitempty"` @@ -3243,6 +3400,9 @@ type ServerInterface interface { // GetQueueItemArtwork A queued song's artwork // (GET /rooms/{roomId}/queue/{itemId}/artwork) GetQueueItemArtwork(w http.ResponseWriter, r *http.Request, roomId RoomId, itemId string, params GetQueueItemArtworkParams) + // GetQueueItemBeatMap A queued song's beat map + // (GET /rooms/{roomId}/queue/{itemId}/beatmap) + GetQueueItemBeatMap(w http.ResponseWriter, r *http.Request, roomId RoomId, itemId string) // UnheartSong Take back a heart // (DELETE /rooms/{roomId}/queue/{itemId}/hearts) UnheartSong(w http.ResponseWriter, r *http.Request, roomId RoomId, itemId string) @@ -6154,6 +6314,41 @@ func (siw *ServerInterfaceWrapper) GetQueueItemArtwork(w http.ResponseWriter, r handler.ServeHTTP(w, r) } +// GetQueueItemBeatMap operation middleware +func (siw *ServerInterfaceWrapper) GetQueueItemBeatMap(w http.ResponseWriter, r *http.Request) { + + var err error + _ = err + + // ------------- Path parameter "roomId" ------------- + var roomId RoomId + + err = runtime.BindStyledParameterWithOptions("simple", "roomId", r.PathValue("roomId"), &roomId, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true, Type: "string", Format: "", ValueIsUnescaped: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "roomId", Err: err}) + return + } + + // ------------- Path parameter "itemId" ------------- + var itemId string + + err = runtime.BindStyledParameterWithOptions("simple", "itemId", r.PathValue("itemId"), &itemId, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true, Type: "string", Format: "", ValueIsUnescaped: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "itemId", Err: err}) + return + } + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.GetQueueItemBeatMap(w, r, roomId, itemId) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + // UnheartSong operation middleware func (siw *ServerInterfaceWrapper) UnheartSong(w http.ResponseWriter, r *http.Request) { @@ -7170,6 +7365,7 @@ func HandlerWithOptions(si ServerInterface, options StdHTTPServerOptions) http.H m.HandleFunc(http.MethodGet+" "+options.BaseURL+"/rooms/{roomId}/queue/{itemId}/similar", wrapper.GetQueueItemSimilar) m.HandleFunc(http.MethodGet+" "+options.BaseURL+"/rooms/{roomId}/queue/{itemId}/artwork", wrapper.GetQueueItemArtwork) m.HandleFunc(http.MethodGet+" "+options.BaseURL+"/rooms/{roomId}/queue/{itemId}/palette", wrapper.GetQueueItemPalette) + m.HandleFunc(http.MethodGet+" "+options.BaseURL+"/rooms/{roomId}/queue/{itemId}/beatmap", wrapper.GetQueueItemBeatMap) m.HandleFunc(http.MethodGet+" "+options.BaseURL+"/rooms/{roomId}/queue/{itemId}/lyrics", wrapper.GetQueueItemLyrics) m.HandleFunc(http.MethodGet+" "+options.BaseURL+"/rooms/{roomId}/queue/{itemId}/liner-notes", wrapper.GetQueueItemLinerNotes) m.HandleFunc(http.MethodPost+" "+options.BaseURL+"/rooms/{roomId}/reactions", wrapper.SendReaction) @@ -11467,6 +11663,46 @@ func (response GetQueueItemArtworkdefaultJSONResponse) VisitGetQueueItemArtworkR return err } +type GetQueueItemBeatMapRequestObject struct { + RoomId RoomId `json:"roomId"` + ItemId string `json:"itemId"` +} + +type GetQueueItemBeatMapResponseObject interface { + VisitGetQueueItemBeatMapResponse(w http.ResponseWriter) error +} + +type GetQueueItemBeatMap200JSONResponse BeatMap + +func (response GetQueueItemBeatMap200JSONResponse) VisitGetQueueItemBeatMapResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, err := buf.WriteTo(w) + return err +} + +type GetQueueItemBeatMapdefaultJSONResponse struct { + Body Error + StatusCode int +} + +func (response GetQueueItemBeatMapdefaultJSONResponse) VisitGetQueueItemBeatMapResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response.Body); err != nil { + return err + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(response.StatusCode) + _, err := buf.WriteTo(w) + return err +} + type UnheartSongRequestObject struct { RoomId RoomId `json:"roomId"` ItemId string `json:"itemId"` @@ -12693,6 +12929,9 @@ type StrictServerInterface interface { // GetQueueItemArtwork A queued song's artwork // (GET /rooms/{roomId}/queue/{itemId}/artwork) GetQueueItemArtwork(ctx context.Context, request GetQueueItemArtworkRequestObject) (GetQueueItemArtworkResponseObject, error) + // GetQueueItemBeatMap A queued song's beat map + // (GET /rooms/{roomId}/queue/{itemId}/beatmap) + GetQueueItemBeatMap(ctx context.Context, request GetQueueItemBeatMapRequestObject) (GetQueueItemBeatMapResponseObject, error) // UnheartSong Take back a heart // (DELETE /rooms/{roomId}/queue/{itemId}/hearts) UnheartSong(ctx context.Context, request UnheartSongRequestObject) (UnheartSongResponseObject, error) @@ -15841,6 +16080,33 @@ func (sh *strictHandler) GetQueueItemArtwork(w http.ResponseWriter, r *http.Requ } } +// GetQueueItemBeatMap operation middleware +func (sh *strictHandler) GetQueueItemBeatMap(w http.ResponseWriter, r *http.Request, roomId RoomId, itemId string) { + var request GetQueueItemBeatMapRequestObject + + request.RoomId = roomId + request.ItemId = itemId + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.GetQueueItemBeatMap(ctx, request.(GetQueueItemBeatMapRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "GetQueueItemBeatMap") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(GetQueueItemBeatMapResponseObject); ok { + if err := validResponse.VisitGetQueueItemBeatMapResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + // UnheartSong operation middleware func (sh *strictHandler) UnheartSong(w http.ResponseWriter, r *http.Request, roomId RoomId, itemId string) { var request UnheartSongRequestObject diff --git a/server/internal/httpapi/browse.go b/server/internal/httpapi/browse.go index 24ee0b4..8649fb1 100644 --- a/server/internal/httpapi/browse.go +++ b/server/internal/httpapi/browse.go @@ -16,6 +16,7 @@ import ( "sync" "time" + "github.com/madeofpendletonwool/syncphony/server/internal/analysis" "github.com/madeofpendletonwool/syncphony/server/internal/artcache" "github.com/madeofpendletonwool/syncphony/server/internal/links" "github.com/madeofpendletonwool/syncphony/server/internal/provider" @@ -361,6 +362,37 @@ func (s *Server) GetQueueItemPalette(ctx context.Context, req GetQueueItemPalett return GetQueueItemPalette200JSONResponse(toPalette(p)), nil } +// GetQueueItemBeatMap returns a queued song's beat map, working it out if +// it hasn't been. +func (s *Server) GetQueueItemBeatMap(ctx context.Context, req GetQueueItemBeatMapRequestObject) (GetQueueItemBeatMapResponseObject, error) { + it, err := s.Queue.Item(ctx, req.RoomId, req.ItemId) + if err != nil { + return nil, err + } + if s.BeatMaps == nil { + return nil, analysis.ErrUnavailable + } + t, _ := queuedTrack(it) + m, err := s.BeatMaps.ForItem(ctx, it, t.Duration) + if err != nil { + return nil, err + } + return GetQueueItemBeatMap200JSONResponse(toBeatMap(m)), nil +} + +func toBeatMap(m analysis.Map) BeatMap { + out := BeatMap{ + DurationMs: m.DurationMs, Bpm: float32(m.BPM), Confidence: float32(m.Confidence), Beats: m.BeatsMs, Downbeat: m.Downbeat, + FrameRate: float32(m.FrameRate), BandCount: analysis.Bands, Bands: m.BandsEnv, Loudness: m.Loudness, + Features: BeatMapFeatures{Energy: float32(m.Features.Energy), Brightness: float32(m.Features.Brightness), Dynamics: float32(m.Features.Dynamics)}, + Sections: make([]BeatMapSection, len(m.Sections)), + } + for i, sec := range m.Sections { + out.Sections[i] = BeatMapSection{StartMs: sec.StartMs, Energy: float32(sec.Energy)} + } + return out +} + // queuedTrack is the provider.Track snapshot taken when it was queued. func queuedTrack(it store.QueueItem) (provider.Track, error) { var t provider.Track diff --git a/server/internal/httpapi/displays.go b/server/internal/httpapi/displays.go index 42c4735..07d2d49 100644 --- a/server/internal/httpapi/displays.go +++ b/server/internal/httpapi/displays.go @@ -35,6 +35,7 @@ var displayOps = map[string]bool{ "GetUserAvatar": true, "GetQueueItemArtwork": true, "GetQueueItemPalette": true, + "GetQueueItemBeatMap": true, "GetQueueItemLyrics": true, "GetQueueItemLinerNotes": true, "GetHearts": true, @@ -44,6 +45,7 @@ var displayOps = map[string]bool{ "ReleasePlayer": true, "ReportPlayback": true, "StreamQueueItem": true, + "ControlPlayback": true, } // audioOps are the displayOps that make a display the room's speaker. They @@ -53,6 +55,7 @@ var audioOps = map[string]bool{ "ReleasePlayer": true, "ReportPlayback": true, "StreamQueueItem": true, + "ControlPlayback": true, } // Display errors. @@ -71,10 +74,14 @@ func displayRoom(req any) (roomID string, ok bool) { return r.RoomId, true case GetPlaybackRequestObject: return r.RoomId, true + case ControlPlaybackRequestObject: + return r.RoomId, true case GetQueueItemArtworkRequestObject: return r.RoomId, true case GetQueueItemPaletteRequestObject: return r.RoomId, true + case GetQueueItemBeatMapRequestObject: + return r.RoomId, true case GetQueueItemLyricsRequestObject: return r.RoomId, true case GetQueueItemLinerNotesRequestObject: diff --git a/server/internal/httpapi/displays_test.go b/server/internal/httpapi/displays_test.go index d60cd2e..72856df 100644 --- a/server/internal/httpapi/displays_test.go +++ b/server/internal/httpapi/displays_test.go @@ -249,6 +249,21 @@ func TestDisplaySpeaker(t *testing.T) { if np.State != httpapi.PlaybackStatePlaying { t.Fatalf("after the display's report: %+v", np) } + // It has a speaker's buttons: play, pause and skip, as bob. Nothing else. + tv.want(http.StatusOK, "POST", base+"/playback", httpapi.PlaybackCommand{Action: httpapi.Pause}).decode(t, &np) + if np.State != httpapi.PlaybackStatePaused { + t.Fatalf("after the display paused: %+v", np) + } + tv.want(http.StatusOK, "POST", base+"/playback", httpapi.PlaybackCommand{Action: httpapi.Play}).decode(t, &np) + if np.State != httpapi.PlaybackStatePlaying { + t.Fatalf("after the display played: %+v", np) + } + if r := tv.do("POST", base+"/playback", httpapi.PlaybackCommand{Action: httpapi.Seek, PositionMs: ptr(int64(0))}); r.status != http.StatusForbidden { + t.Fatalf("display seeking: %d %s", r.status, r.body) + } + if r := tv.do("POST", "/rooms/"+other.Id+"/playback", httpapi.PlaybackCommand{Action: httpapi.Pause}); r.status != http.StatusForbidden { + t.Fatalf("display pausing another room: %d %s", r.status, r.body) + } // Bob's own phone isn't the TV. if r := bob.do("POST", base+"/player/report", httpapi.PlayerReport{DeviceId: "bobs-phone", ItemId: item, Event: "ended"}); r.status != http.StatusConflict { t.Fatalf("bob reporting as his phone: %d %s", r.status, r.body) @@ -267,6 +282,9 @@ func TestDisplaySpeaker(t *testing.T) { if r := tv.do("GET", base+"/stream/"+item, nil); r.status != http.StatusForbidden { t.Fatalf("streaming with audio off: %d", r.status) } + if r := tv.do("POST", base+"/playback", httpapi.PlaybackCommand{Action: httpapi.Play}); r.status != http.StatusForbidden { + t.Fatalf("playing with audio off: %d %s", r.status, r.body) + } // A display can let go itself; unpairing one that's the speaker stops it too. bob.want(http.StatusOK, "PATCH", base+"/displays/"+d.Id, httpapi.UpdateDisplayRequest{Audio: true}) diff --git a/server/internal/httpapi/guests.go b/server/internal/httpapi/guests.go index e2dfee2..f8e58d5 100644 --- a/server/internal/httpapi/guests.go +++ b/server/internal/httpapi/guests.go @@ -55,6 +55,7 @@ var guestOps = map[string]bool{ "ClearLane": true, "GetQueueItemArtwork": true, "GetQueueItemPalette": true, + "GetQueueItemBeatMap": true, "GetQueueItemLyrics": true, "GetQueueItemLinerNotes": true, "GetQueueItemSimilar": true, diff --git a/server/internal/httpapi/playback.go b/server/internal/httpapi/playback.go index 383a586..36f4080 100644 --- a/server/internal/httpapi/playback.go +++ b/server/internal/httpapi/playback.go @@ -26,7 +26,7 @@ func (s *Server) GetPlayback(ctx context.Context, req GetPlaybackRequestObject) return GetPlayback200JSONResponse(toNowPlaying(np)), nil } -// ControlPlayback plays, pauses, skips or seeks. +// ControlPlayback plays, pauses, skips, seeks, or plays a queued song now. func (s *Server) ControlPlayback(ctx context.Context, req ControlPlaybackRequestObject) (ControlPlaybackResponseObject, error) { c := playback.Command{Action: string(req.Body.Action)} if req.Body.PositionMs != nil { @@ -35,7 +35,21 @@ func (s *Server) ControlPlayback(ctx context.Context, req ControlPlaybackRequest if req.Body.ItemId != nil { c.ItemID = *req.Body.ItemId } - np, err := s.Playback.Command(ctx, req.RoomId, sessionFrom(ctx).User.ID, c) + userID := "" + if d := displayFrom(ctx); d != nil { + // A screen playing the room has a speaker's buttons, as whoever paired it. + if c.Action != playback.ActionPlay && c.Action != playback.ActionPause && c.Action != playback.ActionSkip { + return nil, playback.ErrForbidden + } + id, _, err := speakerOf(ctx, "") + if err != nil { + return nil, err + } + userID = id + } else { + userID = sessionFrom(ctx).User.ID + } + np, err := s.Playback.Command(ctx, req.RoomId, userID, c) if err != nil { return nil, err } @@ -53,7 +67,11 @@ func (s *Server) ClaimPlayer(ctx context.Context, req ClaimPlayerRequestObject) if d := displayFrom(ctx); d != nil { name = d.Name } - np, err := s.Playback.Claim(ctx, req.RoomId, userID, device, name) + itemID := "" + if req.Body.PlayItemId != nil { + itemID = *req.Body.PlayItemId + } + np, err := s.Playback.ClaimAndPlay(ctx, req.RoomId, userID, device, name, itemID) if err != nil { return nil, err } @@ -191,5 +209,8 @@ func toNowPlaying(np rooms.NowPlaying) NowPlaying { if v := np.SkipVotes; v != nil { out.SkipVotes = &SkipVotes{Voters: v.Voters, Needed: v.Needed} } + if v := np.PlayNow; v != nil { + out.PlayNow = &PlayNowVote{ItemId: v.ItemID, By: v.By, Voters: v.Voters, Needed: v.Needed, Expires: v.Expires} + } return out } diff --git a/server/internal/httpapi/playback_test.go b/server/internal/httpapi/playback_test.go index 1468fb4..0603207 100644 --- a/server/internal/httpapi/playback_test.go +++ b/server/internal/httpapi/playback_test.go @@ -4,6 +4,7 @@ package httpapi_test import ( "net/http" + "os/exec" "strings" "testing" "time" @@ -132,6 +133,21 @@ func TestPlaybackAPI(t *testing.T) { } item := np.Item.Id + // Anyone in the room can have the song's beat map. + if _, err := exec.LookPath("ffmpeg"); err == nil { + var bm httpapi.BeatMap + bob.want(http.StatusOK, "GET", base+"/queue/"+item+"/beatmap", nil).decode(t, &bm) + frames := len(bm.Loudness) + if len(bm.Sections) == 0 || bm.Sections[0].StartMs != 0 || frames == 0 || len(bm.Bands) != frames*bm.BandCount || bm.DurationMs == 0 { + t.Fatalf("beat map: %v BPM, %d beats, %d sections, %d frames, %d band bytes", bm.Bpm, len(bm.Beats), len(bm.Sections), frames, len(bm.Bands)) + } + } else if r := bob.do("GET", base+"/queue/"+item+"/beatmap", nil); r.status != http.StatusNotFound || r.code() != "no_beat_map" { + t.Fatalf("beat map without ffmpeg: %d %s", r.status, r.body) + } + if r := e.client().do("GET", base+"/queue/"+item+"/beatmap", nil); r.status != http.StatusUnauthorized { + t.Fatalf("beat map signed out: %d", r.status) + } + // The speaker streams the song, from an offset. r := alice.do("GET", base+"/stream/"+item+"?accept=audio/wav,audio/mpeg", nil, "Range", "bytes=100-199") if r.status != http.StatusPartialContent || r.header.Get("Content-Type") != "audio/wav" || r.header.Get("Accept-Ranges") != "bytes" || diff --git a/server/internal/httpapi/rooms.go b/server/internal/httpapi/rooms.go index 89a83e6..2c0b851 100644 --- a/server/internal/httpapi/rooms.go +++ b/server/internal/httpapi/rooms.go @@ -5,6 +5,7 @@ package httpapi import ( "context" "errors" + "math" "github.com/madeofpendletonwool/syncphony/server/internal/rooms" "github.com/madeofpendletonwool/syncphony/server/internal/store" @@ -47,6 +48,9 @@ func (s *Server) CreateRoom(ctx context.Context, req CreateRoomRequestObject) (C if a := req.Body.Autopilot; a != nil { st.Autopilot = fromAutopilot(*a) } + if sc := req.Body.Screens; sc != nil { + st.Screens = fromScreens(*sc) + } if g := req.Body.Guests; g != nil { st.Guests = fromGuests(*g) } @@ -94,6 +98,9 @@ func (s *Server) UpdateRoom(ctx context.Context, req UpdateRoomRequestObject) (U if a := req.Body.Autopilot; a != nil { u.Autopilot = ptr(fromAutopilot(*a)) } + if sc := req.Body.Screens; sc != nil { + u.Screens = ptr(fromScreens(*sc)) + } if g := req.Body.Guests; g != nil { u.Guests = ptr(fromGuests(*g)) } @@ -129,6 +136,8 @@ func toRoom(r store.Room) Room { out.Matching = RoomMatching{Fallback: st.Matching.FallbackOn(), Borrow: st.Matching.Borrow} out.Autopilot = RoomAutopilot{On: st.Autopilot.On, Adventure: RoomAutopilotAdventure(st.Autopilot.Adventure), Explore: new(st.Autopilot.ExploreLevel()), EnergyCurve: new(st.Autopilot.EnergyCurveOn())} out.Guests = RoomGuests{Allowed: st.Guests.Allowed, MaxSongs: st.Guests.SongLimit(), CanVote: st.Guests.CanVote()} + sc := st.Screens.Normal() + out.Screens = RoomScreens{Look: RoomScreensLook(sc.Look), Scene: sc.Scene, Intensity: float32(*sc.Intensity)} if out.Fairness.Weights == nil { out.Fairness.Weights = map[string]int{} } @@ -143,6 +152,10 @@ func fromAutopilot(a RoomAutopilot) rooms.Autopilot { return rooms.Autopilot{On: a.On, Adventure: string(a.Adventure), Explore: a.Explore, EnergyCurve: a.EnergyCurve} } +func fromScreens(s RoomScreens) rooms.Screens { + return rooms.Screens{Look: string(s.Look), Scene: s.Scene, Intensity: ptr(math.Round(float64(s.Intensity)*100) / 100)} +} + func fromGuests(g RoomGuests) rooms.Guests { return rooms.Guests{Allowed: g.Allowed, MaxSongs: &g.MaxSongs, NoVote: !g.CanVote} } diff --git a/server/internal/httpapi/screens_test.go b/server/internal/httpapi/screens_test.go new file mode 100644 index 0000000..504dfbb --- /dev/null +++ b/server/internal/httpapi/screens_test.go @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package httpapi_test + +import ( + "net/http" + "testing" + + "github.com/madeofpendletonwool/syncphony/server/internal/httpapi" +) + +func TestRoomScreens(t *testing.T) { + e := newEnv(t) + alice := e.admin() + bob := e.member(alice, "bob") + var room httpapi.Room + alice.want(http.StatusCreated, "POST", "/rooms", httpapi.CreateRoomRequest{Name: "Living room"}).decode(t, &room) + base := "/rooms/" + room.Id + + // Big screens default to the stage, with visuals for songs with no words. + if room.Screens != (httpapi.RoomScreens{Look: httpapi.Auto, Scene: "", Intensity: 1.5}) { + t.Fatalf("default screens: %+v", room.Screens) + } + want := httpapi.RoomScreens{Look: httpapi.Visualizer, Scene: "fluid", Intensity: 1.75} + alice.want(http.StatusOK, "PATCH", base, httpapi.UpdateRoomRequest{Screens: &want}).decode(t, &room) + if room.Screens != want { + t.Fatalf("screens: %+v", room.Screens) + } + // Only whoever manages the room changes them, and only to something sensible. + if r := bob.do("PATCH", base, httpapi.UpdateRoomRequest{Screens: &httpapi.RoomScreens{Look: httpapi.Stage, Intensity: 1}}); r.status != http.StatusForbidden { + t.Fatalf("bob changed the screens: %d", r.status) + } + if r := alice.do("PATCH", base, httpapi.UpdateRoomRequest{Screens: &httpapi.RoomScreens{Look: httpapi.Stage, Intensity: 9}}); r.status != http.StatusBadRequest { + t.Fatalf("intensity 9: %d %s", r.status, r.body) + } + if r := alice.do("PATCH", base, httpapi.UpdateRoomRequest{Screens: &httpapi.RoomScreens{Look: "disco", Intensity: 1}}); r.status != http.StatusBadRequest { + t.Fatalf("look disco: %d %s", r.status, r.body) + } + alice.want(http.StatusOK, "GET", base, nil).decode(t, &room) + if room.Screens != want { + t.Fatalf("screens after bad changes: %+v", room.Screens) + } +} diff --git a/server/internal/httpapi/server.go b/server/internal/httpapi/server.go index 7c41c61..ce5de80 100644 --- a/server/internal/httpapi/server.go +++ b/server/internal/httpapi/server.go @@ -18,6 +18,7 @@ import ( "time" "github.com/madeofpendletonwool/syncphony/server/internal/admin" + "github.com/madeofpendletonwool/syncphony/server/internal/analysis" "github.com/madeofpendletonwool/syncphony/server/internal/artwork" "github.com/madeofpendletonwool/syncphony/server/internal/auth" "github.com/madeofpendletonwool/syncphony/server/internal/autopilot" @@ -59,6 +60,8 @@ type Server struct { Artwork *artwork.Service // Palettes works out queued songs' artwork colors. Palettes *palette.Service + // BeatMaps works out queued songs' beat maps; nil without ffmpeg. + BeatMaps *analysis.Service // Realtime: room state, the event bus, and who's connected. Rooms *rooms.Service Queue *queue.Service @@ -405,6 +408,7 @@ var errorCodes = []struct { {playback.ErrNothingPlaying, http.StatusConflict, "nothing_playing"}, {playback.ErrNotStreamable, http.StatusConflict, "not_streamable"}, {transcode.ErrNoTranscoder, http.StatusUnsupportedMediaType, "unsupported_format"}, + {analysis.ErrUnavailable, http.StatusNotFound, "no_beat_map"}, {provider.ErrRange, http.StatusRequestedRangeNotSatisfiable, "range_not_satisfiable"}, {queue.ErrNotFound, http.StatusNotFound, "not_found"}, {queue.ErrForbidden, http.StatusForbidden, "forbidden"}, diff --git a/server/internal/httpapi/server_test.go b/server/internal/httpapi/server_test.go index 0ec4585..a88ba4a 100644 --- a/server/internal/httpapi/server_test.go +++ b/server/internal/httpapi/server_test.go @@ -73,9 +73,11 @@ func TestDisplayOpsMatchSpec(t *testing.T) { var displayRequests = map[string]any{ "GetRoom": GetRoomRequestObject{}, "GetQueue": GetQueueRequestObject{}, + "ControlPlayback": ControlPlaybackRequestObject{}, "GetPlayback": GetPlaybackRequestObject{}, "GetQueueItemArtwork": GetQueueItemArtworkRequestObject{}, "GetQueueItemPalette": GetQueueItemPaletteRequestObject{}, + "GetQueueItemBeatMap": GetQueueItemBeatMapRequestObject{}, "GetQueueItemLyrics": GetQueueItemLyricsRequestObject{}, "GetQueueItemLinerNotes": GetQueueItemLinerNotesRequestObject{}, "GetHearts": GetHeartsRequestObject{}, diff --git a/server/internal/playback/playback.go b/server/internal/playback/playback.go index 2d6d0be..75c1161 100644 --- a/server/internal/playback/playback.go +++ b/server/internal/playback/playback.go @@ -214,6 +214,13 @@ type room struct { halted bool // since is when the current state began (for the load timeout). since time.Time + // heard is whether the speaker has said anything about the current + // song since it began loading. A song that never starts on a silent + // speaker isn't the song's fault. + heard bool + // request is someone's request to play a queued song now, while the + // room votes on it. Needed is worked out when it's shown. + request *rooms.PlayNowVote // The remote driver's open session, while a remote song is current. sess provider.Session remote provider.Remote @@ -350,6 +357,7 @@ func (r *room) view(now time.Time) rooms.NowPlaying { func (e *Engine) view(r *room) rooms.NowPlaying { np := r.view(e.cfg.Now()) np.SkipVotes = e.tally(r) + np.PlayNow = e.playNowTally(r) return np } @@ -373,6 +381,15 @@ func (e *Engine) NowPlaying(ctx context.Context, roomID string) (rooms.NowPlayin // Claim makes a device the room's speaker, taking over from any other. // If the room is idle and songs are waiting, the first one starts. func (e *Engine) Claim(ctx context.Context, roomID, userID, deviceID, name string) (rooms.NowPlaying, error) { + return e.ClaimAndPlay(ctx, roomID, userID, deviceID, name, "") +} + +// ClaimAndPlay is Claim, then plays the queued song itemID now, if userID +// may do that outright (see ActionPlayNow): pressing play on a song with +// no speaker makes this device the speaker and plays that song, rather +// than starting another first and skipping it. If they'd have to ask the +// room, or itemID isn't waiting, it's just Claim. +func (e *Engine) ClaimAndPlay(ctx context.Context, roomID, userID, deviceID, name, itemID string) (rooms.NowPlaying, error) { if deviceID == "" || len(deviceID) > 128 || len(name) > 64 { return rooms.NowPlaying{}, &InvalidInputError{"a device needs an ID of up to 128 characters and a name of up to 64"} } @@ -400,19 +417,38 @@ func (e *Engine) Claim(ctx context.Context, roomID, userID, deviceID, name strin r.np.Position, r.np.At = r.position(now), now r.np.Revision++ if r.np.State == StatePlaying && r.np.Driver == string(provider.PlaybackStream) { - r.np.State, r.since = StateLoading, now + r.np.State, r.since, r.heard = StateLoading, now, false } } - if r.np.State == StateIdle && !r.halted { + switch { + case itemID != "" && e.mayPlayNow(ctx, r, userID, itemID): + if err := e.playNow(ctx, r, itemID); err != nil { + return rooms.NowPlaying{}, err + } + case r.np.State == StateIdle && !r.halted: if err := e.next(ctx, r, ""); err != nil { return rooms.NowPlaying{}, err } - } else { + default: e.publish(r) } return e.view(r), nil } +// mayPlayNow says whether userID may play itemID now without asking the +// room, and it's waiting in the queue. +func (e *Engine) mayPlayNow(ctx context.Context, r *room, userID, itemID string) bool { + skip := r.settings.Permissions.Skip + if userID != r.owner && (skip == rooms.Vote || !rooms.Allowed(skip, r.owner, userID)) { + return false + } + if _, err := e.db.GetGuest(ctx, userID); err == nil || !store.IsNotFound(err) { + return false + } + it, err := e.queue.Item(ctx, r.id, itemID) + return err == nil && it.State == store.ItemQueued +} + // Release stops a device being the speaker. The current song pauses. The // speaker's own user or the room's owner may release it. func (e *Engine) Release(ctx context.Context, roomID, userID, deviceID string) (rooms.NowPlaying, error) { @@ -483,8 +519,19 @@ const ( // skip the current song, in rooms that vote on skips. ActionVoteSkip = "vote_skip" ActionUnvoteSkip = "unvote_skip" + // ActionPlayNow plays a queued song straight away, skipping the one + // playing. In a room that votes on skips, it asks the room instead; + // ActionVotePlayNow agrees, and ActionUnvotePlayNow takes it back (or + // withdraws the request, from whoever asked). + ActionPlayNow = "play_now" + ActionVotePlayNow = "vote_play_now" + ActionUnvotePlayNow = "unvote_play_now" ) +// PlayNowTimeout is how long a request to play a song now waits for the +// room to agree. +const PlayNowTimeout = 2 * time.Minute + // Command is a member's request to control playback. type Command struct { Action string @@ -492,6 +539,7 @@ type Command struct { Position time.Duration // ItemID, if set, makes a skip or vote apply only while that item is // current, so two people tapping skip at once skip one song, not two. + // For play_now and its votes, it's the queued song to play. ItemID string } @@ -519,6 +567,12 @@ func (e *Engine) Command(ctx context.Context, roomID, userID string, c Command) level = perms.Seek case ActionSkip: level = perms.Skip + case ActionPlayNow: + // Playing a song now skips the one playing, so it's the skip + // permission's call; a room that votes on skips votes on this too. + if perms.Skip != rooms.Vote { + level = perms.Skip + } } ownSkip := c.Action == ActionSkip && mine if level != "" && !ownSkip && !rooms.Allowed(level, row.OwnerID, userID) { @@ -527,7 +581,7 @@ func (e *Engine) Command(ctx context.Context, roomID, userID string, c Command) // Guests may skip their own songs, and vote if the room lets them; // the rest of the controls are the members'. if _, err := e.db.GetGuest(ctx, userID); err == nil { - vote := c.Action == ActionVoteSkip || c.Action == ActionUnvoteSkip + vote := c.Action == ActionVoteSkip || c.Action == ActionUnvoteSkip || c.Action == ActionVotePlayNow || c.Action == ActionUnvotePlayNow if !ownSkip && (!vote || !r.settings.Guests.CanVote()) { return rooms.NowPlaying{}, ErrForbidden } @@ -591,8 +645,27 @@ func (e *Engine) Command(ctx context.Context, roomID, userID string, c Command) r.votes = slices.DeleteFunc(r.votes, func(id string) bool { return id == userID }) e.publish(r) } + case ActionPlayNow: + err = e.playNowCommand(ctx, r, userID, c.ItemID) + case ActionVotePlayNow, ActionUnvotePlayNow: + q := r.request + if q == nil || (c.ItemID != "" && c.ItemID != q.ItemID) { + break // the request is over + } + voted := slices.Contains(q.Voters, userID) + switch { + case c.Action == ActionUnvotePlayNow && userID == q.By: + r.request = nil + e.publish(r) + case c.Action == ActionVotePlayNow && !voted: + q.Voters = append(q.Voters, userID) + err = e.settlePlayNow(ctx, r) + case c.Action == ActionUnvotePlayNow && voted: + q.Voters = slices.DeleteFunc(q.Voters, func(id string) bool { return id == userID }) + e.publish(r) + } default: - return rooms.NowPlaying{}, &InvalidInputError{"action is play, pause, skip, seek, vote_skip or unvote_skip"} + return rooms.NowPlaying{}, &InvalidInputError{"action is play, pause, skip, seek, vote_skip, unvote_skip, play_now, vote_play_now or unvote_play_now"} } if err != nil { return rooms.NowPlaying{}, err @@ -600,6 +673,58 @@ func (e *Engine) Command(ctx context.Context, roomID, userID string, c Command) return e.view(r), nil } +// playNowCommand plays a queued song now for userID, who passed the +// permission check, or asks the room if it votes on skips and they can't +// skip outright. +func (e *Engine) playNowCommand(ctx context.Context, r *room, userID, itemID string) error { + if itemID == "" { + return &InvalidInputError{"play_now needs the itemId of a queued song"} + } + it, err := e.queue.Item(ctx, r.id, itemID) + if errors.Is(err, queue.ErrNotFound) || (err == nil && it.State != store.ItemQueued) { + return &InvalidInputError{"that song isn't waiting in the queue"} + } else if err != nil { + return err + } + if r.np.Player == nil { + return ErrNoPlayer + } + // The owner doesn't need to ask. + if r.settings.Permissions.Skip != rooms.Vote || userID == r.owner { + return e.playNow(ctx, r, itemID) + } + if q := r.request; q != nil { + if q.ItemID == itemID { + if !slices.Contains(q.Voters, userID) { + q.Voters = append(q.Voters, userID) + } + return e.settlePlayNow(ctx, r) + } + if q.By != userID { + return &InvalidInputError{"someone's already asking to play a song now; wait for the room to decide"} + } + } + r.request = &rooms.PlayNowVote{ItemID: itemID, By: userID, Voters: []string{userID}, Expires: e.cfg.Now().Add(PlayNowTimeout)} + if v := e.playNowTally(r); v != nil && len(v.Voters) < v.Needed { + name := "Someone" + if u, err := e.db.GetUser(ctx, userID); err == nil { + name = u.DisplayName + } + e.notice(r, itemID, fmt.Sprintf("%s wants to play %s now", name, title(&it))) + } + return e.settlePlayNow(ctx, r) +} + +// playNow ends the current song, if any, and starts itemID in its place. +func (e *Engine) playNow(ctx context.Context, r *room, itemID string) error { + r.halted, r.errors = false, 0 + reason := "" + if r.np.Item != nil { + reason = store.EndSkipped + } + return e.nextItem(ctx, r, reason, itemID) +} + func (e *Engine) play(ctx context.Context, r *room) error { if r.np.Player == nil { return ErrNoPlayer @@ -625,6 +750,10 @@ func (e *Engine) play(ctx context.Context, r *room) error { } } r.np.State, r.np.At, r.since = StatePlaying, now, now + if r.np.Driver == string(provider.PlaybackStream) && !r.heard { + // The speaker never started it: wait for it to, under the load timeout. + r.np.State = StateLoading + } r.np.Revision++ e.publish(r) return nil @@ -681,6 +810,7 @@ func (e *Engine) Report(ctx context.Context, roomID, userID string, rep Report) if r.np.Item == nil || r.np.Item.ID != rep.ItemID || r.np.Driver != string(provider.PlaybackStream) { return e.view(r), nil } + r.heard = true switch rep.Event { case EventPlaying: if r.np.State == StateLoading || r.np.State == StatePlaying { @@ -731,14 +861,23 @@ func (e *Engine) Report(ctx context.Context, roomID, userID string, rep Report) // songs that fail to start. With no speaker, or after too many failures, // the room goes idle instead. The caller holds r.mu. func (e *Engine) next(ctx context.Context, r *room, reason string) error { + return e.nextItem(ctx, r, reason, "") +} + +// nextItem is next, starting want first if it's still queued. +func (e *Engine) nextItem(ctx context.Context, r *room, reason, want string) error { for { start := r.np.Player != nil && !r.halted - item, snap, err := e.rotate(ctx, r, reason, start) + item, snap, err := e.rotate(ctx, r, reason, start, want) if err != nil { return err } + want = "" now := e.cfg.Now() r.votes = nil + if r.request != nil && (item == nil || item.ID == r.request.ItemID || !queued(snap, r.request.ItemID)) { + r.request = nil + } r.np.Item, r.np.Position, r.np.At, r.since = item, 0, now, now r.np.Next = nextOf(snap) r.np.Revision++ @@ -774,9 +913,10 @@ func (e *Engine) countFailure(r *room, item *store.QueueItem, msg string, cause } // rotate ends the playing song with reason and, if start, makes the next -// song in fair order the playing one. It returns that song (nil if none) -// and the queue as it now stands. -func (e *Engine) rotate(ctx context.Context, r *room, reason string, start bool) (*store.QueueItem, rooms.QueueSnapshot, error) { +// song the playing one: want if it's queued, or else the next in fair +// order. It returns that song (nil if none) and the queue as it now +// stands. +func (e *Engine) rotate(ctx context.Context, r *room, reason string, start bool, want string) (*store.QueueItem, rooms.QueueSnapshot, error) { r.closeRemote() var next *store.QueueItem snap, err := e.queue.Change(ctx, r.id, func(q *store.Queries, _ store.Room) error { @@ -803,6 +943,9 @@ func (e *Engine) rotate(ctx context.Context, r *room, reason string, start bool) return err } it := nextOf(snap) + if i := slices.IndexFunc(snap.Items, func(it store.QueueItem) bool { return it.ID == want && it.State == store.ItemQueued }); i >= 0 { + it = &snap.Items[i] + } if it == nil { return nil } @@ -911,7 +1054,7 @@ func (e *Engine) start(ctx context.Context, r *room, it store.QueueItem) error { return errors.New("its service can't play songs") } r.np.Driver = string(provider.PlaybackStream) - r.np.State = StateLoading + r.np.State, r.heard = StateLoading, false return nil } @@ -951,9 +1094,12 @@ func (e *Engine) roomUpdated(row store.Room) { was := r.settings.Permissions.Skip r.setRoom(row) if r.settings.Permissions.Skip == rooms.Vote { + if err := e.settlePlayNow(ctx, r); err != nil { + return err + } return e.settleVote(ctx, r) } - r.votes = nil + r.votes, r.request = nil, nil if was == rooms.Vote { e.publish(r) } @@ -983,7 +1129,12 @@ func (e *Engine) roomDeleted(roomID string) { // MembersChanged follows someone joining or leaving a room: a skip vote // needs more or fewer votes, and may now pass. func (e *Engine) MembersChanged(roomID string) { - e.background(roomID, "reacting to members", e.settleVote) + e.background(roomID, "reacting to members", func(ctx context.Context, r *room) error { + if err := e.settlePlayNow(ctx, r); err != nil { + return err + } + return e.settleVote(ctx, r) + }) } // background runs fn on a room the engine has loaded, on another @@ -1021,20 +1172,41 @@ func (e *Engine) tally(r *room) *rooms.SkipVotes { if r.settings.Permissions.Skip != rooms.Vote || r.np.Item == nil { return nil } + exclude := r.np.Item.AddedBy + if r.np.Item.IsAutopilot() { + exclude = "" + } + votes, needed := e.count(r, r.votes, exclude) + return &rooms.SkipVotes{Voters: votes, Needed: needed} +} + +// playNowTally counts the vote on a request to play a song now, or is nil +// if there's none. Everyone in the room may vote. +func (e *Engine) playNowTally(r *room) *rooms.PlayNowVote { + q := r.request + if q == nil || r.settings.Permissions.Skip != rooms.Vote { + return nil + } + v := *q + v.Voters, v.Needed = e.count(r, q.Voters, "") + return &v +} + +// count sizes a room's vote: the votes that count, and how many it needs. +// Everyone connected may vote but exclude; votes count even after the +// voter leaves. +func (e *Engine) count(r *room, cast []string, exclude string) (votes []string, needed int) { voters := map[string]bool{} if e.cfg.Presence != nil { for _, id := range e.cfg.Presence.Members(r.id) { voters[id] = true } } - // Votes count even after the voter leaves. - for _, id := range r.votes { + for _, id := range cast { voters[id] = true } - if !r.np.Item.IsAutopilot() { - delete(voters, r.np.Item.AddedBy) - } - votes := append([]string{}, r.votes...) + delete(voters, exclude) + votes = append([]string{}, cast...) // In rooms where guests don't vote, they don't count toward the votes // needed either. if !r.settings.Guests.CanVote() { @@ -1047,10 +1219,7 @@ func (e *Engine) tally(r *room) *rooms.SkipVotes { } } } - return &rooms.SkipVotes{ - Voters: votes, - Needed: rooms.VotesNeeded(len(voters), *r.settings.SkipVotePercent), - } + return votes, rooms.VotesNeeded(len(voters), *r.settings.SkipVotePercent) } // settleVote skips the current song if enough of the room voted to, and @@ -1068,6 +1237,27 @@ func (e *Engine) settleVote(ctx context.Context, r *room) error { return e.next(ctx, r, store.EndSkipped) } +// settlePlayNow plays the requested song if enough of the room agreed, +// and otherwise publishes the tally. The caller holds r.mu. +func (e *Engine) settlePlayNow(ctx context.Context, r *room) error { + v := e.playNowTally(r) + if v == nil { + return nil + } + if len(v.Voters) < v.Needed || r.np.Player == nil { + e.publish(r) + return nil + } + r.request = nil + if err := e.playNow(ctx, r, v.ItemID); err != nil { + return err + } + if it := r.np.Item; len(v.Voters) > 1 && it != nil && it.ID == v.ItemID { + e.notice(r, it.ID, fmt.Sprintf("Playing %s: the room agreed", title(it))) + } + return nil +} + // refresh starts an idle room when songs arrive, and keeps the preloaded // next song current as the order changes. func (e *Engine) refresh(ctx context.Context, r *room) error { @@ -1079,13 +1269,22 @@ func (e *Engine) refresh(ctx context.Context, r *room) error { return err } next := nextOf(snap) - if idOf(next) != idOf(r.np.Next) { + gone := r.request != nil && !queued(snap, r.request.ItemID) + if gone { + r.request = nil + } + if idOf(next) != idOf(r.np.Next) || gone { r.np.Next = next e.publish(r) } return nil } +// queued reports whether itemID is waiting in snap. +func queued(snap rooms.QueueSnapshot, itemID string) bool { + return slices.Contains(snap.UpNext, itemID) +} + func idOf(it *store.QueueItem) string { if it == nil { return "" @@ -1113,17 +1312,35 @@ func (e *Engine) Tick(ctx context.Context) { } func (e *Engine) tick(ctx context.Context, r *room) error { - if !r.loaded || r.np.Item == nil { + if !r.loaded { return nil } now := e.cfg.Now() + if r.request != nil && !now.Before(r.request.Expires) { + r.request = nil + e.publish(r) + } + if r.np.Item == nil { + return nil + } switch r.np.Driver { case string(provider.PlaybackStream): switch r.np.State { case StateLoading: - if now.Sub(r.since) >= e.cfg.LoadTimeout { - return e.failed(ctx, r, fmt.Sprintf("Skipped %s: it took too long to start", title(r.np.Item)), errors.New("load timeout")) + if now.Sub(r.since) < e.cfg.LoadTimeout { + break + } + if p := r.np.Player; p != nil && !r.heard { + // The speaker never answered: it's asleep or gone, and + // skipping would burn the queue on songs that are fine. + if err := e.pause(ctx, r); err != nil { + return err + } + e.notice(r, r.np.Item.ID, fmt.Sprintf("Paused: %s didn't start %s", playerName(p), title(r.np.Item))) + e.publish(r) + return nil } + return e.failed(ctx, r, fmt.Sprintf("Skipped %s: it took too long to start", title(r.np.Item)), errors.New("load timeout")) case StatePlaying: if p := r.np.Player; p != nil && now.Sub(p.LastSeen) >= e.cfg.PlayerTimeout { if err := e.pause(ctx, r); err != nil { diff --git a/server/internal/playback/playback_test.go b/server/internal/playback/playback_test.go index 0898a2b..cfe9c2a 100644 --- a/server/internal/playback/playback_test.go +++ b/server/internal/playback/playback_test.go @@ -431,7 +431,8 @@ func TestFailuresSkip(t *testing.T) { if !playing(playback.StateLoading, "t02")(np) { t.Fatalf("after error: %s", describe(np)) } - // It never starts: skipped after the load timeout. + // The speaker tries but it never starts: skipped after the load timeout. + e.report(e.alice, "phone", playback.EventProgress, 0) e.advance(11 * time.Second) e.p.Tick(t.Context()) e.waitFor("third song", playing(playback.StateLoading, "t03")) @@ -467,6 +468,175 @@ func TestFailuresSkip(t *testing.T) { } } +func TestSilentSpeaker(t *testing.T) { + e := newEnv(t) + notices := e.notices() + e.claim(e.alice, "phone") + e.add(e.alice, "t01", "t02", "t03", "t04") + e.waitFor("first song", playing(playback.StateLoading, "t01")) + + // The speaker never says a word about the song (its tab is asleep, or + // gone): the song isn't to blame, so the room pauses on it rather than + // skipping through the queue. + e.advance(11 * time.Second) + e.p.Tick(t.Context()) + np := e.np() + if !playing(playback.StatePaused, "t01")(np) || np.Position != 0 { + t.Fatalf("after a silent load: %s at %v", describe(np), np.Position) + } + if got := notices(); len(got) != 1 || !strings.Contains(got[0], "alice's phone didn't start “Reference Tone”") { + t.Fatalf("notices: %q", got) + } + // Play tries the same song again, waiting for the speaker to start it. + if np = e.must(playback.Command{Action: playback.ActionPlay}); !playing(playback.StateLoading, "t01")(np) { + t.Fatalf("play after a silent load: %s", describe(np)) + } + if np = e.report(e.alice, "phone", playback.EventPlaying, 0); !playing(playback.StatePlaying, "t01")(np) { + t.Fatalf("speaker woke up: %s", describe(np)) + } +} + +func TestPlayNow(t *testing.T) { + e := newEnv(t) + ctx := t.Context() + e.add(e.alice, "t01") + e.add(e.bob, "t04", "t05") + e.add(e.carol, "t07") + snap, _ := e.rooms.QueueSnapshot(ctx, e.room.ID) + id := func(track string) string { + for _, it := range snap.Items { + if it.TrackID == track { + return it.ID + } + } + t.Fatalf("no %s", track) + return "" + } + + if _, err := e.command(e.bob, playback.Command{Action: playback.ActionPlayNow, ItemID: id("t05")}); !errors.Is(err, playback.ErrNoPlayer) { + t.Fatalf("play now with no speaker: %v", err) + } + // Pressing play on a song with no speaker: this device becomes the + // speaker and plays it, without starting (and skipping) another first. + np, err := e.p.ClaimAndPlay(ctx, e.room.ID, e.carol.ID, "tablet", "carol's tablet", id("t07")) + if err != nil || !playing(playback.StateLoading, "t07")(np) || np.Player == nil || np.Player.DeviceID != "tablet" { + t.Fatalf("claim and play: %s %+v %v", describe(np), np.Player, err) + } + if h, _ := e.db.ListHistory(ctx, store.ListHistoryParams{RoomID: e.room.ID, Limit: 10}); len(h) != 1 { + t.Fatalf("history after claim and play: %+v", h) + } + e.claim(e.alice, "phone") + e.report(e.alice, "phone", playback.EventPlaying, 0) + + // Bob's second song jumps everyone and the current song is skipped. + np, err = e.command(e.bob, playback.Command{Action: playback.ActionPlayNow, ItemID: id("t05")}) + if err != nil || !playing(playback.StateLoading, "t05")(np) { + t.Fatalf("play now: %s %v", describe(np), err) + } + h, _ := e.db.ListHistory(ctx, store.ListHistoryParams{RoomID: e.room.ID, Limit: 10}) + if len(h) != 2 || h[1].PlayHistory.EndReason.String != store.EndSkipped { + t.Fatalf("history: %+v", h) + } + // Only queued songs can be played now. + if _, err := e.command(e.bob, playback.Command{Action: playback.ActionPlayNow, ItemID: id("t07")}); err == nil { + t.Error("played a song that already played") + } + if _, err := e.command(e.bob, playback.Command{Action: playback.ActionPlayNow}); err == nil { + t.Error("play now with no item") + } + + // It takes the skip permission. + owner := rooms.Update{Permissions: rooms.Permissions{Skip: rooms.Owner}} + if _, err := e.rooms.Update(ctx, rooms.Actor{UserID: e.alice.ID}, e.room.ID, owner); err != nil { + t.Fatal(err) + } + if _, err := e.command(e.carol, playback.Command{Action: playback.ActionPlayNow, ItemID: id("t04")}); !errors.Is(err, playback.ErrForbidden) { + t.Errorf("carol in an owner-skips room: %v", err) + } + if np := e.must(playback.Command{Action: playback.ActionPlayNow, ItemID: id("t04")}); !playing(playback.StateLoading, "t04")(np) { + t.Errorf("the owner: %s", describe(np)) + } +} + +func TestPlayNowVote(t *testing.T) { + e := newEnv(t) + ctx := t.Context() + vote := rooms.Update{Permissions: rooms.Permissions{Skip: rooms.Vote}} + if _, err := e.rooms.Update(ctx, rooms.Actor{UserID: e.alice.ID}, e.room.ID, vote); err != nil { + t.Fatal(err) + } + for _, m := range []member{e.alice, e.bob, e.carol} { + e.presence.Join(e.room.ID, m.ID) + } + notices := e.notices() + e.claim(e.alice, "phone") + e.add(e.alice, "t01") + e.add(e.bob, "t04", "t05") + e.waitFor("alice's song", playing(playback.StateLoading, "t01")) + e.waitFor("bob's song to be next", func(np rooms.NowPlaying) bool { return np.Next != nil && np.Next.TrackID == "t04" }) + snap, _ := e.rooms.QueueSnapshot(ctx, e.room.ID) + var t05 string + for _, it := range snap.Items { + if it.TrackID == "t05" { + t05 = it.ID + } + } + playNow := playback.Command{Action: playback.ActionPlayNow, ItemID: t05} + + // Bob asks; the room has to agree (2 of 3). + np, err := e.command(e.bob, playNow) + if err != nil || !playing(playback.StateLoading, "t01")(np) { + t.Fatalf("asking: %s %v", describe(np), err) + } + if v := np.PlayNow; v == nil || v.ItemID != t05 || v.By != e.bob.ID || len(v.Voters) != 1 || v.Needed != 2 { + t.Fatalf("request: %+v", np.PlayNow) + } + if got := notices(); len(got) != 1 || got[0] != "bob wants to play “Pluge” now" { + t.Fatalf("notices: %q", got) + } + // Only one request at a time. + if _, err := e.command(e.carol, playback.Command{Action: playback.ActionPlayNow, ItemID: np.Next.ID}); err == nil { + t.Error("a second request while one is open") + } + // Carol agrees and agreeing passes it. + np, err = e.command(e.carol, playback.Command{Action: playback.ActionVotePlayNow, ItemID: t05}) + if err != nil || !playing(playback.StateLoading, "t05")(np) || np.PlayNow != nil { + t.Fatalf("after carol agreed: %s %+v %v", describe(np), np.PlayNow, err) + } + if got := notices(); len(got) != 2 || !strings.Contains(got[1], "the room agreed") { + t.Fatalf("notices: %q", got) + } + + // Asking again, then taking it back, ends the request. + e.add(e.carol, "t07") + snap, _ = e.rooms.QueueSnapshot(ctx, e.room.ID) + var t07 string + for _, it := range snap.Items { + if it.TrackID == "t07" { + t07 = it.ID + } + } + if np, _ = e.command(e.carol, playback.Command{Action: playback.ActionPlayNow, ItemID: t07}); np.PlayNow == nil { + t.Fatal("no request") + } + if np, _ = e.command(e.carol, playback.Command{Action: playback.ActionUnvotePlayNow, ItemID: t07}); np.PlayNow != nil { + t.Fatalf("withdrawn request still open: %+v", np.PlayNow) + } + // A request lapses. + if _, err := e.command(e.carol, playback.Command{Action: playback.ActionPlayNow, ItemID: t07}); err != nil { + t.Fatal(err) + } + e.advance(playback.PlayNowTimeout) + e.p.Tick(ctx) + if np = e.np(); np.PlayNow != nil { + t.Fatalf("lapsed request still open: %+v", np.PlayNow) + } + // The owner just plays it. + if np = e.must(playback.Command{Action: playback.ActionPlayNow, ItemID: t07}); !playing(playback.StateLoading, "t07")(np) { + t.Fatalf("the owner: %s", describe(np)) + } +} + func TestUnlinkedSongSkipped(t *testing.T) { e := newEnv(t) ctx := t.Context() @@ -571,8 +741,9 @@ func TestRestart(t *testing.T) { if r, _ := e.db.GetRoom(t.Context(), e.room.ID); r.PlayerDeviceID.Valid { t.Errorf("stale player kept: %+v", r.PlayerDeviceID) } + // The speaker hasn't started it since: play waits for it to. e.claim(e.alice, "phone") - if np, err := e.command(e.alice, playback.Command{Action: playback.ActionPlay}); err != nil || !playing(playback.StatePlaying, "t01")(np) { + if np, err := e.command(e.alice, playback.Command{Action: playback.ActionPlay}); err != nil || !playing(playback.StateLoading, "t01")(np) { t.Fatalf("resume: %s %v", describe(np), err) } } diff --git a/server/internal/rooms/rooms.go b/server/internal/rooms/rooms.go index 062f961..fd40068 100644 --- a/server/internal/rooms/rooms.go +++ b/server/internal/rooms/rooms.go @@ -109,7 +109,65 @@ type Settings struct { Guests Guests `json:"guests"` // ApproveJoins: while the room is private, someone using one of its // invites asks to join, and the owner lets them in. - ApproveJoins bool `json:"approveJoins,omitempty"` + ApproveJoins bool `json:"approveJoins,omitempty"` + Screens Screens `json:"screens"` +} + +// Screens is how the room's big screens show the music (MAD-779). +type Screens struct { + // Look is LookStage, LookVisualizer or LookAuto. Empty means LookAuto. + Look string `json:"look,omitempty"` + // Scene is the visualizer's scene, or "" to pick one per song. Screens + // that don't know it pick one per song too. + Scene string `json:"scene,omitempty"` + // Intensity is how strongly the screens move with the music, + // MinIntensity to MaxIntensity. Nil means DefaultIntensity. + Intensity *float64 `json:"intensity,omitempty"` +} + +// Big screen looks: now playing and lyrics, full-screen visuals, or the +// stage with visuals taking over for songs with no words to sing. +const ( + LookStage = "stage" + LookVisualizer = "visualizer" + LookAuto = "auto" +) + +// Big screen intensity. +const ( + MinIntensity = 0.5 + MaxIntensity = 2.0 + DefaultIntensity = 1.5 + maxSceneLen = 32 +) + +// Normal fills in the defaults. +func (s Screens) Normal() Screens { + if s.Look != LookStage && s.Look != LookVisualizer { + s.Look = LookAuto + } + if s.Intensity == nil || *s.Intensity < MinIntensity || *s.Intensity > MaxIntensity { + s.Intensity = ptr(DefaultIntensity) + } + if len(s.Scene) > maxSceneLen { + s.Scene = "" + } + return s +} + +func (s Screens) validate() error { + switch s.Look { + case "", LookStage, LookVisualizer, LookAuto: + default: + return &InvalidInputError{"a big screen's look is stage, visualizer or auto"} + } + if s.Intensity != nil && (*s.Intensity < MinIntensity || *s.Intensity > MaxIntensity) { + return &InvalidInputError{fmt.Sprintf("big screen intensity is %v to %v", MinIntensity, MaxIntensity)} + } + if len(s.Scene) > maxSceneLen { + return &InvalidInputError{"that scene name is too long"} + } + return nil } // Guests says whether people without an account may join the room with a @@ -310,6 +368,7 @@ func ParseSettings(raw string) Settings { if m := st.Guests.MaxSongs; m != nil && (*m < 0 || *m > MaxGuestSongs) { st.Guests.MaxSongs = nil } + st.Screens = st.Screens.Normal() return st } @@ -386,6 +445,8 @@ type Update struct { // ApproveJoins, if set, is whether the owner lets in each person who // uses an invite to the room while it's private. ApproveJoins *bool + // Screens, if set, replaces how the room's big screens look. + Screens *Screens } // Update changes a room. Only its owner or an admin may. Everyone in the @@ -437,6 +498,9 @@ func (s *Service) Update(ctx context.Context, by Actor, id string, u Update) (st if u.ApproveJoins != nil { st.ApproveJoins = *u.ApproveJoins } + if u.Screens != nil { + st.Screens = *u.Screens + } name, raw, err := validate(name, mode, st) if err != nil { return r, err @@ -597,6 +661,9 @@ func validate(name, mode string, st Settings) (string, string, error) { if m := st.Guests.MaxSongs; m != nil && (*m < 0 || *m > MaxGuestSongs) { return "", "", &InvalidInputError{fmt.Sprintf("a guest may add 0 (no limit) to %d songs", MaxGuestSongs)} } + if err := st.Screens.validate(); err != nil { + return "", "", err + } st.Fairness.clean() st.Version, st.Controls = SettingsVersion, "" raw, err := json.Marshal(st) @@ -639,6 +706,23 @@ type NowPlaying struct { Next *store.QueueItem // SkipVotes is the vote to skip Item, while the room votes on skips. SkipVotes *SkipVotes + // PlayNow is someone's request to play a queued song now, while the + // room votes on it. + PlayNow *PlayNowVote +} + +// PlayNowVote is a request to play a queued song straight away, in a +// room that votes on skips. It passes on the same share of the room. +type PlayNowVote struct { + ItemID string + // By is who asked. Their vote is the first. + By string + // Voters are the IDs of users who agreed, in the order they did. + Voters []string + // Needed is how many agreeing play the song, given who's in the room. + Needed int + // Expires is when the request lapses if the room hasn't agreed. + Expires time.Time } // SkipVotes is a vote to skip the playing song. diff --git a/server/internal/store/beat_maps.sql.gen.go b/server/internal/store/beat_maps.sql.gen.go new file mode 100644 index 0000000..b6256d7 --- /dev/null +++ b/server/internal/store/beat_maps.sql.gen.go @@ -0,0 +1,109 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: beat_maps.sql + +package store + +import ( + "context" + "time" +) + +const getBeatMap = `-- name: GetBeatMap :one +SELECT provider, track_id, version, found, map, analyzed_at, used_at FROM beat_maps WHERE provider = ? AND track_id = ? +` + +type GetBeatMapParams struct { + Provider string + TrackID string +} + +func (q *Queries) GetBeatMap(ctx context.Context, arg GetBeatMapParams) (BeatMap, error) { + row := q.db.QueryRowContext(ctx, getBeatMap, arg.Provider, arg.TrackID) + var i BeatMap + err := row.Scan( + &i.Provider, + &i.TrackID, + &i.Version, + &i.Found, + &i.Map, + &i.AnalyzedAt, + &i.UsedAt, + ) + return i, err +} + +const putBeatMap = `-- name: PutBeatMap :exec +INSERT INTO beat_maps (provider, track_id, version, found, map, analyzed_at, used_at) +VALUES (?, ?, ?, ?, ?, ?, ?) +ON CONFLICT (provider, track_id) DO UPDATE SET + version = excluded.version, found = excluded.found, map = excluded.map, + analyzed_at = excluded.analyzed_at, used_at = excluded.used_at +` + +type PutBeatMapParams struct { + Provider string + TrackID string + Version int64 + Found bool + Map string + AnalyzedAt time.Time + UsedAt time.Time +} + +func (q *Queries) PutBeatMap(ctx context.Context, arg PutBeatMapParams) error { + _, err := q.db.ExecContext(ctx, putBeatMap, + arg.Provider, + arg.TrackID, + arg.Version, + arg.Found, + arg.Map, + arg.AnalyzedAt, + arg.UsedAt, + ) + return err +} + +const sweepBeatMaps = `-- name: SweepBeatMaps :execrows +DELETE FROM beat_maps +WHERE used_at < ?1 OR (NOT found AND analyzed_at < ?2) +` + +type SweepBeatMapsParams struct { + UnusedSince time.Time + MissedBefore time.Time +} + +// SweepBeatMaps forgets maps nobody has played in a long while, and +// misses old enough to try again. +func (q *Queries) SweepBeatMaps(ctx context.Context, arg SweepBeatMapsParams) (int64, error) { + result, err := q.db.ExecContext(ctx, sweepBeatMaps, arg.UnusedSince, arg.MissedBefore) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +const touchBeatMap = `-- name: TouchBeatMap :exec +UPDATE beat_maps SET used_at = ?1 +WHERE provider = ?2 AND track_id = ?3 AND used_at < ?4 +` + +type TouchBeatMapParams struct { + Now time.Time + Provider string + TrackID string + Since time.Time +} + +// TouchBeatMap marks a map as used, at most once a day. +func (q *Queries) TouchBeatMap(ctx context.Context, arg TouchBeatMapParams) error { + _, err := q.db.ExecContext(ctx, touchBeatMap, + arg.Now, + arg.Provider, + arg.TrackID, + arg.Since, + ) + return err +} diff --git a/server/internal/store/migrations/00023_beat_maps.sql b/server/internal/store/migrations/00023_beat_maps.sql new file mode 100644 index 0000000..cc53dcc --- /dev/null +++ b/server/internal/store/migrations/00023_beat_maps.sql @@ -0,0 +1,22 @@ +-- Songs' beat maps (MAD-773): tempo, beats, sections and the spectrum +-- from moment to moment, worked out once from the audio so every screen +-- in a room can move with the music. Songs that can't be analysed are +-- kept too, with found false, so they aren't tried on every play. + +-- +goose Up +CREATE TABLE beat_maps ( + provider TEXT NOT NULL, + track_id TEXT NOT NULL, + -- analysis.Version that made it; older maps are worked out again. + version INTEGER NOT NULL, + found BOOLEAN NOT NULL, + -- analysis.Map as JSON; '{}' when not found. + map TEXT NOT NULL, + analyzed_at TIMESTAMP NOT NULL, + -- When a room last asked for it; maps nobody plays are swept. + used_at TIMESTAMP NOT NULL, + PRIMARY KEY (provider, track_id) +); + +-- +goose Down +DROP TABLE beat_maps; diff --git a/server/internal/store/models.gen.go b/server/internal/store/models.gen.go index 6315ce6..561b6d1 100644 --- a/server/internal/store/models.gen.go +++ b/server/internal/store/models.gen.go @@ -16,6 +16,16 @@ type Avatar struct { UpdatedAt time.Time } +type BeatMap struct { + Provider string + TrackID string + Version int64 + Found bool + Map string + AnalyzedAt time.Time + UsedAt time.Time +} + type CredentialsPasskey struct { ID []byte UserID string diff --git a/server/internal/store/queries/beat_maps.sql b/server/internal/store/queries/beat_maps.sql new file mode 100644 index 0000000..f095514 --- /dev/null +++ b/server/internal/store/queries/beat_maps.sql @@ -0,0 +1,20 @@ +-- name: GetBeatMap :one +SELECT * FROM beat_maps WHERE provider = ? AND track_id = ?; + +-- name: PutBeatMap :exec +INSERT INTO beat_maps (provider, track_id, version, found, map, analyzed_at, used_at) +VALUES (?, ?, ?, ?, ?, ?, ?) +ON CONFLICT (provider, track_id) DO UPDATE SET + version = excluded.version, found = excluded.found, map = excluded.map, + analyzed_at = excluded.analyzed_at, used_at = excluded.used_at; + +-- TouchBeatMap marks a map as used, at most once a day. +-- name: TouchBeatMap :exec +UPDATE beat_maps SET used_at = sqlc.arg(now) +WHERE provider = sqlc.arg(provider) AND track_id = sqlc.arg(track_id) AND used_at < sqlc.arg(since); + +-- SweepBeatMaps forgets maps nobody has played in a long while, and +-- misses old enough to try again. +-- name: SweepBeatMaps :execrows +DELETE FROM beat_maps +WHERE used_at < sqlc.arg(unused_since) OR (NOT found AND analyzed_at < sqlc.arg(missed_before)); diff --git a/web/src/api/schema.gen.ts b/web/src/api/schema.gen.ts index 1ac36dc..9cb61d1 100644 --- a/web/src/api/schema.gen.ts +++ b/web/src/api/schema.gen.ts @@ -1656,6 +1656,35 @@ export interface paths { patch?: never; trace?: never; }; + "/rooms/{roomId}/queue/{itemId}/beatmap": { + parameters: { + query?: never; + header?: never; + path: { + roomId: components["parameters"]["RoomId"]; + itemId: string; + }; + cookie?: never; + }; + /** + * A queued song's beat map + * @description The song's tempo, beats, bars and sections, and its spectrum from + * moment to moment, for visuals that move with the music on every + * screen in the room. Worked out from the audio once, usually soon + * after the song is queued; if it hasn't been yet, it's worked out + * now, which can take a few seconds. 404 `no_beat_map` if the song + * can't have one (its service has no audio, or it couldn't be + * decoded). + */ + get: operations["getQueueItemBeatMap"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/rooms/{roomId}/queue/{itemId}/lyrics": { parameters: { query?: never; @@ -2271,7 +2300,9 @@ export interface paths { * else sends `vote_skip` (and `unvote_skip` to take it back); the * song is skipped once `skipVotes.needed` have voted. * `play` needs a speaker (`no_player`); pausing, skipping, seeking or - * voting with nothing playing is `nothing_playing`. + * voting with nothing playing is `nothing_playing`. A display with + * `audio` on may `play`, `pause` and `skip` in its own room, as + * whoever paired it. */ post: operations["controlPlayback"]; delete?: never; @@ -2453,6 +2484,30 @@ export interface components { */ ended: boolean; }; + /** + * @description How the room's big screens show the music (MAD-779). Set from any + * phone by whoever manages the room; every screen follows. + */ + RoomScreens: { + /** + * @description `stage`: now playing, lyrics and what's next. `visualizer`: + * full-screen visuals with now playing small. `auto`: the stage, + * with visuals taking over for songs with no words to sing. + * @default auto + * @enum {string} + */ + look: "stage" | "visualizer" | "auto"; + /** + * @description The visualizer's scene, or "" to pick one that suits each song. + * @default + */ + scene: string; + /** + * @description How strongly the screens move with the music. + * @default 1.5 + */ + intensity: number; + }; /** * @description Whether people without an account may join the room by scanning a * guest pass, and what they may do. @@ -3372,6 +3427,56 @@ export interface components { dark: components["schemas"]["OklchColor"]; light: components["schemas"]["OklchColor"]; }; + /** + * @description A song's beat map (MAD-773). Times are in ms from the start of the + * song, on the same clock as the playback position. + */ + BeatMap: { + /** Format: int64 */ + durationMs: number; + /** @description The tempo; 0 if the song has no steady beat, and then `beats` is empty. */ + bpm: number; + /** @description How clear the beat is, 0–1. */ + confidence: number; + /** @description The beats. */ + beats: number[]; + /** @description The index in `beats` of the first bar's first beat; every fourth beat after it starts a bar. */ + downbeat: number; + /** @description The song's parts (intro, verse, drop), in order; the first starts at 0. */ + sections: components["schemas"]["BeatMapSection"][]; + /** @description Frames a second in `bands` and `loudness`. */ + frameRate: number; + /** @description Spectrum bands per frame in `bands`. */ + bandCount: number; + /** + * Format: byte + * @description Base64: each frame's level in each band, low to high, a byte + * (0–255) each, `bandCount` bytes a frame. Each band is scaled to + * its own range in the song. + */ + bands: string; + /** + * Format: byte + * @description Base64: each frame's overall level, a byte (0–255) each, scaled to the song. + */ + loudness: string; + features: components["schemas"]["BeatMapFeatures"]; + }; + BeatMapSection: { + /** Format: int64 */ + startMs: number; + /** @description How loud it is next to the rest of the song, 0–1. */ + energy: number; + }; + /** @description The song as a whole, 0–1 each, for choosing how it looks. */ + BeatMapFeatures: { + /** @description How loud it's mastered, from a quiet acoustic recording to a club track. */ + energy: number; + /** @description How much treble there is. */ + brightness: number; + /** @description How much its loudness changes. */ + dynamics: number; + }; Accent: { /** * Format: double @@ -3515,6 +3620,24 @@ export interface components { player?: components["schemas"]["Player"]; next?: components["schemas"]["QueueItem"]; skipVotes?: components["schemas"]["SkipVotes"]; + playNow?: components["schemas"]["PlayNowVote"]; + }; + /** + * @description Someone's request to play a queued song now, in a room whose skip + * permission is `vote`. It passes on the same share of the room as a + * skip, and lapses at `expires`, when the song leaves the queue, or + * when whoever asked takes it back. + */ + PlayNowVote: { + itemId: string; + /** @description Who asked. Their vote is the first. */ + by: string; + /** @description IDs of the users who agreed, in order. */ + voters: string[]; + /** @description How many agreeing play the song. */ + needed: number; + /** Format: date-time */ + expires: string; }; /** * @description The vote to skip the playing song, while the room's skip permission @@ -3542,8 +3665,15 @@ export interface components { lastSeen: string; }; PlaybackCommand: { - /** @enum {string} */ - action: "play" | "pause" | "skip" | "seek" | "vote_skip" | "unvote_skip"; + /** + * @description `play_now` plays the queued song `itemId` straight away, skipping + * the one playing. It takes the skip permission; in a room that + * votes on skips, it asks the room instead (`playNow`), and + * `vote_play_now` agrees. `unvote_play_now` takes a vote back, or + * withdraws the request from whoever asked. + * @enum {string} + */ + action: "play" | "pause" | "skip" | "seek" | "vote_skip" | "unvote_skip" | "play_now" | "vote_play_now" | "unvote_play_now"; /** * Format: int64 * @description Where to seek to. @@ -3551,7 +3681,8 @@ export interface components { positionMs?: number; /** * @description Skip (or vote) only if this is still the current song, so two - * people tapping skip skip one song. + * people tapping skip skip one song. For `play_now` and its votes, + * the queued song to play. */ itemId?: string; }; @@ -3560,6 +3691,13 @@ export interface components { deviceId: string; /** @example Collin's phone */ name: string; + /** + * @description A queued song to play now, as `play_now` would, instead of + * starting the next in fair order: pressing play on a song with no + * speaker. Ignored if the caller would have to ask the room, or + * the song isn't waiting. + */ + playItemId?: string; }; PlayerReport: { deviceId: string; @@ -3765,6 +3903,7 @@ export interface components { matching: components["schemas"]["RoomMatching"]; autopilot: components["schemas"]["RoomAutopilot"]; guests: components["schemas"]["RoomGuests"]; + screens: components["schemas"]["RoomScreens"]; permissions: components["schemas"]["RoomPermissions"]; skipVotePercent: components["schemas"]["SkipVotePercent"]; /** Format: date-time */ @@ -3779,6 +3918,7 @@ export interface components { matching?: components["schemas"]["RoomMatching"]; autopilot?: components["schemas"]["RoomAutopilot"]; guests?: components["schemas"]["RoomGuests"]; + screens?: components["schemas"]["RoomScreens"]; permissions?: components["schemas"]["RoomPermissionsChange"]; skipVotePercent?: components["schemas"]["SkipVotePercent"]; }; @@ -3795,6 +3935,7 @@ export interface components { matching?: components["schemas"]["RoomMatching"]; autopilot?: components["schemas"]["RoomAutopilot"]; guests?: components["schemas"]["RoomGuests"]; + screens?: components["schemas"]["RoomScreens"]; permissions?: components["schemas"]["RoomPermissionsChange"]; skipVotePercent?: components["schemas"]["SkipVotePercent"]; }; @@ -5862,6 +6003,30 @@ export interface operations { default: components["responses"]["Error"]; }; }; + getQueueItemBeatMap: { + parameters: { + query?: never; + header?: never; + path: { + roomId: components["parameters"]["RoomId"]; + itemId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description The beat map */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BeatMap"]; + }; + }; + default: components["responses"]["Error"]; + }; + }; getQueueItemLyrics: { parameters: { query?: never; diff --git a/web/src/components/equalizer.tsx b/web/src/components/equalizer.tsx index 0e03c57..c378b55 100644 --- a/web/src/components/equalizer.tsx +++ b/web/src/components/equalizer.tsx @@ -1,3 +1,4 @@ +import { useBeat } from '@/hooks/use-beat' import { cn } from '@/lib/utils' // Each bar's period and offset differ, so the pattern never visibly repeats. @@ -8,10 +9,14 @@ const bars = [ { duration: '0.95s', delay: '-0.6s' }, ] -/** Bouncing bars: audio is playing here. Settles low while paused. */ +/** + * Bouncing bars: audio is playing here. Settles low while paused. While the + * beat engine is live, each bar follows one of its bands instead. + */ export function Equalizer({ playing, className }: { playing: boolean; className?: string }) { + const beat = useBeat() return ( - + {bars.map((b, i) => ( positionAt(np, Math.max(Date.now(), np.at)) - offset const current = useActiveLine(lines, np, position) const scroller = useRef(null) + // The current line's glow follows the beat. + const beat = useBeat() + const scrollerRef = useCallback( + (el: HTMLDivElement | null) => { + scroller.current = el + return beat(el) + }, + [beat], + ) const lineRefs = useRef<(HTMLElement | null)[]>([]) const reduced = useReducedMotion() // Reading ahead pauses following along for a few seconds. @@ -109,7 +119,7 @@ function SyncedLyrics({ return (
{line.text} diff --git a/web/src/components/room-live.tsx b/web/src/components/room-live.tsx index 9a201cf..c7a66c6 100644 --- a/web/src/components/room-live.tsx +++ b/web/src/components/room-live.tsx @@ -19,7 +19,7 @@ export function RoomLive() { useRoomSocket(room?.id) const playback = useQuery({ ...playbackQuery(room?.id ?? ''), enabled: !!room }) const users = useQuery(usersQuery) - const commands = useRoomControls(room, playback.data, me.id, !!me.guest) + const commands = useRoomControls(room, playback.data, me, !!me.guest) const queryClient = useQueryClient() // Player mode: whatever the server says, the speaker (if this device is @@ -32,8 +32,12 @@ export function RoomLive() { useEffect(() => { if (speaker.active && rooms.isSuccess && speakerRoom() !== room?.id) void speaker.stop() }, [room, rooms.isSuccess]) - // Signing out stops the speaker. - useEffect(() => () => void speaker.stop(), []) + // Signing out stops the speaker. Going to the big screen hands it over + // instead, and coming back keeps whatever it's playing. + useEffect(() => () => speaker.stopSoon(), []) + useEffect(() => { + if (room) speaker.keep(room.id) + }, [room]) // On the next frame, so a burst of updates (a phone waking with songs' // worth of events queued up) shows only the last, and none while hidden. diff --git a/web/src/components/room/big-screen-dialog.tsx b/web/src/components/room/big-screen-dialog.tsx index 8bcea34..7670d0a 100644 --- a/web/src/components/room/big-screen-dialog.tsx +++ b/web/src/components/room/big-screen-dialog.tsx @@ -8,6 +8,7 @@ import { errorMessage } from '@/api/errors' import { Button } from '@/components/ui/button' import { Input } from '@/components/ui/input' import { Switch } from '@/components/ui/switch' +import { BigScreenLook } from './big-screen-look' import { useMe } from '@/lib/auth' import { displaysQuery, pairDisplay, setDisplayAudio, unpairDisplay } from '@/lib/displays' import { easeOutExpo } from '@/lib/motion' @@ -132,6 +133,8 @@ export function BigScreenDialog({ room, open, onOpenChange }: { room: Room; open Or show it on this device + + {(displays.data?.length ?? 0) > 0 && (

Paired screens

diff --git a/web/src/components/room/big-screen-look.tsx b/web/src/components/room/big-screen-look.tsx new file mode 100644 index 0000000..b250613 --- /dev/null +++ b/web/src/components/room/big-screen-look.tsx @@ -0,0 +1,98 @@ +import { useMutation, useQueryClient } from '@tanstack/react-query' +import { useState } from 'react' +import { api } from '@/api/client' +import { errorMessage, unwrap } from '@/api/errors' +import { Button } from '@/components/ui/button' +import { Slider } from '@/components/ui/slider' +import { ToggleGroup, ToggleGroupItem } from '@/components/ui/toggle-group' +import { SHOWS } from '@/hooks/use-scene' +import { roomsQuery, type Room } from '@/lib/room' +import { toast } from '@/lib/toast' + +type Screens = Room['screens'] + +const looks: { value: Screens['look']; label: string; hint: string }[] = [ + { value: 'auto', label: 'Auto', hint: 'Lyrics and what’s next, with the visualizer for songs with no words.' }, + { value: 'stage', label: 'Stage', hint: 'Now playing, lyrics and what’s next.' }, + { value: 'visualizer', label: 'Visualizer', hint: 'Full-screen visuals, with what’s playing in a corner.' }, +] + +/** + * How the room's big screens show the music (MAD-779). Every screen + * follows; whoever manages the room changes it. + */ +export function BigScreenLook({ room, editable }: { room: Room; editable: boolean }) { + const queryClient = useQueryClient() + const [intensity, setIntensity] = useState() + const update = useMutation({ + mutationFn: (screens: Screens) => + unwrap(api.PATCH('/rooms/{roomId}', { params: { path: { roomId: room.id } }, body: { screens } })), + onMutate: (screens) => { + const before = queryClient.getQueryData(roomsQuery.queryKey) + queryClient.setQueryData(roomsQuery.queryKey, (rs) => rs?.map((r) => (r.id === room.id ? { ...r, screens } : r))) + return before + }, + onError: (e, _, before) => { + queryClient.setQueryData(roomsQuery.queryKey, before) + toast({ message: errorMessage(e), tone: 'error' }) + }, + onSettled: () => setIntensity(undefined), + }) + const s = room.screens + const set = (next: Partial) => update.mutate({ ...s, ...next }) + const shown = intensity ?? s.intensity + + return ( +
+

How screens look

+ v && set({ look: v as Screens['look'] })} + className="grid w-full grid-cols-3" + > + {looks.map((l) => ( + + {l.label} + + ))} + +

{looks.find((l) => l.value === s.look)?.hint}

+ +
+ Visualizer +
+ {['', ...SHOWS].map((show) => ( + + ))} +
+
+ + + {!editable &&

Whoever runs the room can change these.

} +
+ ) +} diff --git a/web/src/components/room/play-now-request.tsx b/web/src/components/room/play-now-request.tsx new file mode 100644 index 0000000..caa8800 --- /dev/null +++ b/web/src/components/room/play-now-request.tsx @@ -0,0 +1,93 @@ +import { useQuery } from '@tanstack/react-query' +import { Check, X } from 'lucide-react' +import { AnimatePresence, motion } from 'motion/react' +import { useState } from 'react' +import { Artwork } from '@/components/artwork' +import { Button } from '@/components/ui/button' +import { usePlaybackCommand } from '@/hooks/use-play-now' +import { useMe } from '@/lib/auth' +import { spring } from '@/lib/motion' +import { playbackQuery, queueArtworkUrl } from '@/lib/playback' +import { queueQuery, useCurrentRoom } from '@/lib/room' +import { usersQuery } from '@/lib/users' + +/** + * Someone asked to play a song now, in a room that votes on skips: the + * rest of the room says yes here, on any screen. Whoever asked sees the + * tally and can take it back. + */ +export function PlayNowRequest() { + const me = useMe() + const { room } = useCurrentRoom() + const roomId = room?.id ?? '' + const playback = useQuery({ ...playbackQuery(roomId), enabled: false }) + const queue = useQuery({ ...queueQuery(roomId), enabled: false }) + const users = useQuery(usersQuery) + const command = usePlaybackCommand(roomId) + // "Not now" hides a request here; it lapses on its own. + const [hidden, setHidden] = useState() + + const req = room ? playback.data?.playNow : undefined + const item = req && queue.data?.items.find((i) => i.id === req.itemId) + const mine = req?.by === me.id + const voted = !!req?.voters.includes(me.id) + const mayVote = !me.guest || !!room?.guests.canVote + const show = req && item && mayVote && hidden !== `${req.itemId}:${req.by}` + const by = users.data?.find((u) => u.id === req?.by)?.displayName ?? 'Someone' + + return ( + + {show && ( + + +
+

+ {mine ? 'You asked to play ' : `${by} wants to play `} + {item.track.title} +

+

+ {req.voters.length} of {req.needed} agree +

+
+ {mine ? ( + + ) : ( + <> + + + + )} + + )} + + ) +} diff --git a/web/src/components/room/queue-row.tsx b/web/src/components/room/queue-row.tsx index 37c9daf..a7c8142 100644 --- a/web/src/components/room/queue-row.tsx +++ b/web/src/components/room/queue-row.tsx @@ -1,4 +1,5 @@ -import type { ReactNode } from 'react' +import { Play } from 'lucide-react' +import { useEffect, useRef, useState, type ReactNode } from 'react' import { Artwork } from '@/components/artwork' import { ServiceGlyph } from '@/components/service-tag' import { AutopilotMark } from '@/components/room/autopilot-badge' @@ -22,6 +23,8 @@ export function QueueRow({ mine, hideAvatar, byline, + onPlay, + playLabel = 'Play now', className, }: { roomId: string @@ -35,6 +38,9 @@ export function QueueRow({ hideAvatar?: boolean /** Name who added it under the title, for lists without lane context. */ byline?: boolean + /** Makes the artwork a play button: hover (or tap once on touch) shows it. */ + onPlay?: () => void + playLabel?: string className?: string }) { const autopilot = item.autopilot @@ -54,7 +60,13 @@ export function QueueRow({ )} > {leading} - + {onPlay ? ( + + + + ) : ( + + )}

{item.track.title}

@@ -82,3 +94,50 @@ export function QueueRow({

) } + +// How long a first tap on touch keeps the play button up. +const ARMED_FOR = 3000 + +/** + * Artwork that plays its song. With a mouse, hovering shows the button. On + * touch there's no hover, so the first tap shows it and a second plays: + * one stray tap shouldn't skip the song the room is listening to. + */ +function PlayArt({ label, onPlay, children }: { label: string; onPlay: () => void; children: ReactNode }) { + const [armed, setArmed] = useState(false) + const pointer = useRef('mouse') + useEffect(() => { + if (!armed) return + const t = window.setTimeout(() => setArmed(false), ARMED_FOR) + return () => window.clearTimeout(t) + }, [armed]) + return ( + + ) +} diff --git a/web/src/components/room/speaker-panel.tsx b/web/src/components/room/speaker-panel.tsx index 34c8c64..20b6625 100644 --- a/web/src/components/room/speaker-panel.tsx +++ b/web/src/components/room/speaker-panel.tsx @@ -5,9 +5,9 @@ import { Equalizer } from '@/components/equalizer' import { Switch } from '@/components/ui/switch' import { Button } from '@/components/ui/button' import { useMe } from '@/lib/auth' -import { can, playbackQuery } from '@/lib/playback' +import { can, playbackQuery, sendCommand } from '@/lib/playback' import type { Room } from '@/lib/room' -import { deviceId, speaker, speakerState } from '@/lib/speaker' +import { deviceId, speaker, speakerName, speakerState } from '@/lib/speaker' import { useStore } from '@/lib/store' import { cn } from '@/lib/utils' import { deviceName } from '@/lib/webauthn' @@ -29,11 +29,23 @@ export function SpeakerPanel({ room, prominent }: { room: Room; prominent?: bool // Guests are in the room with the speaker; listening along is for members. const mayListen = !me.guest const thisDevice = deviceNoun() - const name = `${me.displayName.split(' ')[0]}'s ${deviceName()}` + const name = speakerName(me.displayName) const start = () => { setStarting(true) - void speaker.start(room.id, name).finally(() => setStarting(false)) + const takingOver = !!other + void speaker + .start(room.id, name) + .then(async (np) => { + // "Play on this phone" means play: start a paused room too. Taking + // over from another speaker leaves it as it was. + if (!np || takingOver || np.state !== 'paused' || !can(room, me.id, 'playPause')) return + speaker.onState?.(await sendCommand(room.id, { action: 'play' })) + }) + .catch(() => { + // The speaker is set up either way; play is a tap away. + }) + .finally(() => setStarting(false)) } const listen = () => speaker.listen(room.id, name, playback.data, allowed) @@ -141,13 +153,13 @@ function speakerHint(status: string) { return 'Leave this open; it keeps playing locked' } -function listenerHint(status: string, speakerName?: string, roomState?: string) { +function listenerHint(status: string, speakerWho?: string, roomState?: string) { if (status === 'remote') return 'This song plays on its own service' if (status === 'blocked') return 'The browser needs a tap first' if (status === 'paused') return roomState === 'playing' ? 'Paused here; the room plays on' : 'The room is paused' - if (!speakerName) return 'Waiting for a speaker' - if (status === 'waiting') return `Waiting for ${speakerName}` - return `In time with ${speakerName}` + if (!speakerWho) return 'Waiting for a speaker' + if (status === 'waiting') return `Waiting for ${speakerWho}` + return `In time with ${speakerWho}` } /** What to call this device in "play on this …". */ diff --git a/web/src/components/room/up-next.tsx b/web/src/components/room/up-next.tsx index 0531cf5..ef3d8c2 100644 --- a/web/src/components/room/up-next.tsx +++ b/web/src/components/room/up-next.tsx @@ -1,6 +1,7 @@ import { GripVertical } from 'lucide-react' import { AnimatePresence, Reorder, useDragControls } from 'motion/react' import { useState } from 'react' +import { usePlayNow, type PlayNow } from '@/hooks/use-play-now' import { useQueueMove } from '@/hooks/use-queue-move' import { isMine } from '@/lib/autopilot' import { spring } from '@/lib/motion' @@ -42,6 +43,7 @@ export function UpNext({ const mineIds = new Set(items.filter((i) => isMine(i, me) && i.state === 'queued').map((i) => i.id)) const move = useQueueMove(roomId) + const playNow = usePlayNow(roomId) const dropped = (id: string) => { if (draft) { @@ -80,6 +82,7 @@ export function UpNext({ owner={owner} user={userById(item.addedBy)} byline={byline} + playNow={playNow} onDrop={mineIds.has(id) ? () => dropped(id) : undefined} /> ) @@ -99,6 +102,7 @@ function UpNextRow({ owner, user, byline, + playNow, onDrop, }: { roomId: string @@ -109,6 +113,7 @@ function UpNextRow({ owner?: boolean user?: User byline?: boolean + playNow?: PlayNow onDrop?: () => void }) { const controls = useDragControls() @@ -130,6 +135,8 @@ function UpNextRow({ user={user} mine={mine} byline={byline || !!item.autopilot} + onPlay={playNow && (() => playNow.play(item))} + playLabel={playNow?.mode === 'ask' ? 'Ask to play now' : 'Play now'} leading={{index}} trailing={ item.autopilot ? ( diff --git a/web/src/components/shell/album-backdrop.tsx b/web/src/components/shell/album-backdrop.tsx index 81fced3..f2a9e99 100644 --- a/web/src/components/shell/album-backdrop.tsx +++ b/web/src/components/shell/album-backdrop.tsx @@ -1,25 +1,37 @@ import { AnimatePresence, motion } from 'motion/react' +import { useBeat } from '@/hooks/use-beat' import { useLoadedSrc } from '@/hooks/use-loaded-src' import { easeOutExpo } from '@/lib/motion' import { cn } from '@/lib/utils' +import { BackdropScene } from './backdrop-scene' /** * The ambient layer behind everything: a slowly drifting mesh of the * album's palette, a glow in its accent, and the artwork itself, hugely * blurred. The palette colors are registered CSS properties, so the mesh * eases from one song's colors to the next; the artwork crossfades. + * + * It breathes with the music: the mesh and the glow swell once a bar, the + * mesh drifts faster when the song is loud, and each song gets a scene + * drawn over it (see backdrop-scene.tsx). */ -export function AlbumBackdrop({ src: next, className }: { src?: string; className?: string }) { +export function AlbumBackdrop({ src: next, scene = true, className }: { + src?: string + /** Draw the song's scene; off while something covers the backdrop. */ + scene?: boolean + className?: string +}) { // Crossfade to art that's ready, not to an empty image. const src = useLoadedSrc(next) + const beat = useBeat({ drift: true }) return ( -
-
+
+
-
+
{src && ( @@ -36,6 +48,7 @@ export function AlbumBackdrop({ src: next, className }: { src?: string; classNam )}
+ {scene && }
) } diff --git a/web/src/components/shell/app-shell.tsx b/web/src/components/shell/app-shell.tsx index a40e49f..dcc8006 100644 --- a/web/src/components/shell/app-shell.tsx +++ b/web/src/components/shell/app-shell.tsx @@ -1,13 +1,18 @@ import { Outlet, useRouterState } from '@tanstack/react-router' import { WifiOff } from 'lucide-react' import { AnimatePresence, LayoutGroup, motion } from 'motion/react' -import { useState, type ReactNode } from 'react' +import { useEffect, useState, type ReactNode } from 'react' import { useAlbumPalette } from '@/hooks/use-album-palette' +import { useBeatSync } from '@/hooks/use-beat-sync' import { easeOutExpo } from '@/lib/motion' import { usePlayer } from '@/lib/now-playing' import { useOnline } from '@/lib/pwa' +import { PlayNowRequest } from '@/components/room/play-now-request' import { Toaster } from '@/components/toaster' +import { LightsMode } from '@/components/visualizer/lights-mode' +import { VisualizerMode } from '@/components/visualizer/visualizer-mode' import { AlbumBackdrop } from './album-backdrop' +import { BeatLab } from './beat-lab' import { BottomNav } from './bottom-nav' import { MiniPlayer } from './mini-player' import { NowPlayingSheet } from './now-playing-sheet' @@ -20,12 +25,26 @@ import { NowPlayingSheet } from './now-playing-sheet' export function AppShell() { const { nowPlaying } = usePlayer() const [expanded, setExpanded] = useState(false) + // Once now playing has slid all the way up it covers everything, so the + // backdrop behind it stops rendering: two full-screen backdrops of big + // blurred layers, changing at once on a new song, made browsers flash + // black. It's back the moment now playing starts to close. + const [covered, setCovered] = useState(false) + useEffect(() => { + if (!expanded) return + const t = window.setTimeout(() => setCovered(true), 600) + return () => { + window.clearTimeout(t) + setCovered(false) + } + }, [expanded]) useAlbumPalette(nowPlaying) + useBeatSync(nowPlaying) return (
- +
@@ -34,6 +53,7 @@ export function AppShell() {
+ setExpanded(true)} /> @@ -41,6 +61,9 @@ export function AppShell() {
+ {!expanded && } + +
) diff --git a/web/src/components/shell/backdrop-scene.tsx b/web/src/components/shell/backdrop-scene.tsx new file mode 100644 index 0000000..8a09e58 --- /dev/null +++ b/web/src/components/shell/backdrop-scene.tsx @@ -0,0 +1,99 @@ +import { useEffect, useRef } from 'react' +import { useScene } from '@/hooks/use-scene' +import { beatSettings, onBeatFrame } from '@/lib/beat' +import { painters, readColors, type Painter } from '@/lib/scenes' +import { cn } from '@/lib/utils' + +/* + * Backdrop scenes (MAD-782): a canvas over the palette mesh that draws + * something music-shaped in the art's colors (lib/scenes.ts). Each song + * gets one; the pick is in use-scene.ts. + * + * Drawn at half the screen's resolution, then softened with a blur: glow, + * not pixels, and a phone fills a quarter of the pixels a frame. + */ + +/** + * The scene layer. One canvas for good: a new song's scene crossfades in + * on it, rather than on a second full-screen canvas (a pile of big + * blurred layers is what makes a browser flash black). + */ +export function BackdropScene() { + const scene = useScene() + return +} + +/** How long one painter takes to fade into the next. */ +const CROSSFADE_MS = 1600 + +/** + * A canvas a painter draws on every frame the beat engine runs. Drawn at a + * fraction of the element's size (`scale`) and blurred by its class, so it + * reads as glow. A new painter crossfades in on the same canvas. In the + * app it follows the beat lab's scene strength. + */ +export function SceneCanvas({ paint, scale = 1 / 2, app = true, className = 'opacity-70 blur-md dark:opacity-90' }: { + paint: () => Painter + scale?: number + /** In the app's backdrop, rather than the big visualizer. */ + app?: boolean + className?: string +}) { + const ref = useRef(null) + // The painter drawing now, the one fading out, and since when. + const layers = useRef<{ make: () => Painter; draw: Painter; old?: Painter; since: number } | null>(null) + + useEffect(() => { + const l = layers.current + if (!l) layers.current = { make: paint, draw: paint(), since: -Infinity } + else if (l.make !== paint) layers.current = { make: paint, draw: paint(), old: l.draw, since: performance.now() } + }, [paint]) + + useEffect(() => { + const canvas = ref.current + const ctx = canvas?.getContext('2d') + if (!canvas || !ctx) return + + const fit = () => { + canvas.width = Math.max(1, Math.round(canvas.clientWidth * scale)) + canvas.height = Math.max(1, Math.round(canvas.clientHeight * scale)) + } + fit() + const ro = new ResizeObserver(fit) + ro.observe(canvas) + + // The palette eases between songs; follow it a few times a second. + let colors = readColors(canvas) + let readAt = 0 + + const off = onBeatFrame((f) => { + const l = layers.current + if (f.now - readAt > 300) { + colors = readColors(canvas) + readAt = f.now + } + ctx.clearRect(0, 0, canvas.width, canvas.height) + if (!l || f.presence < 0.002) return + const strength = app ? beatSettings.get().effects.scene : 1 + if (strength === 0) return + const t = Math.min(1, (f.now - l.since) / CROSSFADE_MS) + if (t >= 1) l.old = undefined + const layer = (draw: Painter, weight: number) => { + // Painters fade with presence, so scaling it fades the whole layer. + const presence = Math.min(1, f.presence * strength * weight) + ctx.save() + ctx.globalAlpha = presence + draw(ctx, canvas.width, canvas.height, { ...f, presence }, colors) + ctx.restore() + } + if (l.old) layer(l.old, 1 - t) + layer(l.draw, t) + }) + return () => { + off() + ro.disconnect() + } + }, [scale, app]) + + return +} diff --git a/web/src/components/shell/beat-lab.tsx b/web/src/components/shell/beat-lab.tsx new file mode 100644 index 0000000..3707105 --- /dev/null +++ b/web/src/components/shell/beat-lab.tsx @@ -0,0 +1,341 @@ +import { Activity, Lightbulb, RotateCcw, Sparkles, X } from 'lucide-react' +import { Popover } from 'radix-ui' +import { useCallback, useState, type ReactNode } from 'react' +import { Button } from '@/components/ui/button' +import { Slider } from '@/components/ui/slider' +import { Switch } from '@/components/ui/switch' +import { ToggleGroup, ToggleGroupItem } from '@/components/ui/toggle-group' +import { useBeat } from '@/hooks/use-beat' +import { SCENES, useScene } from '@/hooks/use-scene' +import { + BEAT_DEFAULTS, + EFFECTS, + beatSettings, + beatSource, + calibrateTap, + currentBeat, + gridVersion, + lightsOpen, + nudge, + resetGrid, + scaleTempo, + setBeatSettings, + shiftDownbeat, + tap, + visualizerOpen, + type BeatOrigin, + type Effect, +} from '@/lib/beat' +import { useStore } from '@/lib/store' +import { cn } from '@/lib/utils' + +type Tab = 'look' | 'effects' | 'beat' + +/** + * The beat lab (MAD-778): everything about how the app moves with the + * music, on this device. Turn it all off, set the overall intensity, pick + * the backdrop, tune or switch off each effect, tell it how late this + * device's sound is, and fix a song's beat by tapping along. Always a tap + * away: floating top right, or `inline` in a header (now playing's). Its + * button blinks on the beat. + */ +export function BeatLab({ inline = false }: { inline?: boolean }) { + const [open, setOpen] = useState(false) + // Inside now playing (a modal dialog) the panel has to open within the + // dialog: the dialog blocks scrolling everywhere outside it. + const [container, setContainer] = useState(null) + const beat = useBeat() + const trigger = useCallback( + (el: HTMLButtonElement | null) => { + setContainer(el?.closest('[role="dialog"]') ?? null) + return beat(el) + }, + [beat], + ) + const { level } = useStore(beatSettings) + + return ( + + + + + + + + + + + ) +} + +function Panel() { + const settings = useStore(beatSettings) + const [tab, setTab] = useState('look') + const beat = useBeat() + const on = settings.level !== 'off' + + return ( +
+
+
+

Beat lab

+

{on ? 'Visuals move with the music' : 'All visuals off'}

+
+ setBeatSettings({ level: v ? 'subtle' : 'off' })} /> +
+ {!on ? ( +

Everything holds still. Switch it back on to move with the music.

+ ) : ( + <> + v && setTab(v as Tab)} + className="mx-4 mt-3 grid w-auto grid-cols-3" + > + Look + Effects + Beat + +
+ {tab === 'look' && } + {tab === 'effects' && } + {tab === 'beat' && } +
+
+ +
+ + )} +
+ ) +} + +function Look() { + const { intensity, scene: setting } = useStore(beatSettings) + const scene = useScene() + return ( + <> + + setBeatSettings({ intensity: v })} /> + + {setting === 'auto' ? `Auto: ${scene}` : scene}}> +
+ {['auto', ...SCENES].map((s) => ( + + ))} +
+

Auto picks a backdrop that suits each song.

+ + + +

The song's visuals, full screen. ← and → change the show.

+ + + +

+ The screen flashes the song's colors on the downbeat, in time with every other phone in the room doing the same. +

+ + + ) +} + +function Effects() { + const { effects } = useStore(beatSettings) + return ( + <> + {(Object.keys(EFFECTS) as Effect[]).map((k) => ( + +
+ setBeatSettings({ effects: { [k]: v } })} + className="flex-1" + /> + 0} + onChange={(v) => setBeatSettings({ effects: { [k]: v ? 1 : 0 } })} + /> +
+ + ))} + + ) +} + +function Beat() { + const { delayMs } = useStore(beatSettings) + const { mapKey, np } = useStore(beatSource) + useStore(gridVersion) + const now = currentBeat() + const loading = !!np?.itemId && mapKey !== np.itemId + const [heard, setHeard] = useState(0) + const [taps, setTaps] = useState(0) + const canCalibrate = now.origin === 'analysed' && !!np && !np.paused + + return ( + <> +
+
+

+ {!np ? 'Nothing playing' : now.bpm > 0 ? `${Math.round(now.bpm)} BPM` : 'No steady beat'} +

+ {np &&

{loading ? 'Listening to the song…' : origins[now.origin]}

} +
+
+ + +
+
+ + 0 ? '+' : ''}${delayMs} ms`}> + setBeatSettings({ delayMs: v })} /> + +

+ Sound through a Bluetooth speaker arrives late. Tap on the beat you hear and the visuals wait for it. +

+ + + + +

+ Start on a “one” and tap at least 4 times. Fixes this song on this device. +

+
+ + + + + + + + +
+ + + ) +} + +function Field({ label, value, children }: { label: string; value?: ReactNode; children: ReactNode }) { + return ( +
+ + {label} + {value !== undefined && {value}} + + {children} +
+ ) +} + +const origins: Record = { + analysed: 'From the song', + tapped: 'Tapped on this device', + steady: 'Moving with its loudness', + default: 'A guess: this song has no beat map', +} + +function intensityLabel(v: number) { + if (v < 0.8) return 'Calm' + if (v <= 1.2) return 'Normal' + if (v <= 1.5) return 'Strong' + return 'Max' +} + +function Dot({ v, big }: { v: string; big?: boolean }) { + return ( + + ) +} diff --git a/web/src/components/shell/mini-player.tsx b/web/src/components/shell/mini-player.tsx index 56534cc..17b7685 100644 --- a/web/src/components/shell/mini-player.tsx +++ b/web/src/components/shell/mini-player.tsx @@ -2,6 +2,7 @@ import { Link } from '@tanstack/react-router' import { AnimatePresence, motion } from 'motion/react' import { Artwork } from '@/components/artwork' import { Button } from '@/components/ui/button' +import { useBeat } from '@/hooks/use-beat' import { usePosition } from '@/hooks/use-position' import { laneStyle } from '@/lib/lane' import { spring } from '@/lib/motion' @@ -16,6 +17,7 @@ import { PlayPauseButton, SkipButton } from './player-controls' export function MiniPlayer({ expanded, onExpand }: { expanded: boolean; onExpand: () => void }) { const { nowPlaying: np, commands } = usePlayer() const position = usePosition(np) + const beat = useBeat() if (!np) { return ( @@ -36,9 +38,12 @@ export function MiniPlayer({ expanded, onExpand }: { expanded: boolean; onExpand return (
+ {/* Light catches the glass on the one. */} +
{/* The whole bar opens now-playing; the buttons sit above it. */} - + {np.roomId ? ( - +
+ + + + +
) : ( )} @@ -102,12 +128,16 @@ export function NowPlayingSheet({ open, onOpenChange }: { open: boolean; onOpenC onPointerDown={(e) => drag.start(e)} className="flex flex-1 touch-none items-center justify-center px-8 py-4" > - + {/* Lifts on the one, with a bloom of the art's color behind it. */} +
+
+ +
)} @@ -140,6 +170,12 @@ export function NowPlayingSheet({ open, onOpenChange }: { open: boolean; onOpenC
+ (ours && ours.loudness.length > 0 ? waveform(ours.loudness, BARS) : Array(BARS).fill(0.12)), + [ours], + ) + + if (level === 'off' || effects.waveform === 0) return null + const played = durationMs > 0 ? Math.min(1, positionMs / durationMs) : 0 + const bars = ( + + {heights.map((h, i) => ( + + ))} + + ) + return ( +
+
{bars}
+
+ {bars} +
+
+ ) +} diff --git a/web/src/components/tv/tv-audio.tsx b/web/src/components/tv/tv-audio.tsx index 660b558..6bb40a6 100644 --- a/web/src/components/tv/tv-audio.tsx +++ b/web/src/components/tv/tv-audio.tsx @@ -1,9 +1,10 @@ import { useQuery, useQueryClient } from '@tanstack/react-query' -import { Speaker, Volume2 } from 'lucide-react' -import { useEffect, useRef } from 'react' +import { Pause, Play, SkipForward, Speaker, Volume2 } from 'lucide-react' +import { useEffect, useRef, useState } from 'react' +import { errorMessage } from '@/api/errors' import { Equalizer } from '@/components/equalizer' import { Button } from '@/components/ui/button' -import { newer, playbackQuery } from '@/lib/playback' +import { newer, playbackQuery, sendCommand, type PlaybackCommand } from '@/lib/playback' import { speaker, speakerState } from '@/lib/speaker' import { useStore } from '@/lib/store' import { cn } from '@/lib/utils' @@ -15,15 +16,20 @@ const AUDIO_KEY = 'syncphony-tv-audio' * press, so it offers a button to press OK on with the remote. Once it's * playing, it starts again by itself after a reload, unless another device * has taken over in the meantime. If the browser still wants a press, - * the button comes back. + * the button comes back. Opening the big screen on a device that's + * already the speaker carries on playing. It has a speaker's buttons + * too: play, pause and skip. */ -export function TvAudio({ roomId, device, name, onStopped }: { +export function TvAudio({ roomId, device, name, onStopped, canPlayPause = true, canSkip = true }: { roomId: string /** Who the server knows this screen as: its display ID, or this browser's device ID. */ device: string name: string /** It stopped being the speaker without being asked to (its audio was turned off, say). */ onStopped?: () => void + /** Whether to offer play and pause, and skip. The server has the last word. */ + canPlayPause?: boolean + canSkip?: boolean }) { const queryClient = useQueryClient() const state = useStore(speakerState) @@ -31,14 +37,17 @@ export function TvAudio({ roomId, device, name, onStopped }: { const here = state.roomId === roomId && state.status !== 'off' const other = playback.data?.player && playback.data.player.deviceId !== device ? playback.data.player : undefined - // The server's state drives the speaker; its reports' replies flow back into the cache. + // The server's state drives the speaker; its reports' replies flow back + // into the cache. Already playing this room (we came from the app): keep + // going, and pick it up again after a reload. Leaving hands it back. useEffect(() => { speaker.onState = (np) => queryClient.setQueryData(playbackQuery(np.roomId).queryKey, (old) => newer(old, np)) + if (speaker.keep(roomId)) writeFlag(true) return () => { speaker.onState = undefined - void speaker.stop() + speaker.stopSoon() } - }, [queryClient]) + }, [queryClient, roomId]) useEffect(() => speaker.apply(playback.data), [playback.data]) // Pick up where it left off after a reload, unless someone else is playing the room now. @@ -46,6 +55,7 @@ export function TvAudio({ roomId, device, name, onStopped }: { useEffect(() => { if (resumed.current || !playback.data) return resumed.current = true + if (speaker.active) return const player = playback.data.player if (readFlag() && (!player || player.deviceId === device)) void speaker.start(roomId, name, device) }, [playback.data, roomId, name, device]) @@ -57,10 +67,52 @@ export function TvAudio({ roomId, device, name, onStopped }: { was.current = here }, [here, onStopped]) + const [error, setError] = useState() + useEffect(() => { + if (!error) return + const t = window.setTimeout(() => setError(undefined), 5000) + return () => window.clearTimeout(t) + }, [error]) + const command = async (body: PlaybackCommand) => { + try { + const np = await sendCommand(roomId, body) + queryClient.setQueryData(playbackQuery(roomId).queryKey, (old) => newer(old, np)) + } catch (err) { + setError(errorMessage(err)) + } + } + // Playing here means playing: a paused room starts too, unless we're + // taking over from another speaker, which carries on as it was. const start = () => { writeFlag(true) - void speaker.start(roomId, name, device) + const takingOver = !!other + void speaker.start(roomId, name, device).then((np) => { + if (np && !takingOver && np.state === 'paused' && canPlayPause) void command({ action: 'play' }) + }) } + const np = playback.data + const playing = np?.state === 'playing' || np?.state === 'loading' + // Play with no speaker makes this screen it, in the one press. + const toggle = () => (playing ? command({ action: 'pause' }) : np?.player ? command({ action: 'play' }) : start()) + const transport = canPlayPause && np && (np.item || np.next) && ( +
+ + {canSkip && np.item && ( + + )} +
+ ) + const problem = error && {error} const stop = () => { writeFlag(false) void speaker.stop() @@ -80,6 +132,8 @@ export function TvAudio({ roomId, device, name, onStopped }: { if (here) { return (
+ {problem} + {transport} {state.status === 'playing' ? : } {state.status === 'remote' @@ -99,6 +153,8 @@ export function TvAudio({ roomId, device, name, onStopped }: { return (
+ {problem} + {other && transport} {other && Playing on {other.name}} +
+
+

Lights

+

+ {still + ? level === 'off' + ? 'Visuals are off, so the light holds steady.' + : 'Reduce motion is on, so the light holds steady.' + : np + ? 'Hold your phone up. Every phone in lights mode flashes on the same beat.' + : 'Nothing is playing yet.'} +

+
+
+ + ) +} diff --git a/web/src/components/visualizer/visualizer-mode.tsx b/web/src/components/visualizer/visualizer-mode.tsx new file mode 100644 index 0000000..cdca4c4 --- /dev/null +++ b/web/src/components/visualizer/visualizer-mode.tsx @@ -0,0 +1,147 @@ +import { ChevronLeft, ChevronRight, Pause, Play, X } from 'lucide-react' +import { AnimatePresence, motion } from 'motion/react' +import { useEffect, useRef, useState } from 'react' +import { createPortal } from 'react-dom' +import { Artwork } from '@/components/artwork' +import { Button } from '@/components/ui/button' +import { usePosition } from '@/hooks/use-position' +import { SHOWS, useShow } from '@/hooks/use-scene' +import { beatSettings, setBeatSettings, visualizerOpen } from '@/lib/beat' +import { formatDuration, usePlayer } from '@/lib/now-playing' +import { useStore } from '@/lib/store' +import { cn } from '@/lib/utils' +import { Visualizer } from './visualizer' + +/** + * The app's full-window visualizer (MAD-780): the song's visuals filling + * the screen (full screen on a computer), with what's playing small in a + * corner. The controls fade away while the mouse rests. ← and → change + * the show, space plays and pauses, Esc closes. + */ +export function VisualizerMode() { + const open = useStore(visualizerOpen) + return createPortal({open && }, document.body) +} + +function Mode() { + const { nowPlaying: np, commands } = usePlayer() + const position = usePosition(np) + const { show: setting } = useStore(beatSettings) + const show = useShow(setting) + const ref = useRef(null) + const [idle, setIdle] = useState(false) + const close = () => visualizerOpen.set(false) + + // Cycles through the shows; past the last comes back to auto. + const step = (by: number) => { + const list = ['auto', ...SHOWS] + const i = list.indexOf(setting) + setBeatSettings({ show: list[(i + by + list.length) % list.length] }) + } + + // The controls hide after three still seconds. + const timer = useRef(0) + const wake = () => { + setIdle(false) + window.clearTimeout(timer.current) + timer.current = window.setTimeout(() => setIdle(true), 3000) + } + useEffect(() => { + timer.current = window.setTimeout(() => setIdle(true), 3000) + return () => window.clearTimeout(timer.current) + }, []) + + useEffect(() => { + const el = ref.current + // Full screen where the browser allows; a window otherwise. + void el?.requestFullscreen?.().catch(() => {}) + return () => { + if (document.fullscreenElement) void document.exitFullscreen().catch(() => {}) + } + }, []) + + useEffect(() => { + // Leaving full screen (Esc in most browsers) closes it too. + const onFullscreen = () => { + if (!document.fullscreenElement) close() + } + document.addEventListener('fullscreenchange', onFullscreen) + return () => document.removeEventListener('fullscreenchange', onFullscreen) + }, []) + + useEffect(() => { + const onKey = (e: KeyboardEvent) => { + if (e.key === 'Escape') close() + else if (e.key === 'ArrowRight') step(1) + else if (e.key === 'ArrowLeft') step(-1) + else if (e.key === ' ' && commands.toggle) { + e.preventDefault() + commands.toggle() + } else return + wake() + } + window.addEventListener('keydown', onKey) + return () => window.removeEventListener('keydown', onKey) + }) + + const progress = np && np.track.durationMs > 0 ? position / np.track.durationMs : 0 + + return ( + + + +
+
+ +
+ +
+ {np && ( +
+ +
+

{np.track.title}

+

{np.track.artists.join(', ')}

+
+ {formatDuration(position)} + + + + {formatDuration(np.track.durationMs)} +
+
+ {commands.toggle && ( + + )} +
+ )} + +
+ + {setting === 'auto' ? `Auto · ${show}` : show} + +
+
+
+ + ) +} diff --git a/web/src/components/visualizer/visualizer.tsx b/web/src/components/visualizer/visualizer.tsx new file mode 100644 index 0000000..392d78b --- /dev/null +++ b/web/src/components/visualizer/visualizer.tsx @@ -0,0 +1,124 @@ +import { AnimatePresence, motion } from 'motion/react' +import { useEffect, useRef, useState } from 'react' +import { Artwork } from '@/components/artwork' +import { SceneCanvas } from '@/components/shell/backdrop-scene' +import type { Show } from '@/hooks/use-scene' +import { onBeatFrame } from '@/lib/beat' +import { painters, readColors, showPainters } from '@/lib/scenes' +import { cn } from '@/lib/utils' +import { Fluid, toRgb, type FluidColors } from './fluid' + +/** + * The big visualizer (MAD-779, MAD-780): full-screen visuals in the + * album's colors, moving with the song's beat map. For the TV and the + * app's full-window mode. Nothing here is subtle. + */ +export function Visualizer({ show, artworkUrl, className }: { show: Show; artworkUrl?: string; className?: string }) { + return ( +
+ + + + + +
+ ) +} + +// Sharp shows draw at the screen's resolution, up to 1.5 pixels a point. +const sharp = typeof devicePixelRatio === 'number' ? Math.min(devicePixelRatio, 1.5) : 1 + +function ShowLayer({ show, artworkUrl }: { show: Show; artworkUrl?: string }) { + if (show === 'fluid') return + if (show === 'spectrum') { + return ( + <> + + {/* The art sits inside the ring. */} +
+ +
+ + ) + } + if (show === 'tunnel' || show === 'rain' || show === 'sparks') { + return + } + // The backdrop's scenes, full strength. + return +} + +/** The palette fluid, or the aurora where there's no WebGL. */ +function FluidCanvas() { + const ref = useRef(null) + const [webgl] = useState(hasWebGL) + + useEffect(() => { + const box = ref.current + if (!box || !webgl) return + // A canvas of its own each time: a context let go can't be had back, + // and React may mount this twice (strict mode). + const canvas = document.createElement('canvas') + canvas.className = 'absolute inset-0 size-full' + box.append(canvas) + let fluid: Fluid + try { + fluid = new Fluid(canvas) + } catch (err) { + console.warn('visualizer: no fluid', err) + canvas.remove() + return + } + let colors: FluidColors | null = null + let readAt = -Infinity + const off = onBeatFrame((f) => { + if (f.now - readAt > 300) { + const c = readColors(box) + const dark = getComputedStyle(box).getPropertyValue('--pal-dark').trim() || 'black' + colors = { dark: toRgb(dark), dominant: toRgb(c.dominant), vibrant: toRgb(c.vibrant), light: toRgb(c.light) } + readAt = f.now + } + if (colors) fluid.render(f, colors) + }) + return () => { + off() + fluid.dispose() + canvas.remove() + } + }, [webgl]) + + if (!webgl) return + return
+} + +let webglChecked: boolean | undefined + +/** Whether the device can draw WebGL; asked once, and the test context let go. */ +function hasWebGL() { + if (webglChecked === undefined) { + try { + const gl = document.createElement('canvas').getContext('webgl') + webglChecked = !!gl + gl?.getExtension('WEBGL_lose_context')?.loseContext() + } catch { + webglChecked = false + } + } + return webglChecked +} diff --git a/web/src/hooks/use-beat-sync.ts b/web/src/hooks/use-beat-sync.ts new file mode 100644 index 0000000..8fd5953 --- /dev/null +++ b/web/src/hooks/use-beat-sync.ts @@ -0,0 +1,34 @@ +import { useEffect } from 'react' +import { beatSource } from '@/lib/beat' +import { loadBeatMap } from '@/lib/beat-map' +import type { NowPlaying } from '@/lib/now-playing' + +/** + * Points the beat engine at what's playing, and loads the song's beat map. + * Mount it once where the now playing lives: the app shell, the big screen. + */ +export function useBeatSync(np: NowPlaying | null) { + useEffect(() => { + beatSource.set((s) => ({ ...s, np })) + }, [np]) + + const roomId = np?.roomId + const itemId = np?.itemId + useEffect(() => { + if (!roomId || !itemId) { + beatSource.set((s) => ({ ...s, map: null, mapKey: null })) + return + } + let stale = false + // mapKey says the map is known for this song, even when there's none. + void loadBeatMap(roomId, itemId).then((map) => { + if (!stale) beatSource.set((s) => ({ ...s, map, mapKey: itemId })) + }) + return () => { + stale = true + } + }, [roomId, itemId]) + + // Nothing playing here any more (the shell unmounted, say). + useEffect(() => () => beatSource.set({ np: null, map: null, mapKey: null }), []) +} diff --git a/web/src/hooks/use-beat.ts b/web/src/hooks/use-beat.ts new file mode 100644 index 0000000..a5ab585 --- /dev/null +++ b/web/src/hooks/use-beat.ts @@ -0,0 +1,11 @@ +import { useCallback } from 'react' +import { attachBeat } from '@/lib/beat' + +/** + * A ref that lets the beat engine drive an element's --beat, --downbeat, + * --energy and --b0..--b3 (see lib/beat.ts). With `drift`, its CSS + * animations also follow the song's energy. + */ +export function useBeat({ drift = false }: { drift?: boolean } = {}) { + return useCallback((el: T | null) => (el ? attachBeat(el, { drift }) : undefined), [drift]) +} diff --git a/web/src/hooks/use-play-now.ts b/web/src/hooks/use-play-now.ts new file mode 100644 index 0000000..472029b --- /dev/null +++ b/web/src/hooks/use-play-now.ts @@ -0,0 +1,63 @@ +import { useQuery, useQueryClient } from '@tanstack/react-query' +import { errorMessage } from '@/api/errors' +import { useMe } from '@/lib/auth' +import { tap } from '@/lib/haptics' +import { can, playbackQuery, sendCommand, type PlaybackCommand, type QueueItem } from '@/lib/playback' +import { roomsQuery } from '@/lib/room' +import { speaker, speakerName } from '@/lib/speaker' +import { toast } from '@/lib/toast' + +export type PlayNow = { + /** `now` plays it straight away; `ask` asks the room, which votes. */ + mode: 'now' | 'ask' + play: (item: QueueItem) => void +} + +/** + * Playing a queued song now, skipping the one on. It's the skip + * permission's call: in a room that votes on skips, you ask the room + * instead (the owner never has to). Guests can't. With no speaker, this + * device becomes it, in the same tap, if it may. + */ +export function usePlayNow(roomId: string): PlayNow | undefined { + const me = useMe() + const rooms = useQuery(roomsQuery) + const command = usePlaybackCommand(roomId) + const queryClient = useQueryClient() + const room = rooms.data?.find((r) => r.id === roomId) + if (!room || me.guest) return undefined + const skip = room.permissions.skip + const mode = room.ownerId === me.id || skip === 'everyone' ? 'now' : skip === 'vote' ? 'ask' : undefined + if (!mode) return undefined + return { + mode, + play: (item) => { + tap() + const ask = () => + command({ action: 'play_now', itemId: item.id }).then((np) => { + if (np?.playNow?.itemId === item.id) toast({ message: `Asked the room to play “${item.track.title}”` }) + }) + const playback = queryClient.getQueryData(playbackQuery(roomId).queryKey) + if (playback?.player || !can(room, me.id, 'speaker')) return void ask() + // Becoming the speaker plays the song straight away if we may; + // otherwise the room starts and we ask it, as anyone would. + void speaker.start(roomId, speakerName(me.displayName), undefined, item.id).then((np) => { + if (np && np.item?.id !== item.id) void ask() + }) + }, + } +} + +/** Sends a playback command and keeps the cache in step, or says why it failed. */ +export function usePlaybackCommand(roomId: string) { + const queryClient = useQueryClient() + return async (body: PlaybackCommand) => { + try { + const np = await sendCommand(roomId, body) + queryClient.setQueryData(playbackQuery(roomId).queryKey, np) + return np + } catch (err) { + toast({ message: errorMessage(err), tone: 'error' }) + } + } +} diff --git a/web/src/hooks/use-room-controls.ts b/web/src/hooks/use-room-controls.ts index 49c3186..23ea8c6 100644 --- a/web/src/hooks/use-room-controls.ts +++ b/web/src/hooks/use-room-controls.ts @@ -6,6 +6,7 @@ import type { PlayerCommands } from '@/lib/now-playing' import { isMine } from '@/lib/autopilot' import { can, playbackQuery, sendCommand, skipMode, type Permission, type Playback, type PlaybackCommand } from '@/lib/playback' import type { Room } from '@/lib/room' +import { speaker, speakerName } from '@/lib/speaker' import { toast } from '@/lib/toast' /** @@ -14,13 +15,18 @@ import { toast } from '@/lib/toast' * room that votes on skips, everyone else gets a vote instead of `next`. * Guests only skip their own songs, and vote if the room lets them. */ -export function useRoomControls(room: Room | undefined, playback: Playback | undefined, userId: string, guest = false): PlayerCommands { +export function useRoomControls( + room: Room | undefined, + playback: Playback | undefined, + me: { id: string; displayName: string }, + guest = false, +): PlayerCommands { const queryClient = useQueryClient() + const userId = me.id return useMemo(() => { - if (!room || !playback?.item) return {} + if (!room || !playback) return {} const key = playbackQuery(room.id).queryKey - const item = playback.item const send = async (cmd: PlaybackCommand, optimistic?: (p: Playback) => Playback) => { tap() const before = queryClient.getQueryData(key) @@ -34,17 +40,30 @@ export function useRoomControls(room: Room | undefined, playback: Playback | und } const playing = playback.state === 'playing' || playback.state === 'loading' const allowed = (p: Permission) => !guest && can(room, userId, p) + // With no speaker, play makes this device it and starts the room, in + // the one tap (browsers only start audio from a tap). + const play = async () => { + if (!playback.player && allowed('speaker')) { + const np = await speaker.start(room.id, speakerName(me.displayName)) + if (!np || np.state === 'playing' || np.state === 'loading') return + } + await send({ action: 'play' }) + } + const toggle = allowed('playPause') + ? () => + playing + ? send({ action: 'pause' }, (p) => ({ ...p, state: 'paused', positionMs: currentPosition(p), at: new Date().toISOString() })) + : void play() + : undefined + const item = playback.item + // Nothing's on: play is all there is (it starts the next song). + if (!item) return { toggle } let skip = skipMode(room, userId, item) if (guest && !isMine(item, userId)) skip = skip === 'vote' && room.guests.canVote ? 'vote' : undefined const votes = playback.skipVotes const voted = !!votes?.voters.includes(userId) return { - toggle: allowed('playPause') - ? () => - playing - ? send({ action: 'pause' }, (p) => ({ ...p, state: 'paused', positionMs: currentPosition(p), at: new Date().toISOString() })) - : send({ action: 'play' }) - : undefined, + toggle, // There's no going back in a fair queue; "previous" restarts the song, // like most players do past the first few seconds. previous: allowed('seek') ? () => send({ action: 'seek', positionMs: 0 }, (p) => ({ ...p, positionMs: 0, at: new Date().toISOString() })) : undefined, @@ -69,7 +88,7 @@ export function useRoomControls(room: Room | undefined, playback: Playback | und ? (positionMs: number) => send({ action: 'seek', positionMs }, (p) => ({ ...p, positionMs, at: new Date().toISOString() })) : undefined, } - }, [room, playback, userId, guest, queryClient]) + }, [room, playback, userId, me.displayName, guest, queryClient]) } /** Where a playing song is now, from its position at the last report. */ diff --git a/web/src/hooks/use-scene.ts b/web/src/hooks/use-scene.ts new file mode 100644 index 0000000..99f5974 --- /dev/null +++ b/web/src/hooks/use-scene.ts @@ -0,0 +1,63 @@ +import { beatSettings, beatSource } from '@/lib/beat' +import { useStore } from '@/lib/store' + +export const SCENES = ['mesh', 'horizon', 'ripples', 'aurora', 'embers'] as const +export type Scene = (typeof SCENES)[number] + +// Picks follow the song whose beat map has settled (mapKey), not the one +// that just started: while the new song's map is on its way the old pick +// stays, rather than picking now and again when the map lands (two +// crossfades in a row read as a flicker). + +/** + * The backdrop scene for the song playing: the setting's, or one that suits + * the song, from its beat map. A stable pick by the queue item breaks ties, + * so two songs alike don't always look the same. + */ +export function useScene(): Scene { + const { scene } = useStore(beatSettings) + const { np, map, mapKey } = useStore(beatSource) + if (scene !== 'auto') return scene as Scene + const key = mapKey ?? np?.itemId ?? np?.track.trackId + if (!key) return 'mesh' + let h = 0 + for (const ch of key) h = (h * 31 + ch.charCodeAt(0)) | 0 + const pick = (...from: Scene[]) => from[Math.abs(h) % from.length] + if (!map) return pick(...SCENES) + const { energy, brightness } = map.features + // No steady beat (ambient, a drone, spoken word): something that flows. + if (map.bpm === 0) return pick('mesh', 'aurora') + // Quiet and slow: a ballad, acoustic. + if (energy < 0.4 || map.bpm < 90) return pick('mesh', 'aurora', 'embers') + // Loud with a driving beat: dance, rock. + if (energy > 0.65 && map.bpm >= 110) return brightness > 0.5 ? pick('horizon', 'ripples') : pick('horizon', 'embers') + return pick(...SCENES) +} + +/** The big visualizer's shows (MAD-779, MAD-780). */ +export const SHOWS = ['spectrum', 'fluid', 'tunnel', 'horizon', 'rain', 'sparks', 'aurora'] as const +export type Show = (typeof SHOWS)[number] + +export function isShow(s: string): s is Show { + return (SHOWS as readonly string[]).includes(s) +} + +/** + * The visualizer's show: `setting` if it names one, else one that suits + * the song playing, like the backdrop's pick. + */ +export function useShow(setting: string): Show { + const { np, map, mapKey } = useStore(beatSource) + if (isShow(setting)) return setting + const key = mapKey ?? np?.itemId ?? np?.track.trackId + if (!key) return 'fluid' + let h = 7 + for (const ch of key) h = (h * 33 + ch.charCodeAt(0)) | 0 + const pick = (...from: Show[]) => from[Math.abs(h) % from.length] + if (!map) return pick('spectrum', 'fluid', 'tunnel') + const { energy } = map.features + if (map.bpm === 0) return pick('fluid', 'aurora') + if (energy < 0.4 || map.bpm < 90) return pick('fluid', 'aurora', 'rain') + if (energy > 0.65 && map.bpm >= 110) return pick('spectrum', 'tunnel', 'horizon', 'sparks') + return pick(...SHOWS) +} diff --git a/web/src/hooks/use-wake-lock.ts b/web/src/hooks/use-wake-lock.ts new file mode 100644 index 0000000..280409a --- /dev/null +++ b/web/src/hooks/use-wake-lock.ts @@ -0,0 +1,25 @@ +import { useEffect } from 'react' + +/** Keeps the screen from sleeping while it's on show. */ +export function useWakeLock() { + useEffect(() => { + let lock: WakeLockSentinel | undefined + let stopped = false + const request = async () => { + if (document.visibilityState !== 'visible' || !('wakeLock' in navigator)) return + try { + lock = await navigator.wakeLock.request('screen') + if (stopped) void lock.release() + } catch { + // Not allowed (battery saver, or no user gesture yet): the screen may dim. + } + } + void request() + document.addEventListener('visibilitychange', request) + return () => { + stopped = true + document.removeEventListener('visibilitychange', request) + void lock?.release() + } + }, []) +} diff --git a/web/src/index.css b/web/src/index.css index 6b08ea0..1de42e6 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -431,3 +431,65 @@ transform: scaleY(0.6); } } + +/* + * Music-reactive layer (MAD-772). The beat engine (src/lib/beat.ts) writes + * --beat, --downbeat, --swell and --energy on the elements below. Subtle on + * purpose: the app should breathe with the song, never bounce. Light and + * glow move more than size does, and only the downbeat touches anything + * solid. Big surfaces follow --swell (once a bar), never --beat: a + * full-screen change twice a second reads as strobe. + * Every value falls back to 0, so without the engine nothing moves. + * Each effect is scaled by its --fx-* strength from the beat lab (0 is + * off, 1 normal, 2 double), set on the root when the settings change. + */ + +/* The mesh breathes once a bar. */ +@utility beat-swell { + scale: calc(1 + var(--swell, 0) * 0.06 * var(--fx-breath, 1)); +} + +/* So does the accent glow behind the page, brighter when the song is loud. */ +@utility beat-glow { + scale: calc(1 + var(--swell, 0) * 0.1 * var(--fx-breath, 1)); + opacity: calc(0.75 + (var(--energy, 0) * 0.15 + var(--swell, 0) * 0.25) * var(--fx-breath, 1)); +} + +/* Album art lifts slightly on the one. */ +@utility beat-lift { + scale: calc(1 + var(--downbeat, 0) * 0.02 * var(--fx-art, 1)); +} + +/* A soft bloom of the art's color behind the cover, on every beat. */ +@utility beat-bloom { + background: radial-gradient(closest-side, var(--pal-vibrant), transparent); + opacity: calc((var(--beat, 0) * 0.55 + var(--energy, 0) * 0.18) * var(--fx-art, 1)); + scale: calc(1.02 + var(--beat, 0) * 0.07 * var(--fx-art, 1)); +} + +/* Light catching the glass on the one: a faint diagonal sheen and a + brighter top edge. */ +@utility beat-sheen { + background: linear-gradient( + 115deg, + transparent 25%, + color-mix(in oklch, var(--pal-light) 22%, transparent) 50%, + transparent 75% + ); + box-shadow: inset 0 1px 0 0 color-mix(in oklch, var(--pal-light) 60%, transparent); + opacity: calc(var(--downbeat, 0) * 0.9 * var(--fx-glass, 1)); +} + +/* The equalizer follows the engine's bands while it's live. */ +:root:not([data-fx-eq-off]) [data-beat-live] .eq-bar { + animation: none; + transform: scaleY(min(1, calc(0.22 + var(--eq, 0) * 0.78 * var(--fx-eq, 1)))); +} + +/* The lyric being sung glows in the album color, a little more on each beat. */ +@utility beat-lyric { + filter: drop-shadow( + 0 0 calc(24px + var(--beat, 0) * 18px * var(--fx-lyrics, 1)) + color-mix(in oklch, var(--pal-text) calc(35% + var(--beat, 0) * 45% * var(--fx-lyrics, 1)), transparent) + ); +} diff --git a/web/src/lib/beat-map.test.ts b/web/src/lib/beat-map.test.ts new file mode 100644 index 0000000..5ba6229 --- /dev/null +++ b/web/src/lib/beat-map.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest' +import { beatAt, decode, loudnessAt, sectionEnergy, spectrumAt, waveform } from './beat-map' + +const wire = { + durationMs: 10_000, + bpm: 120, + confidence: 0.8, + // A beat every 500 ms from 1 s; the first bar starts on the second beat. + beats: Array.from({ length: 16 }, (_, i) => 1000 + i * 500), + downbeat: 1, + sections: [ + { startMs: 0, energy: 0.2 }, + { startMs: 5000, energy: 0.9 }, + ], + frameRate: 2, + bandCount: 2, + // Two frames a second, two bands: [0, 255], [255, 0], ... + bands: btoa(String.fromCharCode(0, 255, 255, 0, 0, 255)), + loudness: btoa(String.fromCharCode(0, 128, 255)), + features: { energy: 0.5, brightness: 0.5, dynamics: 0.5 }, +} + +describe('beat maps', () => { + const m = decode(wire) + + it('decodes the envelopes', () => { + expect([...m.bands]).toEqual([0, 255, 255, 0, 0, 255]) + expect([...m.loudness]).toEqual([0, 128, 255]) + }) + + it('finds the beat and its place in the bar', () => { + expect(beatAt(m, 999)).toBeNull() + expect(beatAt(m, 1000)).toEqual({ index: 0, into: 0, period: 500, bar: 3 }) + expect(beatAt(m, 1600)).toEqual({ index: 1, into: 100, period: 500, bar: 0 }) + expect(beatAt(m, 3500)?.bar).toBe(0) + // The last beat lasts as long as the one before it. + expect(beatAt(m, 8700)).toEqual({ index: 15, into: 200, period: 500, bar: 2 }) + expect(beatAt(m, 9000)).toBeNull() + }) + + it('reads sections, loudness and the spectrum', () => { + expect(sectionEnergy(m, 4999)).toBe(0.2) + expect(sectionEnergy(m, 5000)).toBe(0.9) + expect(loudnessAt(m, 600)).toBeCloseTo(128 / 255) + expect(loudnessAt(m, 99_000)).toBe(0) + const out = new Float32Array(2) + // Halfway between the first two frames. + expect([...spectrumAt(m, 250, out)]).toEqual([0.5, 0.5]) + expect([...spectrumAt(m, -10, out)]).toEqual([0, 0]) + }) +}) + +describe('waveform', () => { + it('shows the shape of a loud, compressed song instead of filling to the top', () => { + // Verses at 220, choruses at 245: nearly flat out the whole way. + const loud = Uint8Array.from({ length: 400 }, (_, i) => (Math.floor(i / 100) % 2 ? 245 : 220)) + const h = waveform(loud, 4) + expect(Math.max(...h)).toBeCloseTo(0.92) + expect(h[0]).toBeLessThan(0.5) + expect(h[1]).toBeGreaterThan(0.9) + }) + + it('keeps a near-flat song flat, at mid-height', () => { + const flat = Uint8Array.from({ length: 400 }, (_, i) => 240 + Math.floor(i / 100)) + const h = waveform(flat, 8) + expect(Math.max(...h) - Math.min(...h)).toBeLessThan(0.15) + expect(Math.max(...h)).toBeLessThan(0.6) + }) +}) diff --git a/web/src/lib/beat-map.ts b/web/src/lib/beat-map.ts new file mode 100644 index 0000000..9d0ff0a --- /dev/null +++ b/web/src/lib/beat-map.ts @@ -0,0 +1,136 @@ +import { api } from '@/api/client' +import { unwrap } from '@/api/errors' +import type { components } from '@/api/schema.gen' + +// Beat maps (MAD-773): the server works out each song's beats, bars, +// sections and spectrum once, from its audio. This loads them and answers +// "where in the music are we?" for the beat engine (lib/beat.ts). + +type Wire = components['schemas']['BeatMap'] + +export type BeatMap = Omit & { + bands: Uint8Array + loudness: Uint8Array +} + +const loaded = new Map>() + +/** A queued song's beat map, or null if it can't have one. Cached per item. */ +export function loadBeatMap(roomId: string, itemId: string): Promise { + const key = `${roomId}/${itemId}` + let hit = loaded.get(key) + if (!hit) { + hit = unwrap(api.GET('/rooms/{roomId}/queue/{itemId}/beatmap', { params: { path: { roomId, itemId } } })) + .then(decode) + .catch(() => { + // A miss now (no map, or the server was busy) may be a hit later. + setTimeout(() => loaded.delete(key), 60_000) + return null + }) + loaded.set(key, hit) + // Keep the last few songs' maps; they're up to a few hundred KB. + while (loaded.size > 8) loaded.delete(loaded.keys().next().value!) + } + return hit +} + +export function decode(w: Wire): BeatMap { + return { ...w, bands: base64(w.bands), loudness: base64(w.loudness) } +} + +function base64(s: string) { + const bin = atob(s) + const out = new Uint8Array(bin.length) + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i) + return out +} + +/** Where pos (ms) falls among the beats. */ +export type BeatPlace = { + /** Beats since the first, and ms into this one. */ + index: number + into: number + /** This beat's length, ms. */ + period: number + /** Its place in the bar, 0 for the one. */ + bar: number +} + +/** The beat at pos, or null before the first beat and after the last. */ +export function beatAt(m: BeatMap, pos: number): BeatPlace | null { + const b = m.beats + if (b.length < 2 || pos < b[0] || pos >= b[b.length - 1] + (b[b.length - 1] - b[b.length - 2])) return null + // The last beat at or before pos. + let lo = 0 + let hi = b.length - 1 + while (lo < hi) { + const mid = (lo + hi + 1) >> 1 + if (b[mid] <= pos) lo = mid + else hi = mid - 1 + } + const period = lo + 1 < b.length ? b[lo + 1] - b[lo] : b[lo] - b[lo - 1] + return { index: lo, into: pos - b[lo], period, bar: (((lo - m.downbeat) % 4) + 4) % 4 } +} + +/** The section energy at pos, 0–1. */ +export function sectionEnergy(m: BeatMap, pos: number) { + let e = m.sections[0]?.energy ?? 0.5 + for (const s of m.sections) { + if (s.startMs > pos) break + e = s.energy + } + return e +} + +/** Each band's level at pos, 0–1, into out (bandCount long). */ +export function spectrumAt(m: BeatMap, pos: number, out: Float32Array) { + const f = (pos / 1000) * m.frameRate + const i = Math.floor(f) + const frames = m.loudness.length + if (i < 0 || i >= frames) { + out.fill(0) + return out + } + const j = Math.min(i + 1, frames - 1) + const t = f - i + for (let k = 0; k < m.bandCount; k++) { + out[k] = (m.bands[i * m.bandCount + k] * (1 - t) + m.bands[j * m.bandCount + k] * t) / 255 + } + return out +} + +/** The overall level at pos, 0–1. */ +export function loudnessAt(m: BeatMap, pos: number) { + const i = Math.floor((pos / 1000) * m.frameRate) + return i >= 0 && i < m.loudness.length ? m.loudness[i] / 255 : 0 +} + +/** + * The song's shape for the waveform: `bars` heights, 0–1. Each bar is the + * average level of its stretch, scaled against the song's own quiet and + * loud parts, so a loud, compressed song still shows its verses and + * choruses instead of sitting at the top. A song that barely changes sits + * mid-height, so small wobbles don't become big ones. + */ +export function waveform(loudness: Uint8Array, bars: number): number[] { + const per = loudness.length / bars + const means: number[] = [] + for (let b = 0; b < bars; b++) { + const from = Math.floor(b * per) + const to = Math.max(from + 1, Math.min(loudness.length, Math.floor((b + 1) * per))) + let sum = 0 + for (let i = from; i < to; i++) sum += loudness[i] ?? 0 + means.push(sum / (to - from) / 255) + } + const sorted = [...means].sort((a, b) => a - b) + let hi = sorted[sorted.length - 1] ?? 0 + let lo = sorted[Math.floor(sorted.length * 0.1)] ?? 0 + if (hi - lo < 0.08) { + // Barely changes: sit in the middle, wobbling a little. + const mid = (hi + lo) / 2 + lo = mid - 0.04 + hi = mid + 0.04 + } + // Quiet parts still show a sliver, and only the loudest bar reaches the top. + return means.map((m) => 0.12 + 0.8 * Math.min(1, Math.max(0, (m - lo) / (hi - lo))) ** 1.5) +} diff --git a/web/src/lib/beat.ts b/web/src/lib/beat.ts new file mode 100644 index 0000000..9640fd5 --- /dev/null +++ b/web/src/lib/beat.ts @@ -0,0 +1,547 @@ +// The beat engine (MAD-775). One rAF loop turns the room's playback +// position into a few CSS custom properties, so the app can breathe with +// the song without re-rendering anything. +// +// Where the beat comes from, first match wins: +// 1. a grid tapped in the beat lab, for a song the analysis got wrong; +// 2. the song's beat map from the server (lib/beat-map.ts); +// 3. the default tempo, while the map loads or for songs without one: +// in time, but not in phase. +// The map also gives the energy (its sections and loudness) and the +// spectrum. +// +// Variables, written only to the elements that use them (a var on :root +// would restyle the whole tree every frame): +// --beat 0→1 on each beat, easing back down. Scaled by energy. +// --downbeat the same on the first beat of each bar, slower to fade. +// --swell a slow breath once a bar: rises over a beat, fades over the +// rest. For big surfaces, where a per-beat flash reads as strobe. +// --energy how hard the song is going right now, slowly smoothed. +// --b0..--b3 bass, low mids, high mids and treble, for the equalizer. +// Elements also get data-beat-live while the engine drives them, so CSS +// can swap a canned animation for the live one. + +import { beatAt, loudnessAt, sectionEnergy, spectrumAt, type BeatMap } from './beat-map' +import { positionAt, type NowPlaying } from './now-playing' +import { createStore } from './store' + +export type BeatLevel = 'off' | 'subtle' + +/** The effects the beat lab can tune one by one. */ +export const EFFECTS = { + breath: 'Backdrop breathing', + scene: 'Backdrop scene', + drift: 'Backdrop drift', + art: 'Album art lift', + glass: 'Glass shine', + eq: 'Equalizer', + lyrics: 'Lyrics glow', + waveform: 'Waveform scrubber', +} as const +export type Effect = keyof typeof EFFECTS + +export type BeatSettings = { + level: BeatLevel + /** How strongly everything moves, 0.5 (calmer) – 1.75 (stronger). */ + intensity: number + /** The backdrop scene, or 'auto' to pick one per song. */ + scene: string + /** The full-screen visualizer's show, or 'auto'. */ + show: string + /** Each effect's strength, 0 (off) – 2. */ + effects: Record + /** How late this device's sound is, ms: a Bluetooth speaker, say. Visuals wait for it. */ + delayMs: number +} + +const SETTINGS_KEY = 'syncphony-visuals' +export const BEAT_DEFAULTS: BeatSettings = { + level: 'subtle', + intensity: 1, + scene: 'auto', + show: 'auto', + effects: { breath: 1, scene: 1, drift: 1, art: 1, glass: 1, eq: 1, lyrics: 1, waveform: 1 }, + delayMs: 0, +} + +export const beatSettings = createStore(readSettings()) +beatSettings.subscribe(() => { + const s = beatSettings.get() + try { + localStorage.setItem(SETTINGS_KEY, JSON.stringify(s)) + } catch { + // Private mode: settings last until the tab closes. + } + applyEffects(s) +}) +applyEffects(beatSettings.get()) + +/** Changes some settings; effects merge into the current ones. */ +export function setBeatSettings(next: Partial> & { effects?: Partial> }) { + beatSettings.set((s) => ({ ...s, ...next, effects: { ...s.effects, ...next.effects } })) +} + +function readSettings(): BeatSettings { + const clamp = (v: unknown, lo: number, hi: number, d: number) => (typeof v === 'number' && Number.isFinite(v) ? Math.min(hi, Math.max(lo, v)) : d) + try { + const s = JSON.parse(localStorage.getItem(SETTINGS_KEY) ?? '{}') as Partial + const effects = { ...BEAT_DEFAULTS.effects } + for (const k of Object.keys(effects) as Effect[]) effects[k] = clamp(s.effects?.[k], 0, 2, effects[k]) + return { + level: s.level === 'off' ? 'off' : 'subtle', + intensity: clamp(s.intensity, 0.5, 1.75, BEAT_DEFAULTS.intensity), + scene: typeof s.scene === 'string' ? s.scene : BEAT_DEFAULTS.scene, + show: typeof s.show === 'string' ? s.show : BEAT_DEFAULTS.show, + effects, + delayMs: clamp(s.delayMs, -300, 600, 0), + } + } catch { + return BEAT_DEFAULTS + } +} + +/** + * The CSS effects read their strength from --fx-* on the root: set once + * when the settings change, never per frame. + */ +function applyEffects(s: BeatSettings) { + if (typeof document === 'undefined') return + const root = document.documentElement + for (const [k, v] of Object.entries(s.effects)) root.style.setProperty(`--fx-${k}`, String(v)) + root.toggleAttribute('data-fx-eq-off', s.effects.eq === 0) +} + +/** Whether the full-screen visualizer is open in the app. */ +export const visualizerOpen = createStore(false) + +/** Whether lights mode (MAD-781) is on: the phone as a light, flashing on the downbeats. */ +export const lightsOpen = createStore(false) + +/** Tempo for songs with no beat map yet. */ +const DEFAULT_BPM = 120 + +/** + * What's playing, and its beat map once loaded. Set by useBeatSync, which + * the app shell and the big screen each mount once. + */ +export const beatSource = createStore<{ np: NowPlaying | null; map: BeatMap | null; mapKey: string | null }>({ + np: null, + map: null, + mapKey: null, +}) + +/** A song's beat grid: its tempo and the position of one downbeat, in ms. */ +export type Grid = { bpm: number; anchorMs: number } + +/** Grids tapped in the beat lab, by song. They beat the song's map. */ +const tapped = new Map() +export const gridVersion = createStore(0) + +function keyOf(np: NowPlaying) { + return np.itemId ?? np.track.trackId +} + +/** Where the music is now, and which song's grid applies. */ +function clock(): { key: string; pos: number; playing: boolean } | null { + const { np } = beatSource.get() + if (np && !np.paused) return { key: keyOf(np), pos: positionAt(np, Date.now()) - beatSettings.get().delayMs, playing: true } + if (np) return { key: keyOf(np), pos: np.positionMs, playing: false } + return null +} + +/** The current song's map, if it's loaded. */ +function mapFor(key: string) { + const { map, mapKey } = beatSource.get() + return map && mapKey === key ? map : null +} + +/** Whether the song's map is still on its way (mapKey is set once it's known, map or not). */ +function mapPending(key: string) { + const { np, mapKey } = beatSource.get() + return !!np?.itemId && np.itemId === key && mapKey !== key +} + +/** Where the current song's beat comes from, for the beat lab. */ +export type BeatOrigin = 'tapped' | 'analysed' | 'steady' | 'default' + +export function currentBeat(): Grid & { origin: BeatOrigin } { + const c = clock() + const own = c && tapped.get(c.key) + if (own) return { ...own, origin: 'tapped' } + const map = c && mapFor(c.key) + if (map && map.bpm > 0) return { bpm: map.bpm, anchorMs: map.beats[map.downbeat] ?? 0, origin: 'analysed' } + if (map) return { bpm: 0, anchorMs: 0, origin: 'steady' } + return { bpm: DEFAULT_BPM, anchorMs: 0, origin: 'default' } +} + +function setGrid(g: Grid) { + const c = clock() + if (!c) return + tapped.set(c.key, { bpm: Math.min(220, Math.max(50, g.bpm)), anchorMs: g.anchorMs }) + gridVersion.set((v) => v + 1) + wake() +} + +// --- Tap tempo --------------------------------------------------------------- + +let taps: number[] = [] + +/** + * A tap on the beat. Start on a "one": a straight line through the taps + * gives the tempo, and the first tap becomes a downbeat. + */ +export function tap() { + const c = clock() + if (!c) return + const last = taps.at(-1) + if (last === undefined || c.pos - last > 2000 || c.pos < last) taps = [] + taps.push(c.pos) + taps = taps.slice(-12) + if (taps.length < 3) return taps.length + // Least squares: tap i lands at anchor + i·period. + const n = taps.length + const mi = (n - 1) / 2 + const mt = taps.reduce((a, b) => a + b, 0) / n + let num = 0 + let den = 0 + taps.forEach((t, i) => { + num += (i - mi) * (t - mt) + den += (i - mi) ** 2 + }) + const period = num / den + // The first tap is beat 0; move the anchor to the bar of the last one. + const first = mt - mi * period + const bars = Math.floor((n - 1) / 4) + setGrid({ bpm: 60000 / period, anchorMs: first + bars * 4 * period }) + return n +} + +/** The current beat as a grid to adjust: tapped, analysed, or the default. */ +function adjustable(): Grid { + const b = currentBeat() + return b.bpm > 0 ? b : { bpm: DEFAULT_BPM, anchorMs: 0 } +} + +export function nudge(ms: number) { + const g = adjustable() + setGrid({ ...g, anchorMs: g.anchorMs + ms }) +} + +export function scaleTempo(factor: number) { + const g = adjustable() + setGrid({ ...g, bpm: g.bpm * factor }) +} + +/** Moves the "one" along by a beat, for a grid that starts its bars in the wrong place. */ +export function shiftDownbeat() { + const g = adjustable() + setGrid({ ...g, anchorMs: g.anchorMs + 60000 / g.bpm }) +} + +/** Forgets the song's tapped grid, back to its beat map. */ +export function resetGrid() { + const c = clock() + if (!c) return + tapped.delete(c.key) + taps = [] + gridVersion.set((v) => v + 1) +} + +// --- Delay calibration ------------------------------------------------------- + +let heard: number[] = [] + +/** + * A tap on the beat as it's heard here, to work out how late this + * device's sound is (a Bluetooth speaker can add a fifth of a second). + * Compares the taps with the song's beat map; after four, sets delayMs to + * their typical lag. Returns how many taps so far, or null if the song + * playing has no beat to compare with. + */ +export function calibrateTap(): number | null { + const { np } = beatSource.get() + const map = np && !np.paused ? mapFor(keyOf(np)) : null + if (!np || !map || map.bpm <= 0) return null + const pos = positionAt(np, Date.now()) + const last = heard.at(-1) + if (last === undefined || pos - last > 2000 || pos < last) heard = [] + heard.push(pos) + heard = heard.slice(-16) + const lags = heard + .map((t) => beatAt(map, t)) + .filter((p) => p !== null) + .map((p) => (p.into > p.period / 2 ? p.into - p.period : p.into)) + .sort((a, b) => a - b) + if (lags.length >= 4) { + const median = lags[Math.floor(lags.length / 2)] + setBeatSettings({ delayMs: Math.round(Math.min(600, Math.max(-300, median))) }) + } + return heard.length +} + +// --- Targets --------------------------------------------------------------- + +type Target = { el: HTMLElement; drift: boolean } +const targets = new Set() + +/** + * Lets the engine drive el's variables. With `drift`, el's CSS animations + * also speed up and slow down with the energy. Returns the detach. + */ +export function attachBeat(el: HTMLElement, opts: { drift?: boolean } = {}) { + const t = { el, drift: !!opts.drift } + targets.add(t) + wake() + return () => { + targets.delete(t) + clear(t) + } +} + +// --- The loop ---------------------------------------------------------------- + +const reduced = typeof matchMedia === 'function' ? matchMedia('(prefers-reduced-motion: reduce)') : undefined + +// Smoothed state, eased every frame so nothing snaps. +let presence = 0 +// How hard the beat drives things: only while the music actually plays. +let drive = 0 +// When the music stopped playing here. A song change passes through +// "loading" for a moment, sometimes more than once: the backdrop holds +// through that, rather than fading out and back in (it flickers). +let quietSince = 0 +const HOLD_MS = 1500 +let energy = 0 +let loud = 0 +let swell = 0 +let frame = 0 +let lastFrame = 0 +let lastRate = 1 +let lastRateAt = 0 +const written = new WeakMap>() +const BANDS = 12 +const spectrum = new Float32Array(BANDS) +const level = new Float32Array(BANDS) + +/** One frame of the engine, for canvas scenes that draw rather than style. */ +export type BeatFrame = { + beat: number + downbeat: number + swell: number + energy: number + /** Four bands, low to high. */ + bands: [number, number, number, number] + /** Twelve bands, low to high, 0–1 (from the beat map, else made up from the beat). */ + spectrum: Float32Array + /** Fades in when music starts, out when it stops. */ + presence: number + /** Beats since the grid's anchor, and the beat's length in ms. */ + beatIndex: number + /** Where the beat falls in its bar, 0 (the one) – 3. */ + bar: number + periodMs: number + /** Frame time (performance.now) and time since the last frame. */ + now: number + dt: number +} + +const frameListeners = new Set<(f: BeatFrame) => void>() + +/** Calls fn every frame the engine runs. Returns the unsubscribe. */ +export function onBeatFrame(fn: (f: BeatFrame) => void) { + frameListeners.add(fn) + wake() + return () => { + frameListeners.delete(fn) + } +} + +function active() { + return beatSettings.get().level !== 'off' && !reduced?.matches && (targets.size > 0 || frameListeners.size > 0) +} + +function wake() { + if (frame || typeof requestAnimationFrame !== 'function') return + if (!active()) { + targets.forEach(clear) + return + } + lastFrame = performance.now() + frame = requestAnimationFrame(tick) +} + +/** Where pos falls in the song's beat, from whichever source applies. */ +function place(key: string, pos: number): { index: number; into: number; period: number; bar: number } | null { + const own = tapped.get(key) + const map = mapFor(key) + if (!own && map) return map.bpm > 0 ? beatAt(map, pos) : null + // No guessing while the map is on its way: the guess would jump when it lands. + if (!own && mapPending(key)) return null + const { bpm, anchorMs } = own ?? { bpm: DEFAULT_BPM, anchorMs: 0 } + const period = 60000 / bpm + const since = pos - anchorMs + const n = Math.floor(since / period) + return { index: n, into: since - n * period, period, bar: ((n % 4) + 4) % 4 } +} + +function tick(now: number) { + frame = 0 + const dt = Math.min(100, now - lastFrame) + lastFrame = now + if (!active()) { + targets.forEach(clear) + return + } + + const c = clock() + const playing = !!c?.playing && document.visibilityState === 'visible' + const map = c ? mapFor(c.key) : null + const intensity = beatSettings.get().intensity + // Fades in over about a second when the music starts, out when it stops. + if (playing) quietSince = 0 + else if (!quietSince) quietSince = now + const held = playing || (quietSince > 0 && now - quietSince < HOLD_MS && !!c) + presence = approach(presence, held ? 1 : 0, dt, held ? 900 : 700) + drive = approach(drive, playing ? 1 : 0, dt, playing ? 400 : 200) + + // Energy: the section's, nudged by how loud it is right now. + let target = 0.5 + if (map && c) { + loud = approach(loud, loudnessAt(map, c.pos), dt, 600) + target = 0.55 * sectionEnergy(map, c.pos) + 0.45 * loud + } + energy = approach(energy, target, dt, 1500) + + const vars = { beat: 0, downbeat: 0, swell: 0, energy: energy * presence, b0: 0, b1: 0, b2: 0, b3: 0 } + let beatIndex = 0 + let bar = 0 + let periodMs = 500 + let breath = 0 + const p = c && presence > 0.001 ? place(c.key, c.pos) : null + // Louder songs hit harder, but even a quiet one keeps a faint pulse. + const gain = presence * drive * (0.35 + 0.65 * energy) * intensity + if (p) { + beatIndex = p.index + bar = p.bar + periodMs = p.period + // Never more than three flashes a second: at very fast tempos, every other beat. + const flash = p.period >= 333 || p.index % 2 === 0 + vars.beat = flash ? gain * pulse(p.into, Math.min(240, p.period * 0.4)) : 0 + vars.downbeat = p.bar === 0 ? gain * pulse(p.into, Math.min(520, p.period * 0.85)) : 0 + // Rises over the first beat of the bar and sinks over the rest. + const intoBar = p.bar * p.period + p.into + breath = + intoBar < p.period ? Math.sin(((intoBar / p.period) * Math.PI) / 2) : Math.exp(-(intoBar - p.period) / (p.period * 1.2)) + } + + if (map && c && presence > 0.001) { + spectrumAt(map, c.pos, spectrum) + } else if (p) { + // No map: make a spectrum up from the beat, kick low and hats high. + const kick = pulse(p.into, p.period * 0.5) + const snare = p.bar % 2 === 1 ? pulse(p.into, p.period * 0.6) : 0.15 + const hats = 0.35 + 0.65 * pulse(p.into % (p.period / 4), p.period * 0.15) + for (let k = 0; k < BANDS; k++) { + const x = k / (BANDS - 1) + spectrum[k] = kick * Math.max(0, 1 - x * 3) + snare * Math.exp(-(((x - 0.45) / 0.2) ** 2)) + hats * x ** 2 * 0.7 + } + } else { + spectrum.fill(0) + } + // Each band's level: up fast, down slower, and only the top of its range, + // so the bars move rather than sit high. + for (let k = 0; k < BANDS; k++) { + const v = Math.max(0, (spectrum[k] - 0.4) / 0.6) ** 1.3 + level[k] = follow(level[k], v * presence * drive, dt, 40, 220) + } + const group = (from: number) => ((level[from] + level[from + 1] + level[from + 2]) / 3) * Math.min(1.5, intensity) + vars.b0 = group(0) + vars.b1 = group(3) + vars.b2 = group(6) + vars.b3 = group(9) + + swell = approach(swell, breath * presence * drive * (0.45 + 0.55 * energy) * intensity, dt, 160) + vars.swell = swell + + // Calm songs drift slowly, loud ones churn. Only now and then: changing + // an animation's rate is cheap, but not free. + const rate = 1 + presence * (energy - 0.5) * 1.6 * beatSettings.get().effects.drift + const setRate = Math.abs(rate - lastRate) > 0.04 && now - lastRateAt > 250 + if (setRate) { + lastRate = rate + lastRateAt = now + } + + const live = presence > 0.001 + if (frameListeners.size) { + const f: BeatFrame = { + beat: vars.beat, + downbeat: vars.downbeat, + swell: vars.swell, + energy: vars.energy, + bands: [vars.b0, vars.b1, vars.b2, vars.b3], + spectrum: level, + presence, + beatIndex, + bar, + periodMs, + now, + dt, + } + frameListeners.forEach((fn) => fn(f)) + } + for (const t of targets) { + write(t.el, vars) + if (live) t.el.dataset.beatLive = '' + else delete t.el.dataset.beatLive + if (t.drift && setRate) for (const a of t.el.getAnimations({ subtree: true })) a.updatePlaybackRate(rate) + } + + // Idle once everything has settled: nothing playing, nothing fading. + if (playing || presence > 0.001 || drive > 0.001 || swell > 0.001 || Math.abs(energy - target) > 0.001) { + frame = requestAnimationFrame(tick) + } + else { + for (const t of targets) if (t.drift) for (const a of t.el.getAnimations({ subtree: true })) a.updatePlaybackRate(1) + lastRate = 1 + } +} + +/** A soft attack (30 ms), then an exponential fade over about `decay` ms. */ +function pulse(ms: number, decay: number) { + if (ms < 0) return 0 + if (ms < 30) return ms / 30 + return Math.exp(-(ms - 30) / (decay / 3)) +} + +function approach(from: number, to: number, dt: number, ms: number) { + const next = from + (to - from) * (1 - Math.exp(-dt / (ms / 3))) + return Math.abs(next - to) < 0.0005 ? to : next +} + +/** Eases a value: fast up (attack), slower down (release). */ +function follow(from: number, to: number, dt: number, attack: number, release: number) { + return from + (to - from) * (1 - Math.exp(-dt / ((to > from ? attack : release) / 3))) +} + +function write(el: HTMLElement, vars: Record) { + const prev = written.get(el) ?? {} + for (const [k, v] of Object.entries(vars)) { + if (prev[k] !== undefined && Math.abs(prev[k] - v) < 0.003) continue + prev[k] = v + el.style.setProperty(`--${k}`, v.toFixed(3)) + } + written.set(el, prev) +} + +function clear(t: Target) { + for (const k of ['beat', 'downbeat', 'swell', 'energy', 'b0', 'b1', 'b2', 'b3']) t.el.style.removeProperty(`--${k}`) + delete t.el.dataset.beatLive + written.delete(t.el) + if (t.drift) for (const a of t.el.getAnimations({ subtree: true })) a.updatePlaybackRate(1) +} + +beatSource.subscribe(wake) +beatSettings.subscribe(wake) +reduced?.addEventListener('change', wake) +if (typeof document !== 'undefined') document.addEventListener('visibilitychange', wake) diff --git a/web/src/lib/room-socket.ts b/web/src/lib/room-socket.ts index 0e13b8c..6614781 100644 --- a/web/src/lib/room-socket.ts +++ b/web/src/lib/room-socket.ts @@ -4,6 +4,7 @@ import type { components } from '@/api/schema.gen' import { invitesQuery, membersQuery } from './access' import { meQuery } from './auth' import { syncServerClock } from './clock' +import { displayMeQuery } from './displays' import { guestPassQuery, guestsQuery } from './guests' import { crown, heartsQuery, nightsQuery, type Hearts, type Night } from './nights' import { newer, playbackQuery, type Playback } from './playback' @@ -154,6 +155,8 @@ function handle(queryClient: QueryClient, roomId: string, ev: RoomEvent) { case 'room.updated': { const room = ev.data as Room queryClient.setQueryData(roomsQuery.queryKey, (rs) => rs?.map((r) => (r.id === room.id ? room : r))) + // A paired screen knows its room from its own record: its look follows the room's. + queryClient.setQueryData(displayMeQuery.queryKey, (d) => (d && d.room.id === room.id ? { ...d, room } : d)) break } case 'room.deleted': { diff --git a/web/src/lib/scenes.ts b/web/src/lib/scenes.ts new file mode 100644 index 0000000..4f06a0f --- /dev/null +++ b/web/src/lib/scenes.ts @@ -0,0 +1,445 @@ +import type { BeatFrame } from './beat' + +// Scene painters: canvas drawings of the music in the art's colors, for +// the app's backdrop (components/shell/backdrop-scene.tsx) and the big +// visualizer (components/visualizer). Each is a factory, so every canvas +// gets its own state. + +export type Colors = { vibrant: string; dominant: string; muted: string; light: string } + +export type Painter = (ctx: CanvasRenderingContext2D, w: number, h: number, f: BeatFrame, c: Colors) => void + +/** The palette as the element sees it (it eases between songs). */ +export function readColors(el: HTMLElement): Colors { + const s = getComputedStyle(el) + const v = (name: string) => s.getPropertyValue(name).trim() || 'white' + return { vibrant: v('--pal-vibrant'), dominant: v('--pal-dominant'), muted: v('--pal-muted'), light: v('--pal-light') } +} + +/** Eases a value: fast up (attack), slower down (release). */ +export function follow(from: number, to: number, dt: number, attack: number, release: number) { + const ms = to > from ? attack : release + return from + (to - from) * (1 - Math.exp(-dt / (ms / 3))) +} + +/** A background equalizer: soft columns along the bottom, the song's spectrum mirrored, bass in the middle. */ +function horizon(): Painter { + const N = 32 + const level = new Float32Array(N) + return (ctx, w, h, f, c) => { + const t = f.now / 1000 + const gap = w / N + const bands = f.spectrum.length + const grad = ctx.createLinearGradient(0, h, 0, h * 0.35) + grad.addColorStop(0, c.dominant) + grad.addColorStop(0.6, c.vibrant) + grad.addColorStop(1, c.light) + ctx.fillStyle = grad + for (let i = 0; i < N; i++) { + const x = (i + 0.5) / N + const d = Math.abs(x - 0.5) * 2 // 0 middle, 1 edges + // Between the two nearest bands. + const b = d * (bands - 1) + const k = Math.min(Math.floor(b), bands - 2) + const shape = f.spectrum[k] * (1 - (b - k)) + f.spectrum[k + 1] * (b - k) + // Neighbouring columns share a band; a little wander tells them apart. + const wander = f.energy * 0.12 * (0.5 + 0.5 * Math.sin(t * (1.3 + (i % 7) * 0.31) + i * 1.7)) + level[i] = follow(level[i], Math.min(1, shape + wander), f.dt, 50, 320) + const bh = h * (0.04 + level[i] * 0.5) + const bw = gap * 0.62 + roundRect(ctx, i * gap + (gap - bw) / 2, h - bh, bw, bh + bw, bw / 2) + } + } +} + +/** One slow ring a bar, spreading from behind the art like a drop in still water. */ +function ripples(): Painter { + type Ring = { born: number; color: keyof Colors } + let rings: Ring[] = [] + let last = Number.NaN + const palette: (keyof Colors)[] = ['vibrant', 'light'] + return (ctx, w, h, f, c) => { + const bar = Math.floor(f.beatIndex / 4) + if (bar !== last) { + if (!Number.isNaN(last)) rings.push({ born: f.now, color: palette[Math.abs(bar) % palette.length] }) + last = bar + } + const life = 5200 + rings = rings.filter((r) => f.now - r.born < life) + const cx = w / 2 + const cy = h * 0.3 + const reach = Math.hypot(w, h) * 0.7 + for (const r of rings) { + const k = (f.now - r.born) / life + const ease = 1 - (1 - k) ** 2.5 + ctx.globalAlpha = f.presence * (1 - k) ** 2 * (0.35 + 0.35 * f.energy) + ctx.strokeStyle = c[r.color] + ctx.lineWidth = h * 0.02 * (1 - k * 0.5) + ctx.beginPath() + ctx.arc(cx, cy, h * 0.05 + ease * reach, 0, Math.PI * 2) + ctx.stroke() + } + } +} + +/** + * Curtains of light hanging from the top of the screen, like the real + * aurora: soft vertical rays whose hems wave slowly. They reach further + * down on the bar and ripple faster when the song is loud. + */ +function aurora(): Painter { + let phase = 0 + const curtains: { top: number; len: number; color: keyof Colors; f: number; o: number }[] = [ + // Back to front: a tall faint veil, the main curtain, then lower, + // shorter folds in the other colors, so it has depth. + { top: -0.02, len: 0.5, color: 'muted', f: 0.6, o: 5.1 }, + { top: 0.05, len: 0.38, color: 'vibrant', f: 1, o: 0 }, + { top: 0.2, len: 0.28, color: 'light', f: 1.6, o: 2.4 }, + { top: 0.34, len: 0.22, color: 'dominant', f: 2.1, o: 3.7 }, + ] + const COLS = 40 + return (ctx, w, h, f, c) => { + phase += (f.dt / 1000) * (0.12 + f.energy * 0.35) + const colW = w / COLS + for (const k of curtains) { + for (let i = 0; i <= COLS; i++) { + const x = i / COLS + // The hem waves; the rays' brightness shimmers along it. + const wave = Math.sin(x * 4 * k.f + phase + k.o) * 0.5 + Math.sin(x * 9 * k.f - phase * 1.7 + k.o) * 0.25 + const shimmer = 0.55 + 0.45 * Math.sin(x * 23 + phase * 2.3 + k.o) ** 2 + const top = (k.top + wave * 0.05) * h + const len = h * k.len * (0.75 + 0.2 * f.energy + 0.3 * f.swell) * (0.8 + 0.2 * shimmer) + const g = ctx.createLinearGradient(0, top, 0, top + len) + g.addColorStop(0, 'transparent') + g.addColorStop(0.55, c[k.color]) + g.addColorStop(1, 'transparent') + ctx.globalAlpha = f.presence * shimmer * (0.3 + 0.25 * f.swell + 0.15 * f.energy) + ctx.fillStyle = g + ctx.fillRect(i * colW - colW, top, colW * 2.2, len) + } + } + } +} + +/** + * The palette mesh, alive: big soft pools of the art's colors that wander, + * swell and brighten once a bar, and wander faster when the song is loud. + */ +function mesh(): Painter { + let phase = Math.random() * 100 + const pools: { color: keyof Colors; x: number; y: number; ax: number; ay: number; fx: number; fy: number; r: number }[] = [ + { color: 'vibrant', x: 0.3, y: 0.25, ax: 0.18, ay: 0.12, fx: 0.7, fy: 0.9, r: 0.55 }, + { color: 'light', x: 0.72, y: 0.45, ax: 0.15, ay: 0.18, fx: 0.5, fy: 0.6, r: 0.45 }, + { color: 'dominant', x: 0.45, y: 0.8, ax: 0.22, ay: 0.1, fx: 0.4, fy: 0.8, r: 0.6 }, + ] + return (ctx, w, h, f, c) => { + phase += (f.dt / 1000) * (0.08 + f.energy * 0.3) + const m = Math.max(w, h) + for (const p of pools) { + const x = (p.x + p.ax * Math.sin(phase * p.fx)) * w + const y = (p.y + p.ay * Math.cos(phase * p.fy)) * h + const r = m * p.r * (1 + 0.22 * f.swell) + const g = ctx.createRadialGradient(x, y, 0, x, y, r) + g.addColorStop(0, c[p.color]) + g.addColorStop(1, 'transparent') + ctx.globalAlpha = f.presence * (0.28 + 0.4 * f.swell + 0.12 * f.energy) + ctx.fillStyle = g + ctx.fillRect(0, 0, w, h) + } + } +} + +/** Embers rising through the glass; they glow on the beat and quicken with energy. */ +function embers(): Painter { + const N = 70 + const roles: (keyof Colors)[] = ['vibrant', 'light', 'vibrant', 'muted'] + const ps = Array.from({ length: N }, () => ({ + x: Math.random(), + y: Math.random(), + r: 0.8 + Math.random() * 1.8, + s: 0.4 + Math.random() * 0.8, + w: Math.random() * Math.PI * 2, + hit: Math.random(), + color: roles[Math.floor(Math.random() * roles.length)], + })) + return (ctx, w, h, f, c) => { + const rise = (f.dt / 1000) * (0.02 + f.energy * 0.09) + for (const p of ps) { + p.y -= rise * p.s * (1 + f.swell * 0.8) + p.w += f.dt / 1400 + if (p.y < -0.05) { + p.y = 1.05 + p.x = Math.random() + } + const x = (p.x + Math.sin(p.w) * 0.02) * w + const y = p.y * h + // Some embers flare on the beat, the rest just drift. + const flare = p.hit > 0.55 ? f.beat * (p.hit - 0.55) * 2.2 : 0 + ctx.globalAlpha = f.presence * Math.min(1, 0.35 + flare + f.energy * 0.2) + ctx.fillStyle = c[p.color] + ctx.beginPath() + ctx.arc(x, y, p.r * (1 + flare * 0.8), 0, Math.PI * 2) + ctx.fill() + } + } +} + +/** + * The song's spectrum as a ring of bars around the middle of the screen + * (the album art sits inside it), mirrored left and right, bass at the + * top. The ring swells on the bar and the bars flare on the beat. + * For the big visualizer: drawn at full resolution. + */ +function spectrum(): Painter { + const N = 72 + const level = new Float32Array(N) + let spin = 0 + return (ctx, w, h, f, c) => { + const cx = w / 2 + const cy = h / 2 + const m = Math.min(w, h) + const inner = m * (0.2 + 0.012 * f.swell) + const reach = m * 0.2 + const bands = f.spectrum.length + spin += (f.dt / 1000) * (0.03 + f.energy * 0.08) + // A soft glow behind the ring. + const glow = ctx.createRadialGradient(cx, cy, inner * 0.8, cx, cy, inner + reach * 1.4) + glow.addColorStop(0, c.vibrant) + glow.addColorStop(1, 'transparent') + ctx.globalAlpha = f.presence * (0.18 + 0.25 * f.swell) + ctx.fillStyle = glow + ctx.fillRect(0, 0, w, h) + ctx.globalAlpha = f.presence + ctx.lineCap = 'round' + const width = ((Math.PI * 2 * inner) / N) * 0.55 + for (let i = 0; i < N; i++) { + // Mirrored: 0 at the top through the band range down each side. + const half = i < N / 2 ? i / (N / 2) : (N - i) / (N / 2) + const b = half * (bands - 1) + const k = Math.min(Math.floor(b), bands - 2) + const v = f.spectrum[k] * (1 - (b - k)) + f.spectrum[k + 1] * (b - k) + level[i] = follow(level[i], v, f.dt, 40, 260) + const len = reach * (0.06 + level[i] * 0.94) * (1 + f.beat * 0.15) + const a = (i / N) * Math.PI * 2 - Math.PI / 2 + spin + const cos = Math.cos(a) + const sin = Math.sin(a) + ctx.strokeStyle = level[i] > 0.75 ? c.light : c.vibrant + ctx.lineWidth = width + ctx.beginPath() + ctx.moveTo(cx + cos * inner, cy + sin * inner) + ctx.lineTo(cx + cos * (inner + len), cy + sin * (inner + len)) + ctx.stroke() + } + } +} + +/** The spectrum's value at x (0–1 across the bands), between the two nearest. */ +function bandAt(spec: ArrayLike, x: number) { + const b = Math.max(0, Math.min(1, x)) * (spec.length - 1) + const k = Math.min(Math.floor(b), spec.length - 2) + return spec[k] * (1 - (b - k)) + spec[k + 1] * (b - k) +} + +/** + * A tunnel of the song's own spectrum: every few hundredths of a second + * a ring is cast in the shape of the spectrum at that moment and flies at + * you, so the tunnel walls are the song's recent past. Bars' first beats + * cast a bright ring; the whole thing turns faster when the song is loud. + */ +function tunnel(): Painter { + type Ring = { z: number; shape: Float32Array; bright: boolean; hue: number } + const POINTS = 96 + let rings: Ring[] = [] + let castAt = 0 + let lastBar = Number.NaN + let spin = 0 + let wander = 0 + return (ctx, w, h, f, c) => { + const travel = (f.dt / 1000) * (0.32 + f.energy * 0.55 + f.swell * 0.2) + for (const r of rings) r.z -= travel + rings = rings.filter((r) => r.z > 0.02) + const bar = Math.floor(f.beatIndex / 4) + const downbeat = f.downbeat > 0.5 && bar !== lastBar + if (downbeat) lastBar = bar + if (f.now - castAt > 30 || downbeat) { + castAt = f.now + const shape = new Float32Array(POINTS) + for (let i = 0; i < POINTS; i++) { + // Mirrored around the ring: bass on two opposite sides, treble between. + const a = i / POINTS + shape[i] = bandAt(f.spectrum, 1 - Math.abs(Math.cos(a * Math.PI * 2))) + } + rings.push({ z: 1, shape, bright: downbeat, hue: rings.length }) + } + spin += (f.dt / 1000) * (0.05 + f.energy * 0.25) + f.beat * 0.002 + wander += f.dt / 1000 + const m = Math.min(w, h) + const cx = w / 2 + Math.sin(wander * 0.21) * m * 0.06 + const cy = h / 2 + Math.cos(wander * 0.17) * m * 0.05 + ctx.globalCompositeOperation = 'lighter' + ctx.lineJoin = 'round' + // Far to near, so the near rings draw over. + for (let j = 0; j < rings.length; j++) { + const r = rings[j] + const radius = (m * 0.045) / r.z + if (radius > Math.hypot(w, h)) continue + const near = 1 - r.z + ctx.globalAlpha = f.presence * Math.min(1, near * 1.8) * (r.bright ? 1 : 0.32) * Math.min(1, r.z * 5) + ctx.strokeStyle = r.bright ? c.light : near > 0.45 ? c.vibrant : c.dominant + ctx.lineWidth = Math.max(0.75, (r.bright ? 3 : 1) * near * near * 3.5) + ctx.beginPath() + for (let i = 0; i <= POINTS; i++) { + const a = (i / POINTS) * Math.PI * 2 + spin + r.z * 0.8 + const k = 1 + r.shape[i % POINTS] * 0.3 + const x = cx + Math.cos(a) * radius * k + const y = cy + Math.sin(a) * radius * k + if (i === 0) ctx.moveTo(x, y) + else ctx.lineTo(x, y) + } + ctx.stroke() + } + ctx.globalCompositeOperation = 'source-over' + } +} + +/** + * Rain on dark water: ripples open all over the screen, a few on every + * beat and a burst of them on each bar, with the treble flicking tiny + * ones in between. + */ +function rain(): Painter { + type Drop = { x: number; y: number; born: number; size: number; life: number; color: keyof Colors } + let drops: Drop[] = [] + let lastBeat = Number.NaN + let flick = 0 + const roles: (keyof Colors)[] = ['vibrant', 'light', 'vibrant', 'muted'] + const drop = (f: BeatFrame, size: number, life: number, near?: { x: number; y: number }) => { + drops.push({ + x: near ? near.x + (Math.random() - 0.5) * 0.2 : Math.random(), + y: near ? near.y + (Math.random() - 0.5) * 0.2 : Math.random(), + born: f.now, + size: size * (0.7 + Math.random() * 0.6), + life: life * (0.8 + Math.random() * 0.4), + color: roles[Math.floor(Math.random() * roles.length)], + }) + } + return (ctx, w, h, f, c) => { + if (f.beatIndex !== lastBeat) { + lastBeat = f.beatIndex + const one = ((f.beatIndex % 4) + 4) % 4 === 0 + const count = Math.round((one ? 6 : 2) + f.energy * (one ? 6 : 3)) + const at = { x: 0.2 + Math.random() * 0.6, y: 0.2 + Math.random() * 0.6 } + if (one) drop(f, 0.55, 3600, at) + for (let i = 0; i < count; i++) drop(f, one ? 0.22 : 0.16, 2400, one ? at : undefined) + } + // The treble rains between the beats. + const treble = f.bands[3] + flick += (f.dt / 1000) * treble * treble * 14 + while (flick > 1) { + flick -= 1 + drop(f, 0.06, 1400) + } + drops = drops.filter((d) => f.now - d.born < d.life).slice(-140) + const m = Math.min(w, h) + ctx.globalCompositeOperation = 'lighter' + for (const d of drops) { + const k = (f.now - d.born) / d.life + const ease = 1 - (1 - k) ** 2.2 + const r = m * d.size * ease + ctx.strokeStyle = c[d.color] + // A ring and a fainter one inside it, like water. + for (const [scale, alpha] of [ + [1, 0.85], + [0.62, 0.4], + ] as const) { + ctx.globalAlpha = f.presence * (1 - k) ** 1.6 * alpha + ctx.lineWidth = Math.max(1, m * 0.006 * (1 - k)) + ctx.beginPath() + ctx.ellipse(d.x * w, d.y * h, r * scale, r * scale * 0.92, 0, 0, Math.PI * 2) + ctx.stroke() + } + } + ctx.globalCompositeOperation = 'source-over' + } +} + +/** + * Sparks: hundreds of glowing embers swirling up, quicker with the + * energy, flaring on the beat, and a fountain of them from below on each + * bar. + */ +function sparks(): Painter { + type Spark = { x: number; y: number; vx: number; vy: number; size: number; heat: number; color: keyof Colors } + const roles: (keyof Colors)[] = ['vibrant', 'light', 'vibrant', 'dominant'] + const make = (burst: boolean): Spark => ({ + x: burst ? 0.5 + (Math.random() - 0.5) * 0.35 : Math.random(), + y: burst ? 1.02 : Math.random() * 1.1, + vx: burst ? (Math.random() - 0.5) * 0.9 : 0, + vy: burst ? -(0.6 + Math.random() * 0.7) : 0, + size: 0.6 + Math.random() * 1.4, + heat: burst ? 1 : Math.random(), + color: roles[Math.floor(Math.random() * roles.length)], + }) + let sparks = Array.from({ length: 320 }, () => make(false)) + let lastBar = Number.NaN + let t = 0 + // A soft glow per color, drawn once and stamped for every spark. + const sprites = new Map() + const sprite = (color: string) => { + let s = sprites.get(color) + if (!s) { + s = document.createElement('canvas') + s.width = s.height = 64 + const g = s.getContext('2d')! + const grad = g.createRadialGradient(32, 32, 0, 32, 32, 32) + grad.addColorStop(0, color) + grad.addColorStop(0.25, color) + grad.addColorStop(1, 'transparent') + g.fillStyle = grad + g.fillRect(0, 0, 64, 64) + if (sprites.size > 24) sprites.clear() + sprites.set(color, s) + } + return s + } + return (ctx, w, h, f, c) => { + const dt = f.dt / 1000 + t += dt + const bar = Math.floor(f.beatIndex / 4) + if (f.downbeat > 0.5 && bar !== lastBar) { + lastBar = bar + sparks.push(...Array.from({ length: Math.round(30 + f.energy * 40) }, () => make(true))) + } + const lift = 0.05 + f.energy * 0.22 + f.swell * 0.08 + const m = Math.min(w, h) + ctx.globalCompositeOperation = 'lighter' + for (const p of sparks) { + // Rising, swirling in a slow turbulence; fountain sparks arc and slow. + p.vx += Math.sin(p.y * 7 + t * 0.9) * dt * 0.08 + p.vx *= 1 - dt * 0.8 + p.vy += dt * 0.45 * (p.vy < -lift ? 1 : 0) + p.x += p.vx * dt + p.y += (Math.min(p.vy, 0) - lift * (0.5 + p.size * 0.4)) * dt + p.heat = Math.max(0.15, p.heat - dt * 0.25) + if (p.y < -0.05 || p.x < -0.05 || p.x > 1.05) Object.assign(p, make(false), { y: 1.05 }) + const flare = f.beat * (p.size > 1.4 ? 0.9 : 0.35) + const r = m * 0.012 * p.size * (1 + flare + f.bands[0] * 0.5) + ctx.globalAlpha = f.presence * Math.min(1, 0.25 + p.heat * 0.6 + flare) + ctx.drawImage(sprite(c[p.color]), p.x * w - r * 2, p.y * h - r * 2, r * 4, r * 4) + } + // Fountains thin out to the usual crowd. + if (sparks.length > 320) sparks = sparks.filter((p, i) => i < 320 || p.heat > 0.2) + ctx.globalCompositeOperation = 'source-over' + } +} + +export const painters = { mesh, horizon, ripples, aurora, embers } + +/** The visualizer's own canvas shows, drawn sharp at full size. */ +export const showPainters = { spectrum, tunnel, rain, sparks } + +function roundRect(ctx: CanvasRenderingContext2D, x: number, y: number, w: number, h: number, r: number) { + ctx.beginPath() + ctx.roundRect(x, y, w, h, r) + ctx.fill() +} diff --git a/web/src/lib/speaker.ts b/web/src/lib/speaker.ts index 8915df2..5654fd8 100644 --- a/web/src/lib/speaker.ts +++ b/web/src/lib/speaker.ts @@ -4,6 +4,7 @@ import { serverNow } from './clock' import { queueArtworkUrl, sendCommand, type Playback, type QueueItem } from './playback' import { createStore } from './store' import { toast } from './toast' +import { deviceName } from './webauthn' // Player mode: this device plays the room's audio (ADR 0003). The server // decides what plays; the speaker applies each state revision once and @@ -54,6 +55,9 @@ const MAX_DRIFT_RELOAD = 4_000 const RESYNC_EVERY = 15_000 // Retries before a failing stream is reported as an error. const RETRIES = 2 +// How long a screen that let go of the speaker waits for the next one to +// keep it (the big screen loads its room first) before stopping. +const HANDOFF = 5_000 export const speakerState = createStore({ status: 'off', mode: 'speaker', keepAwake: readFlag(AWAKE_KEY, true) }) @@ -72,6 +76,11 @@ export function deviceId() { } let fallbackId: string | undefined +/** What to call this device as the speaker: "Collin's iPhone". */ +export function speakerName(displayName: string) { + return `${displayName.split(' ')[0]}'s ${deviceName()}` +} + /** Content types this browser can decode, for the stream's `accept`. */ export function acceptedTypes(probe: (type: string) => string = (t) => new Audio().canPlayType(t)) { return ['audio/mpeg', 'audio/aac', 'audio/mp4', 'audio/flac', 'audio/ogg', 'audio/wav', 'audio/webm'].filter( @@ -141,6 +150,7 @@ class Speaker { // The room plays on; this device stays quiet until it's played again. private localPause = false private resyncTimer?: ReturnType + private stopping?: ReturnType /** Receives every playback state the server returns to a report. */ onState?: (p: Playback) => void @@ -152,16 +162,18 @@ class Speaker { * Makes this device the room's speaker. Call it from a tap: browsers only * let audio start after a user gesture, so the elements are unlocked * before anything async happens. A paired TV passes its display ID as - * the device, since that's who the server makes the speaker. + * the device, since that's who the server makes the speaker. playItemId + * plays that queued song now, rather than the next in fair order. */ - async start(roomId: string, name: string, device = deviceId()) { + async start(roomId: string, name: string, device = deviceId(), playItemId?: string): Promise { this.begin(roomId, 'speaker', name, device) try { const np = await unwrap( - api.PUT('/rooms/{roomId}/player', { params: { path: { roomId } }, body: { deviceId: device, name } }), + api.PUT('/rooms/{roomId}/player', { params: { path: { roomId } }, body: { deviceId: device, name, playItemId } }), ) this.onState?.(np) this.apply(np) + return np } catch (err) { this.halt() toast({ message: errorMessage(err), tone: 'error' }) @@ -180,8 +192,34 @@ class Speaker { this.apply(np) } + /** + * Stops playing here unless another screen keeps it in a moment: moving + * between the app and the big screen hands the music over rather than + * stopping it. + */ + stopSoon() { + clearTimeout(this.stopping) + this.stopping = setTimeout(() => { + this.stopping = undefined + void this.stop() + }, HANDOFF) + } + + /** + * Keeps playing roomId through a handoff (stopSoon). Says whether this + * device is playing it. + */ + keep(roomId: string) { + if (!this.active || this.roomId !== roomId) return false + clearTimeout(this.stopping) + this.stopping = undefined + return true + } + /** Stops playing here: a speaker gives up the room, a listener just goes quiet. */ async stop() { + clearTimeout(this.stopping) + this.stopping = undefined const roomId = this.roomId const speaking = this.mode === 'speaker' this.halt() @@ -273,6 +311,9 @@ class Speaker { this.keepTime(np) } const listening = this.mode === 'listener' + // Tell the server we're on it: a song that never starts on a speaker + // that's said nothing gets paused, not skipped. + if (!listening && np.state === 'loading') this.report('progress') if (np.state === 'playing' || (!listening && np.state === 'loading')) { // Already playing it (we started the preloaded song when the last one // ended): no new 'playing' event will come, so tell the server now. @@ -298,6 +339,8 @@ class Speaker { /** Sets up for a room, unlocking the audio elements during the tap that started it. */ private begin(roomId: string, mode: SpeakerMode, name: string, device: string) { + clearTimeout(this.stopping) + this.stopping = undefined const [a, b] = this.elements() for (const el of [a, b]) { this.offsets.set(el, 0) diff --git a/web/src/routes/_app/_authed/room.tsx b/web/src/routes/_app/_authed/room.tsx index 3e6367e..e632ce3 100644 --- a/web/src/routes/_app/_authed/room.tsx +++ b/web/src/routes/_app/_authed/room.tsx @@ -1,6 +1,6 @@ import { useQuery } from '@tanstack/react-query' import { createFileRoute, Link } from '@tanstack/react-router' -import { Check, ChevronDown, Crown, History, LogOut, MonitorPlay, Plus, QrCode, Settings2, Sparkles, Speaker, Users } from 'lucide-react' +import { Check, ChevronDown, Crown, History, LogOut, MonitorPlay, Play, Plus, QrCode, Settings2, Sparkles, Speaker, Users } from 'lucide-react' import { AnimatePresence, motion } from 'motion/react' import { DropdownMenu } from 'radix-ui' import { useEffect, useState, type ReactNode } from 'react' @@ -327,14 +327,22 @@ function NowPlayingCard({ room, waiting }: { room: RoomInfo; waiting: number })
{waiting > 0 ? : }
-

{waiting > 0 ? 'Ready when the speaker is' : 'The queue is quiet'}

+

+ {waiting > 0 ? (speaker ? 'Stopped' : 'Ready when the speaker is') : 'The queue is quiet'} +

{waiting > 0 ? speaker - ? `Waiting to start on ${speaker.name}.` + ? `${speaker.name} is the speaker. Press play to start the next song.` : 'Songs are waiting. Start playing on the device connected to the speaker.' : 'Be the first to put something on.'}

+ {waiting > 0 && speaker && commands.toggle && ( + + )}
0 && !speaker} />
diff --git a/web/src/routes/tv.tsx b/web/src/routes/tv.tsx index 23c1a32..723af68 100644 --- a/web/src/routes/tv.tsx +++ b/web/src/routes/tv.tsx @@ -37,6 +37,7 @@ function PairedStage({ me }: { me: DisplayMe }) { queryClient.setQueryData(displayMeQuery.queryKey, null)} audio={ @@ -59,13 +60,20 @@ function SignedInStage() { if (rooms.isPending) return if (!room) return const maySpeak = !!me && !me.guest && can(room, me.id, 'speaker') + const mayPlay = maySpeak && can(room, me.id, 'playPause') + const maySkip = maySpeak && can(room, me.id, 'skip') return ( {}} - audio={maySpeak ? { device: deviceId(), name: `${me.displayName.split(' ')[0]}'s big screen` } : undefined} + audio={ + maySpeak + ? { device: deviceId(), name: `${me.displayName.split(' ')[0]}'s big screen`, canPlayPause: mayPlay, canSkip: maySkip } + : undefined + } /> ) }