diff --git a/wallr-core/shaders/nv12_to_rgb.wgsl b/wallr-core/shaders/nv12_to_rgb.wgsl new file mode 100644 index 0000000..6d3ac76 --- /dev/null +++ b/wallr-core/shaders/nv12_to_rgb.wgsl @@ -0,0 +1,56 @@ +struct VertexOutput { + @builtin(position) clip_position: vec4, + @location(0) uv: vec2, +}; + +struct Conversion { + range: vec4, + red: vec4, + green: vec4, + blue: vec4, +}; + +@group(0) @binding(0) var luma_texture: texture_2d; +@group(0) @binding(1) var chroma_texture: texture_2d; +@group(0) @binding(2) var plane_sampler: sampler; +@group(0) @binding(3) var conversion: Conversion; + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { + var out: VertexOutput; + let u = f32((vertex_index << 1u) & 2u); + let v = f32(vertex_index & 2u); + out.uv = vec2(u, 1.0 - v); + out.clip_position = vec4(u * 2.0 - 1.0, v * 2.0 - 1.0, 0.0, 1.0); + return out; +} + +fn srgb_to_linear(encoded: f32) -> f32 { + let value = clamp(encoded, 0.0, 1.0); + if value <= 0.04045 { + return value / 12.92; + } + return pow((value + 0.055) / 1.055, 2.4); +} + +@fragment +fn fs_main(in: VertexOutput) -> @location(0) vec4 { + let y_sample = textureSample(luma_texture, plane_sampler, in.uv).r; + let uv_sample = textureSample(chroma_texture, plane_sampler, in.uv).rg; + let y = (y_sample - conversion.range.x) * conversion.range.y; + let cbcr = (uv_sample - vec2(conversion.range.z)) * conversion.range.w; + let yuv = vec4(y, cbcr.x, cbcr.y, 1.0); + let encoded = vec3( + dot(conversion.red, yuv), + dot(conversion.green, yuv), + dot(conversion.blue, yuv), + ); + + // The sRGB render target applies its encoding after this linear output. + return vec4( + srgb_to_linear(encoded.r), + srgb_to_linear(encoded.g), + srgb_to_linear(encoded.b), + 1.0, + ); +} diff --git a/wallr-core/src/daemon/mod.rs b/wallr-core/src/daemon/mod.rs index 0a33a6d..b3ae61b 100644 --- a/wallr-core/src/daemon/mod.rs +++ b/wallr-core/src/daemon/mod.rs @@ -656,6 +656,10 @@ struct RenderState { video_playback: std::sync::Arc, /// Hardware backend to request for new decoders (from `video.hw_decode`). hw_accel: crate::video::HwAccel, + /// Maximum decoded frames buffered ahead of presentation. + preload_frames: usize, + /// Optional cap for live video presentation. + max_fps: Option, /// Current scaling mode for live playback. scaling_mode: u32, /// Per-output uniform buffer + bind group (Issue #9 race fix). @@ -687,11 +691,15 @@ struct CommitData { animated: Option, /// Video metadata when committed file is a video. is_video: bool, + /// Plane and conversion resources retained across transition and playback. + video_texture: Option, /// Playback generation captured at commit time; live playback stops when /// it no longer matches `RenderState::playback_gen`. generation: u64, /// Scaling mode: 0=Fill, 1=Fit, 2=Stretch, 3=Center, 4=Tile. scaling_mode: u32, + /// Optional cap for live video presentation. + max_fps: Option, } impl RenderState { @@ -734,7 +742,9 @@ impl RenderState { // Start video playback (replaces any previous playback and joins // its decode thread, releasing the old decoder's buffers). - let metadata = self.video_playback.start(path, self.hw_accel)?; + let metadata = + self.video_playback + .start(path, self.hw_accel, self.preload_frames, generation)?; // Wait for the first frame so the transition's incoming image is // the real first frame, not a black placeholder. @@ -742,22 +752,22 @@ impl RenderState { .video_playback .wait_first_frame(std::time::Duration::from_millis(1000)); - let (new_tex, new_bind, img_width, img_height) = if let Some(frame) = first_frame { - let (tex, bind) = self.renderer.create_texture(frame.width, frame.height); + let (tex_width, tex_height) = first_frame + .as_ref() + .map(|frame| (frame.width, frame.height)) + .unwrap_or((metadata.width, metadata.height)); + let video_texture = self.renderer.create_video_texture(tex_width, tex_height); + let (img_width, img_height) = if let Some(frame) = first_frame { self.renderer - .update_texture(&tex, &frame.data, frame.width, frame.height); - (tex, bind, frame.width, frame.height) + .update_video_texture(&video_texture, &frame.data)?; + (frame.width, frame.height) } else { - // Fallback: create black texture + // WebGPU initializes the output to black when no frame arrives. tracing::warn!("No first frame available, using black texture"); - let (tex, bind) = self - .renderer - .create_texture(metadata.width, metadata.height); - let black = vec![0u8; (metadata.width * metadata.height * 4) as usize]; - self.renderer - .update_texture(&tex, &black, metadata.width, metadata.height); - (tex, bind, metadata.width, metadata.height) + (metadata.width, metadata.height) }; + let new_tex = video_texture.texture().clone(); + let new_bind = video_texture.bind_group().clone(); let old_bind = self.current_bind.take(); let (old_img_width, old_img_height) = if old_bind.is_some() { @@ -785,8 +795,10 @@ impl RenderState { height: self.height, animated: None, is_video: true, + video_texture: Some(video_texture), generation, scaling_mode, + max_fps: self.max_fps, }); } @@ -845,8 +857,10 @@ impl RenderState { height: self.height, animated, is_video: false, + video_texture: None, generation, scaling_mode, + max_fps: self.max_fps, }) } @@ -1227,21 +1241,20 @@ fn play_video( pacer: &LivePacer, per_output_uniforms: &crate::renderer::PerOutputUniforms, ) { - // Get initial frame dimensions - let (width, height) = match video_playback.metadata() { - Some(meta) => (meta.width, meta.height), - None => { - tracing::warn!("No video metadata available"); - return; - } - }; + let (width, height) = (commit.img_width, commit.img_height); - let (texture, bind) = renderer.create_texture(width, height); + let Some(texture) = commit.video_texture.as_ref() else { + tracing::warn!("Video conversion resources unavailable"); + return; + }; let static_effect = crate::animation::Effect::Fade(crate::animation::FadeParams::default()); - // The texture starts empty; present a real frame before the first - // vsync so the surface never flashes black. - let mut uploaded = false; + let min_frame_interval = commit + .max_fps + .filter(|fps| *fps > 0) + .map(|fps| std::time::Duration::from_secs_f64(1.0 / f64::from(fps))); + let mut last_present = None; + let mut warned_size_mismatch = false; loop { // A newer commit superseded us. Do NOT touch the shared @@ -1252,22 +1265,54 @@ fn play_video( return; } + if let (Some(interval), Some(previous)) = (min_frame_interval, last_present) { + pacer.wait_until(previous + interval); + if playback_gen.load(Ordering::SeqCst) != commit.generation { + return; + } + } + // Pull the next displayable frame. The decoder queue is bounded, so - // this never blocks; `None` means "present the current texture". - if let Some(frame) = video_playback.next_frame() { + // this never blocks; unchanged frames need no upload or presentation. + let frame_uploaded = if let Some(frame) = + video_playback.next_frame_in_generation(commit.generation) + { // The shared decoder can be replaced between commits; never // upload a frame whose size does not match this task's texture. if frame.width != width || frame.height != height { + if !warned_size_mismatch { + tracing::warn!( + "Skipping video frame with unexpected size {}x{} (expected {}x{})", + frame.width, + frame.height, + width, + height + ); + warned_size_mismatch = true; + } + pacer.wait_until(std::time::Instant::now() + std::time::Duration::from_millis(2)); continue; } - renderer.update_texture(&texture, &frame.data, frame.width, frame.height); - uploaded = true; - } + if let Err(err) = renderer.update_video_texture(texture, &frame.data) { + tracing::warn!("Video frame upload failed: {err}"); + return; + } + true + } else { + false + }; - if !uploaded { - // No frame yet; wait briefly and try again instead of presenting - // an uninitialized texture. - std::thread::sleep(std::time::Duration::from_millis(2)); + if !frame_uploaded { + if playback_gen.load(Ordering::SeqCst) != commit.generation { + return; + } + let wait = video_playback + .time_until_next_frame_in_generation(commit.generation) + .unwrap_or(std::time::Duration::from_millis(2)); + if playback_gen.load(Ordering::SeqCst) != commit.generation { + return; + } + pacer.wait_until(std::time::Instant::now() + wait); continue; } @@ -1276,8 +1321,8 @@ fn play_video( crate::renderer::FrameRequest { surface, format: commit.format, - bg_bind: &bind, - new_bind: &bind, + bg_bind: texture.bind_group(), + new_bind: texture.bind_group(), effect: &uniforms, width: commit.width, height: commit.height, @@ -2315,6 +2360,8 @@ impl Daemon { format: surf_format, video_playback: std::sync::Arc::new(crate::video::VideoPlayback::new()), hw_accel: crate::video::HwAccel::from_config(&config.video.hw_decode), + preload_frames: config.video.preload_frames, + max_fps: config.daemon.max_fps, scaling_mode: 0, per_output_uniforms: std::sync::Arc::new(renderer.create_per_output_uniforms()), last_wallpaper: None, @@ -2427,6 +2474,8 @@ fn create_render_state_for_output_sync( format: surf_format, video_playback: std::sync::Arc::new(crate::video::VideoPlayback::new()), hw_accel: crate::video::HwAccel::from_config(&config.video.hw_decode), + preload_frames: config.video.preload_frames, + max_fps: config.daemon.max_fps, scaling_mode: 0, per_output_uniforms: std::sync::Arc::new(renderer.create_per_output_uniforms()), last_wallpaper: None, diff --git a/wallr-core/src/preview/mod.rs b/wallr-core/src/preview/mod.rs index d1552e0..243d53d 100644 --- a/wallr-core/src/preview/mod.rs +++ b/wallr-core/src/preview/mod.rs @@ -93,8 +93,7 @@ struct PreviewApp { play_bind: Option, shown_frame: usize, video: Option, - video_tex: Option, - video_bind: Option, + video_texture: Option, video_size: (u32, u32), per_output_uniforms: Option, } @@ -121,8 +120,7 @@ impl PreviewApp { play_bind: None, shown_frame: usize::MAX, video: None, - video_tex: None, - video_bind: None, + video_texture: None, video_size: (1, 1), per_output_uniforms: None, } @@ -207,17 +205,22 @@ impl ApplicationHandler for PreviewApp { match playback.start( &self.target_path, crate::video::HwAccel::from_config("auto"), + crate::config::VideoConfig::default().preload_frames, + 0, ) { Ok(meta) => { let first = playback.wait_first_frame(std::time::Duration::from_millis(2000)); let (w, h) = (meta.width, meta.height); - let (tex, bind) = renderer.create_texture(w, h); + let texture = renderer.create_video_texture(w, h); if let Some(frame) = first { - renderer.update_texture(&tex, &frame.data, frame.width, frame.height); + if let Err(e) = renderer.update_video_texture(&texture, &frame.data) { + eprintln!("failed to upload first video frame: {e}"); + event_loop.exit(); + return; + } } self.video = Some(playback); - self.video_tex = Some(tex); - self.video_bind = Some(bind); + self.video_texture = Some(texture); self.video_size = (w, h); self.window = Some(window); self.per_output_uniforms = Some(renderer.create_per_output_uniforms()); @@ -440,8 +443,9 @@ impl PreviewApp { return; }; let Some(playback) = &self.video else { return }; - let Some(tex) = &self.video_tex else { return }; - let Some(bind) = &self.video_bind else { return }; + let Some(texture) = &self.video_texture else { + return; + }; let size = self .window @@ -454,7 +458,11 @@ impl PreviewApp { && frame.width == w && frame.height == h { - renderer.update_texture(tex, &frame.data, frame.width, frame.height); + if let Err(e) = renderer.update_video_texture(texture, &frame.data) { + eprintln!("video upload error: {e}"); + event_loop.exit(); + return; + } } let uniforms = compute_effect_uniforms(&self.effect, 1.0); @@ -462,8 +470,8 @@ impl PreviewApp { crate::renderer::FrameRequest { surface, format, - bg_bind: bind, - new_bind: bind, + bg_bind: texture.bind_group(), + new_bind: texture.bind_group(), effect: &uniforms, width: size.width.max(1), height: size.height.max(1), diff --git a/wallr-core/src/renderer/mod.rs b/wallr-core/src/renderer/mod.rs index 597276d..9082c31 100644 --- a/wallr-core/src/renderer/mod.rs +++ b/wallr-core/src/renderer/mod.rs @@ -1,6 +1,8 @@ use image::GenericImageView; use wgpu::util::DeviceExt; +use crate::video::{VideoFrameData, YuvColorInfo, YuvMatrix, YuvRange}; + pub struct Renderer { pub instance: wgpu::Instance, pub adapter: wgpu::Adapter, @@ -12,6 +14,59 @@ pub struct Renderer { shader: wgpu::ShaderModule, /// Cached pipeline for a specific surface format. Created lazily. pipeline: std::sync::Mutex>, + nv12_bind_group_layout: wgpu::BindGroupLayout, + nv12_pipeline: wgpu::RenderPipeline, +} + +pub struct VideoTexture { + output: wgpu::Texture, + output_view: wgpu::TextureView, + effects_bind_group: wgpu::BindGroup, + luma: wgpu::Texture, + chroma: wgpu::Texture, + conversion_buffer: wgpu::Buffer, + conversion_bind_group: wgpu::BindGroup, + width: u32, + height: u32, +} + +impl VideoTexture { + pub fn texture(&self) -> &wgpu::Texture { + &self.output + } + + pub fn bind_group(&self) -> &wgpu::BindGroup { + &self.effects_bind_group + } +} + +#[repr(C)] +#[derive(Copy, Clone, Debug, bytemuck::Pod, bytemuck::Zeroable)] +struct YuvConversion { + range: [f32; 4], + red: [f32; 4], + green: [f32; 4], + blue: [f32; 4], +} + +impl YuvConversion { + fn new(color: YuvColorInfo) -> Self { + let range = match color.range { + YuvRange::Limited => [16.0 / 255.0, 255.0 / 219.0, 128.0 / 255.0, 255.0 / 224.0], + YuvRange::Full => [0.0, 1.0, 128.0 / 255.0, 1.0], + }; + let (red_cr, green_cb, green_cr, blue_cb) = match color.matrix { + YuvMatrix::Bt601 => (1.402, -0.344_136, -0.714_136, 1.772), + YuvMatrix::Bt709 => (1.5748, -0.187_324, -0.468_124, 1.8556), + YuvMatrix::Bt2020 => (1.4746, -0.164_553, -0.571_353, 1.8814), + }; + Self { + range, + red: [1.0, 0.0, red_cr, 0.0], + green: [1.0, green_cb, green_cr, 0.0], + blue: [1.0, blue_cb, 0.0, 0.0], + } + } } /// Per-output uniform buffer and bind group. Each output gets its own so @@ -158,6 +213,49 @@ impl Renderer { label: Some("uniform_bind_group_layout"), }); + let nv12_bind_group_layout = + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("NV12 Conversion Bind Group Layout"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + multisampled: false, + view_dimension: wgpu::TextureViewDimension::D2, + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + multisampled: false, + view_dimension: wgpu::TextureViewDimension::D2, + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 3, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + ], + }); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("Render Pipeline Layout"), bind_group_layouts: &[ @@ -168,6 +266,43 @@ impl Renderer { push_constant_ranges: &[], }); + let nv12_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("NV12 to RGB Shader"), + source: wgpu::ShaderSource::Wgsl(std::borrow::Cow::Borrowed( + crate::shader::NV12_TO_RGB_SHADER, + )), + }); + let nv12_pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("NV12 Conversion Pipeline Layout"), + bind_group_layouts: &[&nv12_bind_group_layout], + push_constant_ranges: &[], + }); + let nv12_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("NV12 Conversion Pipeline"), + layout: Some(&nv12_pipeline_layout), + vertex: wgpu::VertexState { + module: &nv12_shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &nv12_shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8UnormSrgb, + blend: Some(wgpu::BlendState::REPLACE), + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }), + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + Ok(Self { instance, adapter, @@ -178,6 +313,8 @@ impl Renderer { bind_group_layout_tex, bind_group_layout_uni, pipeline: std::sync::Mutex::new(None), + nv12_bind_group_layout, + nv12_pipeline, }) } @@ -283,7 +420,6 @@ impl Renderer { mipmap_filter: wgpu::FilterMode::Nearest, ..Default::default() }); - let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { layout: &self.bind_group_layout_tex, entries: &[ @@ -324,6 +460,229 @@ impl Renderer { ); } + pub fn create_video_texture(&self, width: u32, height: u32) -> VideoTexture { + let plane_texture = |label, size, format| { + self.device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }) + }; + let luma = plane_texture( + "Video NV12 Luma", + wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + wgpu::TextureFormat::R8Unorm, + ); + let chroma = plane_texture( + "Video NV12 Chroma", + wgpu::Extent3d { + width: width.div_ceil(2), + height: height.div_ceil(2), + depth_or_array_layers: 1, + }, + wgpu::TextureFormat::Rg8Unorm, + ); + let output = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("Video RGB Output"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8UnormSrgb, + usage: wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_DST + | wgpu::TextureUsages::RENDER_ATTACHMENT, + view_formats: &[], + }); + let sampler = self.device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("Video Plane Sampler"), + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + mipmap_filter: wgpu::FilterMode::Nearest, + ..Default::default() + }); + let output_sampler = self.device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("Video Output Sampler"), + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Nearest, + mipmap_filter: wgpu::FilterMode::Nearest, + ..Default::default() + }); + let conversion = YuvConversion::new(YuvColorInfo { + matrix: YuvMatrix::Bt709, + range: YuvRange::Limited, + }); + let conversion_buffer = self + .device + .create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("Video YUV Conversion Uniform"), + contents: bytemuck::bytes_of(&conversion), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }); + let luma_view = luma.create_view(&wgpu::TextureViewDescriptor::default()); + let chroma_view = chroma.create_view(&wgpu::TextureViewDescriptor::default()); + let output_view = output.create_view(&wgpu::TextureViewDescriptor::default()); + let conversion_bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Video NV12 Conversion Bind Group"), + layout: &self.nv12_bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&luma_view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(&chroma_view), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::Sampler(&sampler), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: conversion_buffer.as_entire_binding(), + }, + ], + }); + let effects_bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Video Effects Bind Group"), + layout: &self.bind_group_layout_tex, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&output_view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&output_sampler), + }, + ], + }); + + VideoTexture { + output, + output_view, + effects_bind_group, + luma, + chroma, + conversion_buffer, + conversion_bind_group, + width, + height, + } + } + + pub fn update_video_texture( + &self, + texture: &VideoTexture, + frame: &VideoFrameData, + ) -> anyhow::Result<()> { + match frame { + VideoFrameData::Rgba(rgba) => { + anyhow::ensure!( + rgba.len() == texture.width as usize * texture.height as usize * 4, + "invalid RGBA video frame size" + ); + self.update_texture(&texture.output, rgba, texture.width, texture.height); + } + VideoFrameData::Nv12 { + y_plane, + uv_plane, + color, + } => { + let chroma_width = texture.width.div_ceil(2); + let chroma_height = texture.height.div_ceil(2); + anyhow::ensure!( + y_plane.len() == texture.width as usize * texture.height as usize, + "invalid NV12 luma plane size" + ); + anyhow::ensure!( + uv_plane.len() == (chroma_width * chroma_height * 2) as usize, + "invalid NV12 chroma plane size" + ); + self.queue.write_texture( + texture.luma.as_image_copy(), + y_plane, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(texture.width), + rows_per_image: Some(texture.height), + }, + wgpu::Extent3d { + width: texture.width, + height: texture.height, + depth_or_array_layers: 1, + }, + ); + self.queue.write_texture( + texture.chroma.as_image_copy(), + uv_plane, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(chroma_width * 2), + rows_per_image: Some(chroma_height), + }, + wgpu::Extent3d { + width: chroma_width, + height: chroma_height, + depth_or_array_layers: 1, + }, + ); + self.queue.write_buffer( + &texture.conversion_buffer, + 0, + bytemuck::bytes_of(&YuvConversion::new(*color)), + ); + + let mut encoder = + self.device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("NV12 Conversion Encoder"), + }); + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("NV12 Conversion Pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &texture.output_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + occlusion_query_set: None, + timestamp_writes: None, + }); + pass.set_pipeline(&self.nv12_pipeline); + pass.set_bind_group(0, &texture.conversion_bind_group, &[]); + pass.draw(0..3, 0..1); + } + self.queue.submit([encoder.finish()]); + } + } + Ok(()) + } + pub fn load_texture( &self, image: &image::DynamicImage, @@ -451,3 +810,28 @@ pub struct FrameRequest<'a> { /// Scaling mode: 0=Fill, 1=Fit, 2=Stretch, 3=Center, 4=Tile. pub scaling_mode: u32, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn selects_yuv_conversion_coefficients_and_range() { + let limited_709 = YuvConversion::new(YuvColorInfo { + matrix: YuvMatrix::Bt709, + range: YuvRange::Limited, + }); + assert_eq!(limited_709.red, [1.0, 0.0, 1.5748, 0.0]); + assert_eq!(limited_709.green, [1.0, -0.187_324, -0.468_124, 0.0]); + assert_eq!(limited_709.range[0], 16.0 / 255.0); + assert_eq!(limited_709.range[1], 255.0 / 219.0); + + let full_2020 = YuvConversion::new(YuvColorInfo { + matrix: YuvMatrix::Bt2020, + range: YuvRange::Full, + }); + assert_eq!(full_2020.red, [1.0, 0.0, 1.4746, 0.0]); + assert_eq!(full_2020.blue, [1.0, 1.8814, 0.0, 0.0]); + assert_eq!(full_2020.range, [0.0, 1.0, 128.0 / 255.0, 1.0]); + } +} diff --git a/wallr-core/src/shader/mod.rs b/wallr-core/src/shader/mod.rs index bb08add..04339cf 100644 --- a/wallr-core/src/shader/mod.rs +++ b/wallr-core/src/shader/mod.rs @@ -1,13 +1,12 @@ pub const EFFECTS_SHADER: &str = include_str!("../../shaders/effects.wgsl"); +pub const NV12_TO_RGB_SHADER: &str = include_str!("../../shaders/nv12_to_rgb.wgsl"); #[cfg(test)] mod tests { - use super::EFFECTS_SHADER; + use super::{EFFECTS_SHADER, NV12_TO_RGB_SHADER}; - #[test] - fn bundled_effect_shader_validates() { - let module = - naga::front::wgsl::parse_str(EFFECTS_SHADER).expect("bundled WGSL should parse"); + fn validate(source: &str) { + let module = naga::front::wgsl::parse_str(source).expect("bundled WGSL should parse"); naga::valid::Validator::new( naga::valid::ValidationFlags::all(), naga::valid::Capabilities::all(), @@ -15,4 +14,14 @@ mod tests { .validate(&module) .expect("bundled WGSL should validate"); } + + #[test] + fn bundled_effect_shader_validates() { + validate(EFFECTS_SHADER); + } + + #[test] + fn bundled_nv12_shader_validates() { + validate(NV12_TO_RGB_SHADER); + } } diff --git a/wallr-core/src/video/decoder.rs b/wallr-core/src/video/decoder.rs index 3fbb243..97e8f69 100644 --- a/wallr-core/src/video/decoder.rs +++ b/wallr-core/src/video/decoder.rs @@ -5,7 +5,7 @@ use ffmpeg_next as ffmpeg; use std::ffi::{CString, c_char}; use std::path::Path; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}; use std::thread; use std::time::Duration; @@ -78,9 +78,53 @@ pub struct VideoMetadata { pub total_frames: u64, } +#[derive(Debug, Clone)] +pub enum VideoFrameData { + Rgba(Vec), + Nv12 { + y_plane: Vec, + uv_plane: Vec, + color: YuvColorInfo, + }, +} + +impl VideoFrameData { + pub fn len(&self) -> usize { + match self { + Self::Rgba(data) => data.len(), + Self::Nv12 { + y_plane, uv_plane, .. + } => y_plane.len() + uv_plane.len(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct YuvColorInfo { + pub matrix: YuvMatrix, + pub range: YuvRange, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum YuvMatrix { + Bt601, + Bt709, + Bt2020, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum YuvRange { + Limited, + Full, +} + #[derive(Debug, Clone)] pub struct VideoFrame { - pub data: Vec, + pub data: VideoFrameData, pub width: u32, pub height: u32, pub pts: Duration, @@ -104,7 +148,7 @@ pub struct DecoderInfo { enum DecoderControl { Pause, Resume, - Seek(Duration), + Seek(Duration, u64), } pub struct VideoDecoder { @@ -112,12 +156,21 @@ pub struct VideoDecoder { frame_rx: Receiver, control_tx: Sender, stop_flag: Arc, + seek_epoch: Arc, hw_in_use: Arc, decode_thread: Option>, } impl VideoDecoder { pub fn new>(path: P, hw_accel: HwAccel) -> VideoResult { + Self::with_preload(path, hw_accel, 2) + } + + pub fn with_preload>( + path: P, + hw_accel: HwAccel, + preload_frames: usize, + ) -> VideoResult { let path = path.as_ref().to_path_buf(); ffmpeg::init() @@ -134,11 +187,13 @@ impl VideoDecoder { metadata.codec ); - let (frame_tx, frame_rx) = crossbeam_channel::bounded(3); + let (frame_tx, frame_rx) = crossbeam_channel::bounded(preload_frames.max(1)); let (control_tx, control_rx) = crossbeam_channel::unbounded(); let stop_flag = Arc::new(AtomicBool::new(false)); let stop_flag_clone = stop_flag.clone(); + let seek_epoch = Arc::new(AtomicU64::new(0)); + let seek_epoch_clone = seek_epoch.clone(); let hw_in_use = Arc::new(AtomicU8::new(0)); let hw_in_use_clone = hw_in_use.clone(); @@ -151,6 +206,7 @@ impl VideoDecoder { frame_tx, control_rx, stop_flag_clone, + seek_epoch_clone, hw_in_use_clone.clone(), ); let used = match used { @@ -171,6 +227,7 @@ impl VideoDecoder { frame_rx, control_tx, stop_flag, + seek_epoch, hw_in_use, decode_thread: Some(decode_thread), }) @@ -350,6 +407,7 @@ impl VideoDecoder { frame_tx: Sender, control_rx: Receiver, stop_flag: Arc, + seek_epoch: Arc, hw_in_use: Arc, ) -> VideoResult { let mut ictx = ffmpeg::format::input(&path).map_err(|e| VideoError::FileOpen { @@ -375,7 +433,8 @@ impl VideoDecoder { let mut scaler_src: Option = None; let mut paused = false; - let mut pending_seek: Option = None; + let mut pending_seek: Option<(Duration, u64)> = None; + let mut applied_seek_epoch = 0; let mut frame_index = 0u64; let mut decoded_frame = ffmpeg::frame::Video::empty(); let mut sw_frame = ffmpeg::frame::Video::empty(); @@ -390,12 +449,13 @@ impl VideoDecoder { match control { DecoderControl::Pause => paused = true, DecoderControl::Resume => paused = false, - DecoderControl::Seek(ts) => pending_seek = Some(ts), + DecoderControl::Seek(ts, epoch) => pending_seek = Some((ts, epoch)), } } - if let Some(ts) = pending_seek.take() { + if let Some((ts, epoch)) = pending_seek.take() { Self::apply_seek(&mut ictx, &mut decoder, time_base, ts); + applied_seek_epoch = epoch; } if paused { @@ -412,7 +472,7 @@ impl VideoDecoder { match control { DecoderControl::Pause => paused = true, DecoderControl::Resume => paused = false, - DecoderControl::Seek(ts) => pending_seek = Some(ts), + DecoderControl::Seek(ts, epoch) => pending_seek = Some((ts, epoch)), } } if paused || pending_seek.is_some() { @@ -432,6 +492,37 @@ impl VideoDecoder { break 'outer; } + // Apply backpressure before GPU readback and color + // conversion, retaining this decoded frame across pause. + let mut interrupted = false; + loop { + if stop_flag.load(Ordering::Relaxed) { + break 'outer; + } + while let Ok(control) = control_rx.try_recv() { + match control { + DecoderControl::Pause => paused = true, + DecoderControl::Resume => paused = false, + DecoderControl::Seek(ts, epoch) => { + pending_seek = Some((ts, epoch)); + } + } + } + if pending_seek.is_some() + || seek_epoch.load(Ordering::Acquire) != applied_seek_epoch + { + interrupted = true; + break; + } + if !paused && !frame_tx.is_full() { + break; + } + thread::sleep(Duration::from_millis(1)); + } + if interrupted { + break; + } + let is_hw_frame = unsafe { !(*decoded_frame.as_ptr()).hw_frames_ctx.is_null() }; let src_frame = if is_hw_frame { @@ -452,29 +543,6 @@ impl VideoDecoder { &decoded_frame }; - let src_format = src_frame.format(); - if scaler_src != Some(src_format) { - scaler = Some( - ffmpeg::software::scaling::context::Context::get( - src_format, - src_frame.width(), - src_frame.height(), - ffmpeg::format::Pixel::RGBA, - src_frame.width(), - src_frame.height(), - ffmpeg::software::scaling::Flags::BILINEAR, - ) - .map_err(|e| VideoError::FormatConversionFailed(e.into()))?, - ); - scaler_src = Some(src_format); - } - - scaler - .as_mut() - .expect("scaler initialized above") - .run(src_frame, &mut rgb_frame) - .map_err(|e| VideoError::FormatConversionFailed(e.into()))?; - let pts_duration = if let Some(pts) = decoded_frame.timestamp() { Duration::from_secs_f64( pts as f64 * time_base.numerator() as f64 @@ -484,10 +552,59 @@ impl VideoDecoder { Duration::from_secs_f64(frame_index as f64 / 30.0) }; + let width = src_frame.width(); + let height = src_frame.height(); + let data = if src_frame.format() == ffmpeg::format::Pixel::NV12 { + let color_space = match decoded_frame.color_space() { + ffmpeg::color::Space::Unspecified => decoder.color_space(), + value => value, + }; + let color_range = match decoded_frame.color_range() { + ffmpeg::color::Range::Unspecified => decoder.color_range(), + value => value, + }; + let color = select_yuv_color(color_space, color_range, width, height); + let (y_plane, uv_plane) = copy_nv12_planes(src_frame); + VideoFrameData::Nv12 { + y_plane, + uv_plane, + color, + } + } else { + let src_format = src_frame.format(); + if scaler_src != Some(src_format) { + scaler = Some( + ffmpeg::software::scaling::context::Context::get( + src_format, + width, + height, + ffmpeg::format::Pixel::RGBA, + width, + height, + ffmpeg::software::scaling::Flags::BILINEAR, + ) + .map_err(|e| VideoError::FormatConversionFailed(e.into()))?, + ); + scaler_src = Some(src_format); + } + + scaler + .as_mut() + .expect("scaler initialized above") + .run(src_frame, &mut rgb_frame) + .map_err(|e| VideoError::FormatConversionFailed(e.into()))?; + VideoFrameData::Rgba(copy_packed_rows( + rgb_frame.data(0), + rgb_frame.stride(0), + rgb_frame.width() as usize * 4, + rgb_frame.height() as usize, + )) + }; + let video_frame = VideoFrame { - data: rgb_frame.data(0).to_vec(), - width: rgb_frame.width(), - height: rgb_frame.height(), + data, + width, + height, pts: pts_duration, index: frame_index, }; @@ -562,7 +679,7 @@ impl VideoDecoder { } else { None }, - pixel_format: "RGBA".to_string(), + pixel_format: "NV12/RGBA".to_string(), } } @@ -586,7 +703,8 @@ impl VideoDecoder { } pub fn seek(&self, timestamp: Duration) { - let _ = self.control_tx.send(DecoderControl::Seek(timestamp)); + let epoch = self.seek_epoch.fetch_add(1, Ordering::AcqRel) + 1; + let _ = self.control_tx.send(DecoderControl::Seek(timestamp, epoch)); } pub fn drain(&mut self) { @@ -606,6 +724,61 @@ impl VideoDecoder { } } +fn copy_packed_rows(source: &[u8], stride: usize, row_bytes: usize, height: usize) -> Vec { + let mut packed = Vec::with_capacity(row_bytes * height); + for row in source.chunks(stride).take(height) { + packed.extend_from_slice(&row[..row_bytes]); + } + packed +} + +fn copy_nv12_planes(frame: &ffmpeg::frame::Video) -> (Vec, Vec) { + copy_nv12_data( + frame.data(0), + frame.stride(0), + frame.data(1), + frame.stride(1), + frame.width() as usize, + frame.height() as usize, + ) +} + +fn copy_nv12_data( + y_source: &[u8], + y_stride: usize, + uv_source: &[u8], + uv_stride: usize, + width: usize, + height: usize, +) -> (Vec, Vec) { + let chroma_width = width.div_ceil(2); + let chroma_height = height.div_ceil(2); + ( + copy_packed_rows(y_source, y_stride, width, height), + copy_packed_rows(uv_source, uv_stride, chroma_width * 2, chroma_height), + ) +} + +fn select_yuv_color( + space: ffmpeg::color::Space, + range: ffmpeg::color::Range, + width: u32, + height: u32, +) -> YuvColorInfo { + let matrix = match space { + ffmpeg::color::Space::BT470BG | ffmpeg::color::Space::SMPTE170M => YuvMatrix::Bt601, + ffmpeg::color::Space::BT709 => YuvMatrix::Bt709, + ffmpeg::color::Space::BT2020NCL => YuvMatrix::Bt2020, + _ if width >= 1280 || height > 576 => YuvMatrix::Bt709, + _ => YuvMatrix::Bt601, + }; + let range = match range { + ffmpeg::color::Range::JPEG => YuvRange::Full, + ffmpeg::color::Range::MPEG | ffmpeg::color::Range::Unspecified => YuvRange::Limited, + }; + YuvColorInfo { matrix, range } +} + impl Drop for VideoDecoder { fn drop(&mut self) { self.stop_flag.store(true, Ordering::Relaxed); @@ -658,4 +831,53 @@ mod tests { assert_eq!(HwAccel::from_config("auto"), HwAccel::Auto); assert_eq!(HwAccel::from_config("unknown"), HwAccel::Auto); } + + #[test] + fn packs_strided_rows() { + let y = [1, 2, 3, 99, 99, 4, 5, 6, 99, 99, 7, 8, 9, 99, 99]; + let uv = [10, 11, 12, 13, 99, 99, 14, 15, 16, 17, 99, 99]; + let (packed_y, packed_uv) = copy_nv12_data(&y, 5, &uv, 6, 3, 3); + assert_eq!(packed_y, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + assert_eq!(packed_uv, [10, 11, 12, 13, 14, 15, 16, 17]); + } + + #[test] + fn selects_yuv_matrix_and_range() { + assert_eq!( + select_yuv_color( + ffmpeg::color::Space::BT2020NCL, + ffmpeg::color::Range::JPEG, + 3840, + 2160, + ), + YuvColorInfo { + matrix: YuvMatrix::Bt2020, + range: YuvRange::Full, + } + ); + assert_eq!( + select_yuv_color( + ffmpeg::color::Space::Unspecified, + ffmpeg::color::Range::Unspecified, + 1920, + 1080, + ), + YuvColorInfo { + matrix: YuvMatrix::Bt709, + range: YuvRange::Limited, + } + ); + assert_eq!( + select_yuv_color( + ffmpeg::color::Space::Unspecified, + ffmpeg::color::Range::MPEG, + 720, + 576, + ), + YuvColorInfo { + matrix: YuvMatrix::Bt601, + range: YuvRange::Limited, + } + ); + } } diff --git a/wallr-core/src/video/mod.rs b/wallr-core/src/video/mod.rs index 2ca635f..9732762 100644 --- a/wallr-core/src/video/mod.rs +++ b/wallr-core/src/video/mod.rs @@ -4,7 +4,10 @@ pub mod gpu; pub mod playback; pub mod scheduler; -pub use decoder::{DecoderInfo, HwAccel, VideoDecoder, VideoFrame, VideoMetadata}; +pub use decoder::{ + DecoderInfo, HwAccel, VideoDecoder, VideoFrame, VideoFrameData, VideoMetadata, YuvColorInfo, + YuvMatrix, YuvRange, +}; pub use error::{VideoError, VideoResult}; pub use gpu::{GpuSelection, detect_adapters, select_adapter}; pub use playback::VideoPlayback; diff --git a/wallr-core/src/video/playback.rs b/wallr-core/src/video/playback.rs index 6100693..53dbc29 100644 --- a/wallr-core/src/video/playback.rs +++ b/wallr-core/src/video/playback.rs @@ -3,13 +3,14 @@ use crate::video::{ }; use std::path::Path; use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; pub struct VideoPlayback { decoder: Mutex>, scheduler: Mutex>, pending_frame: Mutex>, - current_frame: Mutex>, + generation: AtomicU64, } impl VideoPlayback { @@ -18,7 +19,7 @@ impl VideoPlayback { decoder: Mutex::new(None), scheduler: Mutex::new(None), pending_frame: Mutex::new(None), - current_frame: Mutex::new(None), + generation: AtomicU64::new(u64::MAX), } } @@ -26,11 +27,14 @@ impl VideoPlayback { &self, path: &Path, hw_accel: HwAccel, + preload_frames: usize, + generation: u64, ) -> Result { self.stop(); - let decoder = VideoDecoder::new(path, hw_accel)?; + let decoder = VideoDecoder::with_preload(path, hw_accel, preload_frames)?; let metadata = decoder.metadata().clone(); let scheduler = FrameScheduler::new(metadata.duration); + self.generation.store(generation, Ordering::Release); *self.lock_decoder() = Some(decoder); *self.lock_scheduler() = Some(scheduler); tracing::info!( @@ -44,10 +48,25 @@ impl VideoPlayback { } pub fn stop(&self) { + self.generation.store(u64::MAX, Ordering::Release); *self.lock_decoder() = None; *self.lock_scheduler() = None; *self.lock_pending() = None; - *self.lock_current() = None; + } + + /// Stops playback only when `generation` is still active. + /// Superseded callers cannot stop their successor's playback. + pub fn stop_generation(&self, generation: u64) { + let mut decoder = self.lock_decoder(); + let mut scheduler = self.lock_scheduler(); + let mut pending = self.lock_pending(); + if self.generation.load(Ordering::Acquire) != generation { + return; + } + self.generation.store(u64::MAX, Ordering::Release); + *decoder = None; + *scheduler = None; + *pending = None; } pub fn pause(&self) { @@ -72,7 +91,6 @@ impl VideoPlayback { let mut decoder = self.lock_decoder(); let mut scheduler = self.lock_scheduler(); let mut pending = self.lock_pending(); - let mut current = self.lock_current(); let scheduler = scheduler.as_mut().ok_or_else(|| { VideoError::SeekFailed(timestamp, anyhow::anyhow!("no video is playing")) })?; @@ -82,24 +100,31 @@ impl VideoPlayback { d.drain(); } *pending = None; - *current = None; tracing::info!("Video seeked to {:?}", timestamp); Ok(()) } pub fn next_frame(&self) -> Option { + self.next_frame_for_generation(None) + } + + /// Returns a frame only when `generation` is still active. + pub fn next_frame_in_generation(&self, generation: u64) -> Option { + self.next_frame_for_generation(Some(generation)) + } + + fn next_frame_for_generation(&self, generation: Option) -> Option { let mut decoder = self.lock_decoder(); + if generation.is_some_and(|expected| self.generation.load(Ordering::Acquire) != expected) { + return None; + } let mut scheduler = self.lock_scheduler(); let mut pending = self.lock_pending(); let (Some(decoder), Some(scheduler)) = (decoder.as_mut(), scheduler.as_mut()) else { return None; }; - let frame = take_due_frame(scheduler, &mut pending, || decoder.next_frame()); - if let Some(frame) = &frame { - *self.lock_current() = Some(frame.clone()); - } - frame + take_due_frame(scheduler, &mut pending, || decoder.next_frame()) } pub fn wait_first_frame(&self, timeout: Duration) -> Option { @@ -116,8 +141,25 @@ impl VideoPlayback { } } - pub fn current_frame(&self) -> Option { - self.lock_current().clone() + pub fn time_until_next_frame(&self) -> Option { + self.time_until_next_frame_for_generation(None) + } + + /// Returns a deadline only when `generation` is still active. + pub fn time_until_next_frame_in_generation(&self, generation: u64) -> Option { + self.time_until_next_frame_for_generation(Some(generation)) + } + + fn time_until_next_frame_for_generation(&self, generation: Option) -> Option { + let scheduler = self.lock_scheduler(); + let pending = self.lock_pending(); + if generation.is_some_and(|expected| self.generation.load(Ordering::Acquire) != expected) { + return None; + } + let (Some(scheduler), Some(frame)) = (scheduler.as_ref(), pending.as_ref()) else { + return None; + }; + scheduler.time_until_next_frame(frame.pts) } pub fn metadata(&self) -> Option { @@ -161,10 +203,6 @@ impl VideoPlayback { fn lock_pending(&self) -> std::sync::MutexGuard<'_, Option> { self.pending_frame.lock().unwrap_or_else(|p| p.into_inner()) } - - fn lock_current(&self) -> std::sync::MutexGuard<'_, Option> { - self.current_frame.lock().unwrap_or_else(|p| p.into_inner()) - } } fn take_due_frame( @@ -210,7 +248,7 @@ mod tests { fn frame(pts_ms: u64) -> VideoFrame { VideoFrame { - data: vec![pts_ms as u8], + data: crate::video::VideoFrameData::Rgba(vec![pts_ms as u8]), width: 1, height: 1, pts: Duration::from_millis(pts_ms), diff --git a/wallr-core/src/video/scheduler.rs b/wallr-core/src/video/scheduler.rs index 5d4a9b9..4e28888 100644 --- a/wallr-core/src/video/scheduler.rs +++ b/wallr-core/src/video/scheduler.rs @@ -1,9 +1,10 @@ +use crate::video::VideoFrameData; use crate::video::error::{VideoError, VideoResult}; use std::time::{Duration, Instant}; #[derive(Debug, Clone)] pub struct ScheduledFrame { - pub data: Vec, + pub data: VideoFrameData, pub width: u32, pub height: u32, pub pts: Duration, @@ -11,7 +12,7 @@ pub struct ScheduledFrame { } impl ScheduledFrame { - pub fn new(data: Vec, width: u32, height: u32, pts: Duration, index: u64) -> Self { + pub fn new(data: VideoFrameData, width: u32, height: u32, pts: Duration, index: u64) -> Self { Self { data, width,