Add parallax occlusion mapping to PBR materials

Documentation/ArchitectureDecisions/0027-3d-rendering-pipeline-gap-roadmap.md

@@ -295,6 +295,7 @@ Each item needs its own short requirements note before implementation.
 - [ ] Stage 1: motion blur (needs a velocity buffer that exists without temporal upscaling).
 - [ ] Stage 1: quality tiers for depth of field, bloom and auto-exposure.
 - [ ] Stage 2: `KHR_texture_transform`, IOR, specular and clearcoat material extensions.
+- [x] Stage 2: parallax occlusion mapping from `PBRMaterial.heightTexture` (world-unit `heightScale`, per-pixel ray march in `flat3d.glsl` that derives the UV shift from screen-space derivatives so no tangents are needed; the height map is read with the base color sampler because Metal has no free sampler slot; depth, shadows and silhouettes are not displaced; no glTF import yet because no displacement extension is ratified).
 - [ ] Stage 2: sheen, anisotropy and `KHR_lights_punctual` import.
 - [ ] Stage 2: unsupported-extension degradation policy and documented supported list.
 - [ ] Stage 3: morph-target asset, glTF import and GPU evaluation.

Sources/AdaCorePipelines/Assets/Shaders/flat3d.glsl

@@ -139,8 +139,10 @@ layout (binding = 15) uniform PBR3DUniform {
     vec4 u_SurfaceFlags;
     vec4 u_UVSets;
     vec4 u_Emission;
+    vec4 u_Parallax;
 };
 layout (binding = 16) uniform texture2D u_OcclusionTexture;
+layout (binding = 17) uniform texture2D u_HeightTexture;
 layout (binding = 0) uniform sampler u_OcclusionSampler;
 layout (binding = 18) uniform texture2D u_Irradiance;
 layout (binding = 19) uniform texture2D u_PrefilteredEnvironment;
@@ -218,8 +220,56 @@ vec3 fresnelSchlick(float cosine, vec3 reflectanceAtNormal) {
     return reflectanceAtNormal + (vec3(1.0) - reflectanceAtNormal) * pow(clamp(1.0 - cosine, 0.0, 1.0), 5.0);
 }
 
+// UV shift from parallax occlusion mapping, applied to every material texture that uses the height map's UV set.
+vec2 g_ParallaxDelta = vec2(0.0);
+
 vec2 materialUV(float setIndex) {
-    return setIndex > 0.5 ? Input.TextureCoordinate1 : Input.TextureCoordinate;
+    vec2 uv = setIndex > 0.5 ? Input.TextureCoordinate1 : Input.TextureCoordinate;
+    return abs(setIndex - u_Parallax.y) < 0.5 ? uv + g_ParallaxDelta : uv;
+}
+
+// Ray-marches the height map along the view ray. Returns the UV shift that makes the surface appear to have depth.
+// Depth is measured in view units: a surface point `d * scale` below the top shifts by `d * scale` along the view
+// ray's projection onto the surface, converted to UV through the screen-space position/UV derivatives so no
+// tangent frame is needed.
+vec2 parallaxDelta() {
+    vec3 normal = normalize(Input.ViewNormal);
+    vec3 viewDirection = normalize(-Input.ViewPosition);
+    float facing = dot(normal, viewDirection);
+    if (facing < 0.0) { normal = -normal; facing = -facing; }
+    float fadeDistance = u_Parallax.w;
+    float scale = u_Parallax.x * (1.0 - smoothstep(fadeDistance * 0.5, fadeDistance, length(Input.ViewPosition)));
+    vec2 uv = u_Parallax.y > 0.5 ? Input.TextureCoordinate1 : Input.TextureCoordinate;
+    vec3 positionX = dFdx(Input.ViewPosition);
+    vec3 positionY = dFdy(Input.ViewPosition);
+    float xx = dot(positionX, positionX), xy = dot(positionX, positionY), yy = dot(positionY, positionY);
+    float determinant = xx * yy - xy * xy;
+    if (scale <= 0.0 || abs(determinant) < 1e-12) { return vec2(0.0); }
+
+    vec3 shift = -(viewDirection - normal * facing) / max(facing, 0.08);
+    vec2 projected = vec2(dot(shift, positionX), dot(shift, positionY));
+    vec2 coefficients = vec2(yy * projected.x - xy * projected.y, xx * projected.y - xy * projected.x) / determinant;
+    vec2 uvPerDepth = (dFdx(uv) * coefficients.x + dFdy(uv) * coefficients.y) * scale;
+
+    vec2 gradientX = dFdx(uv), gradientY = dFdy(uv);
+    float maximumLayers = u_Parallax.z;
+    float layers = mix(maximumLayers, max(maximumLayers * 0.25, 4.0), clamp(facing, 0.0, 1.0));
+    float previousDepth = 0.0;
+    float previousGap = -(1.0 - textureGrad(sampler2D(u_HeightTexture, u_BaseColorSampler), uv, gradientX, gradientY).r);
+    float hitDepth = 1.0;
+    for (int i = 1; i <= 64; ++i) {
+        if (float(i) > layers) { break; }
+        float depth = float(i) / layers;
+        float height = textureGrad(sampler2D(u_HeightTexture, u_BaseColorSampler), uv + uvPerDepth * depth, gradientX, gradientY).r;
+        float gap = depth - (1.0 - height);
+        if (gap >= 0.0) {
+            hitDepth = mix(previousDepth, depth, previousGap / (previousGap - gap));
+            break;
+        }
+        previousDepth = depth;
+        previousGap = gap;
+    }
+    return uvPerDepth * hitDepth;
 }
 
 vec2 environmentUV(vec3 direction) {
@@ -448,6 +498,7 @@ float reachingLightCount() {
 [[main]]
 void flat3d_fragment()
 {
+    if (u_SurfaceFlags.z > 0.5) { g_ParallaxDelta = parallaxDelta(); }
     if (Input.ShadowFlags.w > 0.0 && visibilityDither(gl_FragCoord.xy) < Input.ShadowFlags.w) { discard; }
     if (u_SurfaceFlags.x < 0.5 && !gl_FrontFacing) { discard; }
     vec4 baseColor = Input.Color;

Sources/AdaCorePipelines/CorePipelines/PBR3DUniforms.swift

@@ -9,6 +9,8 @@ public struct PBR3DUniform: Sendable, Equatable {
     public let flags: Vector4
     public let coordinates: Vector4
     public let emission: Vector4
+    /// Parallax: height scale, UV set, maximum ray-march layers and fade distance.
+    public let parallax: Vector4
 
     public init(material: PBRMaterial?, descriptor _: VertexDescriptor) {
         func coordinate(_ slot: Int) -> Int {
@@ -19,9 +21,16 @@ public struct PBR3DUniform: Sendable, Equatable {
         }
         emissiveFactor = Vector4(material?.emissiveFactor ?? .one, 0)
         properties = Vector4(material?.normalScale ?? 1, material?.occlusionStrength ?? 1, material?.alphaCutoff ?? 0.5, Float(material?.alphaMode.rawValue ?? 0))
-        flags = Vector4(material?.doubleSided ?? true ? 1 : 0, material?.occlusionTexture != nil ? 1 : 0, 0, 0)
+        let heightScale = material?.heightTexture != nil ? max(finiteRenderValue(material?.heightScale ?? 0, fallback: 0, range: 0...1000), 0) : 0
+        flags = Vector4(material?.doubleSided ?? true ? 1 : 0, material?.occlusionTexture != nil ? 1 : 0, heightScale > 0 ? 1 : 0, 0)
         coordinates = Vector4(Float(coordinate(0)), Float(coordinate(1)), Float(coordinate(2)), Float(coordinate(3)))
         emission = Vector4(Float(coordinate(4)), material?.baseColorFactor.w ?? 1, 1, 0)
+        parallax = Vector4(
+            heightScale,
+            Float(min(max(material?.heightTextureCoordinate ?? 0, 0), 1)),
+            Float(min(max(material?.heightMaxLayers ?? 32, 4), 64)),
+            finiteRenderValue(material?.heightFadeDistance ?? 30, fallback: 30, range: 0.1...100000)
+        )
     }
 }
 
@@ -35,6 +44,7 @@ public struct PBR3DUniforms: Resource {
         let flags: SIMD4<UInt32>
         let coordinates: SIMD4<UInt32>
         let emission: SIMD4<UInt32>
+        let parallax: SIMD4<UInt32>
 
         init(_ value: PBR3DUniform) {
             func bits(_ vector: Vector4) -> SIMD4<UInt32> {
@@ -45,6 +55,7 @@ public struct PBR3DUniforms: Resource {
             flags = bits(value.flags)
             coordinates = bits(value.coordinates)
             emission = bits(value.emission)
+            parallax = bits(value.parallax)
         }
     }
     private struct Entry { let buffer: any UniformBuffer; var frame: Int }

Sources/AdaRender/Materials/PBRMaterial.swift

@@ -23,6 +23,20 @@ public class PBRMaterial: Material, @unchecked Sendable {
     public var occlusionTexture: Texture2D? { didSet { uniformRevision &+= 1 } }
     public var occlusionStrength: Float = 1 { didSet { uniformRevision &+= 1 } }
     public var normalScale: Float = 1 { didSet { uniformRevision &+= 1 } }
+    /// Height map for parallax occlusion mapping, read from the red channel (`1` is the top of the surface, `0` the
+    /// deepest point). Use a linear (non-sRGB) texture. Parallax shifts every material texture that shares
+    /// ``heightTextureCoordinate``, so the relief stays consistent with color, normal and occlusion maps.
+    /// The shader has no free sampler slot, so the height map is read with the base color texture's sampler:
+    /// give both the same address and filter modes.
+    public var heightTexture: Texture2D? { didSet { uniformRevision &+= 1 } }
+    /// Depth of the relief in world units, measured from the top of the surface. `0` disables parallax.
+    public var heightScale: Float = 0.05 { didSet { uniformRevision &+= 1 } }
+    /// UV set the height map is sampled with.
+    public var heightTextureCoordinate: Int = 0 { didSet { uniformRevision &+= 1 } }
+    /// Largest number of ray-march steps, used at grazing angles (a quarter of it is used head-on). Clamped to `4...64`.
+    public var heightMaxLayers: Int = 32 { didSet { uniformRevision &+= 1 } }
+    /// Camera distance in view units beyond which parallax has faded out completely. It fades over the last half.
+    public var heightFadeDistance: Float = 30 { didSet { uniformRevision &+= 1 } }
     public enum AlphaMode: UInt8, Sendable { case opaque, mask, blend }
     public var alphaMode: AlphaMode = .opaque { didSet { uniformRevision &+= 1 } }
     public var alphaCutoff: Float = 0.5 { didSet { uniformRevision &+= 1 } }

Sources/AdaScene/3D/Model3DPlugin.swift

@@ -214,6 +214,7 @@ public final class Model3DDrawPass: DrawPass, @unchecked Sendable {
         let metallicRoughnessTexture = pbrMaterial?.metallicRoughnessTexture ?? Texture2D.whiteTexture
         let normalTexture = pbrMaterial?.normalTexture ?? Self.flatNormalTexture
         let emissiveTexture = pbrMaterial?.emissiveTexture ?? Texture2D.whiteTexture
+        let heightTexture = pbrMaterial?.heightTexture ?? Texture2D.whiteTexture
 
         if isMotion {
             let previous = world.getResource(Skinning3DUniforms.self)?.previousBuffer(for: item.entity) ?? palette
@@ -237,6 +238,7 @@ public final class Model3DDrawPass: DrawPass, @unchecked Sendable {
                     .init(binding: 9, shaderStages: .fragment, resource: .sampler(normalTexture.sampler)),
                     .init(binding: 12, shaderStages: .fragment, resource: .texture(emissiveTexture)),
                     .init(binding: 13, shaderStages: .fragment, resource: .sampler(emissiveTexture.sampler)),
+                    .init(binding: 17, shaderStages: .fragment, resource: .texture(heightTexture)),
                 ]
             ),
             index: 0

Tests/AdaEngineTests/Render3DDebugViewTests.swift

@@ -210,6 +210,8 @@ extension Render3DGPUTests {
             let queue: MTLCommandQueue?
             let camera: Entity
             let sun: Entity
+            /// The ground's material, which tests edit between renders.
+            var groundMaterial: PBRMaterial?
 
             /// Scales the sun's intensity, which changes how bright the whole lit scene is.
             func setSunIntensity(_ intensity: Float) throws {
@@ -334,7 +336,7 @@ extension Render3DGPUTests {
                 Mesh3DComponent(mesh: Mesh.generateCube(size: [10, 0.1, 10], renderDevice: device), materials: [ground])
                 Transform(position: [0, -0.05, 0])
             }
-            return Scene(app: app, device: device, target: target, queue: queue, camera: cameraEntity, sun: sunEntity)
+            return Scene(app: app, device: device, target: target, queue: queue, camera: cameraEntity, sun: sunEntity, groundMaterial: ground)
         }
 
         private static func expectGolden(_ image: Image, named name: String) throws {

Tests/AdaEngineTests/Render3DParallaxTests.swift

@@ -0,0 +1,112 @@
+#if os(macOS)
+@_spi(Internal) @testable import AdaRender
+import AdaScene
+import Foundation
+import Math
+import Testing
+
+@testable import AdaCorePipelines
+
+/// Parallax material settings that need no GPU.
+@Suite("3D parallax material settings")
+@MainActor
+struct Render3DParallaxSettingsTests {
+    private func texture() -> Texture2D {
+        let device = HeadlessRenderBackend().renderDevice
+        return Texture2D(
+            gpuTexture: device.createTexture(from: .init(textureUsage: .read, textureType: .texture2D)),
+            sampler: device.createSampler(from: SamplerDescriptor()), size: [1, 1]
+        )
+    }
+
+    @Test func materialsWithoutHeightMapsDisableParallax() {
+        let uniform = PBR3DUniform(material: PBRMaterial(), descriptor: VertexDescriptor())
+        #expect(uniform.flags.z == 0)
+        #expect(uniform.parallax.x == 0)
+    }
+
+    @Test func heightMapEnablesParallaxWithClampedSettings() {
+        let material = PBRMaterial()
+        material.heightTexture = texture()
+        material.heightScale = 0.2
+        material.heightTextureCoordinate = 7
+        material.heightMaxLayers = 1000
+        material.heightFadeDistance = -4
+        let uniform = PBR3DUniform(material: material, descriptor: VertexDescriptor())
+        #expect(uniform.flags.z == 1)
+        #expect(uniform.parallax.x == 0.2)
+        #expect(uniform.parallax.y == 1)
+        #expect(uniform.parallax.z == 64)
+        #expect(uniform.parallax.w == 0.1)
+    }
+
+    @Test func zeroOrInvalidScaleDisablesParallax() {
+        let material = PBRMaterial()
+        material.heightTexture = texture()
+        material.heightScale = 0
+        #expect(PBR3DUniform(material: material, descriptor: VertexDescriptor()).flags.z == 0)
+        material.heightScale = .nan
+        #expect(PBR3DUniform(material: material, descriptor: VertexDescriptor()).flags.z == 0)
+    }
+
+    @Test func editingTheHeightMapInvalidatesTheSharedUniform() {
+        let material = PBRMaterial()
+        let before = material.uniformRevision
+        material.heightTexture = texture()
+        #expect(material.uniformRevision != before)
+    }
+}
+
+extension Render3DGPUTests {
+    /// Renders a relief ground through the real Metal pipeline.
+    ///
+    /// Run alone: `ADAENGINE_GOLDEN_METAL=1 swift test --filter Render3DParallaxGPUTests`.
+    @Suite("3D parallax on GPU")
+    @MainActor
+    struct Parallax {
+        /// A bar of high ground (white) across the V axis on a low floor (black).
+        private static func barTexture() -> Texture2D {
+            let size = 64
+            var bytes = [UInt8](repeating: 255, count: size * size * 4)
+            for row in 0..<size {
+                let value: UInt8 = (28..<36).contains(row) ? 255 : 0
+                for column in 0..<size {
+                    let offset = (row * size + column) * 4
+                    bytes[offset] = value
+                    bytes[offset + 1] = value
+                    bytes[offset + 2] = value
+                }
+            }
+            return Texture2D(image: Image(width: size, height: size, data: Data(bytes), format: .rgba8))
+        }
+
+        private static func brightRange(in image: Image, column: Int) -> ClosedRange<Int>? {
+            let bright = (0..<image.height).filter { image.getPixel(x: column, y: $0).red > 0.5 && image.getPixel(x: column, y: $0).green > 0.5 }
+            guard let first = bright.first, let last = bright.last else { return nil }
+            return first...last
+        }
+
+        @Test(.enabled(if: Render3DDebugViewGPUTests.enabled))
+        func heightMapShiftsTheReliefTowardTheCameraAndZeroScaleChangesNothing() async throws {
+            let scene = try await Render3DDebugViewGPUTests.makeScene()
+            let ground = try #require(scene.groundMaterial)
+            ground.baseColorFactor = [1, 1, 1, 1]
+            ground.baseColorTexture = Self.barTexture()
+            let flat = try await scene.render(.none)
+
+            ground.heightTexture = Self.barTexture()
+            ground.heightScale = 0
+            let zeroScale = try await scene.render(.none)
+            #expect(GoldenFrame.compare(flat, zeroScale, tolerance: GoldenFrame.activeTolerance).differingPixels == 0)
+
+            ground.heightScale = 0.4
+            let relief = try await scene.render(.none)
+            #expect(GoldenFrame.compare(flat, relief, tolerance: GoldenFrame.activeTolerance).differingPixels > 500)
+            let flatBar = try #require(Self.brightRange(in: flat, column: 320), "The flat bar must be visible.")
+            let reliefBar = try #require(Self.brightRange(in: relief, column: 320), "The relief bar must be visible.")
+            // The raised bar's near wall hides floor, so the bright region grows toward the camera (down the image).
+            #expect(reliefBar.upperBound > flatBar.upperBound, "Parallax must extend the bar toward the camera: \(flatBar) vs \(reliefBar).")
+        }
+    }
+}
+#endif