feat(Controller): environment map

README.md

@@ -160,6 +160,10 @@ Controllers can be added with `<Controllers />` for [motion-controllers](https:/
 />
 ```
 
+### Environment map
+
+You can set environment map and/or it's intensity on controller models via props on `<Controllers />`. See [ControllerEnvMap](./examples/src/demos/ControllersEnvMap.tsx) to find out how to do it.
+
 ### useController
 
 `useController` references an `XRController` by handedness, exposing position and orientation info.
@@ -172,14 +176,15 @@ const gazeController = useController('none')
 
 ### XRController
 
-`XRController` is an `Object3D` that represents an [`XRInputSource`](https://developer.mozilla.org/en-US/docs/Web/API/XRInputSource) with the following properties:
+`XRController` is an long-living `Object3D` that represents an [`XRInputSource`](https://developer.mozilla.org/en-US/docs/Web/API/XRInputSource) with the following properties:
 
 ```jsx
 index: number
 controller: THREE.XRTargetRaySpace
 grip: THREE.XRGripSpace
 hand: THREE.XRHandSpace
-inputSource: XRInputSource
+inputSource: XRInputSource | null
+xrControllerModel: XRControllerModel | null
 ```
 
 ## Interactions

examples/src/demos/ControllersEnvMap.tsx

@@ -0,0 +1,41 @@
+import { Canvas, useThree } from '@react-three/fiber'
+import { XR, VRButton, Controllers } from '@react-three/xr'
+import { PMREMGenerator, Texture } from 'three'
+import { RGBELoader } from 'three-stdlib'
+import { useEffect, useState } from 'react'
+import EnvMap from '../assets/brown_photostudio_04_256.hdr'
+
+function ControllersWithEnvMap() {
+  const renderer = useThree(({ gl }) => gl)
+  const [envMap, setEnvMap] = useState<Texture>()
+
+  useEffect(() => {
+    const generateEnvMap = async () => {
+      const rgbeLoader = new RGBELoader()
+      const dataTexture = await rgbeLoader.loadAsync(EnvMap)
+      const pmremGenerator = new PMREMGenerator(renderer)
+      pmremGenerator.compileEquirectangularShader()
+      const rt = pmremGenerator.fromEquirectangular(dataTexture)
+      const radianceMap = rt.texture
+      setEnvMap(radianceMap)
+      pmremGenerator.dispose()
+    }
+
+    generateEnvMap()
+  }, [renderer])
+
+  return <Controllers envMap={envMap} envMapIntensity={1} />
+}
+
+export default function () {
+  return (
+    <>
+      <VRButton onError={(e) => console.error(e)} />
+      <Canvas>
+        <XR>
+          <ControllersWithEnvMap />
+        </XR>
+      </Canvas>
+    </>
+  )
+}

examples/src/demos/index.tsx

@@ -5,7 +5,8 @@ const HitTest = { Component: lazy(() => import('./HitTest')) }
 const Player = { Component: lazy(() => import('./Player')) }
 const Text = { Component: lazy(() => import('./Text')) }
 const Hands = { Component: lazy(() => import('./Hands')) }
+const ControllersEnvMap = { Component: lazy(() => import('./ControllersEnvMap')) }
 const Teleport = { Component: lazy(() => import('./Teleport')) }
 const CameraLinkedObject = { Component: lazy(() => import('./CameraLinkedObject')) }
 
-export { Interactive, HitTest, Player, Text, Hands, Teleport, CameraLinkedObject }
+export { Interactive, HitTest, Player, Text, Hands, Teleport, CameraLinkedObject, ControllersEnvMap }

examples/src/global.d.ts

@@ -0,0 +1,4 @@
+declare module '*.hdr' {
+  const path: string
+  export = path
+}

examples/yarn.lock

@@ -634,10 +634,10 @@
     "@jridgewell/resolve-uri" "3.1.0"
     "@jridgewell/sourcemap-codec" "1.4.14"
 
-"@mediapipe/tasks-vision@0.10.2-rc2":
-  version "0.10.2-rc2"
-  resolved "https://registry.yarnpkg.com/@mediapipe/tasks-vision/-/tasks-vision-0.10.2-rc2.tgz#e3fa5d84d58b9031a0e975d1e5ef8eb8e4a6fc11"
-  integrity sha512-b9ar6TEUo8I07n/jXSuKDu5HgzkDah9pe4H8BYpcubhCEahlfDD5ixE+9SQyJM4HXHXdF9nN/wRQT7rEnLz7Gg==
+"@mediapipe/tasks-vision@^0.10.0":
+  version "0.10.1"
+  resolved "https://registry.yarnpkg.com/@mediapipe/tasks-vision/-/tasks-vision-0.10.1.tgz#68047459352019cc141dc9c1d15c05b8ab689423"
+  integrity sha512-/zIKjOAIABx+KVfqe8hA6X2pxBGsBYlEtvD7/gpXecvzKefo/JQO6XaggmJul7+noaqiPYM0CVGZxmFJ2oTdSQ==
 
 "@nicolo-ribaudo/semver-v6@^6.3.3":
   version "6.3.3"
@@ -718,9 +718,9 @@
     zustand "^3.5.13"
 
 "@react-three/fiber@^8.10.0":
-  version "8.13.4"
-  resolved "https://registry.yarnpkg.com/@react-three/fiber/-/fiber-8.13.4.tgz#27cf964bd1d353884fb9555e21b0460736d173b5"
-  integrity sha512-OmyRKt9JU2i/Rc3uw4A+zERXKkFdu8slJjWQZfacoFNHIzGP9QVQ9XxlJWgTbgTLIOD39cUgnmH3RZZGWJqAoQ==
+  version "8.13.7"
+  resolved "https://registry.yarnpkg.com/@react-three/fiber/-/fiber-8.13.7.tgz#809f63c85effc7dddd3001ee10c2256c53a82b16"
+  integrity sha512-fH1wYi8+A2YZX8uYd9N4hfbAV+kHE565s7f62+SMNmpeynaUsN8NzXACmmJ6BpVKAKdxfvOde6dBGwG1BrWOKQ==
   dependencies:
     "@babel/runtime" "^7.17.8"
     "@types/react-reconciler" "^0.26.7"
@@ -930,10 +930,10 @@ cac@^6.7.14:
   resolved "https://registry.yarnpkg.com/cac/-/cac-6.7.14.tgz#804e1e6f506ee363cb0e3ccbb09cad5dd9870959"
   integrity sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==
 
-camera-controls@^2.4.2:
-  version "2.7.0"
-  resolved "https://registry.yarnpkg.com/camera-controls/-/camera-controls-2.7.0.tgz#13e2895375fbd8fb3353baeada6c8bc267a60d09"
-  integrity sha512-HONMoMYHieOCQOoweS639bdWHP/P/fvVGR08imnECGVUp04mqGfsX/zp1ZufLeiAA5hA6i1JhP6SrnOwh01C0w==
+camera-controls@^2.3.1:
+  version "2.4.2"
+  resolved "https://registry.yarnpkg.com/camera-controls/-/camera-controls-2.4.2.tgz#815aa5d7c4c43054fc55fb8b6cc685a56540fea2"
+  integrity sha512-blYDPECYFT/4egDMNWqKc2lBrpOfIAjPPRUNVswQELPi8naGBXUvZM3sDJSNuIRaHqid+JKPtlcoZk+Cb+X5qg==
 
 caniuse-lite@^1.0.30001503:
   version "1.0.30001512"
@@ -1035,10 +1035,10 @@ deepmerge@^4.2.2:
   resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a"
   integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==
 
-detect-gpu@^5.0.28:
-  version "5.0.31"
-  resolved "https://registry.yarnpkg.com/detect-gpu/-/detect-gpu-5.0.31.tgz#5749bea3aa56bc2ec41383b585f6cbd965619fee"
-  integrity sha512-+ZZr/deA5OvuBxod6kKFUvpZA9YR2r4fRYlAJGL7N5aUSLrY3Xgi+K4U5NHmeuk2mNC044n1YJwsq2Aw6hPmUw==
+detect-gpu@^5.0.14:
+  version "5.0.27"
+  resolved "https://registry.yarnpkg.com/detect-gpu/-/detect-gpu-5.0.27.tgz#821d9331c87e32568c483d85e12a9adee43d7bb2"
+  integrity sha512-IDjjqTkS+f0xm/ntbD21IPYiF0srzpePC/hhUMmctEsoklZwJwStJiMi/KN0pnH0LjSsgjwbP+QwW7y+Qf4/SQ==
   dependencies:
     webgl-constants "^1.1.1"
 
@@ -1251,10 +1251,10 @@ lru-cache@^5.1.1:
   dependencies:
     yallist "^3.0.2"
 
-maath@^0.6.0:
-  version "0.6.0"
-  resolved "https://registry.yarnpkg.com/maath/-/maath-0.6.0.tgz#7841d0fb95bbb37d19b08b7c5458ef70190950d2"
-  integrity sha512-dSb2xQuP7vDnaYqfoKzlApeRcR2xtN8/f7WV/TMAkBC8552TwTLtOO0JTcSygkYMjNDPoo6V01jTw/aPi4JrMw==
+maath@^0.5.2:
+  version "0.5.3"
+  resolved "https://registry.yarnpkg.com/maath/-/maath-0.5.3.tgz#777a1f9b8463c6ffb199ea43406874a357c0cd58"
+  integrity sha512-ut63A4zTd9abtpi+sOHW1fPWPtAFrjK0E17eAthx1k93W/T2cWLKV5oaswyotJVDvvW1EXSdokAqhK5KOu0Qdw==
 
 magic-string@^0.27.0:
   version "0.27.0"
@@ -1532,20 +1532,25 @@ supports-color@^7.1.0:
   dependencies:
     has-flag "^4.0.0"
 
+suspend-react@^0.0.8:
+  version "0.0.8"
+  resolved "https://registry.yarnpkg.com/suspend-react/-/suspend-react-0.0.8.tgz#b0740c1386b4eb652f17affe4339915ee268bd31"
+  integrity sha512-ZC3r8Hu1y0dIThzsGw0RLZplnX9yXwfItcvaIzJc2VQVi8TGyGDlu92syMB5ulybfvGLHAI5Ghzlk23UBPF8xg==
+
 suspend-react@^0.1.3:
   version "0.1.3"
   resolved "https://registry.yarnpkg.com/suspend-react/-/suspend-react-0.1.3.tgz#a52f49d21cfae9a2fb70bd0c68413d3f9d90768e"
   integrity sha512-aqldKgX9aZqpoDp3e8/BZ8Dm7x1pJl+qI3ZKxDN0i/IQTWUwBx/ManmlVJ3wowqbno6c2bmiIfs+Um6LbsjJyQ==
 
-three-mesh-bvh@^0.6.0:
-  version "0.6.0"
-  resolved "https://registry.yarnpkg.com/three-mesh-bvh/-/three-mesh-bvh-0.6.0.tgz#15523c335383df658dc60063a783fdd52d045dc5"
-  integrity sha512-4/oXeqVMLuN9/P0M3L5ezIVrFiXQXKvjVTErkiSYMjSaPoWfNPAwqulSgLf4bIUPn8/Lq3rmIJwxbCuD8qDobA==
+three-mesh-bvh@^0.5.23:
+  version "0.5.23"
+  resolved "https://registry.yarnpkg.com/three-mesh-bvh/-/three-mesh-bvh-0.5.23.tgz#08e5b629144b48b11acbd433519680e457d398ed"
+  integrity sha512-nyk+MskdyDgECqkxdv57UjazqqhrMi+Al9PxJN6yFtx1CTW4r0eCQ27FtyYKY5gCIWhxjtNfWYDPVy8lzx6LkA==
 
-three-stdlib@^2.23.9:
-  version "2.23.12"
-  resolved "https://registry.yarnpkg.com/three-stdlib/-/three-stdlib-2.23.12.tgz#f269398e3125c77bcd374d87f4c1da8d550e7f21"
-  integrity sha512-YFpuCu/ZVHBiK42bzEihZTA3tvEPQhaKE5tYej41AlNYXbwIWxO93fxYYrX7vs275s0yCKr6Zp6y7kI+mOklRQ==
+three-stdlib@^2.23.5:
+  version "2.23.9"
+  resolved "https://registry.yarnpkg.com/three-stdlib/-/three-stdlib-2.23.9.tgz#09c74fc6acced3d124e4f9d695156136c587a355"
+  integrity sha512-fYBClVGQptD7UZcoRZGNlR3sKcUW37hVPoEW1v68E4XuiwD0Ml/VqDUJ0yEMVE2DlooDvqgqv/rIcHC/B4N5pg==
   dependencies:
     "@types/draco3d" "^1.4.0"
     "@types/offscreencanvas" "^2019.6.4"
@@ -1574,7 +1579,7 @@ to-fast-properties@^2.0.0:
   resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e"
   integrity sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==
 
-troika-three-text@^0.47.2:
+troika-three-text@^0.47.1:
   version "0.47.2"
   resolved "https://registry.yarnpkg.com/troika-three-text/-/troika-three-text-0.47.2.tgz#fdf89059c010563bb829262b20c41f69ca79b712"
   integrity sha512-qylT0F+U7xGs+/PEf3ujBdJMYWbn0Qci0kLqI5BJG2kW1wdg4T1XSxneypnF05DxFqJhEzuaOR9S2SjiyknMng==

src/Controllers.test.tsx

@@ -8,6 +8,7 @@ import { XRControllerModel } from './XRControllerModel'
 import { XRControllerModelFactoryMock } from './mocks/XRControllerModelFactoryMock'
 import { XRInputSourceMock } from './mocks/XRInputSourceMock'
 import { act } from '@react-three/test-renderer'
+import { Texture } from 'three'
 
 vi.mock('./XRControllerModelFactory', async () => {
   const { XRControllerModelFactoryMock } = await vi.importActual<typeof import('./mocks/XRControllerModelFactoryMock')>(
@@ -141,4 +142,72 @@ describe('Controllers', () => {
     expect(disconnectSpy).not.toBeCalled()
     expect(xrControllerModelFactory?.initializeControllerModel).toBeCalledTimes(1)
   })
+
+  describe('envMap', () => {
+    it("should not set env map if it's not provided in props", async () => {
+      const store = createStoreMock()
+      const xrControllerMock = new XRControllerMock(0)
+      store.setState({ controllers: [xrControllerMock] })
+
+      await render(<Controllers />, { wrapper: createStoreProvider(store) })
+
+      const xrControllerModel = xrControllerMock.xrControllerModel
+
+      expect(xrControllerModel!.envMap).toBeNull()
+      expect(xrControllerModel!.envMapIntensity).toBe(1)
+    })
+
+    it("should set env map if it's provided in props", async () => {
+      const store = createStoreMock()
+      const xrControllerMock = new XRControllerMock(0)
+      store.setState({ controllers: [xrControllerMock] })
+      const envMap = new Texture()
+
+      await render(<Controllers envMap={envMap} />, { wrapper: createStoreProvider(store) })
+
+      const xrControllerModel = xrControllerMock.xrControllerModel
+
+      expect(xrControllerModel!.envMap).toBe(envMap)
+      expect(xrControllerModel!.envMapIntensity).toBe(1)
+    })
+
+    it("should only set env map intensity if it's provided in props", async () => {
+      const store = createStoreMock()
+      const xrControllerMock = new XRControllerMock(0)
+      store.setState({ controllers: [xrControllerMock] })
+
+      await render(<Controllers envMapIntensity={0.5} />, { wrapper: createStoreProvider(store) })
+
+      const xrControllerModel = xrControllerMock.xrControllerModel
+
+      expect(xrControllerModel!.envMap).toBeNull()
+      expect(xrControllerModel!.envMapIntensity).toBe(0.5)
+    })
+
+    it("should change env map intensity if it's provided in props then updated to a different value", async () => {
+      const store = createStoreMock()
+      const xrControllerMock = new XRControllerMock(0)
+      store.setState({ controllers: [xrControllerMock] })
+
+      const { rerender } = await render(<Controllers envMapIntensity={0.5} />, { wrapper: createStoreProvider(store) })
+      const xrControllerModel = xrControllerMock.xrControllerModel
+      await rerender(<Controllers envMapIntensity={0.6} />)
+
+      expect(xrControllerModel!.envMap).toBeNull()
+      expect(xrControllerModel!.envMapIntensity).toBe(0.6)
+    })
+
+    it("should remove env map if it's provided in props first, and then removed", async () => {
+      const store = createStoreMock()
+      const xrControllerMock = new XRControllerMock(0)
+      store.setState({ controllers: [xrControllerMock] })
+      const envMap = new Texture()
+
+      const { rerender } = await render(<Controllers envMap={envMap} />, { wrapper: createStoreProvider(store) })
+      const xrControllerModel = xrControllerMock.xrControllerModel
+      await rerender(<Controllers />)
+
+      expect(xrControllerModel!.envMap).toBeNull()
+    })
+  })
 })

src/Controllers.tsx

@@ -4,8 +4,8 @@ import { useFrame, Object3DNode, extend, createPortal } from '@react-three/fiber
 import { useXR } from './XR'
 import { XRController } from './XRController'
 import { XRControllerModelFactory } from './XRControllerModelFactory'
-import { useCallback } from 'react'
 import { XRControllerModel } from './XRControllerModel'
+import { useCallbackRef } from './utils'
 
 export interface RayProps extends Partial<JSX.IntrinsicElements['object3D']> {
   /** The XRController to attach the ray to */
@@ -64,13 +64,42 @@ export interface ControllersProps {
   rayMaterial?: JSX.IntrinsicElements['meshBasicMaterial']
   /** Whether to hide controllers' rays on blur. Default is `false` */
   hideRaysOnBlur?: boolean
+  /**
+   * Optional environment map to apply to controllers' models
+   * Useful for make controllers look more realistic
+   * if you don't want to apply env map globally on a scene
+   */
+  envMap?: THREE.Texture
+  /**
+   * Optional environment map intensity to apply to controllers' models
+   * Useful for tweaking the env map intensity if they look too bright or too dark
+   */
+  envMapIntensity?: number
 }
 
-const ControllerModel = ({ target }: { target: XRController }) => {
-  const handleControllerModel = useCallback(
+const ControllerModel = ({
+  target,
+  envMap,
+  envMapIntensity
+}: {
+  target: XRController
+  envMap?: THREE.Texture
+  envMapIntensity?: number
+}) => {
+  const xrControllerModelRef = React.useRef<XRControllerModel | null>(null)
+  const setEnvironmentMapRef = useCallbackRef((xrControllerModel: XRControllerModel) => xrControllerModel.setEnvironmentMap(envMap ?? null))
+  const setEnvironmentMapIntensityRef = useCallbackRef((xrControllerModel: XRControllerModel) => {
+    if (envMapIntensity == null) return
+    xrControllerModel.setEnvironmentMapIntensity(envMapIntensity)
+  })
+
+  const handleControllerModel = React.useCallback(
     (xrControllerModel: XRControllerModel | null) => {
+      xrControllerModelRef.current = xrControllerModel
       if (xrControllerModel) {
         target.xrControllerModel = xrControllerModel
+        setEnvironmentMapRef.current(xrControllerModel)
+        setEnvironmentMapIntensityRef.current(xrControllerModel)
         if (target.inputSource?.hand) {
           return
         }
@@ -87,13 +116,25 @@ const ControllerModel = ({ target }: { target: XRController }) => {
         target.xrControllerModel = null
       }
     },
-    [target]
+    [setEnvironmentMapIntensityRef, setEnvironmentMapRef, target]
   )
 
+  React.useLayoutEffect(() => {
+    if (xrControllerModelRef.current) {
+      setEnvironmentMapRef.current(xrControllerModelRef.current)
+    }
+  }, [envMap, setEnvironmentMapRef])
+
+  React.useLayoutEffect(() => {
+    if (xrControllerModelRef.current) {
+      setEnvironmentMapIntensityRef.current(xrControllerModelRef.current)
+    }
+  }, [envMapIntensity, setEnvironmentMapIntensityRef])
+
   return <xRControllerModel ref={handleControllerModel} />
 }
 
-export function Controllers({ rayMaterial = {}, hideRaysOnBlur = false }: ControllersProps) {
+export function Controllers({ rayMaterial = {}, hideRaysOnBlur = false, envMap, envMapIntensity }: ControllersProps) {
   const controllers = useXR((state) => state.controllers)
   const isHandTracking = useXR((state) => state.isHandTracking)
   const rayMaterialProps = React.useMemo(
@@ -113,7 +154,7 @@ export function Controllers({ rayMaterial = {}, hideRaysOnBlur = false }: Contro
     <>
       {controllers.map((target, i) => (
         <React.Fragment key={i}>
-          {createPortal(<ControllerModel target={target} />, target.grip)}
+          {createPortal(<ControllerModel target={target} envMap={envMap} envMapIntensity={envMapIntensity} />, target.grip)}
           {createPortal(
             <Ray visible={!isHandTracking} hideOnBlur={hideRaysOnBlur} target={target} {...rayMaterialProps} />,
             target.controller