diff --git a/VR/client/app/create_app.js b/VR/client/app/create_app.js index fb9e0ba..01800be 100644 --- a/VR/client/app/create_app.js +++ b/VR/client/app/create_app.js @@ -163,15 +163,21 @@ export function createApp({ document, window }) { dom.characterModeSelect.value = resolvedMode; applyCharacterMode(resolvedMode); } + + // Controlled arms const controlled = message.payload.controlled_arm_ids ?? []; if (state.sessionMode === "vr_client" && controlled.length >= 2) { state.controlledArmByHand.left = controlled[0]; state.controlledArmByHand.right = controlled[1]; } + + // Current user arm IDs state.currentUserRenderArmIds = message.payload.arm_ids ?? []; state.currentUserArmIds = (state.currentUserRenderArmIds || []).filter( (armId) => typeof armId !== "string" || !armId.toLowerCase().includes("head") ); + + // Status message setStatus( `connected: ${configWindow.serverHost} | role=${state.sessionRole} | user=${state.userId ?? "unknown"} | arms=${( message.payload.arm_ids ?? [] @@ -274,6 +280,7 @@ export function createApp({ document, window }) { applyCharacterMode(mode); }; + // Refresh mode UI when character mode select changes dom.characterModeSelect.addEventListener("change", refreshModeUI); refreshModeUI(); diff --git a/VR/client/app/session_bindings.js b/VR/client/app/session_bindings.js new file mode 100644 index 0000000..8f76afb --- /dev/null +++ b/VR/client/app/session_bindings.js @@ -0,0 +1,119 @@ +export function bindArmCameraArms({ + state, + armCameraController, + userArms, + armStates, + filterVisibleArmIds, + isAvatarHeadArmId, +}) { + const { userId: targetUserId, armId: targetArmId } = + armCameraController.syncSelection({ userArms, armStates }); + const selectedUserArmIds = filterVisibleArmIds(userArms?.[targetUserId] ?? []); + const headArmId = Object.values(armStates || {}).find( + (armState) => + armState?.owner_user_id === targetUserId && + isAvatarHeadArmId(armState?.arm_id) + )?.arm_id; + + state.spectatedUserId = targetUserId; + state.spectatedRenderArmIds = [ + ...selectedUserArmIds, + ...(headArmId ? [headArmId] : []), + ]; + state.controlledArmByHand.left = targetArmId || selectedUserArmIds[0] || ""; + state.controlledArmByHand.right = + targetArmId || selectedUserArmIds[1] || selectedUserArmIds[0] || ""; + + if (!targetArmId) { + state.spectatedUserId = ""; + state.spectatedRenderArmIds = []; + } +} + +export function updateArmCamera({ + state, + armCameraController, + camera, + armStates, +}) { + if (state.sessionMode !== "arm_camera") return; + const { armId } = armCameraController.getSelection(); + if (!armId) return; + armCameraController.applyToCamera(camera, armStates?.[armId] ?? null); +} + +export function bindSpectatorArms({ + state, + spectatorAvatarManager, + userArms, + armStates, + filterVisibleArmIds, + isAvatarHeadArmId, + clearRenderedArms, +}) { + const visibleUserArms = Object.fromEntries( + Object.entries(userArms || {}).map(([candidateUserId, armIds]) => [ + candidateUserId, + filterVisibleArmIds(armIds), + ]) + ); + const followedArms = spectatorAvatarManager.pickFollowedArms({ + userArms: visibleUserArms, + armStates, + spectatorUserId: state.userId, + }); + + state.spectatedUserId = followedArms.userId; + const headArmId = Object.values(armStates || {}).find( + (armState) => + armState?.owner_user_id === followedArms.userId && + isAvatarHeadArmId(armState?.arm_id) + )?.arm_id; + state.spectatedRenderArmIds = [ + ...(followedArms.armIds || []), + ...(headArmId ? [headArmId] : []), + ]; + state.controlledArmByHand.left = followedArms.left; + state.controlledArmByHand.right = followedArms.right; + + if (!followedArms.left && !followedArms.right) { + clearRenderedArms(); + } +} + +export function updateSpectatorHead({ + state, + spectatorAvatarManager, + armStates, + isAvatarHeadArmId, +}) { + const allSpectatedArmStates = + state.spectatedUserId && armStates + ? Object.values(armStates).filter( + (armState) => armState?.owner_user_id === state.spectatedUserId + ) + : []; + const headArmState = + allSpectatedArmStates.find((armState) => isAvatarHeadArmId(armState?.arm_id)) ?? + null; + const spectatedArmIds = state.spectatedUserId + ? state.currentUserArmIds.length > 0 && state.spectatedUserId === state.userId + ? state.currentUserArmIds + : null + : null; + + spectatorAvatarManager.updateHead({ + sessionRole: state.sessionRole, + armStates, + headArmState, + spectatedArmIds: + state.spectatedUserId && armStates + ? allSpectatedArmStates + .map((armState) => armState.arm_id) + .filter((armId) => !isAvatarHeadArmId(armId)) + : (spectatedArmIds || [ + state.controlledArmByHand.left, + state.controlledArmByHand.right, + ]).filter((armId) => armId && !isAvatarHeadArmId(armId)), + }); +} diff --git a/VR/client/entities/rod/arm_geometry.js b/VR/client/entities/rod/arm_geometry.js index 63683f7..11e968e 100644 --- a/VR/client/entities/rod/arm_geometry.js +++ b/VR/client/entities/rod/arm_geometry.js @@ -69,7 +69,7 @@ function buildSegmentedPipe(demoArm) { const r1 = Math.max(0.001, radii[Math.min(i + 1, radii.length - 1)]); const geom = new THREE.CylinderGeometry(r1, r0, length, 14, 1, true); const mesh = new THREE.Mesh(geom, material.clone()); - mesh.position.copy(p0.clone().add(p1).multiplyScalar(0.5)); + mesh.position.copy(p0.clone().add(p1).multiplyScalar(0.5)); // midpoint mesh.quaternion.setFromUnitVectors(yAxis, segment.normalize()); demoArm.armBodyGroup.add(mesh); } diff --git a/VR/client/entities/spectators/arm_camera_controller.js b/VR/client/entities/spectators/arm_camera_controller.js new file mode 100644 index 0000000..1546a10 --- /dev/null +++ b/VR/client/entities/spectators/arm_camera_controller.js @@ -0,0 +1,277 @@ +import * as THREE from "three"; + +const DEFAULT_FORWARD_OFFSET = 0.025; +const DEFAULT_POSITION_LERP = 0.2; +const DEFAULT_ROTATION_SLERP = 0.18; +const WORLD_UP = new THREE.Vector3(0.0, 1.0, 0.0); +const ALT_UP = new THREE.Vector3(1.0, 0.0, 0.0); + +function setOptions(select, items, selectedValue, placeholder) { + const previousValue = selectedValue ?? ""; + const signature = items.map((item) => `${item.value}\u0000${item.label}`).join("\u0001"); + const placeholderSignature = `placeholder\u0000${placeholder}`; + const nextSignature = items.length === 0 ? placeholderSignature : signature; + + if (select.dataset.optionSignature !== nextSignature) { + select.innerHTML = ""; + + if (items.length === 0) { + const option = document.createElement("option"); + option.value = ""; + option.textContent = placeholder; + select.append(option); + select.dataset.optionSignature = nextSignature; + select.value = ""; + select.disabled = true; + return ""; + } + + for (const item of items) { + const option = document.createElement("option"); + option.value = item.value; + option.textContent = item.label; + select.append(option); + } + select.dataset.optionSignature = nextSignature; + } + + if (items.length === 0) { + select.value = ""; + select.disabled = true; + return ""; + } + + select.disabled = false; + const nextValue = items.some((item) => item.value === previousValue) + ? previousValue + : items[0].value; + select.value = nextValue; + return nextValue; +} + +function chooseUpHint(forward, preferredUp) { + if (preferredUp.lengthSq() > 1.0e-8) { + const normalized = preferredUp.clone().normalize(); + if (Math.abs(normalized.dot(forward)) < 0.98) { + return normalized; + } + } + + if (Math.abs(WORLD_UP.dot(forward)) < 0.98) { + return WORLD_UP; + } + return ALT_UP; +} + +function derivePoseFromArmState(armState, scratch) { + const { + position, + quaternion, + matrix, + xAxis, + yAxis, + zAxis, + forward, + upHint, + previousPoint, + } = scratch; + const tip = armState?.tip?.translation; + if (!Array.isArray(tip) || tip.length !== 3) { + return false; + } + + let hasForward = false; + upHint.set(0.0, 0.0, 0.0); + const directors = armState?.directors; + if (Array.isArray(directors) && directors.length > 0) { + const lastDirector = directors[directors.length - 1]; + if ( + Array.isArray(lastDirector) && + lastDirector.length === 3 && + Array.isArray(lastDirector[1]) && + Array.isArray(lastDirector[2]) + ) { + forward.set(lastDirector[2][0], lastDirector[2][1], lastDirector[2][2]); + upHint.set(lastDirector[1][0], lastDirector[1][1], lastDirector[1][2]); + hasForward = forward.lengthSq() > 1.0e-8; + } + } + + if (!hasForward && Array.isArray(armState?.centerline) && armState.centerline.length >= 2) { + const { centerline } = armState; + forward.set( + centerline[centerline.length - 1][0], + centerline[centerline.length - 1][1], + centerline[centerline.length - 1][2] + ); + previousPoint.set( + centerline[centerline.length - 2][0], + centerline[centerline.length - 2][1], + centerline[centerline.length - 2][2] + ); + forward.sub(previousPoint); + hasForward = forward.lengthSq() > 1.0e-8; + } + + if (!hasForward) { + return false; + } + + forward.normalize(); + const chosenUp = chooseUpHint(forward, upHint); + zAxis.copy(forward).multiplyScalar(-1.0); + xAxis.crossVectors(chosenUp, zAxis); + if (xAxis.lengthSq() <= 1.0e-8) { + return false; + } + xAxis.normalize(); + yAxis.crossVectors(zAxis, xAxis).normalize(); + + matrix.makeBasis(xAxis, yAxis, zAxis); + quaternion.setFromRotationMatrix(matrix); + position.set(tip[0], tip[1], tip[2]).addScaledVector(forward, DEFAULT_FORWARD_OFFSET); + return true; +} + +export function createArmCameraController({ + panelEl, + userSelectEl, + armSelectEl, + positionLerp = DEFAULT_POSITION_LERP, + rotationSlerp = DEFAULT_ROTATION_SLERP, +} = {}) { + const scratch = { + position: new THREE.Vector3(), + quaternion: new THREE.Quaternion(), + matrix: new THREE.Matrix4(), + xAxis: new THREE.Vector3(), + yAxis: new THREE.Vector3(), + zAxis: new THREE.Vector3(), + forward: new THREE.Vector3(), + upHint: new THREE.Vector3(), + previousPoint: new THREE.Vector3(), + }; + const smoothedPosition = new THREE.Vector3(); + const smoothedQuaternion = new THREE.Quaternion(); + let selectedUserId = ""; + let selectedArmId = ""; + let initialized = false; + + function setPanelVisible(visible) { + if (panelEl) { + panelEl.style.display = visible ? "grid" : "none"; + } + } + + function setSelection(userId, armId) { + selectedUserId = userId ?? ""; + selectedArmId = armId ?? ""; + } + + function clear() { + selectedUserId = ""; + selectedArmId = ""; + initialized = false; + smoothedPosition.set(0.0, 0.0, 0.0); + smoothedQuaternion.identity(); + if (userSelectEl) { + userSelectEl.innerHTML = ""; + userSelectEl.dataset.optionSignature = ""; + userSelectEl.disabled = true; + } + if (armSelectEl) { + armSelectEl.innerHTML = ""; + armSelectEl.dataset.optionSignature = ""; + armSelectEl.disabled = true; + } + } + + function syncSelection({ userArms, armStates }) { + const users = Object.entries(userArms || {}) + .map(([userId, armIds]) => ({ + userId, + armIds: (armIds || []).filter((armId) => { + const state = armStates?.[armId]; + return !!state && !String(armId).toLowerCase().includes("head"); + }), + })) + .filter((entry) => entry.armIds.length > 0); + + const nextUserId = userSelectEl + ? setOptions( + userSelectEl, + users.map((entry) => ({ + value: entry.userId, + label: `${entry.userId} (${entry.armIds.length} arms)`, + })), + selectedUserId, + "No users available" + ) + : users[0]?.userId ?? ""; + + selectedUserId = nextUserId; + const armItems = + users.find((entry) => entry.userId === selectedUserId)?.armIds.map((armId) => ({ + value: armId, + label: armId, + })) ?? []; + const nextArmId = armSelectEl + ? setOptions(armSelectEl, armItems, selectedArmId, "No arms available") + : armItems[0]?.value ?? ""; + selectedArmId = nextArmId; + return { userId: selectedUserId, armId: selectedArmId }; + } + + if (userSelectEl) { + userSelectEl.addEventListener("change", () => { + selectedUserId = userSelectEl.value; + selectedArmId = ""; + }); + } + if (armSelectEl) { + armSelectEl.addEventListener("change", () => { + selectedArmId = armSelectEl.value; + }); + } + + return { + show() { + setPanelVisible(true); + }, + + hide() { + setPanelVisible(false); + }, + + clear, + + getSelection() { + return { userId: selectedUserId, armId: selectedArmId }; + }, + + syncSelection, + + applyToCamera(camera, armState) { + if (!derivePoseFromArmState(armState, scratch)) { + initialized = false; + return false; + } + + if (!initialized) { + smoothedPosition.copy(scratch.position); + smoothedQuaternion.copy(scratch.quaternion); + initialized = true; + } else { + smoothedPosition.lerp(scratch.position, positionLerp); + smoothedQuaternion.slerp(scratch.quaternion, rotationSlerp); + } + + camera.position.copy(smoothedPosition); + camera.quaternion.copy(smoothedQuaternion); + return true; + }, + + setSelection, + }; +} + diff --git a/VR/client/entities/spectators/spectator_avatar_manager.js b/VR/client/entities/spectators/spectator_avatar_manager.js new file mode 100644 index 0000000..91d7b44 --- /dev/null +++ b/VR/client/entities/spectators/spectator_avatar_manager.js @@ -0,0 +1,62 @@ +function armStateLooksRenderable(armState) { + return ( + !!armState && + Array.isArray(armState.centerline) && + armState.centerline.length >= 2 && + Array.isArray(armState.radii) && + armState.radii.length > 0 + ); +} + +function isHeadArmId(armId) { + return typeof armId === "string" && armId.toLowerCase().includes("head"); +} + +export function createSpectatorAvatarManager() { + return { + clear() { + return; + }, + + pickFollowedArms({ userArms, armStates, spectatorUserId }) { + const entries = Object.entries(userArms || {}) + .filter(([candidateUserId, armIds]) => { + return ( + candidateUserId !== spectatorUserId && + Array.isArray(armIds) && + armIds.some((armId) => !isHeadArmId(armId)) + ); + }) + .map(([candidateUserId, armIds]) => { + const visibleArmIds = armIds.filter((armId) => !isHeadArmId(armId)); + const renderableCount = visibleArmIds.filter((armId) => + armStateLooksRenderable(armStates?.[armId]) + ).length; + return { candidateUserId, armIds: visibleArmIds, renderableCount }; + }) + .sort((left, right) => { + if (right.renderableCount !== left.renderableCount) { + return right.renderableCount - left.renderableCount; + } + return right.armIds.length - left.armIds.length; + }); + + if (entries.length === 0) { + return { userId: "", armIds: [], left: "", right: "" }; + } + + const { candidateUserId, armIds } = entries[0]; + return { + userId: candidateUserId, + armIds, + left: armIds[0] ?? "", + right: armIds[1] ?? armIds[0] ?? "", + }; + }, + + updateHead() { + return; + }, + }; +} + diff --git a/VR/client/index.html b/VR/client/index.html index bd5d14f..739f0d5 100644 --- a/VR/client/index.html +++ b/VR/client/index.html @@ -14,6 +14,15 @@ + + + +
+
Experimental arm camera
+ + + +