From 4295476291bb979d9862aa7673fb0dd87dda5980 Mon Sep 17 00:00:00 2001 From: Dorian Date: Thu, 2 Apr 2026 11:10:08 +0100 Subject: [PATCH] feat: frontend remote relay, kiosk hardening, CSS compositor fix Frontend: - Add remote-relay.ts: receives companion input via /ws/remote-relay, dispatches keyboard/mouse/scroll events into browser DOM - Add CompanionIndicator.vue: NES gamepad icon when companion connected - Wire relay start/stop to auth state in App.vue Kiosk: - Move Chromium data dir to /var/lib/archipelago/chromium-kiosk (encrypted) - Disable MetricsReporting, AutofillServerCommunication, PasswordManager - Remove --metrics-recording-only (contradicts disable-metrics) CSS: - Fix Chromium ghost rectangles: only apply preserve-3d + backface-visibility during transitions, not always-on (causes Chromium to skip painting off-viewport cards) Co-Authored-By: Claude Opus 4.6 (1M context) --- image-recipe/build-auto-installer-iso.sh | 2 +- .../configs/archipelago-kiosk-launcher.sh | 5 +- neode-ui/src/App.vue | 9 +- neode-ui/src/api/remote-relay.ts | 199 ++++++++++++++++++ .../src/components/CompanionIndicator.vue | 81 +++++++ .../src/views/dashboard/dashboard-styles.css | 14 +- 6 files changed, 303 insertions(+), 7 deletions(-) create mode 100644 neode-ui/src/api/remote-relay.ts create mode 100644 neode-ui/src/components/CompanionIndicator.vue diff --git a/image-recipe/build-auto-installer-iso.sh b/image-recipe/build-auto-installer-iso.sh index c266349f..b7380a57 100755 --- a/image-recipe/build-auto-installer-iso.sh +++ b/image-recipe/build-auto-installer-iso.sh @@ -2247,7 +2247,7 @@ while true; do --disable-background-networking \ --disable-background-timer-throttling \ --disable-backgrounding-occluded-windows \ - --user-data-dir=/home/archipelago/.config/chromium-kiosk + --user-data-dir=/var/lib/archipelago/chromium-kiosk sleep 3 done diff --git a/image-recipe/configs/archipelago-kiosk-launcher.sh b/image-recipe/configs/archipelago-kiosk-launcher.sh index e860ea01..83392f69 100644 --- a/image-recipe/configs/archipelago-kiosk-launcher.sh +++ b/image-recipe/configs/archipelago-kiosk-launcher.sh @@ -33,7 +33,7 @@ while true; do --disable-translate \ --no-first-run \ --check-for-update-interval=31536000 \ - --disable-features=TranslateUI \ + --disable-features=TranslateUI,MetricsReporting,AutofillServerCommunication,PasswordManagerEnabled \ --disable-session-crashed-bubble \ --disable-save-password-bubble \ --disable-suggestions-service \ @@ -50,10 +50,9 @@ while true; do --disable-breakpad \ --disable-metrics \ --disable-metrics-reporting \ - --metrics-recording-only \ --disable-domain-reliability \ --js-flags="--max-old-space-size=128" \ - --user-data-dir=/home/archipelago/.config/chromium-kiosk + --user-data-dir=/var/lib/archipelago/chromium-kiosk sleep 3 done diff --git a/neode-ui/src/App.vue b/neode-ui/src/App.vue index d17758a6..0e14481e 100644 --- a/neode-ui/src/App.vue +++ b/neode-ui/src/App.vue @@ -37,6 +37,9 @@ + + + @@ -75,6 +78,7 @@ import AppLauncherOverlay from './components/AppLauncherOverlay.vue' import ToastStack from './components/ToastStack.vue' import Screensaver from './components/Screensaver.vue' import HelpGuideModal from './components/HelpGuideModal.vue' +import CompanionIndicator from './components/CompanionIndicator.vue' import { useControllerNav } from '@/composables/useControllerNav' import { playKeyboardTypingSound } from '@/composables/useLoginSounds' import { useSpotlightStore } from '@/stores/spotlight' @@ -83,6 +87,7 @@ import { useMessageToast } from '@/composables/useMessageToast' import { useAppStore } from '@/stores/app' import { useScreensaverStore } from '@/stores/screensaver' import { useUIModeStore } from '@/stores/uiMode' +import { startRemoteRelay, stopRemoteRelay } from '@/api/remote-relay' const router = useRouter() const screensaverStore = useScreensaverStore() @@ -95,16 +100,18 @@ const toastMessage = messageToast.toastMessage useControllerNav() -// Start/stop message polling when auth state changes +// Start/stop message polling and remote relay when auth state changes watch(() => appStore.isAuthenticated, (authenticated) => { if (authenticated) { messageToast.startPolling() screensaverStore.resetInactivityTimer() + startRemoteRelay() } else { messageToast.stopPolling() toastMessage.value = { show: false, text: '' } screensaverStore.clearInactivityTimer() screensaverStore.deactivate() + stopRemoteRelay() } }, { immediate: true }) diff --git a/neode-ui/src/api/remote-relay.ts b/neode-ui/src/api/remote-relay.ts new file mode 100644 index 00000000..c8848b1b --- /dev/null +++ b/neode-ui/src/api/remote-relay.ts @@ -0,0 +1,199 @@ +/** + * Remote Relay — receives companion app input via WebSocket and dispatches + * keyboard/mouse/scroll events into the browser, enabling the NES controller + * or companion keyboard to drive the web UI from another device. + */ + +import { ref } from 'vue' + +// xdotool key name → DOM key mapping +const KEY_MAP: Record = { + Return: 'Enter', + BackSpace: 'Backspace', + Escape: 'Escape', + Tab: 'Tab', + Delete: 'Delete', + space: ' ', + Up: 'ArrowUp', + Down: 'ArrowDown', + Left: 'ArrowLeft', + Right: 'ArrowRight', + Home: 'Home', + End: 'End', + Prior: 'PageUp', + Next: 'PageDown', + F1: 'F1', F2: 'F2', F3: 'F3', F4: 'F4', F5: 'F5', F6: 'F6', + F7: 'F7', F8: 'F8', F9: 'F9', F10: 'F10', F11: 'F11', F12: 'F12', +} + +/** Reactive: relay WebSocket is connected to the server */ +export const relayConnected = ref(false) + +/** Reactive: a companion app is actively sending input (received input in last 30s) */ +export const companionActive = ref(false) + +/** Reactive: input is being received right now (flickers on each event) */ +export const companionInputActive = ref(false) + +let ws: WebSocket | null = null +let shouldReconnect = true +let reconnectTimer: ReturnType | null = null +let cursorEl: HTMLDivElement | null = null +let companionTimeout: ReturnType | null = null +let inputFlickerTimeout: ReturnType | null = null + +let cursorX = typeof window !== 'undefined' ? window.innerWidth / 2 : 0 +let cursorY = typeof window !== 'undefined' ? window.innerHeight / 2 : 0 +let cursorHideTimer: ReturnType | null = null + +function markCompanionActive() { + companionActive.value = true + companionInputActive.value = true + + if (inputFlickerTimeout) clearTimeout(inputFlickerTimeout) + inputFlickerTimeout = setTimeout(() => { companionInputActive.value = false }, 200) + + if (companionTimeout) clearTimeout(companionTimeout) + companionTimeout = setTimeout(() => { companionActive.value = false }, 30_000) +} + +function createCursor(): HTMLDivElement { + if (cursorEl) return cursorEl + const el = document.createElement('div') + el.id = 'remote-relay-cursor' + el.style.cssText = ` + position: fixed; z-index: 999999; pointer-events: none; + width: 20px; height: 20px; border-radius: 50%; + background: rgba(247, 147, 26, 0.7); + border: 2px solid rgba(247, 147, 26, 0.9); + transform: translate(-50%, -50%); + transition: opacity 0.3s; + opacity: 0; display: none; + ` + document.body.appendChild(el) + cursorEl = el + return el +} + +function showCursor() { + const el = createCursor() + el.style.display = 'block' + el.style.opacity = '1' + el.style.left = `${cursorX}px` + el.style.top = `${cursorY}px` + + if (cursorHideTimer) clearTimeout(cursorHideTimer) + cursorHideTimer = setTimeout(() => { + if (cursorEl) cursorEl.style.opacity = '0' + }, 3000) +} + +function moveCursor(dx: number, dy: number) { + cursorX = Math.max(0, Math.min(window.innerWidth, cursorX + dx)) + cursorY = Math.max(0, Math.min(window.innerHeight, cursorY + dy)) + showCursor() +} + +function mapKey(xdotoolKey: string): string { + return KEY_MAP[xdotoolKey] ?? xdotoolKey +} + +function handleMessage(data: string) { + let msg: { t: string; k?: string; x?: number; y?: number; b?: number } + try { + msg = JSON.parse(data) + } catch { + return + } + + if (msg.t === 'ok') return // server ready, not companion input + + markCompanionActive() + + switch (msg.t) { + case 'k': { + if (!msg.k) break + const key = mapKey(msg.k) + document.dispatchEvent(new KeyboardEvent('keydown', { key, bubbles: true })) + document.dispatchEvent(new KeyboardEvent('keyup', { key, bubbles: true })) + break + } + case 'm': { + moveCursor(msg.x ?? 0, msg.y ?? 0) + break + } + case 'c': { + const target = document.elementFromPoint(cursorX, cursorY) + if (target) { + if (cursorEl) { + cursorEl.style.background = 'rgba(247, 147, 26, 1)' + setTimeout(() => { if (cursorEl) cursorEl.style.background = 'rgba(247, 147, 26, 0.7)' }, 150) + } + target.dispatchEvent(new MouseEvent('click', { + bubbles: true, cancelable: true, + clientX: cursorX, clientY: cursorY, + })) + } + break + } + case 's': { + const dy = msg.y ?? 0 + document.dispatchEvent(new WheelEvent('wheel', { + bubbles: true, deltaY: dy * 100, deltaMode: WheelEvent.DOM_DELTA_PIXEL, + })) + break + } + } +} + +function doConnect() { + if (ws && (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING)) { + return + } + + const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:' + const url = `${protocol}//${window.location.host}/ws/remote-relay` + + ws = new WebSocket(url) + + ws.onopen = () => { + relayConnected.value = true + if (import.meta.env.DEV) console.log('[RemoteRelay] Connected') + } + + ws.onmessage = (event) => { + handleMessage(event.data) + } + + ws.onclose = () => { + relayConnected.value = false + ws = null + if (shouldReconnect) { + reconnectTimer = setTimeout(doConnect, 5000) + } + } + + ws.onerror = () => { + // onclose will handle reconnect + } +} + +/** Start the remote relay listener. Connects to /ws/remote-relay. */ +export function startRemoteRelay() { + shouldReconnect = true + doConnect() +} + +/** Stop the remote relay listener and clean up. */ +export function stopRemoteRelay() { + shouldReconnect = false + if (reconnectTimer) { clearTimeout(reconnectTimer); reconnectTimer = null } + if (companionTimeout) { clearTimeout(companionTimeout); companionTimeout = null } + if (inputFlickerTimeout) { clearTimeout(inputFlickerTimeout); inputFlickerTimeout = null } + if (cursorHideTimer) { clearTimeout(cursorHideTimer); cursorHideTimer = null } + if (ws) { ws.onclose = null; ws.close(); ws = null } + if (cursorEl) { cursorEl.remove(); cursorEl = null } + relayConnected.value = false + companionActive.value = false + companionInputActive.value = false +} diff --git a/neode-ui/src/components/CompanionIndicator.vue b/neode-ui/src/components/CompanionIndicator.vue new file mode 100644 index 00000000..f4e01706 --- /dev/null +++ b/neode-ui/src/components/CompanionIndicator.vue @@ -0,0 +1,81 @@ + + + + + diff --git a/neode-ui/src/views/dashboard/dashboard-styles.css b/neode-ui/src/views/dashboard/dashboard-styles.css index 60ebc2a8..776119e3 100644 --- a/neode-ui/src/views/dashboard/dashboard-styles.css +++ b/neode-ui/src/views/dashboard/dashboard-styles.css @@ -394,8 +394,10 @@ aside:not(.sidebar-animate) .sidebar-logout-btn { .view-wrapper { position: absolute; inset: 0; - transform-style: preserve-3d; - backface-visibility: hidden; + /* preserve-3d + backface-visibility only during transitions (applied by + transition classes below). Keeping them always-on causes Chromium to skip + painting cards that start below the viewport — they appear as transparent + ghost rectangles when scrolled into view. */ will-change: transform, opacity; opacity: 1; } @@ -408,6 +410,8 @@ aside:not(.sidebar-animate) .sidebar-logout-btn { .depth-forward-enter-active.view-wrapper, .depth-forward-leave-active.view-wrapper { transition: all 0.8s cubic-bezier(0.25, 0.46, 0.45, 0.94); + transform-style: preserve-3d; + backface-visibility: hidden; } .depth-forward-enter-from.view-wrapper { @@ -438,6 +442,8 @@ aside:not(.sidebar-animate) .sidebar-logout-btn { .depth-back-enter-active.view-wrapper, .depth-back-leave-active.view-wrapper { transition: all 0.8s cubic-bezier(0.25, 0.46, 0.45, 0.94); + transform-style: preserve-3d; + backface-visibility: hidden; } .depth-back-enter-from.view-wrapper { @@ -487,6 +493,8 @@ aside:not(.sidebar-animate) .sidebar-logout-btn { .chat-open-enter-active.view-wrapper, .chat-open-leave-active.view-wrapper { transition: opacity 0.5s cubic-bezier(0.22, 1, 0.36, 1), transform 0.5s cubic-bezier(0.22, 1, 0.36, 1); + transform-style: preserve-3d; + backface-visibility: hidden; } .chat-open-enter-from.view-wrapper { @@ -513,6 +521,8 @@ aside:not(.sidebar-animate) .sidebar-logout-btn { .chat-close-enter-active.view-wrapper, .chat-close-leave-active.view-wrapper { transition: opacity 0.5s cubic-bezier(0.22, 1, 0.36, 1), transform 0.5s cubic-bezier(0.22, 1, 0.36, 1); + transform-style: preserve-3d; + backface-visibility: hidden; } .chat-close-enter-from.view-wrapper {