blanchon commited on
Commit
2d0da8c
·
1 Parent(s): 8eeb37a

Disable Inference

Browse files
Files changed (44) hide show
  1. .env.example +1 -1
  2. README.md +2 -2
  3. SPEC.md +0 -99
  4. external/RobotHub-InferenceServer +1 -1
  5. src/lib/components/3d/elements/compute/ComputeGridItem.svelte +1 -1
  6. src/lib/components/3d/elements/compute/Computes.svelte +13 -2
  7. src/lib/components/3d/elements/compute/modal/AISessionConnectionModal.svelte +101 -79
  8. src/lib/components/3d/elements/compute/status/ComputeBoxUIKit.svelte +1 -1
  9. src/lib/components/3d/elements/compute/status/ComputeInputBoxUIKit.svelte +2 -2
  10. src/lib/components/3d/elements/compute/status/ComputeOutputBoxUIKit.svelte +2 -2
  11. src/lib/components/3d/elements/compute/status/ComputeStatusBillboard.svelte +18 -2
  12. src/lib/components/3d/elements/compute/status/RobotInputBoxUIKit.svelte +2 -2
  13. src/lib/components/3d/elements/compute/status/RobotOutputBoxUIKit.svelte +2 -2
  14. src/lib/components/3d/elements/compute/status/VideoInputBoxUIKit.svelte +2 -2
  15. src/lib/components/3d/elements/robot/RobotGridItem.svelte +11 -5
  16. src/lib/components/3d/elements/robot/Robots.svelte +6 -0
  17. src/lib/components/3d/elements/robot/URDF/primitives/UrdfJoint.svelte +0 -1
  18. src/lib/components/3d/elements/robot/URDF/primitives/UrdfLink.svelte +0 -3
  19. src/lib/components/3d/elements/robot/URDF/primitives/UrdfThree.svelte +0 -1
  20. src/lib/components/3d/elements/robot/modal/InputConnectionModal.svelte +35 -22
  21. src/lib/components/3d/elements/robot/modal/OutputConnectionModal.svelte +27 -16
  22. src/lib/components/3d/elements/robot/status/ConnectionFlowBoxUIkit.svelte +22 -2
  23. src/lib/components/3d/elements/robot/status/OutputBoxUIKit.svelte +1 -1
  24. src/lib/components/3d/elements/robot/status/RobotStatusBillboard.svelte +9 -3
  25. src/lib/components/3d/elements/video/VideoGridItem.svelte +0 -1
  26. src/lib/components/3d/elements/video/Videos.svelte +7 -0
  27. src/lib/components/3d/elements/video/modal/VideoInputConnectionModal.svelte +27 -14
  28. src/lib/components/3d/elements/video/modal/VideoOutputConnectionModal.svelte +29 -16
  29. src/lib/components/3d/elements/video/status/VideoConnectionFlowBoxUIKit.svelte +25 -8
  30. src/lib/components/3d/elements/video/status/VideoStatusBillboard.svelte +3 -3
  31. src/lib/components/interface/overlay/AddAIButton.svelte +50 -90
  32. src/lib/components/interface/overlay/AddRobotButton.svelte +15 -8
  33. src/lib/components/interface/overlay/AddSensorButton.svelte +7 -0
  34. src/lib/components/interface/overlay/Overlay.svelte +20 -25
  35. src/lib/components/interface/overlay/SettingsSheet.svelte +1 -2
  36. src/lib/components/interface/overlay/WorkspaceIdButton.svelte +1 -1
  37. src/lib/configs/robotUrdfConfig.ts +4 -0
  38. src/lib/elements/compute/RemoteCompute.svelte.ts +2 -1
  39. src/lib/elements/compute/RemoteComputeManager.svelte.ts +138 -1
  40. src/lib/elements/compute/index.ts +2 -1
  41. src/lib/elements/robot/RobotManager.svelte.ts +2 -3
  42. src/lib/elements/robot/components/RobotItem.svelte +0 -1
  43. src/lib/elements/video/VideoManager.svelte.ts +8 -0
  44. src/lib/types/urdf.ts +5 -0
.env.example CHANGED
@@ -1,2 +1,2 @@
1
  PUBLIC_TRANSPORT_SERVER_URL=https://blanchon-robothub-transportserver.hf.space/api
2
- PUBLIC_INFERENCE_SERVER_URL=https://blanchon-robothub-inference-server.hf.space/api
 
1
  PUBLIC_TRANSPORT_SERVER_URL=https://blanchon-robothub-transportserver.hf.space/api
2
+ PUBLIC_INFERENCE_SERVER_URL=https://blanchon-robothub-inferenceserver.hf.space/api
README.md CHANGED
@@ -106,7 +106,7 @@ The **workspace-id** in the URL hash ties all three services together. Share `h
106
  2. Click *Add Robot* → spawns an SO-100 6-DoF arm (URDF).
107
  3. Click *Add Sensor → Camera* → creates a virtual camera element.
108
  4. Click *Add Model → ACT* → spawns a *Compute* block.
109
- 5. On the Compute block choose *Create Session* – select model path (`./checkpoints/act_so101_beyond`) and cameras (`front`).
110
  6. Connect:
111
  • *Video Input* – local webcam → `front` room.
112
  • *Robot Input* – robot → *joint-input* room (producer).
@@ -244,7 +244,7 @@ Typical lifecycle:
244
  ```jsonc
245
  {
246
  "session_id": "pick_place_demo",
247
- "policy_path": "./checkpoints/act_so101_beyond",
248
  "camera_names": ["front", "wrist"],
249
  "transport_server_url": "http://localhost:8000",
250
  "workspace_id": "<existing-or-new>" // optional
 
106
  2. Click *Add Robot* → spawns an SO-100 6-DoF arm (URDF).
107
  3. Click *Add Sensor → Camera* → creates a virtual camera element.
108
  4. Click *Add Model → ACT* → spawns a *Compute* block.
109
+ 5. On the Compute block choose *Create Session* – select model path (`LaetusH/act_so101_beyond`) and cameras (`front`).
110
  6. Connect:
111
  • *Video Input* – local webcam → `front` room.
112
  • *Robot Input* – robot → *joint-input* room (producer).
 
244
  ```jsonc
245
  {
246
  "session_id": "pick_place_demo",
247
+ "policy_path": "LaetusH/act_so101_beyond",
248
  "camera_names": ["front", "wrist"],
249
  "transport_server_url": "http://localhost:8000",
250
  "workspace_id": "<existing-or-new>" // optional
SPEC.md DELETED
@@ -1,99 +0,0 @@
1
- # Robot USB Connection System - Summary
2
-
3
- ## Core Components
4
-
5
- ### USBProducer (Output)
6
- **Purpose**: Sends software commands to control physical robot hardware
7
- - Receives normalized joint commands from software
8
- - Converts normalized values to raw servo positions
9
- - Sends position commands to physical servos
10
- - Locks servos for precise software control
11
-
12
- ### USBConsumer (Input)
13
- **Purpose**: Reads physical robot movements and translates to software commands
14
- - Continuously monitors physical servo positions
15
- - Converts raw servo values to normalized percentages
16
- - Detects movement changes and broadcasts updates
17
- - Keeps servos unlocked for manual manipulation
18
-
19
- ## Key Features
20
-
21
- ### Calibration System (`USBCalibrationPanel.svelte`)
22
- - **Requirement**: All USB connections must be calibrated before use
23
- - **Process**: Records min/max physical range for each servo by manual movement
24
- - **Result**: Establishes mapping between raw servo values (0-4095) and normalized values
25
-
26
- ### Normalized Communication Protocol
27
- - **Standard Joints**: -100% to +100% range (bipolar)
28
- - **Gripper/Jaw**: 0% to +100% range (unipolar)
29
- - **Benefits**: Consistent software interface across different robots
30
- - **Conversion**: Automatic normalization/denormalization based on calibration data
31
-
32
- ### Multi-Port Support
33
- - **Capability**: Multiple independent USB connections simultaneously
34
- - **Independence**: Each connection has its own calibration and configuration
35
- - **Use Case**: Control multiple robots or multiple connections to same robot
36
-
37
- ### Batch Operations
38
- - **Sync Read**: Read multiple servo positions simultaneously
39
- - **Sync Write**: Send commands to multiple servos in batched operations
40
- - **Performance**: Reduces USB communication overhead and latency
41
-
42
- ## Key Constraints
43
-
44
- ### Connection Exclusivity
45
- - **Input**: Only one consumer (input source) active per robot at a time
46
- - **Output**: Multiple producers (output destinations) can be active simultaneously
47
- - **Rationale**: Prevents conflicting input commands while allowing broadcast to multiple destinations
48
-
49
- ### Servo Locking Strategy
50
- - **Consumer Mode**: Servos unlocked → manual movement possible + position reading
51
- - **Producer Mode**: Servos locked → software control only, no manual movement
52
- - **Safety**: Prevents mechanical conflicts between manual and software control
53
-
54
- ### Calibration Dependency
55
- - **Mandatory**: USB connections cannot establish without valid calibration
56
- - **Per-Connection**: Each USB port requires independent calibration
57
- - **Safety**: Prevents commanding impossible positions or damaging hardware
58
-
59
- ## Additional System Requirements (from SPEC.md analysis)
60
-
61
- ### Connection Management
62
- - **Auto-Detection**: System should detect available USB ports automatically
63
- - **Status Monitoring**: Real-time connection health and error reporting
64
- - **Graceful Disconnection**: Safe servo unlocking on disconnect/error
65
-
66
- ### Error Handling & Recovery
67
- - **Port Conflicts**: Queue management to prevent "Port is busy" errors
68
- - **Retry Logic**: Automatic retry with backoff for failed servo commands
69
- - **Connection Recovery**: Automatic reconnection attempts after USB disconnect
70
-
71
- ### User Interface Integration
72
- - **Modal Management**: Unified connection setup through calibration panels
73
- - **Status Display**: Visual indicators for connection state and calibration status
74
- - **Manual Control**: Direct joint manipulation when no input consumer active
75
-
76
- ### Performance Optimizations
77
- - **Polling Rates**: Configurable update frequencies for different use cases
78
- - **Change Detection**: Only broadcast updates when values actually change
79
- - **Queueing**: Serial command processing to prevent USB port conflicts
80
-
81
- ## System Architecture Concepts
82
-
83
- ### Bidirectional Data Flow
84
- ```
85
- Physical Robot ←→ USB Hardware ←→ Calibration Layer ←→ Normalized Interface ←→ Software
86
- ```
87
-
88
- ### Connection Patterns
89
- - **Teaching Mode**: USB Consumer only (read robot movements)
90
- - **Control Mode**: USB Producer only (software controls robot)
91
- - **Bidirectional**: Both consumer and producer (full interaction)
92
- - **Broadcasting**: Multiple producers (send to hardware + remote systems)
93
-
94
- ### Value Transformation Pipeline
95
- ```
96
- Raw Servo (0-4095) ←→ Calibrated Range ←→ Normalized (-100/+100) ←→ Software Commands
97
- ```
98
-
99
- This system provides a robust, safe, and flexible interface for robot hardware control while maintaining consistency across different robot configurations and use cases.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
external/RobotHub-InferenceServer CHANGED
@@ -1 +1 @@
1
- Subproject commit e3c6edfc01cbf01b6ca1ac19637a282017ef6c07
 
1
+ Subproject commit be2822ebd8b95c11d982e7f93594016c2285954f
src/lib/components/3d/elements/compute/ComputeGridItem.svelte CHANGED
@@ -16,7 +16,6 @@
16
  $props();
17
 
18
  const { onPointerEnter, onPointerLeave, hovering } = useCursor();
19
- interactivity();
20
 
21
  let isToggled = $state(false);
22
 
@@ -24,6 +23,7 @@
24
  event.stopPropagation();
25
  isToggled = !isToggled;
26
  }
 
27
  </script>
28
 
29
  <T.Group
 
16
  $props();
17
 
18
  const { onPointerEnter, onPointerLeave, hovering } = useCursor();
 
19
 
20
  let isToggled = $state(false);
21
 
 
23
  event.stopPropagation();
24
  isToggled = !isToggled;
25
  }
26
+
27
  </script>
28
 
29
  <T.Group
src/lib/components/3d/elements/compute/Computes.svelte CHANGED
@@ -2,11 +2,13 @@
2
  import { onMount } from "svelte";
3
  import { remoteComputeManager } from "$lib/elements/compute/RemoteComputeManager.svelte";
4
  import AISessionConnectionModal from "@/components/3d/elements/compute/modal/AISessionConnectionModal.svelte";
 
5
  import VideoInputConnectionModal from "@/components/3d/elements/compute/modal/VideoInputConnectionModal.svelte";
6
  import RobotInputConnectionModal from "@/components/3d/elements/compute/modal/RobotInputConnectionModal.svelte";
7
  import RobotOutputConnectionModal from "@/components/3d/elements/compute/modal/RobotOutputConnectionModal.svelte";
8
  import ComputeGridItem from "@/components/3d/elements/compute/ComputeGridItem.svelte";
9
  import type { RemoteCompute } from "$lib/elements/compute/RemoteCompute.svelte";
 
10
 
11
  interface Props {
12
  workspaceId: string;
@@ -14,6 +16,7 @@
14
  let { workspaceId }: Props = $props();
15
 
16
  let isAISessionModalOpen = $state(false);
 
17
  let isVideoInputModalOpen = $state(false);
18
  let isRobotInputModalOpen = $state(false);
19
  let isRobotOutputModalOpen = $state(false);
@@ -64,6 +67,11 @@
64
 
65
  return () => clearInterval(interval);
66
  });
 
 
 
 
 
67
  </script>
68
 
69
  {#each remoteComputeManager.computes as compute (compute.id)}
@@ -76,7 +84,7 @@
76
  {/each}
77
 
78
  {#if selectedCompute}
79
- <!-- Inference Session Creation Modal -->
80
  <AISessionConnectionModal bind:open={isAISessionModalOpen} compute={selectedCompute} {workspaceId} />
81
  <!-- Video Input Connection Modal -->
82
  <VideoInputConnectionModal bind:open={isVideoInputModalOpen} compute={selectedCompute} {workspaceId} />
@@ -84,4 +92,7 @@
84
  <RobotInputConnectionModal bind:open={isRobotInputModalOpen} compute={selectedCompute} {workspaceId} />
85
  <!-- Robot Output Connection Modal -->
86
  <RobotOutputConnectionModal bind:open={isRobotOutputModalOpen} compute={selectedCompute} {workspaceId} />
87
- {/if}
 
 
 
 
2
  import { onMount } from "svelte";
3
  import { remoteComputeManager } from "$lib/elements/compute/RemoteComputeManager.svelte";
4
  import AISessionConnectionModal from "@/components/3d/elements/compute/modal/AISessionConnectionModal.svelte";
5
+ import AIModelConfigurationModal from "@/components/3d/elements/compute/modal/AIModelConfigurationModal.svelte";
6
  import VideoInputConnectionModal from "@/components/3d/elements/compute/modal/VideoInputConnectionModal.svelte";
7
  import RobotInputConnectionModal from "@/components/3d/elements/compute/modal/RobotInputConnectionModal.svelte";
8
  import RobotOutputConnectionModal from "@/components/3d/elements/compute/modal/RobotOutputConnectionModal.svelte";
9
  import ComputeGridItem from "@/components/3d/elements/compute/ComputeGridItem.svelte";
10
  import type { RemoteCompute } from "$lib/elements/compute/RemoteCompute.svelte";
11
+ import { interactivity } from "@threlte/extras";
12
 
13
  interface Props {
14
  workspaceId: string;
 
16
  let { workspaceId }: Props = $props();
17
 
18
  let isAISessionModalOpen = $state(false);
19
+ let isAIConfigModalOpen = $state(false);
20
  let isVideoInputModalOpen = $state(false);
21
  let isRobotInputModalOpen = $state(false);
22
  let isRobotOutputModalOpen = $state(false);
 
67
 
68
  return () => clearInterval(interval);
69
  });
70
+ interactivity({
71
+ filter: (hits, state) => {
72
+ return hits.slice(0, 1);
73
+ }
74
+ });
75
  </script>
76
 
77
  {#each remoteComputeManager.computes as compute (compute.id)}
 
84
  {/each}
85
 
86
  {#if selectedCompute}
87
+ <!-- Inference Session Configuration Modal (for existing computes without sessions) -->
88
  <AISessionConnectionModal bind:open={isAISessionModalOpen} compute={selectedCompute} {workspaceId} />
89
  <!-- Video Input Connection Modal -->
90
  <VideoInputConnectionModal bind:open={isVideoInputModalOpen} compute={selectedCompute} {workspaceId} />
 
92
  <RobotInputConnectionModal bind:open={isRobotInputModalOpen} compute={selectedCompute} {workspaceId} />
93
  <!-- Robot Output Connection Modal -->
94
  <RobotOutputConnectionModal bind:open={isRobotOutputModalOpen} compute={selectedCompute} {workspaceId} />
95
+ {/if}
96
+
97
+ <!-- AI Model Configuration Modal (for creating new models) -->
98
+ <AIModelConfigurationModal bind:open={isAIConfigModalOpen} {workspaceId} />
src/lib/components/3d/elements/compute/modal/AISessionConnectionModal.svelte CHANGED
@@ -5,8 +5,8 @@
5
  import { Badge } from "@/components/ui/badge";
6
  import { Input } from "@/components/ui/input";
7
  import { Label } from "@/components/ui/label";
8
- import * as Alert from "@/components/ui/alert";
9
- import { remoteComputeManager } from "$lib/elements/compute//RemoteComputeManager.svelte";
10
  import type { RemoteCompute } from "$lib/elements/compute//RemoteCompute.svelte";
11
  import type { AISessionConfig } from "$lib/elements/compute//RemoteComputeManager.svelte";
12
  import { settings } from "$lib/runes/settings.svelte";
@@ -22,9 +22,12 @@
22
 
23
  let isConnecting = $state(false);
24
  let sessionId = $state("");
25
- let policyPath = $state("./checkpoints/act_so101_beyond");
26
- let cameraNames = $state("front");
27
- let useProvidedWorkspace = $state(false);
 
 
 
28
 
29
  // Auto-generate session ID when modal opens
30
  $effect(() => {
@@ -33,6 +36,18 @@
33
  }
34
  });
35
 
 
 
 
 
 
 
 
 
 
 
 
 
36
  async function handleCreateSession() {
37
  if (!compute) return;
38
 
@@ -41,6 +56,11 @@
41
  return;
42
  }
43
 
 
 
 
 
 
44
  isConnecting = true;
45
  try {
46
  const cameras = cameraNames
@@ -51,17 +71,19 @@
51
  cameras.push("front");
52
  }
53
 
54
- const config: AISessionConfig = {
55
  sessionId: sessionId.trim(),
 
56
  policyPath: policyPath.trim(),
57
  cameraNames: cameras,
58
  transportServerUrl: settings.transportServerUrl,
59
- workspaceId: useProvidedWorkspace ? workspaceId : undefined
 
60
  };
61
 
62
- const result = await remoteComputeManager.createSession(compute.id, config);
63
  if (result.success) {
64
- toast.success(`Inference Session created: ${sessionId}`);
65
  open = false;
66
  } else {
67
  toast.error(`Failed to create session: ${result.error}`);
@@ -140,11 +162,11 @@
140
  <Dialog.Title
141
  class="flex items-center gap-2 text-lg font-bold text-slate-900 dark:text-slate-100"
142
  >
143
- <span class="icon-[mdi--robot-outline] size-5 text-purple-500 dark:text-purple-400"></span>
144
- AI Compute Session - {compute.name || "No Compute Selected"}
145
  </Dialog.Title>
146
  <Dialog.Description class="text-sm text-slate-600 dark:text-slate-400">
147
- Configure and manage ACT model inference sessions for robot control
148
  </Dialog.Description>
149
  </Dialog.Header>
150
 
@@ -154,7 +176,7 @@
154
  class="flex items-center justify-between rounded-lg border border-purple-300/30 bg-purple-100/20 p-3 dark:border-purple-500/30 dark:bg-purple-900/20"
155
  >
156
  <div class="flex items-center gap-2">
157
- <span class="icon-[mdi--brain] size-4 text-purple-500 dark:text-purple-400"></span>
158
  <span class="text-sm font-medium text-purple-700 dark:text-purple-300"
159
  >Session Status</span
160
  >
@@ -325,84 +347,84 @@
325
  </Card.Header>
326
  <Card.Content>
327
  <div class="space-y-4">
328
- <div class="grid grid-cols-2 gap-4">
329
- <div class="space-y-2">
330
- <Label for="sessionId" class="text-purple-700 dark:text-purple-300"
331
- >Session ID</Label
332
- >
333
- <Input
334
- id="sessionId"
335
- bind:value={sessionId}
336
- placeholder="my-session-01"
337
- class="border-slate-300 bg-slate-50 text-slate-900 dark:border-slate-600 dark:bg-slate-800 dark:text-slate-100"
338
- />
339
- </div>
340
- <div class="space-y-2">
341
- <Label for="policyPath" class="text-purple-700 dark:text-purple-300"
342
- >Policy Path</Label
343
- >
344
- <Input
345
- id="policyPath"
346
- bind:value={policyPath}
347
- placeholder="./checkpoints/act_so101_beyond"
348
- class="border-slate-300 bg-slate-50 text-slate-900 dark:border-slate-600 dark:bg-slate-800 dark:text-slate-100"
349
- />
350
- </div>
351
  </div>
352
 
353
- <div class="grid grid-cols-2 gap-4">
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
354
  <div class="space-y-2">
355
- <Label for="cameraNames" class="text-purple-700 dark:text-purple-300"
356
- >Camera Names</Label
357
- >
358
- <Input
359
- id="cameraNames"
360
- bind:value={cameraNames}
361
- placeholder="front, wrist, overhead"
362
  class="border-slate-300 bg-slate-50 text-slate-900 dark:border-slate-600 dark:bg-slate-800 dark:text-slate-100"
 
363
  />
364
  <p class="text-xs text-slate-600 dark:text-slate-400">
365
- Comma-separated camera names
366
  </p>
367
  </div>
368
- <div class="space-y-2">
369
- <Label for="transportServerUrl" class="text-purple-700 dark:text-purple-300"
370
- >Transport Server URL</Label
371
- >
372
- <Input
373
- id="transportServerUrl"
374
- value={settings.transportServerUrl}
375
- disabled
376
- placeholder="http://localhost:8000"
377
- class="cursor-not-allowed border-slate-300 bg-slate-50 text-slate-900 opacity-60 dark:border-slate-600 dark:bg-slate-800 dark:text-slate-100"
378
- title="Change this value in the settings panel"
379
- />
380
- <p class="text-xs text-slate-600 dark:text-slate-400">
381
- Configure in settings panel
382
- </p>
383
- </div>
384
- </div>
385
 
386
- <div class="flex items-center space-x-2">
387
- <input
388
- type="checkbox"
389
- id="useWorkspace"
390
- bind:checked={useProvidedWorkspace}
391
- class="rounded border-slate-300 bg-slate-50 dark:border-slate-600 dark:bg-slate-800"
392
- />
393
- <Label for="useWorkspace" class="text-sm text-purple-700 dark:text-purple-300">
394
- Use current workspace ({workspaceId})
395
- </Label>
396
  </div>
397
 
398
- <Alert.Root>
399
- <span class="icon-[mdi--information] size-4"></span>
400
- <Alert.Description>
401
- This will create a new ACT inference session with dedicated rooms for camera
 
 
402
  inputs, joint inputs, and joint outputs in the inference server communication
403
  system.
404
- </Alert.Description>
405
- </Alert.Root>
406
 
407
  <Button
408
  variant="default"
@@ -428,7 +450,7 @@
428
  class="rounded border border-slate-300 bg-slate-100/30 p-2 text-xs text-slate-600 dark:border-slate-700 dark:bg-slate-800/30 dark:text-slate-500"
429
  >
430
  <span class="icon-[mdi--information] mr-1 size-3"></span>
431
- Inference Sessions require a trained ACT model and create dedicated communication rooms for video
432
  inputs, robot joint states, and control outputs in the inference server system.
433
  </div>
434
  </div>
 
5
  import { Badge } from "@/components/ui/badge";
6
  import { Input } from "@/components/ui/input";
7
  import { Label } from "@/components/ui/label";
8
+ import { Textarea } from "@/components/ui/textarea";
9
+ import { remoteComputeManager, MODEL_TYPES } from "$lib/elements/compute";
10
  import type { RemoteCompute } from "$lib/elements/compute//RemoteCompute.svelte";
11
  import type { AISessionConfig } from "$lib/elements/compute//RemoteComputeManager.svelte";
12
  import { settings } from "$lib/runes/settings.svelte";
 
22
 
23
  let isConnecting = $state(false);
24
  let sessionId = $state("");
25
+ let policyPath = $state("");
26
+ let cameraNames = $state("");
27
+ let languageInstruction = $state("");
28
+
29
+ // Use the compute's model type (can't be changed here)
30
+ const modelConfig = $derived(MODEL_TYPES[compute.modelType]);
31
 
32
  // Auto-generate session ID when modal opens
33
  $effect(() => {
 
36
  }
37
  });
38
 
39
+ // Set defaults when modal opens
40
+ $effect(() => {
41
+ if (open && modelConfig) {
42
+ if (!policyPath) {
43
+ policyPath = modelConfig.defaultPolicyPath;
44
+ }
45
+ if (!cameraNames) {
46
+ cameraNames = modelConfig.defaultCameraNames.join(", ");
47
+ }
48
+ }
49
+ });
50
+
51
  async function handleCreateSession() {
52
  if (!compute) return;
53
 
 
56
  return;
57
  }
58
 
59
+ if (modelConfig.requiresLanguageInstruction && !languageInstruction.trim()) {
60
+ toast.error("Language instruction is required for this model type");
61
+ return;
62
+ }
63
+
64
  isConnecting = true;
65
  try {
66
  const cameras = cameraNames
 
71
  cameras.push("front");
72
  }
73
 
74
+ const sessionConfig: AISessionConfig = {
75
  sessionId: sessionId.trim(),
76
+ modelType: compute.modelType,
77
  policyPath: policyPath.trim(),
78
  cameraNames: cameras,
79
  transportServerUrl: settings.transportServerUrl,
80
+ workspaceId: workspaceId,
81
+ languageInstruction: languageInstruction.trim() || undefined
82
  };
83
 
84
+ const result = await remoteComputeManager.createSession(compute.id, sessionConfig);
85
  if (result.success) {
86
+ toast.success(`${modelConfig.label} session created: ${sessionId}`);
87
  open = false;
88
  } else {
89
  toast.error(`Failed to create session: ${result.error}`);
 
162
  <Dialog.Title
163
  class="flex items-center gap-2 text-lg font-bold text-slate-900 dark:text-slate-100"
164
  >
165
+ <span class="{modelConfig.icon} size-5 text-purple-500 dark:text-purple-400"></span>
166
+ {modelConfig.label} Session - {compute.name || "No Compute Selected"}
167
  </Dialog.Title>
168
  <Dialog.Description class="text-sm text-slate-600 dark:text-slate-400">
169
+ Configure and manage {modelConfig.label} inference sessions for robot control
170
  </Dialog.Description>
171
  </Dialog.Header>
172
 
 
176
  class="flex items-center justify-between rounded-lg border border-purple-300/30 bg-purple-100/20 p-3 dark:border-purple-500/30 dark:bg-purple-900/20"
177
  >
178
  <div class="flex items-center gap-2">
179
+ <span class="{modelConfig.icon} size-4 text-purple-500 dark:text-purple-400"></span>
180
  <span class="text-sm font-medium text-purple-700 dark:text-purple-300"
181
  >Session Status</span
182
  >
 
347
  </Card.Header>
348
  <Card.Content>
349
  <div class="space-y-4">
350
+ <div class="space-y-2">
351
+ <Label for="sessionId" class="text-purple-700 dark:text-purple-300"
352
+ >Session ID</Label
353
+ >
354
+ <Input
355
+ id="sessionId"
356
+ bind:value={sessionId}
357
+ placeholder="my-session-01"
358
+ class="border-slate-300 bg-slate-50 text-slate-900 dark:border-slate-600 dark:bg-slate-800 dark:text-slate-100"
359
+ />
360
+ </div>
361
+ <div class="space-y-2">
362
+ <Label for="policyPath" class="text-purple-700 dark:text-purple-300"
363
+ >Policy Path</Label
364
+ >
365
+ <Input
366
+ id="policyPath"
367
+ bind:value={policyPath}
368
+ placeholder={modelConfig.defaultPolicyPath}
369
+ class="border-slate-300 bg-slate-50 text-slate-900 dark:border-slate-600 dark:bg-slate-800 dark:text-slate-100"
370
+ />
 
 
371
  </div>
372
 
373
+ <div class="space-y-2">
374
+ <Label for="cameraNames" class="text-purple-700 dark:text-purple-300"
375
+ >Camera Names</Label
376
+ >
377
+ <Input
378
+ id="cameraNames"
379
+ bind:value={cameraNames}
380
+ placeholder="front, wrist, overhead"
381
+ class="border-slate-300 bg-slate-50 text-slate-900 dark:border-slate-600 dark:bg-slate-800 dark:text-slate-100"
382
+ />
383
+ <p class="text-xs text-slate-600 dark:text-slate-400">
384
+ Comma-separated camera names
385
+ </p>
386
+ </div>
387
+
388
+ {#if modelConfig.requiresLanguageInstruction}
389
  <div class="space-y-2">
390
+ <Label for="languageInstruction" class="text-purple-700 dark:text-purple-300">
391
+ Language Instruction
392
+ </Label>
393
+ <Textarea
394
+ id="languageInstruction"
395
+ bind:value={languageInstruction}
396
+ placeholder="Pick up the red cup and place it on the table"
397
  class="border-slate-300 bg-slate-50 text-slate-900 dark:border-slate-600 dark:bg-slate-800 dark:text-slate-100"
398
+ rows={3}
399
  />
400
  <p class="text-xs text-slate-600 dark:text-slate-400">
401
+ Natural language instruction for the task (required for {modelConfig.label})
402
  </p>
403
  </div>
404
+ {/if}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
405
 
406
+ <div class="rounded-lg border border-green-300/30 bg-green-100/20 p-3 dark:border-green-500/30 dark:bg-green-900/20">
407
+ <div class="flex items-center gap-2">
408
+ <span class="icon-[mdi--folder] size-4 text-green-600 dark:text-green-400"></span>
409
+ <span class="text-sm font-medium text-green-700 dark:text-green-300">
410
+ Workspace: {workspaceId}
411
+ </span>
412
+ </div>
413
+ <p class="mt-1 text-xs text-green-600 dark:text-green-400">
414
+ Session will be created in the current workspace
415
+ </p>
416
  </div>
417
 
418
+ <div
419
+ class="rounded-lg border border-slate-300 bg-slate-50 p-3 dark:border-slate-600 dark:bg-slate-800"
420
+ >
421
+ <div class="text-xs text-slate-600 dark:text-slate-400">
422
+ <span class="icon-[mdi--lightbulb] size-3"></span>
423
+ <strong>Tip:</strong> This will create a new {modelConfig.label} inference session with dedicated rooms for camera
424
  inputs, joint inputs, and joint outputs in the inference server communication
425
  system.
426
+ </div>
427
+ </div>
428
 
429
  <Button
430
  variant="default"
 
450
  class="rounded border border-slate-300 bg-slate-100/30 p-2 text-xs text-slate-600 dark:border-slate-700 dark:bg-slate-800/30 dark:text-slate-500"
451
  >
452
  <span class="icon-[mdi--information] mr-1 size-3"></span>
453
+ Inference Sessions require a trained {modelConfig.label} and create dedicated communication rooms for video
454
  inputs, robot joint states, and control outputs in the inference server system.
455
  </div>
456
  </div>
src/lib/components/3d/elements/compute/status/ComputeBoxUIKit.svelte CHANGED
@@ -38,7 +38,7 @@
38
  <StatusContent
39
  title={compute.name}
40
  subtitle={compute.statusInfo.statusText}
41
- color="rgb(221, 214, 254)"
42
  variant="primary"
43
  />
44
  </BaseStatusBox>
 
38
  <StatusContent
39
  title={compute.name}
40
  subtitle={compute.statusInfo.statusText}
41
+ color="rgb(107, 33, 168)"
42
  variant="primary"
43
  />
44
  </BaseStatusBox>
src/lib/components/3d/elements/compute/status/ComputeInputBoxUIKit.svelte CHANGED
@@ -43,7 +43,7 @@
43
  <StatusContent
44
  title={`${Object.keys(compute.inputConnections.cameras).length} Cameras`}
45
  subtitle="Joint States"
46
- color="rgb(187, 247, 208)"
47
  variant="primary"
48
  />
49
 
@@ -60,7 +60,7 @@
60
  fontSize={12}
61
  />
62
 
63
- <StatusContent title="Setup Required" color="rgb(134, 239, 172)" variant="secondary" />
64
 
65
  <StatusButton
66
  text="Add Session"
 
43
  <StatusContent
44
  title={`${Object.keys(compute.inputConnections.cameras).length} Cameras`}
45
  subtitle="Joint States"
46
+ color="rgb(21, 128, 61)"
47
  variant="primary"
48
  />
49
 
 
60
  fontSize={12}
61
  />
62
 
63
+ <StatusContent title="Setup Required" color="rgb(34, 197, 94)" variant="secondary" />
64
 
65
  <StatusButton
66
  text="Add Session"
src/lib/components/3d/elements/compute/status/ComputeOutputBoxUIKit.svelte CHANGED
@@ -42,7 +42,7 @@
42
  <StatusContent
43
  title={compute.isRunning ? "Active" : "Ready"}
44
  subtitle="Commands"
45
- color="rgb(191, 219, 254)"
46
  variant="primary"
47
  />
48
 
@@ -64,7 +64,7 @@
64
 
65
  <StatusContent
66
  title={!compute.hasSession ? "Need Session" : "Configure"}
67
- color="rgb(147, 197, 253)"
68
  variant="secondary"
69
  />
70
 
 
42
  <StatusContent
43
  title={compute.isRunning ? "Active" : "Ready"}
44
  subtitle="Commands"
45
+ color="rgb(37, 99, 235)"
46
  variant="primary"
47
  />
48
 
 
64
 
65
  <StatusContent
66
  title={!compute.hasSession ? "Need Session" : "Configure"}
67
+ color="rgb(59, 130, 246)"
68
  variant="secondary"
69
  />
70
 
src/lib/components/3d/elements/compute/status/ComputeStatusBillboard.svelte CHANGED
@@ -4,6 +4,8 @@
4
  import { Root, Container } from "threlte-uikit";
5
  import ComputeConnectionFlowBoxUIKit from "./ComputeConnectionFlowBoxUIKit.svelte";
6
  import type { RemoteCompute } from "$lib/elements/compute//RemoteCompute.svelte";
 
 
7
 
8
  interface Props {
9
  compute: RemoteCompute;
@@ -11,6 +13,8 @@
11
  onVideoInputBoxClick: (compute: RemoteCompute) => void;
12
  onRobotInputBoxClick: (compute: RemoteCompute) => void;
13
  onRobotOutputBoxClick: (compute: RemoteCompute) => void;
 
 
14
  }
15
 
16
  let {
@@ -18,10 +22,19 @@
18
  visible = true,
19
  onVideoInputBoxClick,
20
  onRobotInputBoxClick,
21
- onRobotOutputBoxClick
 
 
22
  }: Props = $props();
23
 
24
  interactivity();
 
 
 
 
 
 
 
25
  </script>
26
 
27
  <T.Group
@@ -31,7 +44,6 @@
31
  rotation={[-Math.PI / 2, 0, 0]}
32
  scale={[0.1, 0.1, 0.1]}
33
  pointerEvents="listener"
34
- {visible}
35
  >
36
  <Billboard>
37
  <Root name={`compute-status-billboard-${compute.id}`}>
@@ -41,6 +53,10 @@
41
  alignItems="center"
42
  justifyContent="center"
43
  padding={20}
 
 
 
 
44
  >
45
  <ComputeConnectionFlowBoxUIKit
46
  {compute}
 
4
  import { Root, Container } from "threlte-uikit";
5
  import ComputeConnectionFlowBoxUIKit from "./ComputeConnectionFlowBoxUIKit.svelte";
6
  import type { RemoteCompute } from "$lib/elements/compute//RemoteCompute.svelte";
7
+ import { Tween } from "svelte/motion";
8
+ import { cubicOut } from "svelte/easing";
9
 
10
  interface Props {
11
  compute: RemoteCompute;
 
13
  onVideoInputBoxClick: (compute: RemoteCompute) => void;
14
  onRobotInputBoxClick: (compute: RemoteCompute) => void;
15
  onRobotOutputBoxClick: (compute: RemoteCompute) => void;
16
+ duration?: number;
17
+ delay?: number;
18
  }
19
 
20
  let {
 
22
  visible = true,
23
  onVideoInputBoxClick,
24
  onRobotInputBoxClick,
25
+ onRobotOutputBoxClick,
26
+ duration = 100,
27
+ delay = 0
28
  }: Props = $props();
29
 
30
  interactivity();
31
+
32
+ const tweenedScale = Tween.of(() => {
33
+ return visible ? 1 : 0;
34
+ }, { duration: duration, easing: cubicOut, delay: delay });
35
+ const tweenedOpacity = Tween.of(() => {
36
+ return visible ? 1 : 0;
37
+ }, { duration: duration, easing: cubicOut, delay: delay });
38
  </script>
39
 
40
  <T.Group
 
44
  rotation={[-Math.PI / 2, 0, 0]}
45
  scale={[0.1, 0.1, 0.1]}
46
  pointerEvents="listener"
 
47
  >
48
  <Billboard>
49
  <Root name={`compute-status-billboard-${compute.id}`}>
 
53
  alignItems="center"
54
  justifyContent="center"
55
  padding={20}
56
+ transformScaleX={tweenedScale.current}
57
+ transformScaleY={tweenedScale.current}
58
+ transformScaleZ={tweenedScale.current}
59
+ opacity={tweenedOpacity.current}
60
  >
61
  <ComputeConnectionFlowBoxUIKit
62
  {compute}
src/lib/components/3d/elements/compute/status/RobotInputBoxUIKit.svelte CHANGED
@@ -41,7 +41,7 @@
41
  <StatusContent
42
  title="Joint States"
43
  subtitle="6 DOF Robot"
44
- color="rgb(254, 215, 170)"
45
  variant="primary"
46
  />
47
 
@@ -57,7 +57,7 @@
57
  fontSize={11}
58
  />
59
 
60
- <StatusContent title="Setup Robot" color="rgb(254, 215, 170)" variant="secondary" />
61
 
62
  <StatusButton
63
  icon={ICON["icon-[mdi--plus]"].svg}
 
41
  <StatusContent
42
  title="Joint States"
43
  subtitle="6 DOF Robot"
44
+ color="rgb(180, 83, 9)"
45
  variant="primary"
46
  />
47
 
 
57
  fontSize={11}
58
  />
59
 
60
+ <StatusContent title="Setup Robot" color="rgb(245, 158, 11)" variant="secondary" />
61
 
62
  <StatusButton
63
  icon={ICON["icon-[mdi--plus]"].svg}
src/lib/components/3d/elements/compute/status/RobotOutputBoxUIKit.svelte CHANGED
@@ -32,7 +32,7 @@
32
  <StatusContent
33
  title={compute.isRunning ? "AI Commands Active" : "Session Ready"}
34
  subtitle="Motor Control"
35
- color="rgb(191, 219, 254)"
36
  variant="primary"
37
  />
38
 
@@ -55,7 +55,7 @@
55
 
56
  <StatusContent
57
  title={!compute.hasSession ? 'Need Session' : 'Click to Configure'}
58
- color="rgb(147, 197, 253)"
59
  variant="secondary"
60
  />
61
 
 
32
  <StatusContent
33
  title={compute.isRunning ? "AI Commands Active" : "Session Ready"}
34
  subtitle="Motor Control"
35
+ color="rgb(37, 99, 235)"
36
  variant="primary"
37
  />
38
 
 
55
 
56
  <StatusContent
57
  title={!compute.hasSession ? 'Need Session' : 'Click to Configure'}
58
+ color="rgb(59, 130, 246)"
59
  variant="secondary"
60
  />
61
 
src/lib/components/3d/elements/compute/status/VideoInputBoxUIKit.svelte CHANGED
@@ -35,7 +35,7 @@
35
  <!-- Camera Streams -->
36
  <StatusContent
37
  title={`${Object.keys(compute.inputConnections.cameras).length} Cameras`}
38
- color="rgb(187, 247, 208)"
39
  variant="primary"
40
  />
41
 
@@ -53,7 +53,7 @@
53
 
54
  <StatusContent
55
  title="Setup Video"
56
- color="rgb(134, 239, 172)"
57
  variant="secondary"
58
  />
59
 
 
35
  <!-- Camera Streams -->
36
  <StatusContent
37
  title={`${Object.keys(compute.inputConnections.cameras).length} Cameras`}
38
+ color="rgb(21, 128, 61)"
39
  variant="primary"
40
  />
41
 
 
53
 
54
  <StatusContent
55
  title="Setup Video"
56
+ color="rgb(34, 197, 94)"
57
  variant="secondary"
58
  />
59
 
src/lib/components/3d/elements/robot/RobotGridItem.svelte CHANGED
@@ -6,7 +6,7 @@
6
  import RobotStatusBillboard from "@/components/3d/elements/robot/status/RobotStatusBillboard.svelte";
7
  import type { Robot } from "$lib/elements/robot/Robot.svelte.js";
8
  import { createUrdfRobot } from "$lib/elements/robot/createRobot.svelte";
9
- import { interactivity, type IntersectionEvent, useCursor } from "@threlte/extras";
10
  import type { RobotUrdfConfig } from "$lib/types/urdf";
11
  import { onMount } from "svelte";
12
  import type IUrdfRobot from "@/components/3d/elements/robot/URDF/interfaces/IUrdfRobot.js";
@@ -66,6 +66,7 @@
66
  // Batch update all joints that have changed (or all joints on initial sync)
67
  let updatedCount = 0;
68
  robot.jointArray.forEach((joint) => {
 
69
  if (isInitialSync || Math.abs((lastJointValues[joint.name] || 0) - joint.value) > threshold) {
70
  lastJointValues[joint.name] = joint.value;
71
  const urdfJoint = findUrdfJoint(urdfRobotState, joint.name);
@@ -92,7 +93,7 @@
92
  });
93
  });
94
 
95
- function findUrdfJoint(robot: Robot, jointName: string): any {
96
  // Search through the robot's joints array
97
  if (robot.joints && Array.isArray(robot.joints)) {
98
  for (const joint of robot.joints) {
@@ -105,7 +106,7 @@
105
  }
106
 
107
  const { onPointerEnter, onPointerLeave, hovering } = useCursor();
108
- interactivity();
109
 
110
  let isToggled = $state(false);
111
 
@@ -123,7 +124,13 @@
123
  scale={[10, 10, 10]}
124
  rotation={[-Math.PI / 2, 0, 0]}
125
  >
126
- <T.Group onpointerenter={onPointerEnter} onpointerleave={onPointerLeave} onclick={handleClick}>
 
 
 
 
 
 
127
  {#if urdfRobotState}
128
  {#each getRootLinks(urdfRobotState) as link}
129
  <UrdfLink
@@ -144,7 +151,6 @@
144
  nameHeight={0.1}
145
  showLine={$hovering || isToggled}
146
  opacity={1}
147
- isInteractive={false}
148
  />
149
  {/each}
150
  {:else}
 
6
  import RobotStatusBillboard from "@/components/3d/elements/robot/status/RobotStatusBillboard.svelte";
7
  import type { Robot } from "$lib/elements/robot/Robot.svelte.js";
8
  import { createUrdfRobot } from "$lib/elements/robot/createRobot.svelte";
9
+ import { type IntersectionEvent, useCursor } from "@threlte/extras";
10
  import type { RobotUrdfConfig } from "$lib/types/urdf";
11
  import { onMount } from "svelte";
12
  import type IUrdfRobot from "@/components/3d/elements/robot/URDF/interfaces/IUrdfRobot.js";
 
66
  // Batch update all joints that have changed (or all joints on initial sync)
67
  let updatedCount = 0;
68
  robot.jointArray.forEach((joint) => {
69
+ if (!urdfRobotState) return;
70
  if (isInitialSync || Math.abs((lastJointValues[joint.name] || 0) - joint.value) > threshold) {
71
  lastJointValues[joint.name] = joint.value;
72
  const urdfJoint = findUrdfJoint(urdfRobotState, joint.name);
 
93
  });
94
  });
95
 
96
+ function findUrdfJoint(robot: IUrdfRobot, jointName: string): any {
97
  // Search through the robot's joints array
98
  if (robot.joints && Array.isArray(robot.joints)) {
99
  for (const joint of robot.joints) {
 
106
  }
107
 
108
  const { onPointerEnter, onPointerLeave, hovering } = useCursor();
109
+
110
 
111
  let isToggled = $state(false);
112
 
 
124
  scale={[10, 10, 10]}
125
  rotation={[-Math.PI / 2, 0, 0]}
126
  >
127
+ <T.Group onpointerenter={(event) => {
128
+ event.stopPropagation();
129
+ onPointerEnter();
130
+ }} onpointerleave={(event) => {
131
+ event.stopPropagation();
132
+ onPointerLeave();
133
+ }} onclick={handleClick}>
134
  {#if urdfRobotState}
135
  {#each getRootLinks(urdfRobotState) as link}
136
  <UrdfLink
 
151
  nameHeight={0.1}
152
  showLine={$hovering || isToggled}
153
  opacity={1}
 
154
  />
155
  {/each}
156
  {:else}
src/lib/components/3d/elements/robot/Robots.svelte CHANGED
@@ -7,6 +7,7 @@
7
  import type { Robot } from "$lib/elements/robot/Robot.svelte.js";
8
  import { generateName } from "$lib/utils/generateName";
9
  import RobotGridItem from "@/components/3d/elements/robot/RobotGridItem.svelte";
 
10
 
11
  interface Props {
12
  workspaceId: string;
@@ -64,6 +65,11 @@
64
  console.error("❌ Error during cleanup:", error);
65
  });
66
  });
 
 
 
 
 
67
  </script>
68
 
69
  {#each robotManager.robots as robot (robot.id)}
 
7
  import type { Robot } from "$lib/elements/robot/Robot.svelte.js";
8
  import { generateName } from "$lib/utils/generateName";
9
  import RobotGridItem from "@/components/3d/elements/robot/RobotGridItem.svelte";
10
+ import { interactivity } from "@threlte/extras";
11
 
12
  interface Props {
13
  workspaceId: string;
 
65
  console.error("❌ Error during cleanup:", error);
66
  });
67
  });
68
+ interactivity({
69
+ filter: (hits, state) => {
70
+ return hits.slice(0, 1);
71
+ }
72
+ });
73
  </script>
74
 
75
  {#each robotManager.robots as robot (robot.id)}
src/lib/components/3d/elements/robot/URDF/primitives/UrdfJoint.svelte CHANGED
@@ -145,7 +145,6 @@
145
  {nameHeight}
146
  {showLine}
147
  opacity={1}
148
- isInteractive={true}
149
  />
150
  {/if}
151
 
 
145
  {nameHeight}
146
  {showLine}
147
  opacity={1}
 
148
  />
149
  {/if}
150
 
src/lib/components/3d/elements/robot/URDF/primitives/UrdfLink.svelte CHANGED
@@ -28,7 +28,6 @@
28
  nameHeight?: number;
29
  showLine?: boolean;
30
  opacity?: number;
31
- isInteractive?: boolean;
32
  }
33
 
34
  let {
@@ -49,7 +48,6 @@
49
  nameHeight = 0.1,
50
  showLine = true,
51
  opacity = 0.7,
52
- isInteractive = false
53
  }: Props = $props();
54
 
55
  let showPointCloud = false;
@@ -85,7 +83,6 @@
85
  {jointIndicatorColor}
86
  {showLine}
87
  {opacity}
88
- {isInteractive}
89
  {showVisual}
90
  {showCollision}
91
  {visualOpacity}
 
28
  nameHeight?: number;
29
  showLine?: boolean;
30
  opacity?: number;
 
31
  }
32
 
33
  let {
 
48
  nameHeight = 0.1,
49
  showLine = true,
50
  opacity = 0.7,
 
51
  }: Props = $props();
52
 
53
  let showPointCloud = false;
 
83
  {jointIndicatorColor}
84
  {showLine}
85
  {opacity}
 
86
  {showVisual}
87
  {showCollision}
88
  {visualOpacity}
src/lib/components/3d/elements/robot/URDF/primitives/UrdfThree.svelte CHANGED
@@ -62,7 +62,6 @@
62
  {nameHeight}
63
  showLine={false}
64
  opacity={1}
65
- isInteractive={false}
66
  />
67
  {/each}
68
  </T.Group>
 
62
  {nameHeight}
63
  showLine={false}
64
  opacity={1}
 
65
  />
66
  {/each}
67
  </T.Group>
src/lib/components/3d/elements/robot/modal/InputConnectionModal.svelte CHANGED
@@ -9,6 +9,7 @@
9
  import type { Robot } from "$lib/elements/robot/Robot.svelte.js";
10
  import { robotManager } from "$lib/elements/robot/RobotManager.svelte.js";
11
  import USBCalibrationPanel from "$lib/elements/robot/calibration/USBCalibrationPanel.svelte";
 
12
 
13
  interface Props {
14
  workspaceId: string;
@@ -181,9 +182,9 @@
181
  showUSBCalibration = false;
182
  pendingUSBConnection = null;
183
  isConnecting = false;
184
-
185
  // Clean up the uncalibrated USB consumer
186
- robot.removeConsumer().catch(err => {
187
  console.error("Failed to clean up USB consumer after calibration cancel:", err);
188
  });
189
  }
@@ -261,9 +262,7 @@
261
  onCancel={onCalibrationCancel}
262
  />
263
  {:else}
264
- <div class="text-center text-slate-400">
265
- No USB drivers require calibration
266
- </div>
267
  {/each}
268
  </Card.Content>
269
  </Card.Root>
@@ -370,13 +369,14 @@
370
  <div class="flex items-center justify-between">
371
  <div>
372
  <Card.Title
373
- class="flex items-center gap-2 text-base text-purple-700 dark:text-purple-200"
374
  >
375
  <span class="icon-[mdi--cloud-sync] size-4"></span>
376
- Remote Collaboration (Rooms)
377
  </Card.Title>
378
  <Card.Description class="text-xs text-purple-600/70 dark:text-purple-300/70">
379
- Receive commands from AI systems, remote users, or other software
 
380
  </Card.Description>
381
  </div>
382
  <Button
@@ -499,11 +499,36 @@
499
  >
500
  {room.id}
501
  </p>
502
- <div class="flex gap-3 text-xs text-slate-600 dark:text-slate-400">
503
  <span>{room.has_producer ? "📤 Has Output" : "📥 No Output"}</span>
504
  <span>👥 {room.participants?.total || 0} users</span>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
505
  </div>
 
506
  </div>
 
507
  <Button
508
  variant="secondary"
509
  size="sm"
@@ -514,7 +539,7 @@
514
  disabled={isConnecting || robot.hasConsumer}
515
  class="h-6 shrink-0 bg-purple-500 px-2 text-xs hover:bg-purple-600 disabled:opacity-50 dark:bg-purple-600 dark:hover:bg-purple-700"
516
  >
517
- <span class="icon-[mdi--login] mr-1 size-3"></span>
518
  Join as Input
519
  </Button>
520
  </div>
@@ -532,18 +557,6 @@
532
  {/if}
533
  </Card.Content>
534
  </Card.Root>
535
-
536
- <!-- Help Information -->
537
- <Alert.Root
538
- class="border-slate-300 bg-slate-100/30 dark:border-slate-700 dark:bg-slate-800/30"
539
- >
540
- <span class="icon-[mdi--help-circle] size-4 text-slate-600 dark:text-slate-400"></span>
541
- <Alert.Title class="text-slate-700 dark:text-slate-300">Input Sources</Alert.Title>
542
- <Alert.Description class="text-xs text-slate-600 dark:text-slate-400">
543
- <strong>USB:</strong> Read physical movements • <strong>Remote:</strong> Receive network
544
- commands • Only one active at a time
545
- </Alert.Description>
546
- </Alert.Root>
547
  {/if}
548
  </div>
549
  </div>
 
9
  import type { Robot } from "$lib/elements/robot/Robot.svelte.js";
10
  import { robotManager } from "$lib/elements/robot/RobotManager.svelte.js";
11
  import USBCalibrationPanel from "$lib/elements/robot/calibration/USBCalibrationPanel.svelte";
12
+ import { settings } from "@/runes/settings.svelte";
13
 
14
  interface Props {
15
  workspaceId: string;
 
182
  showUSBCalibration = false;
183
  pendingUSBConnection = null;
184
  isConnecting = false;
185
+
186
  // Clean up the uncalibrated USB consumer
187
+ robot.removeConsumer().catch((err) => {
188
  console.error("Failed to clean up USB consumer after calibration cancel:", err);
189
  });
190
  }
 
262
  onCancel={onCalibrationCancel}
263
  />
264
  {:else}
265
+ <div class="text-center text-slate-400">No USB drivers require calibration</div>
 
 
266
  {/each}
267
  </Card.Content>
268
  </Card.Root>
 
369
  <div class="flex items-center justify-between">
370
  <div>
371
  <Card.Title
372
+ class="flex items-center gap-2 pb-1 text-base text-purple-700 dark:text-purple-200"
373
  >
374
  <span class="icon-[mdi--cloud-sync] size-4"></span>
375
+ Remote Control
376
  </Card.Title>
377
  <Card.Description class="text-xs text-purple-600/70 dark:text-purple-300/70">
378
+ Receive commands from AI systems, remote users, or other software from anywhere
379
+ in the world
380
  </Card.Description>
381
  </div>
382
  <Button
 
499
  >
500
  {room.id}
501
  </p>
502
+ <div class="flex items-center gap-3 text-xs text-slate-600 dark:text-slate-400">
503
  <span>{room.has_producer ? "📤 Has Output" : "📥 No Output"}</span>
504
  <span>👥 {room.participants?.total || 0} users</span>
505
+ <!-- Monitoring links -->
506
+ <div class="flex gap-1">
507
+ <a
508
+ href={`${settings.transportServerUrl.replace('/api', '')}/${workspaceId}/robotics/consumer?room=${room.id}`}
509
+ target="_blank"
510
+ rel="noopener noreferrer"
511
+ class="inline-flex items-center gap-1 rounded bg-blue-500/10 px-1.5 py-0.5 text-xs text-blue-600 hover:bg-blue-500/20 dark:bg-blue-400/10 dark:text-blue-400 dark:hover:bg-blue-400/20"
512
+ title="Monitor Consumer"
513
+ >
514
+ <span class="icon-[mdi--monitor-eye] size-3"></span>
515
+ Consumer
516
+ </a>
517
+ <a
518
+ href={`${settings.transportServerUrl.replace('/api', '')}/${workspaceId}/robotics/producer?room=${room.id}`}
519
+ target="_blank"
520
+ rel="noopener noreferrer"
521
+ class="inline-flex items-center gap-1 rounded bg-green-500/10 px-1.5 py-0.5 text-xs text-green-600 hover:bg-green-500/20 dark:bg-green-400/10 dark:text-green-400 dark:hover:bg-green-400/20"
522
+ title="Monitor Producer"
523
+ >
524
+ <span class="icon-[mdi--monitor-eye] size-3"></span>
525
+ Producer
526
+ </a>
527
+ </div>
528
  </div>
529
+
530
  </div>
531
+
532
  <Button
533
  variant="secondary"
534
  size="sm"
 
539
  disabled={isConnecting || robot.hasConsumer}
540
  class="h-6 shrink-0 bg-purple-500 px-2 text-xs hover:bg-purple-600 disabled:opacity-50 dark:bg-purple-600 dark:hover:bg-purple-700"
541
  >
542
+ <span class="icon-[mdi--login] size-5 h-full w-full"></span>
543
  Join as Input
544
  </Button>
545
  </div>
 
557
  {/if}
558
  </Card.Content>
559
  </Card.Root>
 
 
 
 
 
 
 
 
 
 
 
 
560
  {/if}
561
  </div>
562
  </div>
src/lib/components/3d/elements/robot/modal/OutputConnectionModal.svelte CHANGED
@@ -340,13 +340,13 @@
340
  <div class="flex items-center justify-between">
341
  <div>
342
  <Card.Title
343
- class="flex items-center gap-2 text-base text-orange-700 dark:text-orange-200"
344
  >
345
  <span class="icon-[mdi--cloud-sync] size-4"></span>
346
- Remote Collaboration (Rooms)
347
  </Card.Title>
348
  <Card.Description class="text-xs text-orange-600/70 dark:text-orange-300/70">
349
- Broadcast robot movements to remote systems and AI
350
  </Card.Description>
351
  </div>
352
  <Button
@@ -441,9 +441,32 @@
441
  >
442
  {room.id}
443
  </p>
444
- <div class="flex gap-3 text-xs text-slate-600 dark:text-slate-400">
445
  <span>{room.has_producer ? "🔴 Occupied" : "🟢 Available"}</span>
446
  <span>👥 {room.participants?.total || 0} users</span>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
447
  </div>
448
  </div>
449
  {#if !room.has_producer}
@@ -525,18 +548,6 @@
525
  </Card.Content>
526
  </Card.Root>
527
  {/if}
528
-
529
- <!-- Help Information -->
530
- <Alert.Root
531
- class="border-slate-300 bg-slate-100/30 dark:border-slate-700 dark:bg-slate-800/30"
532
- >
533
- <span class="icon-[mdi--help-circle] size-4 text-slate-600 dark:text-slate-400"></span>
534
- <Alert.Title class="text-slate-700 dark:text-slate-300">Output Sources</Alert.Title>
535
- <Alert.Description class="text-xs text-slate-600 dark:text-slate-400">
536
- <strong>USB:</strong> Control physical hardware • <strong>Remote:</strong> Broadcast to
537
- network • Multiple outputs can be active
538
- </Alert.Description>
539
- </Alert.Root>
540
  {/if}
541
  </div>
542
  </div>
 
340
  <div class="flex items-center justify-between">
341
  <div>
342
  <Card.Title
343
+ class="flex items-center gap-2 text-base text-orange-700 dark:text-orange-200 pb-1"
344
  >
345
  <span class="icon-[mdi--cloud-sync] size-4"></span>
346
+ Remote Control
347
  </Card.Title>
348
  <Card.Description class="text-xs text-orange-600/70 dark:text-orange-300/70">
349
+ Broadcast robot movements to remote robots or AI systems from anywhere in the world
350
  </Card.Description>
351
  </div>
352
  <Button
 
441
  >
442
  {room.id}
443
  </p>
444
+ <div class="flex items-center gap-3 text-xs text-slate-600 dark:text-slate-400">
445
  <span>{room.has_producer ? "🔴 Occupied" : "🟢 Available"}</span>
446
  <span>👥 {room.participants?.total || 0} users</span>
447
+ <!-- Monitoring links -->
448
+ <div class="flex gap-1">
449
+ <a
450
+ href={`${settings.transportServerUrl.replace('/api', '')}/${workspaceId}/robotics/consumer?room=${room.id}`}
451
+ target="_blank"
452
+ rel="noopener noreferrer"
453
+ class="inline-flex items-center gap-1 rounded bg-blue-500/10 px-1.5 py-0.5 text-xs text-blue-600 hover:bg-blue-500/20 dark:bg-blue-400/10 dark:text-blue-400 dark:hover:bg-blue-400/20"
454
+ title="Monitor Consumer"
455
+ >
456
+ <span class="icon-[mdi--monitor-eye] size-3"></span>
457
+ Consumer
458
+ </a>
459
+ <a
460
+ href={`${settings.transportServerUrl.replace('/api', '')}/${workspaceId}/robotics/producer?room=${room.id}`}
461
+ target="_blank"
462
+ rel="noopener noreferrer"
463
+ class="inline-flex items-center gap-1 rounded bg-green-500/10 px-1.5 py-0.5 text-xs text-green-600 hover:bg-green-500/20 dark:bg-green-400/10 dark:text-green-400 dark:hover:bg-green-400/20"
464
+ title="Monitor Producer"
465
+ >
466
+ <span class="icon-[mdi--monitor-eye] size-3"></span>
467
+ Producer
468
+ </a>
469
+ </div>
470
  </div>
471
  </div>
472
  {#if !room.has_producer}
 
548
  </Card.Content>
549
  </Card.Root>
550
  {/if}
 
 
 
 
 
 
 
 
 
 
 
 
551
  {/if}
552
  </div>
553
  </div>
src/lib/components/3d/elements/robot/status/ConnectionFlowBoxUIkit.svelte CHANGED
@@ -5,21 +5,41 @@
5
  import InputBoxUIKit from "./InputBoxUIKit.svelte";
6
  import RobotBoxUIKit from "./RobotBoxUIKit.svelte";
7
  import OutputBoxUIKit from "./OutputBoxUIKit.svelte";
 
 
8
 
9
  interface Props {
 
10
  robot: Robot;
11
  onInputBoxClick: (robot: Robot) => void;
12
  onRobotBoxClick: (robot: Robot) => void;
13
  onOutputBoxClick: (robot: Robot) => void;
 
 
14
  }
15
 
16
- let { robot, onInputBoxClick, onRobotBoxClick, onOutputBoxClick }: Props = $props();
17
 
18
  const inputColor = "rgb(34, 197, 94)";
19
  const outputColor = "rgb(59, 130, 246)";
 
 
 
 
 
 
 
20
  </script>
21
 
22
- <Container flexDirection="row" alignItems="center" gap={12}>
 
 
 
 
 
 
 
 
23
  <!-- Input Box -->
24
  <InputBoxUIKit {robot} {onInputBoxClick} />
25
 
 
5
  import InputBoxUIKit from "./InputBoxUIKit.svelte";
6
  import RobotBoxUIKit from "./RobotBoxUIKit.svelte";
7
  import OutputBoxUIKit from "./OutputBoxUIKit.svelte";
8
+ import { Tween } from "svelte/motion";
9
+ import { cubicOut } from "svelte/easing";
10
 
11
  interface Props {
12
+ visible: boolean;
13
  robot: Robot;
14
  onInputBoxClick: (robot: Robot) => void;
15
  onRobotBoxClick: (robot: Robot) => void;
16
  onOutputBoxClick: (robot: Robot) => void;
17
+ duration?: number;
18
+ delay?: number;
19
  }
20
 
21
+ let { visible, robot, onInputBoxClick, onRobotBoxClick, onOutputBoxClick, duration = 100, delay = 0 }: Props = $props();
22
 
23
  const inputColor = "rgb(34, 197, 94)";
24
  const outputColor = "rgb(59, 130, 246)";
25
+
26
+ const tweenedScale = Tween.of(() => {
27
+ return visible ? 1 : 0;
28
+ }, { duration: duration, easing: cubicOut, delay: delay });
29
+ const tweenedOpacity = Tween.of(() => {
30
+ return visible ? 1 : 0;
31
+ }, { duration: duration, easing: cubicOut, delay: delay });
32
  </script>
33
 
34
+ <Container
35
+ flexDirection="row"
36
+ alignItems="center"
37
+ gap={12}
38
+ transformScaleX={tweenedScale.current}
39
+ transformScaleY={tweenedScale.current}
40
+ transformScaleZ={tweenedScale.current}
41
+ opacity={tweenedOpacity.current}
42
+ >
43
  <!-- Input Box -->
44
  <InputBoxUIKit {robot} {onInputBoxClick} />
45
 
src/lib/components/3d/elements/robot/status/OutputBoxUIKit.svelte CHANGED
@@ -41,7 +41,7 @@
41
  <!-- Outputs Count -->
42
  <StatusContent
43
  title={`${robot.outputDriverCount} Outputs Active`}
44
- color="rgb(191, 219, 254)"
45
  variant="primary"
46
  />
47
 
 
41
  <!-- Outputs Count -->
42
  <StatusContent
43
  title={`${robot.outputDriverCount} Outputs Active`}
44
+ color={outputColor}
45
  variant="primary"
46
  />
47
 
src/lib/components/3d/elements/robot/status/RobotStatusBillboard.svelte CHANGED
@@ -26,7 +26,7 @@
26
  interactivity();
27
  </script>
28
 
29
- {#if visible}
30
  <T.Group
31
  position.z={0.35}
32
  rotation={[Math.PI / 2, 0, 0]}
@@ -43,7 +43,13 @@
43
  justifyContent="center"
44
  padding={20}
45
  >
46
- <ConnectionFlowBoxUIkit {robot} {onInputBoxClick} {onRobotBoxClick} {onOutputBoxClick} />
 
 
 
 
 
 
47
  </Container>
48
  </Root>
49
  </Billboard>
@@ -81,4 +87,4 @@
81
  </HTML>
82
  </Billboard> -->
83
  </T.Group>
84
- {/if}
 
26
  interactivity();
27
  </script>
28
 
29
+ <!-- {#if visible} -->
30
  <T.Group
31
  position.z={0.35}
32
  rotation={[Math.PI / 2, 0, 0]}
 
43
  justifyContent="center"
44
  padding={20}
45
  >
46
+ <ConnectionFlowBoxUIkit
47
+ {visible}
48
+ {robot}
49
+ {onInputBoxClick}
50
+ {onRobotBoxClick}
51
+ {onOutputBoxClick}
52
+ />
53
  </Container>
54
  </Root>
55
  </Billboard>
 
87
  </HTML>
88
  </Billboard> -->
89
  </T.Group>
90
+ <!-- {/if} -->
src/lib/components/3d/elements/video/VideoGridItem.svelte CHANGED
@@ -16,7 +16,6 @@
16
  let { video, workspaceId, onCameraMove, onInputBoxClick, onOutputBoxClick }: Props = $props();
17
 
18
  const { onPointerEnter, onPointerLeave } = useCursor();
19
- interactivity();
20
 
21
  let isToggled = $state(false);
22
  let hovering = $state(false);
 
16
  let { video, workspaceId, onCameraMove, onInputBoxClick, onOutputBoxClick }: Props = $props();
17
 
18
  const { onPointerEnter, onPointerLeave } = useCursor();
 
19
 
20
  let isToggled = $state(false);
21
  let hovering = $state(false);
src/lib/components/3d/elements/video/Videos.svelte CHANGED
@@ -7,6 +7,7 @@
7
  import type { VideoInstance } from "$lib/elements/video/VideoManager.svelte";
8
  import { generateName } from "$lib/utils/generateName";
9
  import VideoGridItem from "@/components/3d/elements/video/VideoGridItem.svelte";
 
10
 
11
  interface Props {
12
  workspaceId: string;
@@ -27,6 +28,12 @@
27
  selectedVideo = video;
28
  isOutputModalOpen = true;
29
  }
 
 
 
 
 
 
30
  </script>
31
 
32
  {#each videoManager.videos as video (video.id)}
 
7
  import type { VideoInstance } from "$lib/elements/video/VideoManager.svelte";
8
  import { generateName } from "$lib/utils/generateName";
9
  import VideoGridItem from "@/components/3d/elements/video/VideoGridItem.svelte";
10
+ import { interactivity } from "@threlte/extras";
11
 
12
  interface Props {
13
  workspaceId: string;
 
28
  selectedVideo = video;
29
  isOutputModalOpen = true;
30
  }
31
+
32
+ interactivity({
33
+ filter: (hits, state) => {
34
+ return hits.slice(0, 1);
35
+ }
36
+ });
37
  </script>
38
 
39
  {#each videoManager.videos as video (video.id)}
src/lib/components/3d/elements/video/modal/VideoInputConnectionModal.svelte CHANGED
@@ -7,6 +7,7 @@
7
  import { toast } from "svelte-sonner";
8
  import { videoManager } from "$lib/elements/video/VideoManager.svelte";
9
  import type { VideoInstance } from "$lib/elements/video/VideoManager.svelte";
 
10
 
11
  interface Props {
12
  workspaceId: string;
@@ -355,10 +356,10 @@
355
  <div class="flex items-center justify-between">
356
  <div>
357
  <Card.Title
358
- class="flex items-center gap-2 text-base text-purple-700 dark:text-purple-200"
359
  >
360
  <span class="icon-[mdi--cloud-download] size-4"></span>
361
- Remote Collaboration (Rooms)
362
  </Card.Title>
363
  <Card.Description class="text-xs text-purple-600/70 dark:text-purple-300/70">
364
  Receive video streams from remote cameras or AI systems
@@ -483,13 +484,36 @@
483
  >
484
  {room.id}
485
  </p>
486
- <div class="flex gap-3 text-xs text-slate-600 dark:text-slate-400">
487
  <span
488
  >{room.participants?.producer
489
  ? "📹 Has Output"
490
  : "📭 No Output"}</span
491
  >
492
  <span>👥 {room.participants?.consumers?.length || 0} inputs</span>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
493
  </div>
494
  </div>
495
  {#if room.participants?.producer}
@@ -529,17 +553,6 @@
529
  </Card.Content>
530
  </Card.Root>
531
 
532
- <!-- Help Information -->
533
- <Alert.Root
534
- class="border-slate-300 bg-slate-100/30 dark:border-slate-700 dark:bg-slate-800/30"
535
- >
536
- <span class="icon-[mdi--help-circle] size-4 text-slate-600 dark:text-slate-400"></span>
537
- <Alert.Title class="text-slate-700 dark:text-slate-300">Video Input Sources</Alert.Title>
538
- <Alert.Description class="text-xs text-slate-600 dark:text-slate-400">
539
- <strong>Camera:</strong> Local device camera • <strong>Remote:</strong> Video streams from
540
- rooms • Only one active at a time
541
- </Alert.Description>
542
- </Alert.Root>
543
  </div>
544
  </div>
545
  </Dialog.Content>
 
7
  import { toast } from "svelte-sonner";
8
  import { videoManager } from "$lib/elements/video/VideoManager.svelte";
9
  import type { VideoInstance } from "$lib/elements/video/VideoManager.svelte";
10
+ import { settings } from "$lib/runes/settings.svelte";
11
 
12
  interface Props {
13
  workspaceId: string;
 
356
  <div class="flex items-center justify-between">
357
  <div>
358
  <Card.Title
359
+ class="flex items-center gap-2 text-base text-purple-700 dark:text-purple-200 pb-1"
360
  >
361
  <span class="icon-[mdi--cloud-download] size-4"></span>
362
+ Remote Control
363
  </Card.Title>
364
  <Card.Description class="text-xs text-purple-600/70 dark:text-purple-300/70">
365
  Receive video streams from remote cameras or AI systems
 
484
  >
485
  {room.id}
486
  </p>
487
+ <div class="flex items-center gap-3 text-xs text-slate-600 dark:text-slate-400">
488
  <span
489
  >{room.participants?.producer
490
  ? "📹 Has Output"
491
  : "📭 No Output"}</span
492
  >
493
  <span>👥 {room.participants?.consumers?.length || 0} inputs</span>
494
+ <!-- Monitoring links -->
495
+ <div class="flex gap-1">
496
+ <a
497
+ href={`${settings.transportServerUrl.replace('/api', '')}/${workspaceId}/video/consumer?room=${room.id}`}
498
+ target="_blank"
499
+ rel="noopener noreferrer"
500
+ class="inline-flex items-center gap-1 rounded bg-blue-500/10 px-1.5 py-0.5 text-xs text-blue-600 hover:bg-blue-500/20 dark:bg-blue-400/10 dark:text-blue-400 dark:hover:bg-blue-400/20"
501
+ title="Monitor Consumer"
502
+ >
503
+ <span class="icon-[mdi--monitor-eye] size-3"></span>
504
+ Consumer
505
+ </a>
506
+ <a
507
+ href={`${settings.transportServerUrl.replace('/api', '')}/${workspaceId}/video/producer?room=${room.id}`}
508
+ target="_blank"
509
+ rel="noopener noreferrer"
510
+ class="inline-flex items-center gap-1 rounded bg-green-500/10 px-1.5 py-0.5 text-xs text-green-600 hover:bg-green-500/20 dark:bg-green-400/10 dark:text-green-400 dark:hover:bg-green-400/20"
511
+ title="Monitor Producer"
512
+ >
513
+ <span class="icon-[mdi--monitor-eye] size-3"></span>
514
+ Producer
515
+ </a>
516
+ </div>
517
  </div>
518
  </div>
519
  {#if room.participants?.producer}
 
553
  </Card.Content>
554
  </Card.Root>
555
 
 
 
 
 
 
 
 
 
 
 
 
556
  </div>
557
  </div>
558
  </Dialog.Content>
src/lib/components/3d/elements/video/modal/VideoOutputConnectionModal.svelte CHANGED
@@ -7,6 +7,7 @@
7
  import { toast } from "svelte-sonner";
8
  import { videoManager } from "$lib/elements/video/VideoManager.svelte";
9
  import type { VideoInstance } from "$lib/elements/video/VideoManager.svelte";
 
10
 
11
  interface Props {
12
  workspaceId: string;
@@ -92,7 +93,6 @@
92
  try {
93
  isConnecting = true;
94
  error = null;
95
- const roomId = customRoomId.trim() || video.id;
96
  const result = await videoManager.startVideoOutputAsProducer(workspaceId, video.id);
97
  if (result.success) {
98
  customRoomId = "";
@@ -366,10 +366,10 @@
366
  <div class="flex items-center justify-between">
367
  <div>
368
  <Card.Title
369
- class="flex items-center gap-2 text-base text-purple-700 dark:text-purple-200"
370
  >
371
  <span class="icon-[mdi--cloud-upload] size-4"></span>
372
- Remote Collaboration (Rooms)
373
  </Card.Title>
374
  <Card.Description class="text-xs text-purple-600/70 dark:text-purple-300/70">
375
  Broadcast video stream to remote systems and users
@@ -483,7 +483,7 @@
483
  : "No rooms available. Create one to get started."}
484
  </div>
485
  {:else}
486
- {#each videoManager.rooms as room}
487
  <div
488
  class="rounded border border-slate-300 bg-slate-50/50 p-2 dark:border-slate-600 dark:bg-slate-800/50"
489
  >
@@ -494,13 +494,36 @@
494
  >
495
  {room.id}
496
  </p>
497
- <div class="flex gap-3 text-xs text-slate-600 dark:text-slate-400">
498
  <span
499
  >{room.participants?.producer
500
  ? "🔴 Has Output"
501
  : "🟢 Available"}</span
502
  >
503
  <span>👥 {room.participants?.consumers?.length || 0} inputs</span>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
504
  </div>
505
  </div>
506
  {#if !room.participants?.producer}
@@ -540,17 +563,7 @@
540
  </Card.Content>
541
  </Card.Root>
542
 
543
- <!-- Help Information -->
544
- <Alert.Root
545
- class="border-slate-300 bg-slate-100/30 dark:border-slate-700 dark:bg-slate-800/30"
546
- >
547
- <span class="icon-[mdi--help-circle] size-4 text-slate-600 dark:text-slate-400"></span>
548
- <Alert.Title class="text-slate-700 dark:text-slate-300">Video Output Options</Alert.Title>
549
- <Alert.Description class="text-xs text-slate-600 dark:text-slate-400">
550
- <strong>Recording:</strong> Save locally • <strong>Remote:</strong> Broadcast to rooms •
551
- Only one active at a time
552
- </Alert.Description>
553
- </Alert.Root>
554
  </div>
555
  </div>
556
  </Dialog.Content>
 
7
  import { toast } from "svelte-sonner";
8
  import { videoManager } from "$lib/elements/video/VideoManager.svelte";
9
  import type { VideoInstance } from "$lib/elements/video/VideoManager.svelte";
10
+ import { settings } from "$lib/runes/settings.svelte";
11
 
12
  interface Props {
13
  workspaceId: string;
 
93
  try {
94
  isConnecting = true;
95
  error = null;
 
96
  const result = await videoManager.startVideoOutputAsProducer(workspaceId, video.id);
97
  if (result.success) {
98
  customRoomId = "";
 
366
  <div class="flex items-center justify-between">
367
  <div>
368
  <Card.Title
369
+ class="flex items-center gap-2 text-base text-purple-700 dark:text-purple-200 pb-1"
370
  >
371
  <span class="icon-[mdi--cloud-upload] size-4"></span>
372
+ Remote Control
373
  </Card.Title>
374
  <Card.Description class="text-xs text-purple-600/70 dark:text-purple-300/70">
375
  Broadcast video stream to remote systems and users
 
483
  : "No rooms available. Create one to get started."}
484
  </div>
485
  {:else}
486
+ {#each videoManager.rooms as room (room.id)}
487
  <div
488
  class="rounded border border-slate-300 bg-slate-50/50 p-2 dark:border-slate-600 dark:bg-slate-800/50"
489
  >
 
494
  >
495
  {room.id}
496
  </p>
497
+ <div class="flex items-center gap-3 text-xs text-slate-600 dark:text-slate-400">
498
  <span
499
  >{room.participants?.producer
500
  ? "🔴 Has Output"
501
  : "🟢 Available"}</span
502
  >
503
  <span>👥 {room.participants?.consumers?.length || 0} inputs</span>
504
+ <!-- Monitoring links -->
505
+ <div class="flex gap-1">
506
+ <a
507
+ href={`${settings.transportServerUrl.replace('/api', '')}/${workspaceId}/video/consumer?room=${room.id}`}
508
+ target="_blank"
509
+ rel="noopener noreferrer"
510
+ class="inline-flex items-center gap-1 rounded bg-blue-500/10 px-1.5 py-0.5 text-xs text-blue-600 hover:bg-blue-500/20 dark:bg-blue-400/10 dark:text-blue-400 dark:hover:bg-blue-400/20"
511
+ title="Monitor Consumer"
512
+ >
513
+ <span class="icon-[mdi--monitor-eye] size-3"></span>
514
+ Consumer
515
+ </a>
516
+ <a
517
+ href={`${settings.transportServerUrl.replace('/api', '')}/${workspaceId}/video/producer?room=${room.id}`}
518
+ target="_blank"
519
+ rel="noopener noreferrer"
520
+ class="inline-flex items-center gap-1 rounded bg-green-500/10 px-1.5 py-0.5 text-xs text-green-600 hover:bg-green-500/20 dark:bg-green-400/10 dark:text-green-400 dark:hover:bg-green-400/20"
521
+ title="Monitor Producer"
522
+ >
523
+ <span class="icon-[mdi--monitor-eye] size-3"></span>
524
+ Producer
525
+ </a>
526
+ </div>
527
  </div>
528
  </div>
529
  {#if !room.participants?.producer}
 
563
  </Card.Content>
564
  </Card.Root>
565
 
566
+
 
 
 
 
 
 
 
 
 
 
567
  </div>
568
  </div>
569
  </Dialog.Content>
src/lib/components/3d/elements/video/status/VideoConnectionFlowBoxUIKit.svelte CHANGED
@@ -5,38 +5,55 @@
5
  import type { VideoInstance } from "$lib/elements/video/VideoManager.svelte";
6
  import { Container } from "threlte-uikit";
7
  import { StatusArrow } from "$lib/components/3d/ui";
 
 
8
 
9
  interface Props {
 
10
  video: VideoInstance;
11
  onInputBoxClick: (video: VideoInstance) => void;
12
  onOutputBoxClick: (video: VideoInstance) => void;
 
 
13
  }
14
 
15
- let { video, onInputBoxClick, onOutputBoxClick }: Props = $props();
16
 
17
  const inputColor = "rgb(34, 197, 94)";
18
  const outputColor = "rgb(59, 130, 246)";
 
 
 
 
 
 
 
19
  </script>
20
 
21
- <Container flexDirection="row" alignItems="center" gap={12}>
 
 
 
 
 
 
 
 
22
  <!-- Input Video Box -->
23
  <InputVideoBoxUIKit {video} handleClick={() => onInputBoxClick(video)} />
24
 
25
  <!-- Arrow 1: Input to Video -->
26
- <StatusArrow
27
- color={inputColor}
28
- opacity={video.hasInput ? 1 : 0.5}
29
- />
30
 
31
  <!-- Video Box -->
32
  <VideoBoxUIKit {video} />
33
 
34
  <!-- Arrow 2: Video to Output -->
35
- <StatusArrow
36
  color={outputColor}
37
  opacity={video.hasInput && video.hasOutput ? 1 : video.hasInput && video.canOutput ? 0.7 : 0.5}
38
  />
39
 
40
  <!-- Output Box -->
41
  <OutputVideoBoxUIKit {video} handleClick={() => onOutputBoxClick(video)} />
42
- </Container>
 
5
  import type { VideoInstance } from "$lib/elements/video/VideoManager.svelte";
6
  import { Container } from "threlte-uikit";
7
  import { StatusArrow } from "$lib/components/3d/ui";
8
+ import { Tween } from "svelte/motion";
9
+ import { cubicOut } from "svelte/easing";
10
 
11
  interface Props {
12
+ visible: boolean;
13
  video: VideoInstance;
14
  onInputBoxClick: (video: VideoInstance) => void;
15
  onOutputBoxClick: (video: VideoInstance) => void;
16
+ duration?: number;
17
+ delay?: number;
18
  }
19
 
20
+ let { visible, video, onInputBoxClick, onOutputBoxClick, duration = 100, delay = 0 }: Props = $props();
21
 
22
  const inputColor = "rgb(34, 197, 94)";
23
  const outputColor = "rgb(59, 130, 246)";
24
+
25
+ const tweenedScale = Tween.of(() => {
26
+ return visible ? 1 : 0;
27
+ }, { duration: duration, easing: cubicOut, delay: delay });
28
+ const tweenedOpacity = Tween.of(() => {
29
+ return visible ? 1 : 0;
30
+ }, { duration: duration, easing: cubicOut, delay: delay });
31
  </script>
32
 
33
+ <Container
34
+ flexDirection="row"
35
+ alignItems="center"
36
+ gap={12}
37
+ transformScaleX={tweenedScale.current}
38
+ transformScaleY={tweenedScale.current}
39
+ transformScaleZ={tweenedScale.current}
40
+ opacity={tweenedOpacity.current}
41
+ >
42
  <!-- Input Video Box -->
43
  <InputVideoBoxUIKit {video} handleClick={() => onInputBoxClick(video)} />
44
 
45
  <!-- Arrow 1: Input to Video -->
46
+ <StatusArrow color={inputColor} opacity={video.hasInput ? 1 : 0.5} />
 
 
 
47
 
48
  <!-- Video Box -->
49
  <VideoBoxUIKit {video} />
50
 
51
  <!-- Arrow 2: Video to Output -->
52
+ <StatusArrow
53
  color={outputColor}
54
  opacity={video.hasInput && video.hasOutput ? 1 : video.hasInput && video.canOutput ? 0.7 : 0.5}
55
  />
56
 
57
  <!-- Output Box -->
58
  <OutputVideoBoxUIKit {video} handleClick={() => onOutputBoxClick(video)} />
59
+ </Container>
src/lib/components/3d/elements/video/status/VideoStatusBillboard.svelte CHANGED
@@ -18,7 +18,7 @@
18
  interactivity();
19
  </script>
20
 
21
- {#if visible}
22
  <T.Group
23
  onpointerdown={(e) => e.stopPropagation()}
24
  onpointerup={(e) => e.stopPropagation()}
@@ -39,7 +39,7 @@
39
  justifyContent="center"
40
  padding={20}
41
  >
42
- <VideoConnectionFlowBoxUIKit {video} {onInputBoxClick} {onOutputBoxClick} />
43
  </Container>
44
  </Root>
45
  </Billboard>
@@ -73,4 +73,4 @@
73
  </HTML>
74
  </Billboard> -->
75
  </T.Group>
76
- {/if}
 
18
  interactivity();
19
  </script>
20
 
21
+ <!-- {#if visible} -->
22
  <T.Group
23
  onpointerdown={(e) => e.stopPropagation()}
24
  onpointerup={(e) => e.stopPropagation()}
 
39
  justifyContent="center"
40
  padding={20}
41
  >
42
+ <VideoConnectionFlowBoxUIKit {visible} {video} {onInputBoxClick} {onOutputBoxClick} />
43
  </Container>
44
  </Root>
45
  </Billboard>
 
73
  </HTML>
74
  </Billboard> -->
75
  </T.Group>
76
+ <!-- {/if} -->
src/lib/components/interface/overlay/AddAIButton.svelte CHANGED
@@ -1,81 +1,55 @@
1
  <script lang="ts">
2
  import { Button } from "@/components/ui/button";
3
  import * as DropdownMenu from "@/components/ui/dropdown-menu";
4
- import { toast } from "svelte-sonner";
5
  import { cn } from "$lib/utils";
6
- import { generateName } from "@/utils/generateName";
7
- import { remoteComputeManager } from "$lib/elements/compute//RemoteComputeManager.svelte";
8
 
9
  interface Props {
 
10
  open?: boolean;
11
  }
12
 
13
- let { open = $bindable() }: Props = $props();
14
 
15
- const aiOptions = [
16
- { id: "act", label: "ACT Model", icon: "icon-[mdi--brain]", enabled: true },
17
- { id: "pi0", label: "Pi0", icon: "icon-[mdi--brain]", enabled: false },
18
- { id: "nano-vla", label: "Nano VLA", icon: "icon-[mdi--brain]", enabled: false },
19
- { id: "nvidia-groot", label: "Nvidia Groot", icon: "icon-[mdi--robot-outline]", enabled: false }
20
- ];
21
 
22
- async function addAI(aiType: string) {
23
- try {
24
- // Basic validation
25
- if (!aiType) return;
26
 
27
- const computeId = generateName();
28
- const computeName = `${formatAIType(aiType)} ${computeId}`;
29
-
30
- // Create a new compute instance
31
- const compute = remoteComputeManager.createCompute(computeId, computeName);
32
-
33
- toast.success(`Created ${formatAIType(aiType)} compute: ${computeName}`);
34
 
35
- // Close the dropdown
36
- open = false;
37
- } catch (error) {
38
- console.error("AI creation failed:", error);
39
- toast.error("Failed to create AI compute");
40
- }
41
  }
42
 
43
- async function quickAddAI() {
44
- await addAI("act");
 
45
  }
46
 
47
- function formatAIType(aiType: string): string {
48
- switch (aiType) {
49
- case "pi0":
50
- return "Pi0";
51
- case "nano-vla":
52
- return "Nano VLA";
53
- case "nvidia-groot":
54
- return "Nvidia Groot";
55
- default:
56
- return aiType;
57
- }
58
  }
59
 
60
- function getAIDescription(aiType: string): string {
61
- switch (aiType) {
62
- case "pi0":
63
- return "Lightweight AI model";
64
- case "nano-vla":
65
- return "Vision-language-action model";
66
- case "nvidia-groot":
67
- return "Humanoid robotics model";
68
- default:
69
- return "AI model";
70
- }
71
  }
72
  </script>
73
 
74
- <!-- Main Add Button (Neural Network) -->
75
  <Button
76
  variant="default"
77
  size="sm"
78
- onclick={quickAddAI}
 
79
  class="group rounded-r-none border-0 bg-purple-500 text-white transition-all duration-200 hover:bg-purple-400 dark:bg-purple-600 dark:hover:bg-purple-500"
80
  >
81
  <span
@@ -93,6 +67,7 @@
93
  {#snippet child({ props })}
94
  <Button
95
  {...props}
 
96
  variant="default"
97
  size="sm"
98
  class={cn(
@@ -112,44 +87,29 @@
112
  {/snippet}
113
  </DropdownMenu.Trigger>
114
 
115
- <DropdownMenu.Content
116
- class="w-56 border-purple-400/30 bg-purple-500 backdrop-blur-sm dark:border-purple-500/30 dark:bg-purple-600"
117
- align="center"
118
- >
119
- <DropdownMenu.Group>
120
- <DropdownMenu.GroupHeading
121
- class="text-xs font-semibold tracking-wider text-purple-100 uppercase dark:text-purple-200"
122
  >
123
- AI Types
124
- </DropdownMenu.GroupHeading>
125
-
126
- {#each aiOptions as ai}
127
- <DropdownMenu.Item
128
- class={[
129
- "group group cursor-pointer bg-purple-500 text-white transition-all duration-200",
130
- "data-highlighted:bg-purple-600 dark:bg-purple-600 dark:data-highlighted:bg-purple-700"
131
- ]}
132
- onclick={async () => await addAI(ai.id)}
133
- disabled={!ai.enabled}
134
- >
135
- <span
136
- class={[
137
- ai.icon,
138
- "mr-3 size-4 text-purple-100 transition-colors duration-200 dark:text-purple-200"
139
- ]}
140
- ></span>
141
- <div class="flex flex-1 flex-col">
142
- <span class="font-medium text-white transition-colors duration-200"
143
- >{formatAIType(ai.id)}</span
144
- >
145
- <span
146
- class="text-xs text-purple-100 transition-colors duration-200 dark:text-purple-200"
147
- >
148
- {getAIDescription(ai.id)}
149
- </span>
150
  </div>
151
- </DropdownMenu.Item>
152
- {/each}
153
- </DropdownMenu.Group>
154
  </DropdownMenu.Content>
155
  </DropdownMenu.Root>
 
 
 
 
 
 
 
 
1
  <script lang="ts">
2
  import { Button } from "@/components/ui/button";
3
  import * as DropdownMenu from "@/components/ui/dropdown-menu";
 
4
  import { cn } from "$lib/utils";
5
+ import { MODEL_TYPES, type ModelType } from "$lib/elements/compute";
6
+ import AIModelConfigurationModal from "@/components/3d/elements/compute/modal/AIModelConfigurationModal.svelte";
7
 
8
  interface Props {
9
+ workspaceId: string;
10
  open?: boolean;
11
  }
12
 
13
+ let { open = $bindable(), workspaceId }: Props = $props();
14
 
15
+ let isConfigModalOpen = $state(false);
16
+ let selectedModelType = $state<ModelType>('act');
 
 
 
 
17
 
18
+ // Get available model types
19
+ const availableModels = Object.values(MODEL_TYPES).filter(model => model.enabled);
 
 
20
 
21
+ function openConfigModal(modelType: ModelType) {
22
+ selectedModelType = modelType;
23
+ isConfigModalOpen = true;
24
+ open = false; // Close the dropdown
25
+ }
 
 
26
 
27
+ function quickAddACT() {
28
+ openConfigModal('act');
 
 
 
 
29
  }
30
 
31
+ function formatModelType(modelType: string): string {
32
+ const config = MODEL_TYPES[modelType as ModelType];
33
+ return config ? config.label : modelType;
34
  }
35
 
36
+ function getModelDescription(modelType: string): string {
37
+ const config = MODEL_TYPES[modelType as ModelType];
38
+ return config ? config.description : "AI model";
 
 
 
 
 
 
 
 
39
  }
40
 
41
+ function getModelIcon(modelType: string): string {
42
+ const config = MODEL_TYPES[modelType as ModelType];
43
+ return config ? config.icon : "icon-[mdi--brain]";
 
 
 
 
 
 
 
 
44
  }
45
  </script>
46
 
47
+ <!-- Main Add Button (ACT Model - Quick Add) -->
48
  <Button
49
  variant="default"
50
  size="sm"
51
+ onclick={quickAddACT}
52
+ disabled={true}
53
  class="group rounded-r-none border-0 bg-purple-500 text-white transition-all duration-200 hover:bg-purple-400 dark:bg-purple-600 dark:hover:bg-purple-500"
54
  >
55
  <span
 
67
  {#snippet child({ props })}
68
  <Button
69
  {...props}
70
+ disabled={true}
71
  variant="default"
72
  size="sm"
73
  class={cn(
 
87
  {/snippet}
88
  </DropdownMenu.Trigger>
89
 
90
+ <DropdownMenu.Content class="w-64 bg-slate-100 border-slate-300 dark:bg-slate-900 dark:border-slate-600">
91
+ {#each availableModels as model}
92
+ <DropdownMenu.Item
93
+ class="flex items-center gap-3 p-3 cursor-pointer hover:bg-purple-100 dark:hover:bg-purple-900/30"
94
+ onclick={() => openConfigModal(model.id)}
 
 
95
  >
96
+ <span class="{model.icon} size-5 text-purple-500 dark:text-purple-400"></span>
97
+ <div class="flex-1">
98
+ <div class="font-medium text-slate-900 dark:text-slate-100">
99
+ {model.label}
100
+ </div>
101
+ <div class="text-xs text-slate-600 dark:text-slate-400">
102
+ {model.description}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
  </div>
104
+ </div>
105
+ </DropdownMenu.Item>
106
+ {/each}
107
  </DropdownMenu.Content>
108
  </DropdownMenu.Root>
109
+
110
+ <!-- Configuration Modal -->
111
+ <AIModelConfigurationModal
112
+ bind:open={isConfigModalOpen}
113
+ {workspaceId}
114
+ initialModelType={selectedModelType}
115
+ />
src/lib/components/interface/overlay/AddRobotButton.svelte CHANGED
@@ -44,17 +44,23 @@
44
  }
45
  }
46
 
47
- async function quickAddSO100() {
48
- await addRobot("so-arm100");
 
 
 
 
 
 
49
  }
50
 
51
  </script>
52
 
53
- <!-- Main Add Button (SO-100) -->
54
  <Button
55
  variant="default"
56
  size="sm"
57
- onclick={quickAddSO100}
58
  class="group rounded-r-none border-0 bg-emerald-500 text-white transition-all duration-200 hover:bg-emerald-400 dark:bg-emerald-600 dark:hover:bg-emerald-500"
59
  >
60
  <span
@@ -103,6 +109,7 @@
103
  </DropdownMenu.GroupHeading>
104
 
105
  {#each robotTypes as robotType}
 
106
  <DropdownMenu.Item
107
  class={[
108
  "group group cursor-pointer bg-emerald-500 text-white transition-all duration-200",
@@ -112,19 +119,19 @@
112
  >
113
  <span
114
  class={[
115
- robotType === "so-arm100" ? "icon-[ix--robotic-arm]" : "icon-[mdi--robot-industrial]",
116
  "mr-3 size-4 text-emerald-100 transition-colors duration-200 dark:text-emerald-200"
117
  ]}
118
  ></span>
119
  <div class="flex flex-1 flex-col">
120
  <span class="font-medium text-white transition-colors duration-200"
121
- >{robotType.replace(/-/g, " ").toUpperCase()}</span
122
  >
123
  <span class="text-xs text-emerald-100 transition-colors duration-200 dark:text-emerald-200">
124
- {robotType === "so-arm100" ? "6-DOF Robotic Arm" : "Industrial Robot"}
125
  </span>
126
  </div>
127
- {#if robotType === "so-arm100"}
128
  <Badge
129
  variant="secondary"
130
  class="ml-2 bg-emerald-600 text-xs text-emerald-100 group-data-highlighted:bg-emerald-300 group-data-highlighted:text-emerald-900 dark:bg-emerald-700 dark:text-emerald-100 dark:group-data-highlighted:bg-emerald-400 dark:group-data-highlighted:text-emerald-900"
 
44
  }
45
  }
46
 
47
+ async function quickAddDefault() {
48
+ const defaultRobotType = robotTypes.find(type => robotUrdfConfigMap[type].isDefault);
49
+ if (defaultRobotType) {
50
+ await addRobot(defaultRobotType);
51
+ } else {
52
+ // Fallback to first robot type if no default is set
53
+ await addRobot(robotTypes[0]);
54
+ }
55
  }
56
 
57
  </script>
58
 
59
+ <!-- Main Add Button (Default Robot) -->
60
  <Button
61
  variant="default"
62
  size="sm"
63
+ onclick={quickAddDefault}
64
  class="group rounded-r-none border-0 bg-emerald-500 text-white transition-all duration-200 hover:bg-emerald-400 dark:bg-emerald-600 dark:hover:bg-emerald-500"
65
  >
66
  <span
 
109
  </DropdownMenu.GroupHeading>
110
 
111
  {#each robotTypes as robotType}
112
+ {@const urdfConfig = robotUrdfConfigMap[robotType]}
113
  <DropdownMenu.Item
114
  class={[
115
  "group group cursor-pointer bg-emerald-500 text-white transition-all duration-200",
 
119
  >
120
  <span
121
  class={[
122
+ urdfConfig.icon || "icon-[mdi--robot-industrial]",
123
  "mr-3 size-4 text-emerald-100 transition-colors duration-200 dark:text-emerald-200"
124
  ]}
125
  ></span>
126
  <div class="flex flex-1 flex-col">
127
  <span class="font-medium text-white transition-colors duration-200"
128
+ >{urdfConfig.displayName || robotType.replace(/-/g, " ").toUpperCase()}</span
129
  >
130
  <span class="text-xs text-emerald-100 transition-colors duration-200 dark:text-emerald-200">
131
+ {urdfConfig.description || "Robot"}
132
  </span>
133
  </div>
134
+ {#if urdfConfig.isDefault}
135
  <Badge
136
  variant="secondary"
137
  class="ml-2 bg-emerald-600 text-xs text-emerald-100 group-data-highlighted:bg-emerald-300 group-data-highlighted:text-emerald-900 dark:bg-emerald-700 dark:text-emerald-100 dark:group-data-highlighted:bg-emerald-400 dark:group-data-highlighted:text-emerald-900"
src/lib/components/interface/overlay/AddSensorButton.svelte CHANGED
@@ -29,6 +29,13 @@
29
  enabled: true,
30
  isDefault: true
31
  },
 
 
 
 
 
 
 
32
  {
33
  id: 'lidar',
34
  label: 'Lidar',
 
29
  enabled: true,
30
  isDefault: true
31
  },
32
+ {
33
+ id: 'depth-camera',
34
+ label: 'Depth Camera',
35
+ description: 'Depth Camera Sensor',
36
+ icon: 'icon-[mdi--camera]',
37
+ enabled: false
38
+ },
39
  {
40
  id: 'lidar',
41
  label: 'Lidar',
src/lib/components/interface/overlay/Overlay.svelte CHANGED
@@ -26,49 +26,44 @@
26
  </script>
27
 
28
  <div class="select-none">
29
- <!-- Button Bar Container -->
30
- <div class="fixed top-4 left-4 z-50 flex gap-2 select-none">
31
- <!-- Add Robot Button Group -->
32
- <div class="flex items-center justify-center gap-2 overflow-hidden rounded-lg">
33
  <!-- Logo/Favicon -->
34
  <div class="flex items-center justify-center">
35
- <!-- From /favicon_1024.png -->
36
  <img
37
  src="/favicon_1024.png"
38
  alt="Logo"
39
  draggable="false"
40
- class="h-10 w-10 invert-0 filter dark:invert"
41
  />
42
  </div>
43
- <!-- Add robot button and dropdown menu (Top Left) -->
44
- <div class="flex items-center justify-center">
 
45
  <AddRobotButton bind:open={addRobotDropdownMenuOpen} />
46
  </div>
47
- </div>
48
 
49
- <!-- Add Sensor Button Group -->
50
- <div class="flex items-center justify-center overflow-hidden rounded-lg">
51
- <!-- Add sensor button and dropdown menu -->
52
- <div class="flex items-center justify-center">
53
  <AddSensorButton bind:open={addSensorDropdownMenuOpen} />
54
  </div>
55
- </div>
56
 
57
- <!-- Add AI Button Group -->
58
- <div class="flex items-center justify-center overflow-hidden rounded-lg">
59
- <!-- Add AI button and dropdown menu -->
60
- <div class="flex items-center justify-center">
61
- <AddAIButton bind:open={addAIDropdownMenuOpen} />
62
  </div>
63
  </div>
64
- </div>
65
 
66
- <div class="fixed top-4 right-4 z-50 flex gap-2">
67
- <!-- Workspace ID Button -->
68
- <WorkspaceIdButton {workspaceId} bind:open={workspaceIdMenuOpen} />
 
69
 
70
- <!-- Settings Button and Sheet (Top Right, Left Side) -->
71
- <SettingsButton bind:open={settingsOpen} />
 
72
  </div>
73
  </div>
74
  <SettingsSheet bind:open={settingsOpen} />
 
26
  </script>
27
 
28
  <div class="select-none">
29
+ <!-- Responsive Button Bar Container -->
30
+ <div class="fixed top-2 left-2 right-2 z-50 flex flex-wrap items-center justify-between gap-1 select-none md:top-4 md:left-4 md:right-4 md:gap-2">
31
+ <!-- Left Group: Logo + Add Buttons -->
32
+ <div class="flex items-center gap-1 flex-wrap md:gap-2">
33
  <!-- Logo/Favicon -->
34
  <div class="flex items-center justify-center">
 
35
  <img
36
  src="/favicon_1024.png"
37
  alt="Logo"
38
  draggable="false"
39
+ class="h-8 w-8 invert-0 filter dark:invert md:h-10 md:w-10"
40
  />
41
  </div>
42
+
43
+ <!-- Add Robot Button Group -->
44
+ <div class="flex items-center justify-center overflow-hidden rounded-lg">
45
  <AddRobotButton bind:open={addRobotDropdownMenuOpen} />
46
  </div>
 
47
 
48
+ <!-- Add Sensor Button Group - Hidden on very small screens -->
49
+ <div class="hidden min-[480px]:flex items-center justify-center overflow-hidden rounded-lg">
 
 
50
  <AddSensorButton bind:open={addSensorDropdownMenuOpen} />
51
  </div>
 
52
 
53
+ <!-- Add AI Button Group - Hidden on small screens -->
54
+ <div class="hidden min-[560px]:flex items-center justify-center overflow-hidden rounded-lg">
55
+ <AddAIButton bind:open={addAIDropdownMenuOpen} workspaceId={workspaceId} />
 
 
56
  </div>
57
  </div>
 
58
 
59
+ <!-- Right Group: Workspace ID + Settings -->
60
+ <div class="flex items-center gap-1 md:gap-2">
61
+ <!-- Workspace ID Button -->
62
+ <WorkspaceIdButton {workspaceId} bind:open={workspaceIdMenuOpen} />
63
 
64
+ <!-- Settings Button -->
65
+ <SettingsButton bind:open={settingsOpen} />
66
+ </div>
67
  </div>
68
  </div>
69
  <SettingsSheet bind:open={settingsOpen} />
src/lib/components/interface/overlay/SettingsSheet.svelte CHANGED
@@ -130,8 +130,7 @@
130
  >Inference Server URL</Label
131
  >
132
  <p class="text-xs text-slate-600 dark:text-slate-400">
133
- URL for the remote AI inference server that runs ACT models and manages robot
134
- sessions
135
  </p>
136
  </div>
137
  <div class="flex gap-2">
 
130
  >Inference Server URL</Label
131
  >
132
  <p class="text-xs text-slate-600 dark:text-slate-400">
133
+ URL for the remote AI inference server to run policies using remote compute resources
 
134
  </p>
135
  </div>
136
  <div class="flex gap-2">
src/lib/components/interface/overlay/WorkspaceIdButton.svelte CHANGED
@@ -128,7 +128,7 @@
128
  class="rounded-lg border border-slate-300 bg-slate-50 p-2 dark:border-slate-600 dark:bg-slate-800"
129
  >
130
  <div class="font-mono text-sm break-all text-slate-800 dark:text-slate-200">
131
- https://blanchon-robothub-frontend.hf.space<span
132
  class="rounded bg-blue-100 px-1 text-blue-800 dark:bg-blue-900 dark:text-blue-200"
133
  >#{workspaceId}</span
134
  >
 
128
  class="rounded-lg border border-slate-300 bg-slate-50 p-2 dark:border-slate-600 dark:bg-slate-800"
129
  >
130
  <div class="font-mono text-sm break-all text-slate-800 dark:text-slate-200">
131
+ https://blanchon-robothub-frontend.hf.space/<span
132
  class="rounded bg-blue-100 px-1 text-blue-800 dark:bg-blue-900 dark:text-blue-200"
133
  >#{workspaceId}</span
134
  >
src/lib/configs/robotUrdfConfig.ts CHANGED
@@ -3,6 +3,10 @@ import type { RobotUrdfConfig } from "$lib/types/urdf";
3
  export const robotUrdfConfigMap: { [key: string]: RobotUrdfConfig } = {
4
  "so-arm100": {
5
  urdfUrl: "/robots/so-100/so_arm100.urdf",
 
 
 
 
6
  jointNameIdMap: {
7
  Rotation: 1,
8
  Pitch: 2,
 
3
  export const robotUrdfConfigMap: { [key: string]: RobotUrdfConfig } = {
4
  "so-arm100": {
5
  urdfUrl: "/robots/so-100/so_arm100.urdf",
6
+ displayName: "SO ARM 100",
7
+ description: "6-DOF Robotic Arm",
8
+ icon: "icon-[ix--robotic-arm]",
9
+ isDefault: true,
10
  jointNameIdMap: {
11
  Rotation: 1,
12
  Pitch: 2,
src/lib/elements/compute/RemoteCompute.svelte.ts CHANGED
@@ -1,5 +1,5 @@
1
  import type { Positionable, Position3D } from '$lib/types/positionable.js';
2
- import type { AISessionConfig, AISessionResponse } from './RemoteComputeManager.svelte';
3
 
4
  export type ComputeStatus = 'disconnected' | 'ready' | 'running' | 'stopped' | 'initializing';
5
 
@@ -10,6 +10,7 @@ export class RemoteCompute implements Positionable {
10
  position = $state<Position3D>({ x: 0, y: 0, z: 0 });
11
  name = $state<string>('');
12
  status = $state<ComputeStatus>('disconnected');
 
13
 
14
  // Session data
15
  sessionId = $state<string | null>(null);
 
1
  import type { Positionable, Position3D } from '$lib/types/positionable.js';
2
+ import type { AISessionConfig, AISessionResponse, ModelType } from './RemoteComputeManager.svelte';
3
 
4
  export type ComputeStatus = 'disconnected' | 'ready' | 'running' | 'stopped' | 'initializing';
5
 
 
10
  position = $state<Position3D>({ x: 0, y: 0, z: 0 });
11
  name = $state<string>('');
12
  status = $state<ComputeStatus>('disconnected');
13
+ modelType = $state<ModelType>('act'); // Default to ACT model
14
 
15
  // Session data
16
  sessionId = $state<string | null>(null);
src/lib/elements/compute/RemoteComputeManager.svelte.ts CHANGED
@@ -17,12 +17,86 @@ import type {
17
  CreateSessionResponse
18
  } from '@robothub/inference-server-client';
19
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
  export interface AISessionConfig {
21
  sessionId: string;
 
22
  policyPath: string;
23
  cameraNames: string[];
24
  transportServerUrl: string;
25
  workspaceId?: string;
 
26
  }
27
 
28
  export interface AISessionResponse {
@@ -84,7 +158,68 @@ export class RemoteComputeManager {
84
  }
85
 
86
  /**
87
- * Create a new AI compute instance
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
  */
89
  createCompute(id?: string, name?: string, position?: Position3D): RemoteCompute {
90
  const computeId = id || generateName();
@@ -150,6 +285,8 @@ export class RemoteComputeManager {
150
  camera_names: config.cameraNames,
151
  transport_server_url: config.transportServerUrl,
152
  workspace_id: config.workspaceId || undefined,
 
 
153
  };
154
 
155
  const response = await createSessionSessionsPost({
 
17
  CreateSessionResponse
18
  } from '@robothub/inference-server-client';
19
 
20
+ export type ModelType = 'act' | 'diffusion' | 'smolvla' | 'pi0' | 'groot' | 'custom';
21
+
22
+ export interface ModelTypeConfig {
23
+ id: ModelType;
24
+ label: string;
25
+ icon: string;
26
+ description: string;
27
+ defaultPolicyPath: string;
28
+ defaultCameraNames: string[];
29
+ requiresLanguageInstruction?: boolean;
30
+ enabled: boolean;
31
+ }
32
+
33
+ export const MODEL_TYPES: Record<ModelType, ModelTypeConfig> = {
34
+ act: {
35
+ id: 'act',
36
+ label: 'ACT Model',
37
+ icon: 'icon-[mdi--brain]',
38
+ description: 'Action Chunking with Transformers',
39
+ defaultPolicyPath: 'LaetusH/act_so101_beyond',
40
+ defaultCameraNames: ['front'],
41
+ enabled: true
42
+ },
43
+ diffusion: {
44
+ id: 'diffusion',
45
+ label: 'Diffusion Policy',
46
+ icon: 'icon-[mdi--creation]',
47
+ description: 'Diffusion-based robot control',
48
+ defaultPolicyPath: 'diffusion_policy/default',
49
+ defaultCameraNames: ['front', 'wrist'],
50
+ enabled: true
51
+ },
52
+ smolvla: {
53
+ id: 'smolvla',
54
+ label: 'SmolVLA',
55
+ icon: 'icon-[mdi--eye-outline]',
56
+ description: 'Small Vision-Language-Action model',
57
+ defaultPolicyPath: 'smolvla/latest',
58
+ defaultCameraNames: ['front'],
59
+ requiresLanguageInstruction: true,
60
+ enabled: true
61
+ },
62
+ pi0: {
63
+ id: 'pi0',
64
+ label: 'Pi0',
65
+ icon: 'icon-[mdi--pi]',
66
+ description: 'Lightweight robotics model',
67
+ defaultPolicyPath: 'pi0/base',
68
+ defaultCameraNames: ['front'],
69
+ enabled: true
70
+ },
71
+ groot: {
72
+ id: 'groot',
73
+ label: 'NVIDIA Groot',
74
+ icon: 'icon-[mdi--robot-outline]',
75
+ description: 'Humanoid robotics foundation model',
76
+ defaultPolicyPath: 'nvidia/groot',
77
+ defaultCameraNames: ['front', 'left', 'right'],
78
+ requiresLanguageInstruction: true,
79
+ enabled: false // Not yet implemented
80
+ },
81
+ custom: {
82
+ id: 'custom',
83
+ label: 'Custom Model',
84
+ icon: 'icon-[mdi--cog]',
85
+ description: 'Custom model configuration',
86
+ defaultPolicyPath: '',
87
+ defaultCameraNames: ['front'],
88
+ enabled: true
89
+ }
90
+ };
91
+
92
  export interface AISessionConfig {
93
  sessionId: string;
94
+ modelType: ModelType;
95
  policyPath: string;
96
  cameraNames: string[];
97
  transportServerUrl: string;
98
  workspaceId?: string;
99
+ languageInstruction?: string;
100
  }
101
 
102
  export interface AISessionResponse {
 
158
  }
159
 
160
  /**
161
+ * Get available model types
162
+ */
163
+ get availableModelTypes(): ModelTypeConfig[] {
164
+ return Object.values(MODEL_TYPES).filter(model => model.enabled);
165
+ }
166
+
167
+ /**
168
+ * Get model type configuration
169
+ */
170
+ getModelTypeConfig(modelType: ModelType): ModelTypeConfig | undefined {
171
+ return MODEL_TYPES[modelType];
172
+ }
173
+
174
+ /**
175
+ * Create a new AI compute instance with full configuration
176
+ */
177
+ async createComputeWithSession(
178
+ config: AISessionConfig,
179
+ computeId?: string,
180
+ computeName?: string,
181
+ position?: Position3D
182
+ ): Promise<{ success: boolean; error?: string; compute?: RemoteCompute }> {
183
+ const finalComputeId = computeId || generateName();
184
+
185
+ // Check if compute already exists
186
+ if (this._computes.find(c => c.id === finalComputeId)) {
187
+ return { success: false, error: `Compute with ID ${finalComputeId} already exists` };
188
+ }
189
+
190
+ try {
191
+ // Create compute instance
192
+ const compute = new RemoteCompute(finalComputeId, computeName);
193
+ compute.modelType = config.modelType;
194
+
195
+ // Set position (from position manager if not provided)
196
+ compute.position = position || positionManager.getNextPosition();
197
+
198
+ // Add to reactive array
199
+ this._computes.push(compute);
200
+
201
+ // Create the session immediately
202
+ const sessionResult = await this.createSession(compute.id, config);
203
+ if (!sessionResult.success) {
204
+ // Remove compute if session creation failed
205
+ await this.removeCompute(compute.id);
206
+ return { success: false, error: sessionResult.error };
207
+ }
208
+
209
+ console.log(`Created compute ${finalComputeId} with ${config.modelType} model at position (${compute.position.x.toFixed(1)}, ${compute.position.y.toFixed(1)}, ${compute.position.z.toFixed(1)}). Total computes: ${this._computes.length}`);
210
+
211
+ return { success: true, compute };
212
+ } catch (error) {
213
+ console.error('Failed to create compute with session:', error);
214
+ return {
215
+ success: false,
216
+ error: error instanceof Error ? error.message : String(error)
217
+ };
218
+ }
219
+ }
220
+
221
+ /**
222
+ * Create a new AI compute instance (legacy method)
223
  */
224
  createCompute(id?: string, name?: string, position?: Position3D): RemoteCompute {
225
  const computeId = id || generateName();
 
285
  camera_names: config.cameraNames,
286
  transport_server_url: config.transportServerUrl,
287
  workspace_id: config.workspaceId || undefined,
288
+ policy_type: config.modelType, // Use model type as policy type
289
+ language_instruction: config.languageInstruction || undefined,
290
  };
291
 
292
  const response = await createSessionSessionsPost({
src/lib/elements/compute/index.ts CHANGED
@@ -1,4 +1,5 @@
1
  export { RemoteComputeManager, remoteComputeManager } from './RemoteComputeManager.svelte.js';
2
  export { RemoteCompute } from './RemoteCompute.svelte.js';
3
- export type { AISessionConfig, AISessionResponse, AISessionStatus } from './RemoteComputeManager.svelte.js';
 
4
  export type { ComputeStatus } from './RemoteCompute.svelte.js';
 
1
  export { RemoteComputeManager, remoteComputeManager } from './RemoteComputeManager.svelte.js';
2
  export { RemoteCompute } from './RemoteCompute.svelte.js';
3
+ export type { AISessionConfig, AISessionResponse, AISessionStatus, ModelType, ModelTypeConfig } from './RemoteComputeManager.svelte.js';
4
+ export { MODEL_TYPES } from './RemoteComputeManager.svelte.js';
5
  export type { ComputeStatus } from './RemoteCompute.svelte.js';
src/lib/elements/robot/RobotManager.svelte.ts CHANGED
@@ -8,6 +8,7 @@ import { positionManager } from '$lib/utils/positionManager.js';
8
  import { settings } from '$lib/runes/settings.svelte';
9
  import { robotics } from '@robothub/transport-server-client';
10
  import type { robotics as roboticsTypes } from '@robothub/transport-server-client';
 
11
 
12
  export class RobotManager {
13
  private _robots = $state<Robot[]>([]);
@@ -135,9 +136,7 @@ export class RobotManager {
135
  */
136
  async createSO100Robot(id?: string, position?: Position3D): Promise<Robot> {
137
  const robotId = id || `so100-${Date.now()}`;
138
- const urdfConfig: RobotUrdfConfig = {
139
- urdfUrl: "/robots/so-100/so_arm100.urdf"
140
- };
141
 
142
  return this.createRobotFromUrdf(robotId, urdfConfig, position);
143
  }
 
8
  import { settings } from '$lib/runes/settings.svelte';
9
  import { robotics } from '@robothub/transport-server-client';
10
  import type { robotics as roboticsTypes } from '@robothub/transport-server-client';
11
+ import { robotUrdfConfigMap } from "$lib/configs/robotUrdfConfig";
12
 
13
  export class RobotManager {
14
  private _robots = $state<Robot[]>([]);
 
136
  */
137
  async createSO100Robot(id?: string, position?: Position3D): Promise<Robot> {
138
  const robotId = id || `so100-${Date.now()}`;
139
+ const urdfConfig = robotUrdfConfigMap["so-arm100"];
 
 
140
 
141
  return this.createRobotFromUrdf(robotId, urdfConfig, position);
142
  }
src/lib/elements/robot/components/RobotItem.svelte CHANGED
@@ -137,7 +137,6 @@
137
  nameHeight={0.1}
138
  showLine={isHovered || isSelected}
139
  opacity={1}
140
- isInteractive={false}
141
  />
142
  {/each}
143
  </T.Group>
 
137
  nameHeight={0.1}
138
  showLine={isHovered || isSelected}
139
  opacity={1}
 
140
  />
141
  {/each}
142
  </T.Group>
src/lib/elements/video/VideoManager.svelte.ts CHANGED
@@ -34,6 +34,8 @@ export class VideoInstance implements Positionable {
34
  // Output state (what this video is broadcasting)
35
  output = $state({
36
  active: false,
 
 
37
  client: null as videoTypes.VideoProducer | null,
38
  roomId: null as string | null,
39
  });
@@ -270,6 +272,8 @@ export class VideoManager {
270
 
271
  // Update output state
272
  video.output.active = true;
 
 
273
  video.output.client = producer;
274
  video.output.roomId = roomId;
275
 
@@ -528,6 +532,8 @@ export class VideoManager {
528
 
529
  // Update output state
530
  video.output.active = true;
 
 
531
  video.output.client = producer;
532
  video.output.roomId = result.roomId;
533
 
@@ -552,6 +558,8 @@ export class VideoManager {
552
  }
553
 
554
  video.output.active = false;
 
 
555
  video.output.client = null;
556
  video.output.roomId = null;
557
 
 
34
  // Output state (what this video is broadcasting)
35
  output = $state({
36
  active: false,
37
+ type: null as 'recording' | 'remote-broadcast' | null,
38
+ stream: null as MediaStream | null,
39
  client: null as videoTypes.VideoProducer | null,
40
  roomId: null as string | null,
41
  });
 
272
 
273
  // Update output state
274
  video.output.active = true;
275
+ video.output.type = 'remote-broadcast';
276
+ video.output.stream = video.input.stream;
277
  video.output.client = producer;
278
  video.output.roomId = roomId;
279
 
 
532
 
533
  // Update output state
534
  video.output.active = true;
535
+ video.output.type = 'remote-broadcast';
536
+ video.output.stream = video.input.stream;
537
  video.output.client = producer;
538
  video.output.roomId = result.roomId;
539
 
 
558
  }
559
 
560
  video.output.active = false;
561
+ video.output.type = null;
562
+ video.output.stream = null;
563
  video.output.client = null;
564
  video.output.roomId = null;
565
 
src/lib/types/urdf.ts CHANGED
@@ -23,4 +23,9 @@ export type RobotUrdfConfig = {
23
  restPosition?: {
24
  [jointName: string]: number;
25
  };
 
 
 
 
 
26
  };
 
23
  restPosition?: {
24
  [jointName: string]: number;
25
  };
26
+ // Display metadata for UI
27
+ displayName?: string;
28
+ description?: string;
29
+ icon?: string;
30
+ isDefault?: boolean;
31
  };