OpenXR Extension: XR_EXT_overlay_protocol

Extension Name: XR_EXT_overlay_protocol
Extension Type: Instance extension
Registered Extension Number: TBD
Revision: 1
Extension and Version Dependencies: Requires OpenXR 1.0
Last Modified Date: 2025-01-18
Contributors:

• catnet Team
• [Additional contributors to be added]

Overview

This extension introduces a standardized protocol for creating and managing overlay applications in OpenXR. It enables developers to create 2D and 3D overlays that persist across applications using a message-based protocol (VROP - VR Overlay Protocol).

Problem Statement

Current OpenXR specification lacks standardized overlay support, forcing developers to use platform-specific APIs (e.g., OpenVR's overlay system) or implement custom solutions. This fragmentation increases development complexity and reduces application portability.

Solution

XR_EXT_overlay_protocol provides:

1. Standardized overlay lifecycle management
2. Message-based communication protocol
3. Capability-based security model
4. Cross-application overlay persistence
5. Language-agnostic plugin architecture

Extension Details

New Object Types

// Handle to an overlay protocol session
XR_DEFINE_HANDLE(XrOverlayProtocolSessionEXT)

// Handle to an individual overlay
XR_DEFINE_HANDLE(XrOverlayEXT)

New Enums

// Overlay types
typedef enum XrOverlayTypeEXT {
    XR_OVERLAY_TYPE_2D_EXT = 0,
    XR_OVERLAY_TYPE_3D_EXT = 1,
    XR_OVERLAY_TYPE_DASHBOARD_EXT = 2,
    XR_OVERLAY_TYPE_MAX_ENUM_EXT = 0x7FFFFFFF
} XrOverlayTypeEXT;

// Message types following VROP specification
typedef enum XrOverlayMessageTypeEXT {
    // Lifecycle messages (0x0000-0x00FF)
    XR_OVERLAY_MESSAGE_INIT_EXT = 0x0001,
    XR_OVERLAY_MESSAGE_SHUTDOWN_EXT = 0x0002,
    XR_OVERLAY_MESSAGE_CAPABILITY_QUERY_EXT = 0x0003,
    
    // Overlay messages (0x0100-0x01FF)
    XR_OVERLAY_MESSAGE_CREATE_OVERLAY_EXT = 0x0100,
    XR_OVERLAY_MESSAGE_UPDATE_OVERLAY_EXT = 0x0101,
    XR_OVERLAY_MESSAGE_DESTROY_OVERLAY_EXT = 0x0102,
    
    // Rendering messages (0x0200-0x02FF)
    XR_OVERLAY_MESSAGE_SUBMIT_FRAME_EXT = 0x0200,
    XR_OVERLAY_MESSAGE_FRAME_TIMING_EXT = 0x0201,
    
    // Input messages (0x0300-0x03FF)
    XR_OVERLAY_MESSAGE_INPUT_EVENT_EXT = 0x0300,
    XR_OVERLAY_MESSAGE_HAPTIC_FEEDBACK_EXT = 0x0301,
    
    XR_OVERLAY_MESSAGE_MAX_ENUM_EXT = 0x7FFFFFFF
} XrOverlayMessageTypeEXT;

// Capability flags
typedef enum XrOverlayCapabilityFlagBitsEXT {
    XR_OVERLAY_CAPABILITY_CREATE_BIT_EXT = 0x00000001,
    XR_OVERLAY_CAPABILITY_GPU_DIRECT_BIT_EXT = 0x00000002,
    XR_OVERLAY_CAPABILITY_INPUT_RECEIVE_BIT_EXT = 0x00000004,
    XR_OVERLAY_CAPABILITY_HAPTIC_SEND_BIT_EXT = 0x00000008,
    XR_OVERLAY_CAPABILITY_NETWORK_ACCESS_BIT_EXT = 0x00000010,
} XrOverlayCapabilityFlagBitsEXT;
typedef XrFlags64 XrOverlayCapabilityFlagsEXT;

New Structures

// Protocol session creation info
typedef struct XrOverlayProtocolSessionCreateInfoEXT {
    XrStructureType type;
    const void* next;
    const char* applicationName;
    uint32_t applicationVersion;
    XrOverlayCapabilityFlagsEXT requestedCapabilities;
    XrOverlayCapabilityFlagsEXT requiredCapabilities;
} XrOverlayProtocolSessionCreateInfoEXT;

// Message structure following VROP format
typedef struct XrOverlayMessageEXT {
    XrStructureType type;
    const void* next;
    XrOverlayMessageTypeEXT messageType;
    uint64_t requestId;  // Optional, 0 if not request/response
    XrTime timestamp;
    uint32_t payloadSize;
    const void* payload;  // Serialized message data (format negotiated at runtime)
} XrOverlayMessageEXT;

// Overlay creation info
typedef struct XrOverlayCreateInfoEXT {
    XrStructureType type;
    const void* next;
    XrOverlayTypeEXT overlayType;
    const char* overlayName;
    XrExtent2Df size;  // For 2D overlays
    uint32_t width;    // Texture dimensions
    uint32_t height;
} XrOverlayCreateInfoEXT;

// Frame submission info
typedef struct XrOverlayFrameSubmitInfoEXT {
    XrStructureType type;
    const void* next;
    XrOverlayEXT overlay;
    uint64_t frameId;
    XrCompositionLayerProjection* layers;
    uint32_t layerCount;
} XrOverlayFrameSubmitInfoEXT;

New Functions

// Create overlay protocol session
typedef XrResult (XRAPI_PTR *PFN_xrCreateOverlayProtocolSessionEXT)(
    XrInstance instance,
    const XrOverlayProtocolSessionCreateInfoEXT* createInfo,
    XrOverlayProtocolSessionEXT* session);

// Send message through protocol
typedef XrResult (XRAPI_PTR *PFN_xrSendOverlayMessageEXT)(
    XrOverlayProtocolSessionEXT session,
    const XrOverlayMessageEXT* message);

// Receive message (non-blocking)
typedef XrResult (XRAPI_PTR *PFN_xrReceiveOverlayMessageEXT)(
    XrOverlayProtocolSessionEXT session,
    XrOverlayMessageEXT* message);

// Destroy protocol session
typedef XrResult (XRAPI_PTR *PFN_xrDestroyOverlayProtocolSessionEXT)(
    XrOverlayProtocolSessionEXT session);

Usage Example

// 1. Create overlay protocol session
XrOverlayProtocolSessionCreateInfoEXT sessionInfo = {
    .type = XR_TYPE_OVERLAY_PROTOCOL_SESSION_CREATE_INFO_EXT,
    .next = NULL,
    .applicationName = "MyOverlayApp",
    .applicationVersion = XR_MAKE_VERSION(1, 0, 0),
    .requestedCapabilities = XR_OVERLAY_CAPABILITY_CREATE_BIT_EXT |
                             XR_OVERLAY_CAPABILITY_INPUT_RECEIVE_BIT_EXT,
    .requiredCapabilities = XR_OVERLAY_CAPABILITY_CREATE_BIT_EXT
};

XrOverlayProtocolSessionEXT overlaySession;
xrCreateOverlayProtocolSessionEXT(instance, &sessionInfo, &overlaySession);

// 2. Send overlay creation message
uint8_t payload[256];
size_t payloadSize = vrop_encode_create_overlay(&payload, "my_overlay", 
                                                 1920, 1080, 1.0f, 0.8f);

XrOverlayMessageEXT createMsg = {
    .type = XR_TYPE_OVERLAY_MESSAGE_EXT,
    .messageType = XR_OVERLAY_MESSAGE_CREATE_OVERLAY_EXT,
    .requestId = 1,
    .timestamp = xrGetTime(),
    .payloadSize = payloadSize,
    .payload = payload
};

xrSendOverlayMessageEXT(overlaySession, &createMsg);

// 3. Receive response
XrOverlayMessageEXT response;
while (xrReceiveOverlayMessageEXT(overlaySession, &response) == XR_SUCCESS) {
    // Process response
    if (response.requestId == 1) {
        // Overlay created successfully
        break;
    }
}

Security Considerations

1. Capability-based permissions: Applications must request capabilities at session creation
2. Sandboxed execution: Runtime implementations should isolate overlay processes
3. Resource quotas: Runtimes should enforce limits on overlay count, memory, and GPU usage
4. Message validation: All messages must be validated before processing

Conformance Testing

Implementations must:

1. Support all defined message types
2. Maintain message ordering within 1ms accuracy
3. Achieve <100μs message processing latency (excluding payload parsing)
4. Pass the VROP conformance test suite

Issues

ISSUE 1: Message size limits

Resolution: Runtime-defined limits with minimum of 64KB per message

ISSUE 2: Direct GPU texture sharing

Resolution: Optional capability, fallback to CPU copy

ISSUE 3: Multi-runtime coordination

Resolution: Instance-level coordination through shared protocol state

Version History

• Revision 1, 2025-01-18: Initial draft based on VROP 0.1.0

Dependencies on External Specifications

• VROP Specification v0.1.0
• Serialization format specifications (implementation-dependent)

Intellectual Property

This extension follows the Khronos IP Framework. The VROP protocol referenced by this extension is available under MIT license.