Add stereoscopic rendering through multiview

This commit is contained in:
Bastiaan Olij 2021-05-07 23:19:04 +10:00
parent 600b4c9c7b
commit 15c1a76361
53 changed files with 1062 additions and 467 deletions

View file

@ -5068,7 +5068,7 @@
},
{
"name": "godot_xr_get_worldscale",
"return_type": "godot_float",
"return_type": "godot_real_t",
"arguments": []
},
{
@ -5189,7 +5189,7 @@
"p_exis"
],
[
"godot_float",
"godot_real_t",
"p_value"
],
[
@ -5200,7 +5200,7 @@
},
{
"name": "godot_xr_get_controller_rumble",
"return_type": "godot_float",
"return_type": "godot_real_t",
"arguments": [
[
"godot_int",

View file

@ -41,8 +41,8 @@ extern "C" {
// version info to detect whether a call is available
// Use these to populate version in your plugin
#define GODOTVR_API_MAJOR 1
#define GODOTVR_API_MINOR 1
#define GODOTVR_API_MAJOR 4
#define GODOTVR_API_MINOR 0
typedef struct {
godot_gdnative_api_version version; /* version of our API */
@ -52,24 +52,31 @@ typedef struct {
godot_int (*get_capabilities)(const void *);
godot_bool (*get_anchor_detection_is_enabled)(const void *);
void (*set_anchor_detection_is_enabled)(void *, godot_bool);
godot_bool (*is_stereo)(const void *);
godot_int (*get_view_count)(const void *);
godot_bool (*is_initialized)(const void *);
godot_bool (*initialize)(void *);
void (*uninitialize)(void *);
godot_vector2 (*get_render_targetsize)(const void *);
godot_transform3d (*get_transform_for_eye)(void *, godot_int, godot_transform3d *);
void (*fill_projection_for_eye)(void *, godot_float *, godot_int, godot_float, godot_float, godot_float);
void (*commit_for_eye)(void *, godot_int, godot_rid *, godot_rect2 *);
godot_transform3d (*get_camera_transform)(void *);
godot_transform3d (*get_transform_for_view)(void *, godot_int, godot_transform3d *);
void (*fill_projection_for_view)(void *, godot_real_t *, godot_int, godot_real_t, godot_real_t, godot_real_t);
void (*commit_views)(void *, godot_rid *, godot_rect2 *);
void (*process)(void *);
godot_int (*get_external_texture_for_eye)(void *, godot_int);
void (*notification)(void *, godot_int);
godot_int (*get_camera_feed_id)(void *);
// possibly deprecate but adding/keeping as a reminder these are in Godot 3
void (*commit_for_eye)(void *, godot_int, godot_rid *, godot_rect2 *);
godot_int (*get_external_texture_for_eye)(void *, godot_int);
godot_int (*get_external_depth_for_eye)(void *, godot_int);
} godot_xr_interface_gdnative;
void GDAPI godot_xr_register_interface(const godot_xr_interface_gdnative *p_interface);
// helper functions to access XRServer data
godot_float GDAPI godot_xr_get_worldscale();
godot_real_t GDAPI godot_xr_get_worldscale();
godot_transform3d GDAPI godot_xr_get_reference_frame();
// helper functions for rendering
@ -81,8 +88,8 @@ godot_int GDAPI godot_xr_add_controller(char *p_device_name, godot_int p_hand, g
void GDAPI godot_xr_remove_controller(godot_int p_controller_id);
void GDAPI godot_xr_set_controller_transform(godot_int p_controller_id, godot_transform3d *p_transform, godot_bool p_tracks_orientation, godot_bool p_tracks_position);
void GDAPI godot_xr_set_controller_button(godot_int p_controller_id, godot_int p_button, godot_bool p_is_pressed);
void GDAPI godot_xr_set_controller_axis(godot_int p_controller_id, godot_int p_axis, godot_float p_value, godot_bool p_can_be_negative);
godot_float GDAPI godot_xr_get_controller_rumble(godot_int p_controller_id);
void GDAPI godot_xr_set_controller_axis(godot_int p_controller_id, godot_int p_axis, godot_real_t p_value, godot_bool p_can_be_negative);
godot_real_t GDAPI godot_xr_get_controller_rumble(godot_int p_controller_id);
#ifdef __cplusplus
}

View file

@ -70,8 +70,13 @@ void XRInterfaceGDNative::set_interface(const godot_xr_interface_gdnative *p_int
// this should only be called once, just being paranoid..
if (interface) {
cleanup();
interface = NULL;
}
// validate
ERR_FAIL_NULL(p_interface);
ERR_FAIL_COND_MSG(p_interface->version.major < 4, "This is an incompatible GDNative XR plugin.");
// bind to our interface
interface = p_interface;
@ -119,14 +124,14 @@ int XRInterfaceGDNative::get_camera_feed_id() {
return (unsigned int)interface->get_camera_feed_id(data);
}
bool XRInterfaceGDNative::is_stereo() {
bool stereo;
uint32_t XRInterfaceGDNative::get_view_count() {
uint32_t view_count;
ERR_FAIL_COND_V(interface == nullptr, false);
ERR_FAIL_COND_V(interface == nullptr, 1);
stereo = interface->is_stereo(data);
view_count = interface->get_view_count(data);
return stereo;
return view_count;
}
bool XRInterfaceGDNative::is_initialized() const {
@ -173,28 +178,52 @@ Size2 XRInterfaceGDNative::get_render_targetsize() {
return *vec;
}
Transform3D XRInterfaceGDNative::get_transform_for_eye(XRInterface::Eyes p_eye, const Transform3D &p_cam_transform) {
Transform3D XRInterfaceGDNative::get_camera_transform() {
Transform3D *ret;
ERR_FAIL_COND_V(interface == nullptr, Transform3D());
godot_transform3d t = interface->get_transform_for_eye(data, (int)p_eye, (godot_transform3d *)&p_cam_transform);
godot_transform3d t = interface->get_camera_transform(data);
ret = (Transform3D *)&t;
return *ret;
}
CameraMatrix XRInterfaceGDNative::get_projection_for_eye(XRInterface::Eyes p_eye, real_t p_aspect, real_t p_z_near, real_t p_z_far) {
Transform3D XRInterfaceGDNative::get_transform_for_view(uint32_t p_view, const Transform3D &p_cam_transform) {
Transform3D *ret;
ERR_FAIL_COND_V(interface == nullptr, Transform3D());
godot_transform3d t = interface->get_transform_for_view(data, (int)p_view, (godot_transform3d *)&p_cam_transform);
ret = (Transform3D *)&t;
return *ret;
}
CameraMatrix XRInterfaceGDNative::get_projection_for_view(uint32_t p_view, real_t p_aspect, real_t p_z_near, real_t p_z_far) {
CameraMatrix cm;
ERR_FAIL_COND_V(interface == nullptr, CameraMatrix());
interface->fill_projection_for_eye(data, (godot_float *)cm.matrix, (godot_int)p_eye, p_aspect, p_z_near, p_z_far);
interface->fill_projection_for_view(data, (godot_real_t *)cm.matrix, (godot_int)p_view, p_aspect, p_z_near, p_z_far);
return cm;
}
Vector<BlitToScreen> XRInterfaceGDNative::commit_views(RID p_render_target, const Rect2 &p_screen_rect) {
// possibly move this as a member variable and add a callback to populate?
Vector<BlitToScreen> blit_to_screen;
ERR_FAIL_COND_V(interface == nullptr, blit_to_screen);
// must implement
interface->commit_views(data, (godot_rid *)&p_render_target, (godot_rect2 *)&p_screen_rect);
return blit_to_screen;
}
unsigned int XRInterfaceGDNative::get_external_texture_for_eye(XRInterface::Eyes p_eye) {
ERR_FAIL_COND_V(interface == nullptr, 0);
@ -234,7 +263,7 @@ void GDAPI godot_xr_register_interface(const godot_xr_interface_gdnative *p_inte
XRServer::get_singleton()->add_interface(new_interface);
}
godot_float GDAPI godot_xr_get_worldscale() {
godot_real_t GDAPI godot_xr_get_worldscale() {
XRServer *xr_server = XRServer::get_singleton();
ERR_FAIL_NULL_V(xr_server, 1.0);
@ -388,7 +417,7 @@ void GDAPI godot_xr_set_controller_button(godot_int p_controller_id, godot_int p
}
}
void GDAPI godot_xr_set_controller_axis(godot_int p_controller_id, godot_int p_axis, godot_float p_value, godot_bool p_can_be_negative) {
void GDAPI godot_xr_set_controller_axis(godot_int p_controller_id, godot_int p_axis, godot_real_t p_value, godot_bool p_can_be_negative) {
XRServer *xr_server = XRServer::get_singleton();
ERR_FAIL_NULL(xr_server);
@ -407,7 +436,7 @@ void GDAPI godot_xr_set_controller_axis(godot_int p_controller_id, godot_int p_a
}
}
godot_float GDAPI godot_xr_get_controller_rumble(godot_int p_controller_id) {
godot_real_t GDAPI godot_xr_get_controller_rumble(godot_int p_controller_id) {
XRServer *xr_server = XRServer::get_singleton();
ERR_FAIL_NULL_V(xr_server, 0.0);

View file

@ -72,20 +72,24 @@ public:
/** rendering and internal **/
virtual Size2 get_render_targetsize() override;
virtual bool is_stereo() override;
virtual Transform3D get_transform_for_eye(XRInterface::Eyes p_eye, const Transform3D &p_cam_transform) override;
virtual uint32_t get_view_count() override;
virtual Transform3D get_camera_transform() override;
virtual Transform3D get_transform_for_view(uint32_t p_view, const Transform3D &p_cam_transform) override;
// we expose a Vector<float> version of this function to GDNative
Vector<float> _get_projection_for_eye(XRInterface::Eyes p_eye, real_t p_aspect, real_t p_z_near, real_t p_z_far);
// and a CameraMatrix version to XRServer
virtual CameraMatrix get_projection_for_eye(XRInterface::Eyes p_eye, real_t p_aspect, real_t p_z_near, real_t p_z_far) override;
virtual CameraMatrix get_projection_for_view(uint32_t p_view, real_t p_aspect, real_t p_z_near, real_t p_z_far) override;
virtual unsigned int get_external_texture_for_eye(XRInterface::Eyes p_eye) override;
virtual void commit_for_eye(XRInterface::Eyes p_eye, RID p_render_target, const Rect2 &p_screen_rect) override;
virtual Vector<BlitToScreen> commit_views(RID p_render_target, const Rect2 &p_screen_rect) override;
virtual void process() override;
virtual void notification(int p_what) override;
// deprecated
virtual void commit_for_eye(XRInterface::Eyes p_eye, RID p_render_target, const Rect2 &p_screen_rect) override;
virtual unsigned int get_external_texture_for_eye(XRInterface::Eyes p_eye) override;
};
#endif // XR_INTERFACE_GDNATIVE_H

View file

@ -116,6 +116,10 @@ static Vector<uint8_t> _compile_shader_glsl(RenderingDevice::ShaderStage p_stage
}
}
if (p_capabilities->supports_multiview) {
preamble += "#define has_VK_KHR_multiview 1\n";
}
if (preamble != "") {
shader.setPreamble(preamble.c_str());
}

View file

@ -300,9 +300,9 @@ real_t MobileVRInterface::get_k2() const {
return k2;
};
bool MobileVRInterface::is_stereo() {
uint32_t MobileVRInterface::get_view_count() {
// needs stereo...
return true;
return 2;
};
bool MobileVRInterface::is_initialized() const {
@ -361,7 +361,29 @@ Size2 MobileVRInterface::get_render_targetsize() {
return target_size;
};
Transform3D MobileVRInterface::get_transform_for_eye(XRInterface::Eyes p_eye, const Transform3D &p_cam_transform) {
Transform3D MobileVRInterface::get_camera_transform() {
_THREAD_SAFE_METHOD_
Transform3D transform_for_eye;
XRServer *xr_server = XRServer::get_singleton();
ERR_FAIL_NULL_V(xr_server, transform_for_eye);
if (initialized) {
float world_scale = xr_server->get_world_scale();
// just scale our origin point of our transform
Transform3D hmd_transform;
hmd_transform.basis = orientation;
hmd_transform.origin = Vector3(0.0, eye_height * world_scale, 0.0);
transform_for_eye = (xr_server->get_reference_frame()) * hmd_transform;
}
return transform_for_eye;
};
Transform3D MobileVRInterface::get_transform_for_view(uint32_t p_view, const Transform3D &p_cam_transform) {
_THREAD_SAFE_METHOD_
Transform3D transform_for_eye;
@ -374,12 +396,12 @@ Transform3D MobileVRInterface::get_transform_for_eye(XRInterface::Eyes p_eye, co
// we don't need to check for the existence of our HMD, doesn't affect our values...
// note * 0.01 to convert cm to m and * 0.5 as we're moving half in each direction...
if (p_eye == XRInterface::EYE_LEFT) {
if (p_view == 0) {
transform_for_eye.origin.x = -(intraocular_dist * 0.01 * 0.5 * world_scale);
} else if (p_eye == XRInterface::EYE_RIGHT) {
} else if (p_view == 1) {
transform_for_eye.origin.x = intraocular_dist * 0.01 * 0.5 * world_scale;
} else {
// for mono we don't reposition, we want our center position.
// should not have any other values..
};
// just scale our origin point of our transform
@ -396,21 +418,13 @@ Transform3D MobileVRInterface::get_transform_for_eye(XRInterface::Eyes p_eye, co
return transform_for_eye;
};
CameraMatrix MobileVRInterface::get_projection_for_eye(XRInterface::Eyes p_eye, real_t p_aspect, real_t p_z_near, real_t p_z_far) {
CameraMatrix MobileVRInterface::get_projection_for_view(uint32_t p_view, real_t p_aspect, real_t p_z_near, real_t p_z_far) {
_THREAD_SAFE_METHOD_
CameraMatrix eye;
if (p_eye == XRInterface::EYE_MONO) {
///@TODO for now hardcode some of this, what is really needed here is that this needs to be in sync with the real camera's properties
// which probably means implementing a specific class for iOS and Android. For now this is purely here as an example.
// Note also that if you use a normal viewport with AR/VR turned off you can still use the tracker output of this interface
// to position a stock standard Godot camera and have control over this.
// This will make more sense when we implement ARkit on iOS (probably a separate interface).
eye.set_perspective(60.0, p_aspect, p_z_near, p_z_far, false);
} else {
eye.set_for_hmd(p_eye == XRInterface::EYE_LEFT ? 1 : 2, p_aspect, intraocular_dist, display_width, display_to_lens, oversample, p_z_near, p_z_far);
};
aspect = p_aspect;
eye.set_for_hmd(p_view + 1, p_aspect, intraocular_dist, display_width, display_to_lens, oversample, p_z_near, p_z_far);
return eye;
};
@ -440,6 +454,45 @@ void MobileVRInterface::commit_for_eye(XRInterface::Eyes p_eye, RID p_render_tar
eye_center.y = 0.0;
}
Vector<BlitToScreen> MobileVRInterface::commit_views(RID p_render_target, const Rect2 &p_screen_rect) {
_THREAD_SAFE_METHOD_
Vector<BlitToScreen> blit_to_screen;
// We must have a valid render target
ERR_FAIL_COND_V(!p_render_target.is_valid(), blit_to_screen);
// Because we are rendering to our device we must use our main viewport!
ERR_FAIL_COND_V(p_screen_rect == Rect2(), blit_to_screen);
// and add our blits
BlitToScreen blit;
blit.render_target = p_render_target;
blit.multi_view.use_layer = true;
blit.lens_distortion.apply = true;
blit.lens_distortion.k1 = k1;
blit.lens_distortion.k2 = k2;
blit.lens_distortion.upscale = oversample;
blit.lens_distortion.aspect_ratio = aspect;
// left eye
blit.rect = p_screen_rect;
blit.rect.size.width *= 0.5;
blit.multi_view.layer = 0;
blit.lens_distortion.eye_center.x = ((-intraocular_dist / 2.0) + (display_width / 4.0)) / (display_width / 2.0);
blit_to_screen.push_back(blit);
// right eye
blit.rect = p_screen_rect;
blit.rect.size.width *= 0.5;
blit.rect.position.x = blit.rect.size.width;
blit.multi_view.layer = 1;
blit.lens_distortion.eye_center.x = ((intraocular_dist / 2.0) - (display_width / 4.0)) / (display_width / 2.0);
blit_to_screen.push_back(blit);
return blit_to_screen;
}
void MobileVRInterface::process() {
_THREAD_SAFE_METHOD_

View file

@ -63,9 +63,9 @@ private:
real_t display_to_lens = 4.0;
real_t oversample = 1.5;
//@TODO not yet used, these are needed in our distortion shader...
real_t k1 = 0.215;
real_t k2 = 0.215;
real_t aspect = 1.0;
/*
logic for processing our sensor data, this was originally in our positional tracker logic but I think
@ -138,16 +138,20 @@ public:
virtual void uninitialize() override;
virtual Size2 get_render_targetsize() override;
virtual bool is_stereo() override;
virtual Transform3D get_transform_for_eye(XRInterface::Eyes p_eye, const Transform3D &p_cam_transform) override;
virtual CameraMatrix get_projection_for_eye(XRInterface::Eyes p_eye, real_t p_aspect, real_t p_z_near, real_t p_z_far) override;
virtual void commit_for_eye(XRInterface::Eyes p_eye, RID p_render_target, const Rect2 &p_screen_rect) override;
virtual uint32_t get_view_count() override;
virtual Transform3D get_camera_transform() override;
virtual Transform3D get_transform_for_view(uint32_t p_view, const Transform3D &p_cam_transform) override;
virtual CameraMatrix get_projection_for_view(uint32_t p_view, real_t p_aspect, real_t p_z_near, real_t p_z_far) override;
virtual Vector<BlitToScreen> commit_views(RID p_render_target, const Rect2 &p_screen_rect) override;
virtual void process() override;
virtual void notification(int p_what) override {}
MobileVRInterface();
~MobileVRInterface();
// deprecated
virtual void commit_for_eye(XRInterface::Eyes p_eye, RID p_render_target, const Rect2 &p_screen_rect) override;
};
#endif // !MOBILE_VR_INTERFACE_H

View file

@ -265,7 +265,7 @@ void WebXRInterfaceJS::uninitialize() {
};
};
Transform WebXRInterfaceJS::_js_matrix_to_transform(float *p_js_matrix) {
Transform3D WebXRInterfaceJS::_js_matrix_to_transform(float *p_js_matrix) {
Transform3D transform;
transform.basis.elements[0].x = p_js_matrix[0];
@ -305,13 +305,30 @@ Size2 WebXRInterfaceJS::get_render_targetsize() {
return render_targetsize;
};
Transform WebXRInterfaceJS::get_transform_for_eye(XRInterface::Eyes p_eye, const Transform3D &p_cam_transform) {
Transform3D WebXRInterfaceJS::get_camera_transform() {
Transform3D transform_for_eye;
XRServer *xr_server = XRServer::get_singleton();
ERR_FAIL_NULL_V(xr_server, transform_for_eye);
float *js_matrix = godot_webxr_get_transform_for_eye(p_eye);
float *js_matrix = godot_webxr_get_transform_for_eye(0);
if (!initialized || js_matrix == nullptr) {
return transform_for_eye;
}
transform_for_eye = _js_matrix_to_transform(js_matrix);
free(js_matrix);
return xr_server->get_reference_frame() * transform_for_eye;
};
Transform3D WebXRInterfaceJS::get_transform_for_view(uint32_t p_view, const Transform3D &p_cam_transform) {
Transform3D transform_for_eye;
XRServer *xr_server = XRServer::get_singleton();
ERR_FAIL_NULL_V(xr_server, transform_for_eye);
float *js_matrix = godot_webxr_get_transform_for_eye(p_view + 1);
if (!initialized || js_matrix == nullptr) {
transform_for_eye = p_cam_transform;
return transform_for_eye;
@ -323,10 +340,10 @@ Transform WebXRInterfaceJS::get_transform_for_eye(XRInterface::Eyes p_eye, const
return p_cam_transform * xr_server->get_reference_frame() * transform_for_eye;
};
CameraMatrix WebXRInterfaceJS::get_projection_for_eye(XRInterface::Eyes p_eye, real_t p_aspect, real_t p_z_near, real_t p_z_far) {
CameraMatrix WebXRInterfaceJS::get_projection_for_view(uint32_t p_view, real_t p_aspect, real_t p_z_near, real_t p_z_far) {
CameraMatrix eye;
float *js_matrix = godot_webxr_get_projection_for_eye(p_eye);
float *js_matrix = godot_webxr_get_projection_for_eye(p_view + 1);
if (!initialized || js_matrix == nullptr) {
return eye;
}

View file

@ -84,8 +84,9 @@ public:
virtual Size2 get_render_targetsize() override;
virtual bool is_stereo() override;
virtual Transform get_transform_for_eye(XRInterface::Eyes p_eye, const Transform3D &p_cam_transform) override;
virtual CameraMatrix get_projection_for_eye(XRInterface::Eyes p_eye, real_t p_aspect, real_t p_z_near, real_t p_z_far) override;
virtual Transform3D get_camera_transform() override;
virtual Transform3D get_transform_for_view(uint32_t p_view, const Transform3D &p_cam_transform) override;
virtual CameraMatrix get_projection_for_view(uint32_t p_view, real_t p_aspect, real_t p_z_near, real_t p_z_far) override;
virtual unsigned int get_external_texture_for_eye(XRInterface::Eyes p_eye) override;
virtual void commit_for_eye(XRInterface::Eyes p_eye, RID p_render_target, const Rect2 &p_screen_rect) override;