From 574f5126a2025589ce69242b74d33e83ed57b683 Mon Sep 17 00:00:00 2001 From: clayjohn Date: Mon, 3 Mar 2025 18:48:26 -0800 Subject: [PATCH] Optimize _fill_instance_data function in Forward+ and Mobile renderers. This batches together a couple of micro-optimizations that were discovered in profiling and analyzing disassembly. Importantly, this reduces the amount of instructions in a heavy loop. This has the biggest impact when there are lots of objects and lights in the scene as the function is called once per objects + once per light that touches the object --- .../render_forward_clustered.cpp | 42 ++++------- .../render_forward_clustered.h | 70 ++++++++++++++---- .../forward_mobile/render_forward_mobile.cpp | 32 +++------ .../forward_mobile/render_forward_mobile.h | 72 +++++++++++++++---- 4 files changed, 135 insertions(+), 81 deletions(-) diff --git a/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.cpp b/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.cpp index 7acaeb506b..d8071dd624 100644 --- a/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.cpp +++ b/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.cpp @@ -786,7 +786,7 @@ void RenderForwardClustered::_fill_instance_data(RenderListType p_render_list, i if (p_render_info) { p_render_info[RS::VIEWPORT_RENDER_INFO_OBJECTS_IN_FRAME] += element_total; } - uint64_t frame = RSG::rasterizer->get_frame_number(); + uint32_t repeats = 0; GeometryInstanceSurfaceDataCache *prev_surface = nullptr; for (uint32_t i = 0; i < element_total; i++) { @@ -795,12 +795,7 @@ void RenderForwardClustered::_fill_instance_data(RenderListType p_render_list, i SceneState::InstanceData &instance_data = scene_state.instance_data[p_render_list][i + p_offset]; - if (inst->prev_transform_dirty && frame > inst->prev_transform_change_frame + 1 && inst->prev_transform_change_frame) { - inst->prev_transform = inst->transform; - inst->prev_transform_dirty = false; - } - - if (inst->store_transform_cache) { + if (likely(inst->store_transform_cache)) { RendererRD::MaterialStorage::store_transform(inst->transform, instance_data.transform); RendererRD::MaterialStorage::store_transform(inst->prev_transform, instance_data.prev_transform); @@ -820,10 +815,7 @@ void RenderForwardClustered::_fill_instance_data(RenderListType p_render_list, i instance_data.gi_offset = inst->gi_offset_cache; instance_data.layer_mask = inst->layer_mask; instance_data.instance_uniforms_ofs = uint32_t(inst->shader_uniforms_offset); - instance_data.lightmap_uv_scale[0] = inst->lightmap_uv_scale.position.x; - instance_data.lightmap_uv_scale[1] = inst->lightmap_uv_scale.position.y; - instance_data.lightmap_uv_scale[2] = inst->lightmap_uv_scale.size.x; - instance_data.lightmap_uv_scale[3] = inst->lightmap_uv_scale.size.y; + instance_data.set_lightmap_uv_scale(inst->lightmap_uv_scale); AABB surface_aabb = AABB(Vector3(0.0, 0.0, 0.0), Vector3(1.0, 1.0, 1.0)); uint64_t format = RendererRD::MeshStorage::get_singleton()->mesh_surface_get_format(surface->surface); @@ -834,18 +826,8 @@ void RenderForwardClustered::_fill_instance_data(RenderListType p_render_list, i uv_scale = RendererRD::MeshStorage::get_singleton()->mesh_surface_get_uv_scale(surface->surface); } - instance_data.compressed_aabb_position[0] = surface_aabb.position.x; - instance_data.compressed_aabb_position[1] = surface_aabb.position.y; - instance_data.compressed_aabb_position[2] = surface_aabb.position.z; - - instance_data.compressed_aabb_size[0] = surface_aabb.size.x; - instance_data.compressed_aabb_size[1] = surface_aabb.size.y; - instance_data.compressed_aabb_size[2] = surface_aabb.size.z; - - instance_data.uv_scale[0] = uv_scale.x; - instance_data.uv_scale[1] = uv_scale.y; - instance_data.uv_scale[2] = uv_scale.z; - instance_data.uv_scale[3] = uv_scale.w; + instance_data.set_compressed_aabb(surface_aabb); + instance_data.set_uv_scale(uv_scale); bool cant_repeat = instance_data.flags & INSTANCE_DATA_FLAG_MULTIMESH || inst->mesh_instance.is_valid(); @@ -866,11 +848,7 @@ void RenderForwardClustered::_fill_instance_data(RenderListType p_render_list, i RenderElementInfo &element_info = rl->element_info[p_offset + i]; - element_info.lod_index = surface->sort.lod_index; - element_info.uses_forward_gi = surface->sort.uses_forward_gi; - element_info.uses_lightmap = surface->sort.uses_lightmap; - element_info.uses_softshadow = surface->sort.uses_softshadow; - element_info.uses_projector = surface->sort.uses_projector; + element_info.value = uint32_t(surface->sort.sort_key1 & 0xFFF); if (cant_repeat) { prev_surface = nullptr; @@ -1068,6 +1046,11 @@ void RenderForwardClustered::_fill_render_list(RenderListType p_render_list, con lod_distance = surface_distance.length(); } + if (unlikely(inst->prev_transform_dirty && frame > inst->prev_transform_change_frame + 1 && inst->prev_transform_change_frame)) { + inst->prev_transform = inst->transform; + inst->prev_transform_dirty = false; + } + while (surf) { surf->sort.uses_forward_gi = 0; surf->sort.uses_lightmap = 0; @@ -4039,8 +4022,7 @@ void RenderForwardClustered::_geometry_instance_add_surface_with_material(Geomet sdcache->sort.sort_key2 = 0; sdcache->sort.surface_index = p_surface; - sdcache->sort.material_id_low = p_material_id & 0xFFFF; - sdcache->sort.material_id_hi = p_material_id >> 16; + sdcache->sort.material_id = p_material_id; sdcache->sort.shader_id = p_shader_id; sdcache->sort.geometry_id = p_mesh.get_local_index(); //only meshes can repeat anyway sdcache->sort.uses_forward_gi = ginstance->can_sdfgi; diff --git a/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.h b/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.h index bfde315c68..c8045df65c 100644 --- a/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.h +++ b/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.h @@ -326,6 +326,48 @@ private: float compressed_aabb_position[4]; float compressed_aabb_size[4]; float uv_scale[4]; + + // These setters allow us to copy the data over with operation when using floats. + inline void set_lightmap_uv_scale(const Rect2 &p_rect) { +#ifdef REAL_T_IS_DOUBLE + lightmap_uv_scale[0] = p_rect.position.x; + lightmap_uv_scale[1] = p_rect.position.y; + lightmap_uv_scale[2] = p_rect.size.x; + lightmap_uv_scale[3] = p_rect.size.y; +#else + Rect2 *rect = reinterpret_cast(lightmap_uv_scale); + *rect = p_rect; +#endif + } + + inline void set_compressed_aabb(const AABB &p_aabb) { +#ifdef REAL_T_IS_DOUBLE + compressed_aabb_position[0] = p_aabb.position.x; + compressed_aabb_position[1] = p_aabb.position.y; + compressed_aabb_position[2] = p_aabb.position.z; + + compressed_aabb_size[0] = p_aabb.size.x; + compressed_aabb_size[1] = p_aabb.size.y; + compressed_aabb_size[2] = p_aabb.size.z; +#else + Vector3 *compressed_aabb_position_vec3 = reinterpret_cast(compressed_aabb_position); + Vector3 *compressed_aabb_size_vec3 = reinterpret_cast(compressed_aabb_size); + *compressed_aabb_position_vec3 = p_aabb.position; + *compressed_aabb_size_vec3 = p_aabb.size; +#endif + } + + inline void set_uv_scale(const Vector4 &p_uv_scale) { +#ifdef REAL_T_IS_DOUBLE + uv_scale[0] = p_uv_scale.x; + uv_scale[1] = p_uv_scale.y; + uv_scale[2] = p_uv_scale.z; + uv_scale[3] = p_uv_scale.w; +#else + Vector4 *uv_scale_vec4 = reinterpret_cast(uv_scale); + *uv_scale_vec4 = p_uv_scale; +#endif + } }; UBO ubo; @@ -384,12 +426,17 @@ private: struct RenderElementInfo { enum { MAX_REPEATS = (1 << 20) - 1 }; - uint32_t repeat : 20; - uint32_t uses_projector : 1; - uint32_t uses_softshadow : 1; - uint32_t uses_lightmap : 1; - uint32_t uses_forward_gi : 1; - uint32_t lod_index : 8; + union { + struct { + uint32_t lod_index : 8; + uint32_t uses_softshadow : 1; + uint32_t uses_projector : 1; + uint32_t uses_forward_gi : 1; + uint32_t uses_lightmap : 1; + }; + uint32_t value; + }; + uint32_t repeat; }; template @@ -434,18 +481,17 @@ private: }; struct { uint64_t lod_index : 8; - uint64_t surface_index : 8; - uint64_t geometry_id : 32; - uint64_t material_id_low : 16; - - uint64_t material_id_hi : 16; - uint64_t shader_id : 32; uint64_t uses_softshadow : 1; uint64_t uses_projector : 1; uint64_t uses_forward_gi : 1; uint64_t uses_lightmap : 1; uint64_t depth_layer : 4; + uint64_t surface_index : 8; uint64_t priority : 8; + uint64_t geometry_id : 32; + + uint64_t material_id : 32; + uint64_t shader_id : 32; }; } sort; diff --git a/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.cpp b/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.cpp index 9289c21a38..a554e5c7d1 100644 --- a/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.cpp +++ b/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.cpp @@ -1862,10 +1862,7 @@ void RenderForwardMobile::_fill_instance_data(RenderListType p_render_list, uint instance_data.gi_offset = inst->gi_offset_cache; instance_data.layer_mask = inst->layer_mask; instance_data.instance_uniforms_ofs = uint32_t(inst->shader_uniforms_offset); - instance_data.lightmap_uv_scale[0] = inst->lightmap_uv_scale.position.x; - instance_data.lightmap_uv_scale[1] = inst->lightmap_uv_scale.position.y; - instance_data.lightmap_uv_scale[2] = inst->lightmap_uv_scale.size.x; - instance_data.lightmap_uv_scale[3] = inst->lightmap_uv_scale.size.y; + instance_data.set_lightmap_uv_scale(inst->lightmap_uv_scale); AABB surface_aabb = AABB(Vector3(0.0, 0.0, 0.0), Vector3(1.0, 1.0, 1.0)); uint64_t format = RendererRD::MeshStorage::get_singleton()->mesh_surface_get_format(surface->surface); @@ -1878,23 +1875,13 @@ void RenderForwardMobile::_fill_instance_data(RenderListType p_render_list, uint fill_push_constant_instance_indices(&instance_data, inst); - instance_data.compressed_aabb_position[0] = surface_aabb.position.x; - instance_data.compressed_aabb_position[1] = surface_aabb.position.y; - instance_data.compressed_aabb_position[2] = surface_aabb.position.z; - - instance_data.compressed_aabb_size[0] = surface_aabb.size.x; - instance_data.compressed_aabb_size[1] = surface_aabb.size.y; - instance_data.compressed_aabb_size[2] = surface_aabb.size.z; - - instance_data.uv_scale[0] = uv_scale.x; - instance_data.uv_scale[1] = uv_scale.y; - instance_data.uv_scale[2] = uv_scale.z; - instance_data.uv_scale[3] = uv_scale.w; + instance_data.set_compressed_aabb(surface_aabb); + instance_data.set_uv_scale(uv_scale); RenderElementInfo &element_info = rl->element_info[p_offset + i]; - element_info.lod_index = surface->lod_index; - element_info.uses_lightmap = surface->sort.uses_lightmap; + // Sets lod_index and uses_lightmap at once. + element_info.value = uint32_t(surface->sort.sort_key1 & 0x1FF); } if (p_update_buffer) { @@ -2026,7 +2013,7 @@ void RenderForwardMobile::_fill_render_list(RenderListType p_render_list, const if (p_render_data->scene_data->screen_mesh_lod_threshold > 0.0 && mesh_storage->mesh_surface_has_lod(surf->surface)) { uint32_t indices = 0; - surf->lod_index = mesh_storage->mesh_surface_get_lod(surf->surface, inst->lod_model_scale * inst->lod_bias, lod_distance * p_render_data->scene_data->lod_distance_multiplier, p_render_data->scene_data->screen_mesh_lod_threshold, indices); + surf->sort.lod_index = mesh_storage->mesh_surface_get_lod(surf->surface, inst->lod_model_scale * inst->lod_bias, lod_distance * p_render_data->scene_data->lod_distance_multiplier, p_render_data->scene_data->screen_mesh_lod_threshold, indices); if (p_render_data->render_info) { indices = _indices_to_primitives(surf->primitive, indices); if (p_render_list == RENDER_LIST_OPAQUE) { //opaque @@ -2036,7 +2023,7 @@ void RenderForwardMobile::_fill_render_list(RenderListType p_render_list, const } } } else { - surf->lod_index = 0; + surf->sort.lod_index = 0; if (p_render_data->render_info) { uint32_t to_draw = mesh_storage->mesh_surface_get_vertices_drawn_count(surf->surface); to_draw = _indices_to_primitives(surf->primitive, to_draw); @@ -2312,7 +2299,6 @@ void RenderForwardMobile::_render_list_template(RenderingDevice::DrawListID p_dr mesh_storage->mesh_surface_get_vertex_arrays_and_format(mesh_surface, input_mask, false, vertex_array_rd, vertex_format); } - index_array_rd = mesh_storage->mesh_surface_get_index_array(mesh_surface, element_info.lod_index); pipeline_key.vertex_format_id = vertex_format; if (pipeline_key.ubershader) { @@ -2640,11 +2626,9 @@ void RenderForwardMobile::_geometry_instance_add_surface_with_material(GeometryI sdcache->sort.sort_key2 = 0; sdcache->sort.surface_index = p_surface; - sdcache->sort.material_id_low = p_material_id & 0x0000FFFF; - sdcache->sort.material_id_hi = p_material_id >> 16; + sdcache->sort.material_id = p_material_id; sdcache->sort.shader_id = p_shader_id; sdcache->sort.geometry_id = p_mesh.get_local_index(); - // sdcache->sort.uses_forward_gi = ginstance->can_sdfgi; sdcache->sort.priority = p_material->priority; uint64_t format = RendererRD::MeshStorage::get_singleton()->mesh_surface_get_format(sdcache->surface); diff --git a/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.h b/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.h index 6e4d33520e..70297fa5dd 100644 --- a/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.h +++ b/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.h @@ -220,6 +220,48 @@ private: float compressed_aabb_position[4]; float compressed_aabb_size[4]; float uv_scale[4]; + + // These setters allow us to copy the data over with operation when using floats. + inline void set_lightmap_uv_scale(const Rect2 &p_rect) { +#ifdef REAL_T_IS_DOUBLE + lightmap_uv_scale[0] = p_rect.position.x; + lightmap_uv_scale[1] = p_rect.position.y; + lightmap_uv_scale[2] = p_rect.size.x; + lightmap_uv_scale[3] = p_rect.size.y; +#else + Rect2 *rect = reinterpret_cast(lightmap_uv_scale); + *rect = p_rect; +#endif + } + + inline void set_compressed_aabb(const AABB &p_aabb) { +#ifdef REAL_T_IS_DOUBLE + compressed_aabb_position[0] = p_aabb.position.x; + compressed_aabb_position[1] = p_aabb.position.y; + compressed_aabb_position[2] = p_aabb.position.z; + + compressed_aabb_size[0] = p_aabb.size.x; + compressed_aabb_size[1] = p_aabb.size.y; + compressed_aabb_size[2] = p_aabb.size.z; +#else + Vector3 *compressed_aabb_position_vec3 = reinterpret_cast(compressed_aabb_position); + Vector3 *compressed_aabb_size_vec3 = reinterpret_cast(compressed_aabb_size); + *compressed_aabb_position_vec3 = p_aabb.position; + *compressed_aabb_size_vec3 = p_aabb.size; +#endif + } + + inline void set_uv_scale(const Vector4 &p_uv_scale) { +#ifdef REAL_T_IS_DOUBLE + uv_scale[0] = p_uv_scale.x; + uv_scale[1] = p_uv_scale.y; + uv_scale[2] = p_uv_scale.z; + uv_scale[3] = p_uv_scale.w; +#else + Vector4 *uv_scale_vec4 = reinterpret_cast(uv_scale); + *uv_scale_vec4 = p_uv_scale; +#endif + } }; RID instance_buffer[RENDER_LIST_MAX]; @@ -322,9 +364,14 @@ private: }; struct RenderElementInfo { - uint32_t uses_lightmap : 1; - uint32_t lod_index : 8; - uint32_t reserved : 23; + union { + struct { + uint32_t lod_index : 8; + uint32_t uses_lightmap : 1; + uint32_t reserved : 23; + }; + uint32_t value; + }; }; template @@ -402,27 +449,22 @@ protected: uint64_t sort_key2; }; struct { - // !BAS! CHECK BITS!!! - - uint64_t surface_index : 10; - uint64_t geometry_id : 32; - uint64_t material_id_low : 16; - - uint64_t material_id_hi : 16; - uint64_t shader_id : 32; - uint64_t uses_lightmap : 4; // sort by lightmap id here, not whether its yes/no (is 4 bits enough?) + uint64_t lod_index : 8; + uint64_t uses_lightmap : 1; + uint64_t pad : 3; uint64_t depth_layer : 4; + uint64_t surface_index : 8; uint64_t priority : 8; + uint64_t geometry_id : 32; - // uint64_t lod_index : 8; // no need to sort on LOD - // uint64_t uses_forward_gi : 1; // no GI here, remove + uint64_t material_id : 32; + uint64_t shader_id : 32; }; } sort; RS::PrimitiveType primitive = RS::PRIMITIVE_MAX; uint32_t flags = 0; uint32_t surface_index = 0; - uint32_t lod_index = 0; void *surface = nullptr; RID material_uniform_set;