Jolt Physics: Swapping vertices of triangle if it is scaled inside out

Fixed an issue where collide/cast shape against a triangle would return a hit result with mShape2Face in incorrect winding order. This caused an incorrect normal in the enhanced internal edge removal algorithm. This in turn resulted in objects not settling properly on dense triangle grids.

thirdparty/jolt_physics/Jolt/Physics/Collision/CastConvexVsTriangles.cpp

@@ -92,11 +92,14 @@ void CastConvexVsTriangles::Cast(Vec3Arg inV0, Vec3Arg inV1, Vec3Arg inV2, uint8
 			static_cast<const ConvexShape *>(mShapeCast.mShape)->GetSupportingFace(SubShapeID(), transform_1_to_2.Multiply3x3Transposed(-contact_normal), mShapeCast.mScale, mCenterOfMassTransform2 * transform_1_to_2, result.mShape1Face);
 
 			// Get face of the triangle
-			triangle.GetSupportingFace(contact_normal, result.mShape2Face);
+			result.mShape2Face.resize(3);
+			result.mShape2Face[0] = mCenterOfMassTransform2 * v0;
+			result.mShape2Face[1] = mCenterOfMassTransform2 * v1;
+			result.mShape2Face[2] = mCenterOfMassTransform2 * v2;
 
-			// Convert to world space
-			for (Vec3 &p : result.mShape2Face)
-				p = mCenterOfMassTransform2 * p;
+			// When inside out, we need to swap the triangle winding
+			if (mScaleSign < 0.0f)
+				std::swap(result.mShape2Face[1], result.mShape2Face[2]);
 		}
 
 		JPH_IF_TRACK_NARROWPHASE_STATS(TrackNarrowPhaseCollector track;)

thirdparty/jolt_physics/Jolt/Physics/Collision/CollideConvexVsTriangles.cpp

@@ -145,6 +145,10 @@ void CollideConvexVsTriangles::Collide(Vec3Arg inV0, Vec3Arg inV1, Vec3Arg inV2,
 		result.mShape2Face[0] = mTransform1 * v0;
 		result.mShape2Face[1] = mTransform1 * v1;
 		result.mShape2Face[2] = mTransform1 * v2;
+
+		// When inside out, we need to swap the triangle winding
+		if (mScaleSign2 < 0.0f)
+			std::swap(result.mShape2Face[1], result.mShape2Face[2]);
 	}
 
 	// Notify the collector

thirdparty/jolt_physics/Jolt/Physics/Collision/CollideSphereVsTriangles.cpp

@@ -111,6 +111,10 @@ void CollideSphereVsTriangles::Collide(Vec3Arg inV0, Vec3Arg inV1, Vec3Arg inV2,
 		result.mShape2Face[0] = mTransform2 * (mSphereCenterIn2 + v0);
 		result.mShape2Face[1] = mTransform2 * (mSphereCenterIn2 + v1);
 		result.mShape2Face[2] = mTransform2 * (mSphereCenterIn2 + v2);
+
+		// When inside out, we need to swap the triangle winding
+		if (mScaleSign2 < 0.0f)
+			std::swap(result.mShape2Face[1], result.mShape2Face[2]);
 	}
 
 	// Notify the collector