buggy - add custom btOverlapFilterCallback

5 files changed, 93 insertions, 7 deletions

diff --git a/src/components.hpp b/src/components.hpp
index 0de9592..4223845 100644
--- a/src/components.hpp
+++ b/src/components.hpp
@@ -1,8 +1,9 @@
 #ifndef PKE_COMPONENTS_HPP
 #define PKE_COMPONENTS_HPP
 
-#include "macros.hpp"
 #include "dynamic-array.hpp"
+#include "macros.hpp"
+#include "physics.hpp"
 
 #include <BulletDynamics/Dynamics/btRigidBody.h>
 #include <LinearMath/btDefaultMotionState.h>
@@ -76,6 +77,8 @@ struct CompInstance {
 	GrBindsHandle grBindsHandle = GrBindsHandle_MAX;
 	InstanceHandle instanceHandle = InstanceHandle_MAX;
 	uint64_t index = ECS_UNSET_VAL;
+	PhysicsCollision physicsLayer = PhysicsCollision{1};
+	PhysicsCollision physicsMask = PhysicsCollision{1};
 	InstBt bt;
 	bool isNeedingUpdated = false;
 };
diff --git a/src/entities.hpp b/src/entities.hpp
index 522fe68..85feb04 100644
--- a/src/entities.hpp
+++ b/src/entities.hpp
@@ -35,6 +35,8 @@ struct EntityType {
 	struct {
 		btCollisionShape *shape = nullptr;
 		btScalar startingMass = 1.f;
+		PhysicsCollision startingCollisionLayer = PhysicsCollision{1};
+		PhysicsCollision startingCollisionMask = PhysicsCollision{1};
 	} bt;
 };
 extern DynArray<EntityType> GlobalEntityTypes;
diff --git a/src/game.cpp b/src/game.cpp
index ea0c3bc..d43e366 100644
--- a/src/game.cpp
+++ b/src/game.cpp
@@ -1,7 +1,6 @@
 
 #include "game.hpp"
 
-#include "BulletCollision/CollisionShapes/btConvexHullShape.h"
 #include "camera.hpp"
 #include "components.hpp"
 #include "dynamic-array.hpp"
@@ -15,6 +14,7 @@
 #include "vendor/glm_include.hpp"
 #include "window.hpp"
 
+#include <BulletCollision/CollisionShapes/btConvexHullShape.h>
 #include <GLFW/glfw3.h>
 #include <cstring>
 #include <glm/gtc/quaternion.hpp>
@@ -79,13 +79,17 @@ const char *PKE_FILE_ENTITY_TYPE_IMPORTER_GLTF_ACCESSOR_INDEX_NORMAL = "Importer
 const char *PKE_FILE_ENTITY_TYPE_IMPORTER_GLTF_ACCESSOR_INDEX_UV = "Importer_GLTF::AccessorIndexUV: ";
 const char *PKE_FILE_ENTITY_TYPE_IMPORTER_GLTF_ACCESSOR_INDEX_INDEX = "Importer_GLTF::AccessorIndexIndex: ";
 const char *PKE_FILE_ENTITY_TYPE_PHYSICS_STARTING_MASS = "BT::StartingMass: ";
+const char *PKE_FILE_ENTITY_TYPE_PHYSICS_STARTING_COLLISION_LAYER = "BT::StartingCollisionLayer: ";
+const char *PKE_FILE_ENTITY_TYPE_PHYSICS_STARTING_COLLISION_MASK = "BT::StartingCollisionMask: ";
 
 const char *PKE_FILE_INSTANCE_ENTITY_HANDLE = "EntityHandle: ";
 const char *PKE_FILE_INSTANCE_ENTITY_TYPE_CODE = "EntityTypeCode: ";
 const char *PKE_FILE_INSTANCE_POS_POS = "InstPos::Pos: ";
 const char *PKE_FILE_INSTANCE_POS_ROT = "InstPos::Rot: ";
 const char *PKE_FILE_INSTANCE_POS_SCALE = "InstPos::Scale: ";
-const char *PKE_FILE_INSTANCE_MASS = "InstPos::Mass: ";
+const char *PKE_FILE_INSTANCE_PHYSICS_MASS = "InstPos::Mass: ";
+const char *PKE_FILE_INSTANCE_PHYSICS_COLLISION_LAYER = "InstPos::CollisionLayer: ";
+const char *PKE_FILE_INSTANCE_PHYSICS_COLLISION_MASK = "InstPos::CollisionMask: ";
 
 char consoleBuffer[consoleBufferCount][consoleLineLength];
 long consoleBufferIndex = 0;
@@ -123,6 +127,12 @@ void SerializeEntityType(std::ofstream &stream, const EntityType &et) {
 	if (et.bt.startingMass != e.bt.startingMass) {
 		stream << PKE_FILE_ENTITY_TYPE_PHYSICS_STARTING_MASS << et.bt.startingMass << std::endl;
 	}
+	if (et.bt.startingCollisionLayer != e.bt.startingCollisionLayer) {
+		stream << PKE_FILE_ENTITY_TYPE_PHYSICS_STARTING_COLLISION_LAYER << static_cast<PhysicsCollision_T>(et.bt.startingCollisionLayer) << std::endl;
+	}
+	if (et.bt.startingCollisionMask != e.bt.startingCollisionMask) {
+		stream << PKE_FILE_ENTITY_TYPE_PHYSICS_STARTING_COLLISION_MASK << static_cast<PhysicsCollision_T>(et.bt.startingCollisionMask) << std::endl;
+	}
 }
 
 void SerializeInstance(std::ofstream &stream, const CompInstance &comp) {
@@ -149,6 +159,8 @@ void SerializeInstance(std::ofstream &stream, const CompInstance &comp) {
 	comp.bt.motionState->getWorldTransform(trans);
 	btVector3 scale = comp.bt.rigidBody->getCollisionShape()->getLocalScaling();
 	btScalar mass = comp.bt.rigidBody->getMass();
+	PhysicsCollision collisionLayer{static_cast<PhysicsCollision_T>(comp.bt.rigidBody->getBroadphaseProxy()->m_collisionFilterGroup)};
+	PhysicsCollision collisionMask{static_cast<PhysicsCollision_T>(comp.bt.rigidBody->getBroadphaseProxy()->m_collisionFilterMask)};
 	if (trans != baseInst.posRot) {
 		btVector3 pos = trans.getOrigin();
 		btQuaternion rot = trans.getRotation();
@@ -168,7 +180,13 @@ void SerializeInstance(std::ofstream &stream, const CompInstance &comp) {
 			<< std::setw(10) << scale[1] << ","
 			<< std::setw(10) << scale[2] << "]" << std::endl;
 	if (mass != baseInst.mass) {
-		stream << PKE_FILE_INSTANCE_MASS << mass << std::endl;
+		stream << PKE_FILE_INSTANCE_PHYSICS_MASS << mass << std::endl;
+	}
+	if (collisionLayer != c.physicsLayer) {
+		stream << PKE_FILE_INSTANCE_PHYSICS_COLLISION_LAYER << static_cast<PhysicsCollision_T>(collisionLayer) << std::endl;
+	}
+	if (collisionMask != c.physicsMask) {
+		stream << PKE_FILE_INSTANCE_PHYSICS_COLLISION_MASK << static_cast<PhysicsCollision_T>(collisionMask) << std::endl;
 	}
 }
 
@@ -256,6 +274,22 @@ void ParseEntityType(std::ifstream &stream) {
 			assert(result == STR2NUM_ERROR::SUCCESS);
 			continue;
 		}
+		if (strstr(readLine, PKE_FILE_ENTITY_TYPE_PHYSICS_STARTING_COLLISION_LAYER)) {
+			uint64_t prefixLen = strlen(PKE_FILE_ENTITY_TYPE_PHYSICS_STARTING_COLLISION_LAYER);
+			PhysicsCollision_T val = static_cast<PhysicsCollision_T>(et.bt.startingCollisionLayer);
+			STR2NUM_ERROR result = str2num(val, readLine + prefixLen);
+			et.bt.startingCollisionLayer = PhysicsCollision{val};
+			assert(result == STR2NUM_ERROR::SUCCESS);
+			continue;
+		}
+		if (strstr(readLine, PKE_FILE_ENTITY_TYPE_PHYSICS_STARTING_COLLISION_MASK)) {
+			uint64_t prefixLen = strlen(PKE_FILE_ENTITY_TYPE_PHYSICS_STARTING_COLLISION_MASK);
+			PhysicsCollision_T val = static_cast<PhysicsCollision_T>(et.bt.startingCollisionMask);
+			STR2NUM_ERROR result = str2num(val, readLine + prefixLen);
+			et.bt.startingCollisionMask = PhysicsCollision{val};
+			assert(result == STR2NUM_ERROR::SUCCESS);
+			continue;
+		}
 	}
 }
 
@@ -293,6 +327,9 @@ void ParseInstance(std::ifstream &stream) {
 			compInst.bt.rigidBody->setAngularVelocity(btVector3(0,0,0));
 			compInst.bt.rigidBody->getCollisionShape()->setLocalScaling(instPos.scale);
 			BtDynamicsWorld->addRigidBody(compInst.bt.rigidBody);
+			auto *broadphaseProxy = compInst.bt.rigidBody->getBroadphaseProxy();
+			broadphaseProxy->m_collisionFilterGroup = static_cast<PhysicsCollision_T>(comp.physicsLayer);
+			broadphaseProxy->m_collisionFilterMask = static_cast<PhysicsCollision_T>(comp.physicsMask);
 			compInst.bt.rigidBody->setUserPointer(reinterpret_cast<void *>(compInst.entHandle));
 			break;
 		}
@@ -359,12 +396,28 @@ void ParseInstance(std::ifstream &stream) {
 			} while (*pEnd != ']');
 			continue;
 		}
-		if (strstr(readLine, PKE_FILE_INSTANCE_MASS)) {
-			uint64_t prefixLen = strlen(PKE_FILE_INSTANCE_MASS);
+		if (strstr(readLine, PKE_FILE_INSTANCE_PHYSICS_MASS)) {
+			uint64_t prefixLen = strlen(PKE_FILE_INSTANCE_PHYSICS_MASS);
 			STR2NUM_ERROR result = str2num(instPos.mass, readLine + prefixLen);
 			assert(result == STR2NUM_ERROR::SUCCESS);
 			continue;
 		}
+		if (strstr(readLine, PKE_FILE_INSTANCE_PHYSICS_COLLISION_LAYER)) {
+			uint64_t prefixLen = strlen(PKE_FILE_INSTANCE_PHYSICS_COLLISION_LAYER);
+			PhysicsCollision_T val = static_cast<PhysicsCollision_T>(comp.physicsLayer);
+			STR2NUM_ERROR result = str2num(val, readLine + prefixLen);
+			comp.physicsLayer = PhysicsCollision{val};
+			assert(result == STR2NUM_ERROR::SUCCESS);
+			continue;
+		}
+		if (strstr(readLine, PKE_FILE_INSTANCE_PHYSICS_COLLISION_MASK)) {
+			uint64_t prefixLen = strlen(PKE_FILE_INSTANCE_PHYSICS_COLLISION_MASK);
+			PhysicsCollision_T val = static_cast<PhysicsCollision_T>(comp.physicsMask);
+			STR2NUM_ERROR result = str2num(val, readLine + prefixLen);
+			comp.physicsMask = PhysicsCollision{val};
+			assert(result == STR2NUM_ERROR::SUCCESS);
+			continue;
+		}
 	}
 }
 
@@ -493,6 +546,8 @@ void Game_Tick(double delta) {
 		compInst.bt.rigidBody->setAngularVelocity(btVector3(0,0,0));
 		compInst.bt.rigidBody->getCollisionShape()->setLocalScaling(btVector3(1, 1, 1));
 		BtDynamicsWorld->addRigidBody(compInst.bt.rigidBody);
+		compInst.bt.rigidBody->getBroadphaseProxy()->m_collisionFilterGroup = static_cast<PhysicsCollision_T>(et.bt.startingCollisionLayer);
+		compInst.bt.rigidBody->getBroadphaseProxy()->m_collisionFilterMask = static_cast<PhysicsCollision_T>(et.bt.startingCollisionMask);
 		compInst.bt.rigidBody->setUserPointer(reinterpret_cast<void *>(compInst.entHandle));
 	}
 
@@ -783,6 +838,8 @@ void RecordImGuiModalCreateEntityType() {
 		ImGui::InputScalar("GLTF Import Index - UV", ImGuiDataType_S16, &entityTypeToCreate->Importer_GLTF.AccessorIndexUV);
 		ImGui::InputScalar("GLTF Import Index - Index", ImGuiDataType_S16, &entityTypeToCreate->Importer_GLTF.AccessorIndexIndex);
 		ImGui::InputFloat("Physics - Mass", &entityTypeToCreate->bt.startingMass);
+		ImGui::InputScalar("Physics - Collision Layer", ImGuiDataType_U16, &entityTypeToCreate->bt.startingCollisionLayer);
+		ImGui::InputScalar("Physics - Collision Mask", ImGuiDataType_U16, &entityTypeToCreate->bt.startingCollisionMask);
 
 		ImGui::Separator();
 
@@ -898,6 +955,9 @@ void RecordImGui_CompInstPos(bool readonly, CompInstance *component) {
 	changed = ImGui::InputScalarN("scale", ImGuiDataType_Float, &instPos.scale, 3, nullptr, nullptr, nullptr, inputTextFlags) || changed;
 	changed = ImGui::InputFloat("mass", &instPos.mass, 0.0, 0.0, "%.3f", inputTextFlags) || changed;
 
+	changed = ImGui::InputScalar("Phys - Collision Layer", ImGuiDataType_U16, &component->physicsLayer, nullptr, nullptr, nullptr, inputTextFlags) || changed;
+	changed = ImGui::InputScalar("Phys - Collision Mask", ImGuiDataType_U16, &component->physicsMask, nullptr, nullptr, nullptr, inputTextFlags) || changed;
+
 	ImGui::InputScalar("Phys - Rigid Body", ImGuiDataType_U64, &component->bt.rigidBody, nullptr, nullptr, "0x%016lX", ImGuiInputTextFlags_ReadOnly);
 	ImGui::InputScalar("Phys - Motion State", ImGuiDataType_U64, &component->bt.motionState, nullptr, nullptr, "0x%016lX", ImGuiInputTextFlags_ReadOnly);
 
diff --git a/src/physics.cpp b/src/physics.cpp
index b8d093d..f0d8b8e 100644
--- a/src/physics.cpp
+++ b/src/physics.cpp
@@ -1,11 +1,16 @@
 
 #include "physics.hpp"
 
+#include "components.hpp"
 #include "dynamic-array.hpp"
 #include "game-settings.hpp"
 
-#include <btBulletDynamicsCommon.h>
 #include <LinearMath/btAlignedAllocator.h>
+#include <btBulletDynamicsCommon.h>
+#include <BulletCollision/BroadphaseCollision/btOverlappingPairCache.h>
+#include <BulletDynamics/Dynamics/btRigidBody.h>
+
+TypeSafeInt_B(PhysicsCollision);
 
 MemBucket *MemBkt_Bullet = nullptr;
 btDiscreteDynamicsWorld *BtDynamicsWorld = nullptr;
@@ -60,6 +65,14 @@ void pke_btFreeFunc(void *memBlock) {
 	bulletAllocs->Remove(index);
 }
 
+struct CollisionHandlerStruct : public btOverlapFilterCallback {
+	~CollisionHandlerStruct() override {}
+	bool needBroadphaseCollision(btBroadphaseProxy* proxy0, btBroadphaseProxy* proxy1) const override {
+		return (proxy0->m_collisionFilterGroup & proxy1->m_collisionFilterMask) ||
+			(proxy1->m_collisionFilterGroup & proxy0->m_collisionFilterMask);
+	}
+} collisionHandlerStruct;
+
 void Physics_Init() {
 	MemBkt_Bullet = Pke_BeginTransientBucket();
 	bulletAllocs = Pke_New<DynArray<AllocedData>>(MemBkt_Bullet);
@@ -71,8 +84,13 @@ void Physics_Init() {
 	btConfiguration = Pke_New<btDefaultCollisionConfiguration>(MemBkt_Bullet);
 	btDispatcher = Pke_New<btCollisionDispatcher>(MemBkt_Bullet);
 	new (btDispatcher) btCollisionDispatcher(btConfiguration);
+
+	btHashedOverlappingPairCache *overlappingPairCache = Pke_New<btHashedOverlappingPairCache>(MemBkt_Bullet);
+	overlappingPairCache->setOverlapFilterCallback(&collisionHandlerStruct);
 	btBroadphase = Pke_New<btDbvtBroadphase>(MemBkt_Bullet);
+	new (btBroadphase) btDbvtBroadphase(overlappingPairCache);
 	btSolver = Pke_New<btSequentialImpulseConstraintSolver>(MemBkt_Bullet);
+
 	BtDynamicsWorld = Pke_New<btDiscreteDynamicsWorld>(MemBkt_Bullet);
 	new (BtDynamicsWorld) btDiscreteDynamicsWorld(btDispatcher, btBroadphase, btSolver, btConfiguration);
 	auto grav = BtDynamicsWorld->getGravity();
diff --git a/src/physics.hpp b/src/physics.hpp
index 3d41ab7..67ac60c 100644
--- a/src/physics.hpp
+++ b/src/physics.hpp
@@ -1,10 +1,13 @@
 #ifndef PKE_PHYSICS_HPP
 #define PKE_PHYSICS_HPP
 
+#include "macros.hpp"
 #include "memory.hpp"
 
 #include <BulletDynamics/Dynamics/btDiscreteDynamicsWorld.h>
 
+TypeSafeInt_H(PhysicsCollision, uint64_t, 0xFFFFFFFFFFFFFFFF);
+
 extern MemBucket *MemBkt_Bullet;
 extern btDiscreteDynamicsWorld *BtDynamicsWorld;