Modelo 3D do Collide RealityKit com malha LiDAR

Question

Passei muitos dias tentando entender e seguir exemplos sem sucesso. Meu objetivo é colocar um objeto AR virtual no mundo real digitalizado anteriormente com o LiDAR. Com o showSceneUnderstandingeu posso ver a malha em tempo real criada ok que bom. Com uma função de toque, posso inserir um arquivo usdz, que também está bom. Porque eu tenho toyModel.physicsBody?.mode = .kinematice self.arView.installGestures(for: toyRobot)posso mover / dimensionar o modelo. Agora você deseja mover o modelo E colidir com a malha gerada pelo LiDAR. Quando movo o modelo para uma parede digitalizada, a malha é interrompida, por exemplo.

Aqui está meu código completo:

import UIKit
import RealityKit
import ARKit

class ViewController: UIViewController, ARSessionDelegate {
    
    @IBOutlet var arView: ARView!
    var tapRecognizer = UITapGestureRecognizer()
    
    override func viewDidLoad() {
        super.viewDidLoad()
        
        self.arView.session.delegate = self
        
        //Scene Understanding options
        self.arView.environment.sceneUnderstanding.options.insert([.physics, .collision, .occlusion])
        
        //Only for dev
        self.arView.debugOptions.insert(.showSceneUnderstanding)
        
        self.tapRecognizer = UITapGestureRecognizer(target: self, action: #selector(placeObject(_:)))
        self.arView.addGestureRecognizer(self.tapRecognizer)
        
    }
    
    @objc func placeObject(_ sender: UITapGestureRecognizer) {
        
        // Perform a ray cast against the mesh (sceneUnderstanding)
        // Note: Ray-cast option ".estimatedPlane" with alignment ".any" also takes the mesh into account.
        let tapLocation = sender.location(in: arView)
        if let result = arView.raycast(from: tapLocation, allowing: .estimatedPlane, alignment: .any).first {
            
            // Load the "Toy robot"
            let toyRobot = try! ModelEntity.loadModel(named: "toy_robot_vintage.usdz")
                
            // Add gestures to the toy (only available is physicsBody mode == kinematic)
            self.arView.installGestures(for: toyRobot)
            
            // Toy Anchor to place the toy on surface
            let toyAnchor = AnchorEntity(world: result.worldTransform)
                toyAnchor.addChild(toyRobot)
            
            // Create a "Physics" model of the toy in order to add physics mode
            guard let toyModel = toyAnchor.children.first as? HasPhysics else {
                return
            }
            
            // Because toyModel is a fresh new model we need to init physics
            toyModel.generateCollisionShapes(recursive: true)
            toyModel.physicsBody = .init()
            
            // Add the physics body mode
            toyModel.physicsBody?.mode = .kinematic
            
            let test = ShapeResource.generateConvex(from: toyRobot.model!.mesh)
            
            toyModel.components[CollisionComponent] = CollisionComponent(shapes: [test], mode: .default, filter: .default)
            
            // Finally add the toy anchor to the scene
            self.arView.scene.addAnchor(toyAnchor)
        }
        
    }
}

Alguém sabe se é possível conseguir isso? Muito obrigado antecipadamente!

Answer 1

Tente reduzir para Entity & HasPhysics:

guard let model = anchor.children.first as? (Entity & HasPhysics) 
else { return }

model.generateCollisionShapes(recursive: true)
model.physicsBody = PhysicsBodyComponent(shapes: [.generateBox(size: .one)],
                                           mass: 1.0, 
                                       material: .default, 
                                           mode: .kinematic)

A documentação oficial diz :

Para entidades não modelo , ogenerateCollisionShapes(recursive:)método não tem efeito. No entanto, o método é definido para todas as entidades de forma que você possa chamá-lo em qualquer entidade e fazer com que o cálculo se propague recursivamente para todos os descendentes dessa entidade.

Answer 2

Seguindo a discussão anterior com @AndyFedoroff eu adicionei raycast convext para, sempre, colidir o objeto 3d colocado com a malha criada por LiDAR. Aqui está meu código completo. Não sei se estou bem ... Em todo caso ainda não funciona.

import UIKit
import RealityKit
import ARKit

class ViewController: UIViewController, ARSessionDelegate {
    
    @IBOutlet var arView: ARView!
    
    var tapRecognizer = UITapGestureRecognizer()
    var panGesture = UIPanGestureRecognizer()
    var panGestureEntity: Entity? = nil
    
    override func viewDidLoad() {
        super.viewDidLoad()
        
        self.arView.session.delegate = self
        
        //Scene Understanding options
        self.arView.environment.sceneUnderstanding.options.insert([.physics, .collision, .occlusion])
        
        //Only for dev
        self.arView.debugOptions.insert(.showSceneUnderstanding)
        
        self.tapRecognizer = UITapGestureRecognizer(target: self, action: #selector(self.placeObject))
        self.arView.addGestureRecognizer(self.tapRecognizer)
        
        self.panGesture = UIPanGestureRecognizer(target: self, action: #selector(self.didPan))
        self.arView.addGestureRecognizer(self.panGesture)
    }
    
    @objc func placeObject(_ sender: UITapGestureRecognizer) {
        
        // Perform a ray cast against the mesh (sceneUnderstanding)
        // Note: Ray-cast option ".estimatedPlane" with alignment ".any" also takes the mesh into account.
        let tapLocation = sender.location(in: arView)
        if let result = self.arView.raycast(from: tapLocation, allowing: .estimatedPlane, alignment: .any).first {

            // Load the "Toy robot"
            let toyRobot = try! ModelEntity.loadModel(named: "toy_robot_vintage.usdz")

            // Add gestures to the toy (only available is physicsBody mode == kinematic)
            self.arView.installGestures(for: toyRobot)

            // Toy Anchor to place the toy on surface
            let toyAnchor = AnchorEntity(world: result.worldTransform)
                toyAnchor.addChild(toyRobot)

            // Create a "Physics" model of the toy in order to add physics mode
            guard let model = toyAnchor.children.first as? (Entity & HasPhysics)
            else { return }

            model.generateCollisionShapes(recursive: true)
            model.physicsBody = PhysicsBodyComponent(shapes: [.generateBox(size: .one)],
                                                       mass: 1.0,
                                                   material: .default,
                                                   mode: .kinematic)

            // Finally add the toy anchor to the scene
            self.arView.scene.addAnchor(toyAnchor)
            
        }
    
    }
    
    @objc public func didPan(_ sender: UIPanGestureRecognizer) {
        print(sender.state)
        let point = sender.location(in: self.arView)
        switch sender.state {
        case .began:
            if let entity = self.arView.hitTest(point).first?.entity {
                self.panGestureEntity = entity
            } else {
                self.panGestureEntity = nil
            }
        case .changed:
            if let entity = self.panGestureEntity {
                if let raycast = arView
                    .scene
                    .raycast(origin: .zero, direction: .one, length: 1, query: .all, mask: .all, relativeTo: entity)
                    .first
                {
                    print("hit", raycast.entity, raycast.distance)
                }

          }
        case .ended:
            self.panGestureEntity = nil
        default: break;
        }

    }
    
    
}