Collide RealityKit modello 3D con mesh LiDAR

Question

Ho passato molti giorni cercando di capire e seguire gli esempi senza successo. Il mio obiettivo è posizionare un oggetto AR virtuale nel mondo reale scansionato in precedenza con LiDAR. Con showSceneUnderstandingposso vedere la mesh in tempo reale creata ok così bene. Con una funzione tap posso inserire un file usdz, anche questo va bene. Perché ho toyModel.physicsBody?.mode = .kinematice self.arView.installGestures(for: toyRobot)posso spostare / ridimensionare il modello. Ora voglio essere in grado di muovere il modello E scontrarsi con la mesh generata dal LiDAR. Quando sposto il modello su una parete scansionata, la mesh viene interrotta, ad esempio.

Ecco il mio codice completo:

import UIKit
import RealityKit
import ARKit

class ViewController: UIViewController, ARSessionDelegate {
    
    @IBOutlet var arView: ARView!
    var tapRecognizer = UITapGestureRecognizer()
    
    override func viewDidLoad() {
        super.viewDidLoad()
        
        self.arView.session.delegate = self
        
        //Scene Understanding options
        self.arView.environment.sceneUnderstanding.options.insert([.physics, .collision, .occlusion])
        
        //Only for dev
        self.arView.debugOptions.insert(.showSceneUnderstanding)
        
        self.tapRecognizer = UITapGestureRecognizer(target: self, action: #selector(placeObject(_:)))
        self.arView.addGestureRecognizer(self.tapRecognizer)
        
    }
    
    @objc func placeObject(_ sender: UITapGestureRecognizer) {
        
        // Perform a ray cast against the mesh (sceneUnderstanding)
        // Note: Ray-cast option ".estimatedPlane" with alignment ".any" also takes the mesh into account.
        let tapLocation = sender.location(in: arView)
        if let result = arView.raycast(from: tapLocation, allowing: .estimatedPlane, alignment: .any).first {
            
            // Load the "Toy robot"
            let toyRobot = try! ModelEntity.loadModel(named: "toy_robot_vintage.usdz")
                
            // Add gestures to the toy (only available is physicsBody mode == kinematic)
            self.arView.installGestures(for: toyRobot)
            
            // Toy Anchor to place the toy on surface
            let toyAnchor = AnchorEntity(world: result.worldTransform)
                toyAnchor.addChild(toyRobot)
            
            // Create a "Physics" model of the toy in order to add physics mode
            guard let toyModel = toyAnchor.children.first as? HasPhysics else {
                return
            }
            
            // Because toyModel is a fresh new model we need to init physics
            toyModel.generateCollisionShapes(recursive: true)
            toyModel.physicsBody = .init()
            
            // Add the physics body mode
            toyModel.physicsBody?.mode = .kinematic
            
            let test = ShapeResource.generateConvex(from: toyRobot.model!.mesh)
            
            toyModel.components[CollisionComponent] = CollisionComponent(shapes: [test], mode: .default, filter: .default)
            
            // Finally add the toy anchor to the scene
            self.arView.scene.addAnchor(toyAnchor)
        }
        
    }
}

Qualcuno sa se è possibile farlo? Molte grazie in anticipo!

Answer 1

Prova il downcasting a Entity & HasPhysics:

guard let model = anchor.children.first as? (Entity & HasPhysics) 
else { return }

model.generateCollisionShapes(recursive: true)
model.physicsBody = PhysicsBodyComponent(shapes: [.generateBox(size: .one)],
                                           mass: 1.0, 
                                       material: .default, 
                                           mode: .kinematic)

La documentazione ufficiale dice :

Per le entità non modello , ilgenerateCollisionShapes(recursive:)metodo non ha effetto. Tuttavia, il metodo è definito per tutte le entità in modo che tu possa chiamarlo su qualsiasi entità e fare in modo che il calcolo si propaghi ricorsivamente a tutti i discendenti di quell'entità.

Answer 2

In seguito alla discussione precedente con @AndyFedoroff ho aggiunto il raycast convesso in modo da far collidere sempre l'oggetto 3d posizionato con la mesh creata da LiDAR. Ecco il mio codice completo. Non so se sto bene ... In ogni caso ancora non funziona.

import UIKit
import RealityKit
import ARKit

class ViewController: UIViewController, ARSessionDelegate {
    
    @IBOutlet var arView: ARView!
    
    var tapRecognizer = UITapGestureRecognizer()
    var panGesture = UIPanGestureRecognizer()
    var panGestureEntity: Entity? = nil
    
    override func viewDidLoad() {
        super.viewDidLoad()
        
        self.arView.session.delegate = self
        
        //Scene Understanding options
        self.arView.environment.sceneUnderstanding.options.insert([.physics, .collision, .occlusion])
        
        //Only for dev
        self.arView.debugOptions.insert(.showSceneUnderstanding)
        
        self.tapRecognizer = UITapGestureRecognizer(target: self, action: #selector(self.placeObject))
        self.arView.addGestureRecognizer(self.tapRecognizer)
        
        self.panGesture = UIPanGestureRecognizer(target: self, action: #selector(self.didPan))
        self.arView.addGestureRecognizer(self.panGesture)
    }
    
    @objc func placeObject(_ sender: UITapGestureRecognizer) {
        
        // Perform a ray cast against the mesh (sceneUnderstanding)
        // Note: Ray-cast option ".estimatedPlane" with alignment ".any" also takes the mesh into account.
        let tapLocation = sender.location(in: arView)
        if let result = self.arView.raycast(from: tapLocation, allowing: .estimatedPlane, alignment: .any).first {

            // Load the "Toy robot"
            let toyRobot = try! ModelEntity.loadModel(named: "toy_robot_vintage.usdz")

            // Add gestures to the toy (only available is physicsBody mode == kinematic)
            self.arView.installGestures(for: toyRobot)

            // Toy Anchor to place the toy on surface
            let toyAnchor = AnchorEntity(world: result.worldTransform)
                toyAnchor.addChild(toyRobot)

            // Create a "Physics" model of the toy in order to add physics mode
            guard let model = toyAnchor.children.first as? (Entity & HasPhysics)
            else { return }

            model.generateCollisionShapes(recursive: true)
            model.physicsBody = PhysicsBodyComponent(shapes: [.generateBox(size: .one)],
                                                       mass: 1.0,
                                                   material: .default,
                                                   mode: .kinematic)

            // Finally add the toy anchor to the scene
            self.arView.scene.addAnchor(toyAnchor)
            
        }
    
    }
    
    @objc public func didPan(_ sender: UIPanGestureRecognizer) {
        print(sender.state)
        let point = sender.location(in: self.arView)
        switch sender.state {
        case .began:
            if let entity = self.arView.hitTest(point).first?.entity {
                self.panGestureEntity = entity
            } else {
                self.panGestureEntity = nil
            }
        case .changed:
            if let entity = self.panGestureEntity {
                if let raycast = arView
                    .scene
                    .raycast(origin: .zero, direction: .one, length: 1, query: .all, mask: .all, relativeTo: entity)
                    .first
                {
                    print("hit", raycast.entity, raycast.distance)
                }

          }
        case .ended:
            self.panGestureEntity = nil
        default: break;
        }

    }
    
    
}