IOS UIkit ViewDidLayout Calls On Handle Drag

I am a new uikit developer. I have a problem on my app. App has 2 different parts. Rectangle view and button. Rectangle view can drag with pan touch gesture. Button can change own image with tap gestures. When user taps the button, it changes own image. But when I tap the button, rectangle view moves to the at the beginning position.(view did load position).

I checked logs and I see viewDidLayout methods calling when I tap the button. How can I handle this problem, is there anyone help me ? Thanks a lot.

import UIKit


final class SampleViewController: UIViewController {
    
    private var isTapped: Bool = false
    
    let button: UIButton = {
        let button = UIButton()
        let image = UIImageView(image: UIImage(systemName: "play"))
        button.translatesAutoresizingMaskIntoConstraints = false
        button.backgroundColor = .yellow
        return button
    }()
    
    let littleView: UIView = {
        let view = UIView()
        view.translatesAutoresizingMaskIntoConstraints = false
        view.layer.borderColor = UIColor.red.cgColor
        view.layer.borderWidth = 4
        view.isUserInteractionEnabled = true
        view.backgroundColor = .red
        return view
    }()
    
    
    override func viewDidLoad() {
        super.viewDidLoad()
        self.view.addSubview(button)
        self.view.addSubview(littleView)
        
        littleView.addGestureRecognizer(UIPanGestureRecognizer(target: self, action: #selector(handlePan(_:))))
        
        button.addTarget(self, action: #selector(buttonTapped), for: .touchUpInside)
        
        
        NSLayoutConstraint.activate([
            
            button.bottomAnchor.constraint(equalTo: self.view.safeAreaLayoutGuide.bottomAnchor),
            
            button.centerXAnchor.constraint(equalTo: self.view.centerXAnchor),
            littleView.heightAnchor.constraint(equalTo: self.view.heightAnchor, multiplier: 1/5),
            littleView.widthAnchor.constraint(equalTo: self.view.heightAnchor, multiplier: 1/3),
        
        ])
    }
    override func viewDidLayoutSubviews() {
        littleView.frame = CGRect(x: 0, y: 0, width: littleView.frame.width, height: littleView.frame.height)
    }
    
    override func viewWillLayoutSubviews() {
        littleView.frame = CGRect(x: 0, y: 0, width: littleView.frame.width, height: littleView.frame.height)
    }
    @objc private func buttonTapped() {
        button.setImage(UIImage(systemName: isTapped ? "play.slash" : "play"), for: .normal)
        self.view.layoutIfNeeded()
        isTapped.toggle()
        print("Button tapped")
    }
    
    @objc private func handlePan(_ gesture: UIPanGestureRecognizer) {
        guard let viewToMove = gesture.view else { return }
        if gesture.state == .began || gesture.state == .changed {
            let translation = gesture.translation(in: view)
            
            let newX = max(0, min(view.frame.width - viewToMove.frame.width, viewToMove.frame.origin.x + translation.x))
            let newY = max(0, min((view.frame.height - viewToMove.frame.height), viewToMove.frame.origin.y + translation.y))
            
            viewToMove.frame.origin = CGPoint(x: newX, y: newY)
            gesture.setTranslation(CGPoint.zero, in: view)
            
        }
    }
}

Github link