How to prevent camera from adjusting brightness in manual mode?

In case when I have locked white balance and custom exposure, on black background when I introduce new object in view, both objects become brighter. How to turn off this feature or compensate for that change in a performant way?

This is how I configure the session, note that Im setting a video format which supports at least 180 fps which is required for my needs.

private func configureSession() {
        
        self.sessionQueue.async { [self] in
            
            //MARK: Init session
            guard let session = try? validSession() else { fatalError("Session is unexpectedly nil") }
            session.beginConfiguration()
            guard let device = AVCaptureDevice.default(AVCaptureDevice.DeviceType.builtInWideAngleCamera, for:AVMediaType.video, position: .back) else { fatalError("Video Device is unexpectedly nil") }
            guard let videoDeviceInput: AVCaptureDeviceInput = try? AVCaptureDeviceInput(device:device) else { fatalError("videoDeviceInput is unexpectedly nil")  }
            guard session.canAddInput(videoDeviceInput) else { fatalError("videoDeviceInput could not be added") }
            
            session.addInput(videoDeviceInput)
            self.videoDeviceInput = videoDeviceInput
            self.videoDevice = device
           
            //MARK: Connect session IO
            let dataOutput = AVCaptureVideoDataOutput()
            dataOutput.setSampleBufferDelegate(self, queue: sampleBufferQueue)
            session.automaticallyConfiguresCaptureDeviceForWideColor = false
            guard session.canAddOutput(dataOutput) else { fatalError("Could not add video data output") }
            session.addOutput(dataOutput)
            dataOutput.alwaysDiscardsLateVideoFrames = true
            dataOutput.videoSettings = [
                    String(kCVPixelBufferPixelFormatTypeKey): pixelFormat.rawValue
            ]
            if let captureConnection = dataOutput.connection(with: .video) {
                captureConnection.preferredVideoStabilizationMode = .off
                captureConnection.isEnabled = true
            } else {
                fatalError("No Capture Connection for the session")
            }
            
            //MARK: Configure AVCaptureDevice
            do { try device.lockForConfiguration() } catch { fatalError(error.localizedDescription) }
            if let format  = format(fps: fps, minWidth: minWidth, format: pixelFormat) { // 180FPS, YUV layout
                
                device.activeFormat = format
                device.activeVideoMinFrameDuration = CMTime(value: 1, timescale: CMTimeScale(fps))
                device.activeVideoMaxFrameDuration = CMTime(value: 1, timescale: CMTimeScale(fps))
                
            } else {
                fatalError("Compatible format not found")
            }
            
            device.activeColorSpace = .sRGB
            device.isGlobalToneMappingEnabled = false
            device.automaticallyAdjustsVideoHDREnabled = false
            device.automaticallyAdjustsFaceDrivenAutoExposureEnabled = false
            device.isFaceDrivenAutoExposureEnabled = false
            device.setFocusModeLocked(lensPosition: 0.4)
            device.isSubjectAreaChangeMonitoringEnabled = false
            device.exposureMode = AVCaptureDevice.ExposureMode.custom
            
            
            let exp = CMTime(value: Int64(40), timescale: 100_000)
            let isoValue = min(max(40, device.activeFormat.minISO), device.activeFormat.maxISO)
            device.setExposureModeCustom(duration: exp, iso: isoValue) { t in }
           
            device.setWhiteBalanceModeLocked(with: AVCaptureDevice.WhiteBalanceGains(redGain: 1.0, greenGain: 1.0, blueGain: 1.0)) {
                      (timestamp:CMTime) -> Void in }
            

            device.unlockForConfiguration()
            session.commitConfiguration()
            onAVSessionReady()
        }
    }

This post (https://stackoverflow.com/questions/34511431/ios-avfoundation-different-photo-brightness-with-the-same-manual-exposure-set) suggests that the effect can be mitigated by settings camera exposure to .locked right after setting device.setExposureModeCustom(). This works properly only if used with async api and still does not influence the effect.

Async approach:

 private func onAVSessionReady() {
        
        guard let device = device() else { fatalError("Device is unexpectedly nil") }
        guard let sesh = try? validSession() else { fatalError("Device is unexpectedly nil") }
        
        MCamSession.shared.activeFormat = device.activeFormat
        MCamSession.shared.currentDevice = device
        self.observer = SPSDeviceKVO(device: device, session: sesh)
        self.start()
        
        Task {
            await lockCamera(device)

        }

    }
    
    private func lockCamera(_ device: AVCaptureDevice) async {
        
        do { try device.lockForConfiguration() } catch { fatalError(error.localizedDescription) }
        
        _ = await device.setFocusModeLocked(lensPosition: 0.4)
        
        let exp = CMTime(value: Int64(40), timescale: 100_000)
        let isoValue = min(max(40, device.activeFormat.minISO), device.activeFormat.maxISO)
        
        
        _ = await device.setExposureModeCustom(duration: exp, iso: isoValue)
        _ = await device.setWhiteBalanceModeLocked(with: AVCaptureDevice.WhiteBalanceGains(redGain: 1.0, greenGain: 1.0, blueGain: 1.0))
        
        device.exposureMode = AVCaptureDevice.ExposureMode.locked
        device.unlockForConfiguration()
    }
    
    private func configureSession() {
          // same session init as before
           ...         
            onAVSessionReady()
    }