Post not yet marked as solved
I'm doing random access sampling from AVAsset of local h264 video file
let track = asset.tracks(withMediaType: .video)[0]
let assetReader = try! AVAssetReader(asset: asset)
let trackOutput = AVAssetReaderTrackOutput(track: track, outputSettings: nil)
trackOutput.supportsRandomAccess = true
assetReader.add(trackOutput)
assetReader.startReading()
...
let targetFrameDTS = CMTime(value: 56, timescale: 30)
let timeRange = CMTimeRange(
start: CMTimeAdd(time, CMTime(value: -1, timescale: 30)),
duration: CMTime(value: 2, timescale: 30)
)
// reset output to be near target frame decoding time
trackOutput.reset(forReadingTimeRanges: [NSValue(timeRange: timeRange)])
while assetReader.status == .reading {
guard let sample = trackOutput.copyNextSampleBuffer() else { break }
let dts = CMSampleBufferGetDecodeTimeStamp(sample)
print("\(dts.value)/\(dts.timescale)")
}
for some reason with some targetFrameDTS assetReader copyNextSampleBuffer will skip samples.
in my particular case the output is
...
47/30
48/30
50/30
51/30
54/30
55/30
57/30
why is it so?
Post not yet marked as solved
As the title describe, I cause a crash when call [AVCaptureSession stopRunning] on macos 14.4.1. the crash stack is as below:
Process: Nebula for Mac [31922]
Path: /Applications/Nebula for Mac.app/Contents/MacOS/Nebula for Mac
Identifier: ai.nreal.nebula.mac
Version: 0.8.0.1098 (0.8.0)
Code Type: ARM-64 (Native)
Parent Process: launchd [1]
User ID: 501
Date/Time: 2024-04-11 14:12:34.6474 +0800
OS Version: macOS 14.4.1 (23E224)
Report Version: 12
Anonymous UUID: C438684A-95E7-7DA1-D063-81E1A5FBF5DC
Sleep/Wake UUID: 3EB85031-82AC-4BDB-8F28-FAF4CBD28CA1
Time Awake Since Boot: 110000 seconds
Time Since Wake: 1108 seconds
System Integrity Protection: enabled
Crashed Thread: 0 Dispatch queue: com.apple.avfoundation.proprietarydefaults.singleton.source_queue.0x202f8b460
Exception Type: EXC_CRASH (SIGTRAP)
Exception Codes: 0x0000000000000000, 0x0000000000000000
Termination Reason: Namespace SIGNAL, Code 5 Trace/BPT trap: 5
Terminating Process: Nebula for Mac [31922]
Thread 0 Crashed:: Dispatch queue: com.apple.avfoundation.proprietarydefaults.singleton.source_queue.0x202f8b460
0 libsystem_kernel.dylib 0x19c1a61f4 mach_msg2_trap + 8
1 libsystem_kernel.dylib 0x19c1b8b24 mach_msg2_internal + 80
2 libsystem_kernel.dylib 0x19c1aee34 mach_msg_overwrite + 476
3 libsystem_kernel.dylib 0x19c1a6578 mach_msg + 24
4 libdispatch.dylib 0x19c0513b0 _dispatch_mach_send_and_wait_for_reply + 544
5 libdispatch.dylib 0x19c051740 dispatch_mach_send_with_result_and_wait_for_reply + 60
6 libxpc.dylib 0x19bef2af0 xpc_connection_send_message_with_reply_sync + 288
7 AVFCapture 0x1b9e5565c -[CMIOProprietaryDefaultsSource setObject:forKey:] + 140
8 AVFCapture 0x1b9e57044 __58-[AVCaptureProprietaryDefaultsSingleton setObject:forKey:]_block_invoke + 36
9 libdispatch.dylib 0x19c0363e8 _dispatch_client_callout + 20
10 libdispatch.dylib 0x19c0458d8 _dispatch_lane_barrier_sync_invoke_and_complete + 56
11 AVFCapture 0x1b9e597e0 -[AVCaptureProprietaryDefaultsSingleton _runBlockOnProprietaryDefaultsSourceQueueSync:] + 136
12 AVFCapture 0x1b9e56fbc -[AVCaptureProprietaryDefaultsSingleton setObject:forKey:] + 180
13 AVFCapture 0x1b9e776a0 -[AVCaptureDALDevice _refreshCenterStageUnavailableReasons] + 400
14 AVFCapture 0x1b9e7d0fc -[AVCaptureDALDevice updateActivelyProvidingInputCountForActiveUseState:] + 488
15 AVFCapture 0x1b9e33474 -[AVCaptureSession_Tundra _updateNewActiveUseState:forConnection:] + 196
16 AVFCapture 0x1b9e32e4c -[AVCaptureSession_Tundra _setRunning:] + 428
17 AVFCapture 0x1b9e32a28 -[AVCaptureSession_Tundra stopRunning] + 432
18 libnr_api.dylib 0x1446d7514 0x144478000 + 2487572
19 libnr_api.dylib 0x14468a690 0x144478000 + 2172560
20 libnr_api.dylib 0x14468bcb0 0x144478000 + 2178224
21 libnr_api.dylib 0x1444d0268 0x144478000 + 361064
22 libnr_api.dylib 0x1444ecb00 0x144478000 + 477952
23 libnr_api.dylib 0x1444ec724 0x144478000 + 476964
24 libnr_api.dylib 0x144541bcc 0x144478000 + 826316
25 libnr_api.dylib 0x144543e00 0x144478000 + 835072
26 libnr_api.dylib 0x144543f88 0x144478000 + 835464
27 libnr_api.dylib 0x144542ca8 0x144478000 + 830632
28 GameAssembly.dylib 0x12117d4c4 0x120000000 + 18339012
29 GameAssembly.dylib 0x1211894e0 0x120000000 + 18388192
30 GameAssembly.dylib 0x121165fe4 0x120000000 + 18243556
31 GameAssembly.dylib 0x1202e4248 0x120000000 + 3031624
32 GameAssembly.dylib 0x12116931c 0x120000000 + 18256668
33 GameAssembly.dylib 0x1201dcdf0 0x120000000 + 1953264
34 GameAssembly.dylib 0x1201dcd2c 0x120000000 + 1953068
35 UnityPlayer.dylib 0x10428dc60 0x103c38000 + 6642784
36 UnityPlayer.dylib 0x104295170 0x103c38000 + 6672752
37 UnityPlayer.dylib 0x1042b1620 0x103c38000 + 6788640
38 UnityPlayer.dylib 0x103f788d0 0x103c38000 + 3410128
39 UnityPlayer.dylib 0x1040d8c4c 0x103c38000 + 4852812
40 UnityPlayer.dylib 0x1040d8c98 0x103c38000 + 4852888
41 UnityPlayer.dylib 0x1040d8f2c 0x103c38000 + 4853548
42 UnityPlayer.dylib 0x104b104b8 0x103c38000 + 15566008
43 UnityPlayer.dylib 0x104b10304 0x103c38000 + 15565572
44 Foundation 0x19d430224 __NSFireTimer + 104
45 CoreFoundation 0x19c2e1f90 CFRUNLOOP_IS_CALLING_OUT_TO_A_TIMER_CALLBACK_FUNCTION + 32
46 CoreFoundation 0x19c2e1c34 __CFRunLoopDoTimer + 972
47 CoreFoundation 0x19c2e176c __CFRunLoopDoTimers + 356
48 CoreFoundation 0x19c2c4ba4 __CFRunLoopRun + 1856
49 CoreFoundation 0x19c2c3e0c CFRunLoopRunSpecific + 608
50 HIToolbox 0x1a6a5f000 RunCurrentEventLoopInMode + 292
51 HIToolbox 0x1a6a5ec90 ReceiveNextEventCommon + 220
52 HIToolbox 0x1a6a5eb94 _BlockUntilNextEventMatchingListInModeWithFilter + 76
53 AppKit 0x19fb1c970 _DPSNextEvent + 660
54 AppKit 0x1a030edec -[NSApplication(NSEventRouting) _nextEventMatchingEventMask:untilDate:inMode:dequeue:] + 700
55 AppKit 0x19fb0fcb8 -[NSApplication run] + 476
56 AppKit 0x19fae6f54 NSApplicationMain + 880
57 UnityPlayer.dylib 0x104b0ffe4 PlayerMain(int, char const**) + 944
58 dyld 0x19be5e0e0 start + 2360
Post not yet marked as solved
Hi guys,
I'm designing a customized camera based on avfoundation. I can output Live Photo from avCaptureDeviceInput for now. I expect to take still and live Photos with different aspect ratio, just like the apple's camera app does (1:1, 4:3, 16:9).
I didn't find any useful infos from docs, any suggestion?
Post not yet marked as solved
On a Vision Pro I load an HDR video served over HLS using AVPlayer.
Per FFMPEG the video has:
pixel format: yuv420p10le
color space / ycbcr matrix: bt2020nc
color primaries: bt2020
transfer function: smte2084
I wanted to try out letting AVFoundation do all of the color conversion instead of making my own YUV -> RGB shader.
To display a 10-bit texture in a drawable queue, the destination Metal texture format must be MTLPixelFormat.rgba16Float (no other formats above 8-bit are supported). So the pixel format I am capturing in is kCVPixelFormatType_64RGBAHalf since it's pretty close.
It's worth noting that the AVAsset shows no track information...must be because it's HLS?
I am using AVPlayerItemVideoOutput to get pixel buffers:
AVPlayerItemVideoOutput(outputSettings: [
AVVideoColorPropertiesKey: [
AVVideoColorPrimariesKey: AVVideoColorPrimaries_ITU_R_2020,
AVVideoTransferFunctionKey: AVVideoTransferFunction_SMPTE_ST_2084_PQ,
AVVideoYCbCrMatrixKey: AVVideoYCbCrMatrix_ITU_R_2020
],
kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_64RGBAHalf),
kCVPixelBufferMetalCompatibilityKey as String: true
])
I can change these settings in real time and see they are having an effect on my drawable queue.
The BT.2020 primaries do not look correct to me, it's very bright and washed out. When I switch to BT.709 it looks closer to the output of the AVPlayer. The AVPlayer by itself doesn't look terrible, just a little dark maybe.
When I leave out the outputSettings and let the AVPlayerItemVideoOutput choose its own color settings, it appears to choose BT.2020 also.
Is it enough to put in these outputSettings and expect an RGB pixelBuffer that perfectly matches those settings? Or do I have to just capture in YUV and do all of the conversion manually?
Am I misunderstanding something related to color settings here? I am definitely not an expert.
Thanks
Post not yet marked as solved
https://developer.apple.com/videos/play/wwdc2023/10235/ - In this WWDC session,
at 3:19 - Apple has introduced **Other audio ducking ** feature
In iOS17, we can control the amount of 'other audio' ducking through the AVAudioEngine. Is this also possible on AVAudioSession ?
We are using an AVAudioSession for a VOIP call while concurrently attempting to play a video through an AVPlayer. However, the volume of the AVPlayer is considerably low.
Does anyone have any ideas on how to achieve the level of control that AVAudioEngine offers?
Post not yet marked as solved
I added VideoPlayer view inside my project, but I noticed that during loading or with different aspect ratio the default color of this view is black.
I would like to change it according to to my app background.
Unfortunately using modifiers such as .background or .foregroundColor doesn't seem to change it.
Is there a way to customize this color?
struct PlayerLooperView: View {
private let queuePlayer: AVQueuePlayer!
private let playerLooper: AVPlayerLooper!
init(url: URL) {
let playerItem = AVPlayerItem(url: url)
self.queuePlayer = AVQueuePlayer(items: [playerItem])
self.queuePlayer.isMuted = true
self.playerLooper = AVPlayerLooper(player: queuePlayer,
templateItem: playerItem)
self.queuePlayer.play()
}
var body: some View {
VideoPlayer(player: queuePlayer)
.disabled(true)
.scaledToFit()
}
}
Post not yet marked as solved
I have built a camera application which uses a AVCaptureSession with the AVCaptureDevice set to .builtInDualWideCamera and isVirtualDeviceConstituentPhotoDeliveryEnabled=true to enable delivery of "simultaneous" photos (AVCapturePhoto) for a single capture request.
Our app ideally would have the timestamp difference between the photos in a single capture request as short as possible, but we don't have a good idea of what the theoretical or practical limits of this timestamp difference are.
In my testing on an iPhone 12 Pro, with a frame rate of 33Hz and the preset set to hd1920x1080, I get the timestamp difference between photos at approx 0.3ms, which seems smaller than I would expect, unless the frames are being synchronised incredibly well under the hood.
This leaves the following unanswered questions:
What sort of ranges of values should we expect to come out of these timestamp differences between photos?
What factors influence this?
Is there any way to control these values to ensure they are as small as possible? (Will likely be answered by (2))
Post not yet marked as solved
After my iPad 6 upgrades from iOS 17.3 to 17.4, the AVCaptureMetadataOutput delegate is not called anymore. I find there is the same problem in a stackoverflow post:
https://stackoverflow.com/questions/78128010/ipados-17-4-avcapturemetadataoutput-delegate-not-called-qrscanner
An Apple webpage said the "QR code scanning" issue is fixed in iPadOS 17.4.1:
If your iPad is unable to scan QR codes after updating to iPadOS 17.4 - Apple Support - https://support.apple.com/en-lamr/118614
That's true, I confirm that on my iPad 6.
But, unfortunately, iPadOS 17.4.1 does fix ONLY QR code scanning! It doesn't fix barcode scanning, like PDF 417
Happening on
iPad (7th Generation)
iPad (6th Generation)
iPad Pro 12.9-inch (2nd Generation)
iPad Pro 10.5-inch
Post not yet marked as solved
Hey!
I'm trying to open the front camera on my demo app, and from what I read on the Apple docs and forums if you have configured your Persona you will get that image.
But I'm having some issues with it, this is my code:
struct ContentView: View {
@Environment(\.presentationMode) var presentationMode
var body: some View {
ZStack {
VStack {
Image("logo")
.resizable()
.frame(width: 337, height: 211)
.accessibilityHidden(true)
Text("My first Vision Pro app.")
.multilineTextAlignment(.center)
.font(.headline)
.frame(width: 340)
.padding(.bottom, 10)
Button {
// Add camera functionality here
} label: {
Text("Open Camera")
.frame(maxWidth: .infinity)
}
.onAppear {
requestCameraAccess()
}
.onTapGesture {
// Check if camera permission is granted
if AVCaptureDevice.authorizationStatus(for: .video) == .authorized {
openFrontCamera()
} else {
requestCameraAccess()
}
}
}
}
}
func requestCameraAccess() {
AVCaptureDevice.requestAccess(for: .video) { authorized in
DispatchQueue.main.async {
if authorized {
// Permission granted, open camera if needed
openFrontCamera()
} else {
// Handle permission denied case (optional)
}
}
}
}
func openFrontCamera() {
}
}```
On the openFrontCamera() function I tried using .devices() .default() and other methods like you would use for other Apple devices but this doesn't work with Vision Pro and I can't find anything that tells me how to open it.
Has anyone been able to work this out?
Post not yet marked as solved
For our application, we are aiming to have full control over setting and locking the camera exposure settings when taking a video. We’re working with Apple’s AVFoundation framework for a range of devices, but most of the development is focused on the iPad 8 front camera. The manual settings are specific to our use, so we aim to use the custom exposure mode with e.g ISO = 100, exposureDuration = 1/60, and a fixed white balance. The duration, ISO, and white balance are all set in advance of recording, but when we begin we can see that something is still adjusting and compensating for lighting changes.
We then also tried locking the exposure mode after setting the custom values, but there appears to be a delay in this lock taking effect. While tracking the ISO during a recording, we see that the ISO values change in the first second of the recording, leading to oversaturated images, despite our efforts to keep it locked.
This is our attempt to lock the settings using custom mode, which we don’t adjust ourselves during the recording, but it does not work as expected:
func setCameraSettings(newValueISO: Float, newValueDuration: CMTime){
do {
try cameraDevice?.lockForConfiguration()
cameraDevice?.automaticallyAdjustsVideoHDREnabled = false
cameraDevice?.setExposureModeCustom(duration: newValueDuration, iso: newValueISO, completionHandler: { [self] _ in
cameraDevice?.setWhiteBalanceModeLocked(with: cameraDevice!.deviceWhiteBalanceGains)
if ((cameraDevice!.isFocusModeSupported(.locked))) {
do {
cameraDevice!.focusMode = .locked
debugPrint("Focus mode set to locked.")
}
}
cameraDevice?.unlockForConfiguration()
})
} catch {
debugPrint("Error adjusting the exposure")
cameraDevice?.unlockForConfiguration()
}
}
We then tried to lock the exposure mode after setting the custom values, but it then changes during the first second of the recording. We also explicitly tried setting exposureTargetBias to 0, but this made no difference.
func setCameraSettings(newValueISO: Float, newValueDuration: CMTime){
guard let camera = cameraDevice else { return }
do {
if camera.isExposureModeSupported(.custom) {
do {
try camera.lockForConfiguration()
let customExposureBias: Float = 0
//camera.setExposureTargetBias(customExposureBias, completionHandler: nil)
if camera.isExposureModeSupported(.custom) {
camera.setExposureModeCustom(duration: newValueDuration, iso: newValueISO) { [weak camera ] _ in
guard let camera = camera else { return }
if camera.isExposureModeSupported(.locked) {
camera.exposureMode = .locked
}
}
}
camera.unlockForConfiguration()
print("Exposure settings locked with custom values.")
} catch {
print("Failed to lock configuration for capture device: \(error.localizedDescription)")
camera.unlockForConfiguration()
}
} else {
print("Custom exposure mode is not supported.")
}
}
}
We would very much appreciate input on how to keep the manually selected camera settings fixed throughout the video recording.
Post not yet marked as solved
Hey all!
I'm building a Camera app using AVFoundation, and I am using AVCaptureVideoDataOutput and AVCaptureAudioDataOutput delegates. (I cannot use AVCaptureMovieFileOutput because I am doing some processing inbetween)
When recording the audio CMSampleBuffers to the AVAssetWriter, I noticed that compared to the stock iOS camera app, they are mono-audio, not stereo audio.
I wonder how recording in stereo audio works, are there any guides or documentation available for that?
Is a stereo audio frame still one CMSampleBuffer, or will it be multiple CMSampleBuffers? Do I need to synchronize them? Do I need to set up the AVAssetWriter/AVAssetWriterInput differently?
This is my Audio Session code:
func configureAudioSession(configuration: CameraConfiguration) throws {
ReactLogger.log(level: .info, message: "Configuring Audio Session...")
// Prevent iOS from automatically configuring the Audio Session for us
audioCaptureSession.automaticallyConfiguresApplicationAudioSession = false
let enableAudio = configuration.audio != .disabled
// Check microphone permission
if enableAudio {
let audioPermissionStatus = AVCaptureDevice.authorizationStatus(for: .audio)
if audioPermissionStatus != .authorized {
throw CameraError.permission(.microphone)
}
}
// Remove all current inputs
for input in audioCaptureSession.inputs {
audioCaptureSession.removeInput(input)
}
audioDeviceInput = nil
// Audio Input (Microphone)
if enableAudio {
ReactLogger.log(level: .info, message: "Adding Audio input...")
guard let microphone = AVCaptureDevice.default(for: .audio) else {
throw CameraError.device(.microphoneUnavailable)
}
let input = try AVCaptureDeviceInput(device: microphone)
guard audioCaptureSession.canAddInput(input) else {
throw CameraError.parameter(.unsupportedInput(inputDescriptor: "audio-input"))
}
audioCaptureSession.addInput(input)
audioDeviceInput = input
}
// Remove all current outputs
for output in audioCaptureSession.outputs {
audioCaptureSession.removeOutput(output)
}
audioOutput = nil
// Audio Output
if enableAudio {
ReactLogger.log(level: .info, message: "Adding Audio Data output...")
let output = AVCaptureAudioDataOutput()
guard audioCaptureSession.canAddOutput(output) else {
throw CameraError.parameter(.unsupportedOutput(outputDescriptor: "audio-output"))
}
output.setSampleBufferDelegate(self, queue: CameraQueues.audioQueue)
audioCaptureSession.addOutput(output)
audioOutput = output
}
}
This is how I activate the audio session just before I start recording:
let audioSession = AVAudioSession.sharedInstance()
try audioSession.updateCategory(AVAudioSession.Category.playAndRecord,
mode: .videoRecording,
options: [.mixWithOthers,
.allowBluetoothA2DP,
.defaultToSpeaker,
.allowAirPlay])
if #available(iOS 14.5, *) {
// prevents the audio session from being interrupted by a phone call
try audioSession.setPrefersNoInterruptionsFromSystemAlerts(true)
}
if #available(iOS 13.0, *) {
// allow system sounds (notifications, calls, music) to play while recording
try audioSession.setAllowHapticsAndSystemSoundsDuringRecording(true)
}
audioCaptureSession.startRunning()
And this is how I set up the AVAssetWriter:
let audioSettings = audioOutput.recommendedAudioSettingsForAssetWriter(writingTo: options.fileType)
let format = audioInput.device.activeFormat.formatDescription
audioWriter = AVAssetWriterInput(mediaType: .audio, outputSettings: audioSettings, sourceFormatHint: format)
audioWriter!.expectsMediaDataInRealTime = true
assetWriter.add(audioWriter!)
ReactLogger.log(level: .info, message: "Initialized Audio AssetWriter.")
The rest is trivial - I receive CMSampleBuffers of the audio in my delegate's callback, write them to the audioWriter, and it ends up in the .mov file - but it is not stereo, it's mono.
Is there anything I'm missing here?
Post not yet marked as solved
Hi folks,
iOS 17.4 - iPhone 14 Pro - Real device
I'm facing an error when I try to implement a simple VideoPlayer view:
Main thread blocked by synchronous property query on not-yet-loaded property (PreferredTransform) for HTTP(S) asset. This could have been a problem if this asset were being read from a slow network.
This is the code I'm using to show the video player:
struct ContentView: View {
let player = AVPlayer(url: URL(string: "https://video.twimg.com/amplify_video/1760315643142750208/vid/avc1/640x360/-1etorSK7w2g9Nlc.mp4?tag=16")!)
var body: some View {
VideoPlayer(player: player)
}
}
I noticed that the bundle initi method of AVPlayer doesn't trigger this error:
AVPlayer(url: Bundle.main.url(forResource: "video", withExtension: "mp4")!)
I also tried to load metadata's asset with:
try await AVAsset.load(_:)
Before giving It to AVPlayer via AVPlayerItem with no success :/
Does anyone is facing this error too ? Or is It a bug from the SDK ?
Post not yet marked as solved
I'm creating an app that uses AVCaptureSession to pass camera input to AVCaptureMetadataOutput and scan QRCode.
After updating to iPadOS 17.4, an issue has occurred where the delegate method of AVCaptureMetadataOutputObjectsDelegate is not called on some devices.
The following devices are experiencing this issue.
iPad (7th Gen)
iPad (6th Gen)
iPad Pro (10.5)
iPad Pro (12.9 2nd Gen)
This issue has not occur on any other devices I have.
This may only occur on devices with model number "iPad7,x".
I tried running the AVFoundation sample code on the Apple Developer site on the above device. The same problem still occurs.
https://developer.apple.com/documentation/avfoundation/capture_setup/avcambarcode_detecting_barcodes_and_faces
Are any additional settings required after iPadOS17.4?
Or is there some problem on the OS side?
Post not yet marked as solved
We utilized AVFragmentedAssetMinder to refresh the player data. While notifications for AVAssetDurationDidChange were consistently received whenever the player duration changed.
However, following the release of iOS 17, notifications for AVAssetDurationDidChange ceased to be received.
Could you please advise anyone why this notification is not being triggered? what we have to change
NotificationCenter.default.addObserver(self, selector: #selector(self.onVideoUpdate), name: .AVAssetDurationDidChange, object: nil)
#AVPLAyer, #AVMUtableMovie
Post not yet marked as solved
We have a local video file whose data keeps updating every second from web socket connection. An AVAssetItem is created from this local file and an AVFragmentedAssetMinder is used to monitor the changes to the asset with mindingInterval: 2.
We are observing for AVAssetDurationDidChange and AVAssetContainsFragmentsDidChange notifications
NotificationCenter.default.addObserver(self, selector: #selector(onVideoUpdate), name: .AVAssetDurationDidChange, object: nil)
NotificationCenter.default.addObserver(self, selector: #selector(assetContainsFragments), name: .AVAssetContainsFragmentsDidChange, object: nil)
The above notifications are not called in iOS 17 iPhone devices. These notifications are called in iOS 17 simulator and in previous iOS devices.
Post not yet marked as solved
Adding AVPlayer as attachment on the side using RealityKit. The video in it thought is not aligned. And thoughts on what could be going wrong?
RealityView { content, attachments in
let url = self.video.resolvedURL
let asset = AVURLAsset(url: url)
let playerItem = AVPlayerItem(asset: asset)
var videoPlayerComponent = VideoPlayerComponent(avPlayer: player)
videoPlayerComponent.isPassthroughTintingEnabled = true
// entity.components[VideoPlayerComponent.self] = videoPlayerComponent
entity.position = [0, 0, 0]
entity.scale *= 0.50
player.replaceCurrentItem(with: playerItem)
player.play()
content.add(entity)
} update: { content, attachments in
// if content.entities.count < 2 {
if showAnotherPlayer {
if let attachment = attachments.entity(for: "Attachment") {
playerModel.loadVideo(library.selectedVideo!, presentation: .fullWindow)
//4. Position the Attachment and add it to the RealityViewContent
attachment.position = [1.0, 0, 0]
attachment.scale *= 1.0
//let radians = -45.0 * Float.pi / 180.0
//attachment.transform.rotation += simd_quatf(angle: radians, axis: SIMD3<Float>(0,1,0))
let entity = content.entities.first
attachment.setParent(entity)
content.add(attachment)
}
}
if showLibrary {
if let attachment = attachments.entity(for: "Featured") {
//4. Position the Attachment and add it to the RealityViewContent
attachment.position = [0.0, -0.3, 0]
attachment.scale *= 0.7
//let radians = -45.0 * Float.pi / 180.0
//attachment.transform.rotation += simd_quatf(angle: radians, axis: SIMD3<Float>(0,1,0))
let entity = content.entities.first
attachment.setParent(entity)
viewModel.attachment = attachment
content.add(attachment)
}
} else {
if let scene = content.entities.first?.scene {
let _ = print("found scene")
}
if let featuredEntity = content.entities.first?.scene?.findEntity(named: "Featured") {
let _ = print("featured entity found")
}
if let attachment = viewModel.attachment {
let _ = print("-- removing attachment")
if let anchor = attachment.anchor {
let _ = print("-- removing anchor")
anchor.removeFromParent()
}
attachment.removeFromParent()
content.remove(attachment)
} else {
let _ = print("the attachment is missing")
}
}
// }
} attachments: {
Attachment(id: "Attachment") {
PlayerView()
.frame(width: 2048, height: 1024)
.environment(library)
.environment(playerModel)
.onAppear {
DispatchQueue.main.asyncAfter(deadline: .now()+1) {
playerModel.play()
}
}
.onDisappear {
}
}
if showLibrary {
Attachment(id: "Featured") {
VideoListView(title: "Featured",
videos: library.videos,
cardStyle: .full,
cardSpacing: 20) { video in
library.selectedVideo = video
showAnotherPlayer = true
}
.frame(width: 2048, height: 1024)
}
}
}
PlayerView
Post not yet marked as solved
Hello, I fetch Live Photo AVAsset using: PHImageManager and PHAssetResourceManager for getting Data. And then I want to wrap it to AVAsset using fileURL, and everything works fine, but also I want to trim this AVAsset using AVMutableComposition. I use insertTimeRange of AVMutableCompositionTrack method, and I don't not why but naturalSize of originalVideoTrack and newVideoTrack are different, and this happening only with Live Photo, default Videos work fine. Seems like this is AVMutableCompositionTrack bug inside AVFoundation. Please give me some info. Thanks)
PHImageManager.default().requestLivePhoto(
for: phAsset,
targetSize: size,
contentMode: .aspectFill,
options: livePhotoOptions
) { [weak self] livePhoto, info in
guard let livePhoto else {
return
}
self?.writeAVAsset(livePhoto: livePhoto, fileURL: fileURL)
}
private func writeAVAsset(livePhoto: PHLivePhoto, fileURL: URL) {
let resources = PHAssetResource.assetResources(for: livePhoto)
guard let videoResource = resources.first(where: { $0.type == .pairedVideo }) else {
return
}
let requestOptions = PHAssetResourceRequestOptions()
var data = Data()
dataRequestID = PHAssetResourceManager.default().requestData(
for: videoResource,
options: requestOptions,
dataReceivedHandler: { chunk in
data.append(chunk)
},
completionHandler: { [weak self] error in
try data.write(to: fileURL)
let avAsset = AVAsset(url: fileURL)
let composition = AVMutableComposition()
if let originalVideoTrack = tracks(withMediaType: .video).first,
let videoTrack = composition.addMutableTrack(withMediaType: .video, preferredTrackID: 0)
{
// originalVideoTrack has naturalSize (width: 1744, height: 1308)
try? videoTrack.insertTimeRange(timeRange, of: originalVideoTrack, at: .zero)
videoTrack.preferredTransform = originalVideoTrack.preferredTransform
// videoTrack has naturalSize (width: 1920.0, height: 1440.0)
}
)
}
I'm attempting to record from a device's microphone (under iOS) using AVAudioRecorder. The examples are all quite simple, and I'm following the same method. But I'm getting error messages on attempts to record, and the resulting M4A file (after several seconds of recording) is only 552 bytes long and won't load. Here's the recorder usage:
func startRecording()
{
let settings = [
AVFormatIDKey: Int(kAudioFormatMPEG4AAC),
AVSampleRateKey: 22050,
AVNumberOfChannelsKey: 1,
AVEncoderAudioQualityKey: AVAudioQuality.high.rawValue
]
do
{
recorder = try AVAudioRecorder(url: tempFileURL(), settings: settings)
recorder?.delegate = self
recorder!.record()
recording = true
}
catch
{
recording = false
recordingFinished(success: false)
}
}
The immediate sign of trouble appears to be the following, in the console. Note the 0 bits per channel and irrelevant 8K sample rate:
AudioQueueObject.cpp:1580 BuildConverter: AudioConverterNew returned -50 from: 0 ch, 8000 Hz, .... (0x00000000) 0 bits/channel, 0 bytes/packet, 0 frames/packet, 0 bytes/frame to: 1 ch, 8000 Hz, Int16
A subsequent attempt to load the file into AVAudioPlayer results in:
MP4_BoxParser.cpp:1089 DataSource read failed MP4AudioFile.cpp:4365 MP4Parser_PacketProvider->GetASBD() failed AudioFileObject.cpp:105 OpenFromDataSource failed AudioFileObject.cpp:80 Open failed
But that's not surprising given that it's only 500+ bytes and we had the earlier error. Anybody have an idea here? Every example on the Web shows essentially this exact method.
I've also tried constructing the recorder with
let audioFormat = AVAudioFormat.init(standardFormatWithSampleRate: 44100, channels: 1)
if audioFormat == nil
{
print("Audio format failed.")
}
else
{
do
{
recorder = try AVAudioRecorder(url: tempFileURL(), format: audioFormat!)
...
with mostly the same result. In that case the instantiation error message was the following, which at least mentions the requested sample rate:
AudioQueueObject.cpp:1580 BuildConverter: AudioConverterNew returned -50 from: 0 ch, 44100 Hz, .... (0x00000000) 0 bits/channel, 0 bytes/packet, 0 frames/packet, 0 bytes/frame to: 1 ch, 44100 Hz, Int32
Post not yet marked as solved
Similar post on StackOverflow and multiple people reported this (you will encounter it if you run the app for like 10 minutes). I'm hoping this could get Apple's attention somehow
After downloading the project code (https://developer.apple.com/documentation/avfoundation/additional_data_capture/capturing_depth_using_the_lidar_camera) and running the Swift sample code on an iPhone 14 Pro, the app crashes intermittently, throwing this error:
Execution of the command buffer was aborted due to an error during execution. Caused GPU Timeout Error (00000002:kIOGPUCommandBufferCallbackErrorTimeout)
Sometimes it will crash within a few seconds, sometimes it can take around 10 minutes.
Has anyone here experienced this crash from the sample code, or using the LiDAR camera?
I have spent a long time trying to solve this issue, I have searched the web high and low and submitted (I think) a report to Apple about it. I am unable to get xcode to show me the line of code where the crash is happening.
Any help would be greatly appreciated.
Post not yet marked as solved
So I've been trying for weeks now to implement a compression mechanism into my app project that compresses MV-HEVC video files in-app without stripping videos of their 3D properties, but every single implementation I have tried has either stripped the encoded MV-HEVC video file of its 3D properties (making the video monoscopic), or has crashed with a fatal error. I've read the Reading multiview 3D video files and Converting side-by-side 3D video to multiview HEVC documentation files, but was unable to myself come out with anything useful.
My question therefore is: How do you go about compressing/encoding an MV-HEVC video file in-app whilst preserving the stereoscopic/3D properties of that MV-HEVC video file? Below is the best implementation I was able to come up with (which simply compresses uploaded MV-HEVC videos with an arbitrary bit rate). With this implementation (my compressVideo function), the MV-HEVC files that go through it are compressed fine, but the final result is the loss of that MV-HEVC video file's stereoscopic/3D properties.
If anyone could point me in the right direction with anything it would be greatly, greatly appreciated.
My current implementation (that strips MV-HEVC videos of their stereoscopic/3D properties):
static func compressVideo(sourceUrl: URL, bitrate: Int, completion: @escaping (Result<URL, Error>) -> Void) {
let asset = AVAsset(url: sourceUrl)
asset.loadTracks(withMediaType: .video) { videoTracks, videoError in
guard let videoTrack = videoTracks?.first, videoError == nil else {
completion(.failure(videoError ?? NSError(domain: "VideoUploader", code: -1, userInfo: [NSLocalizedDescriptionKey: "Failed to load video track"])))
return
}
asset.loadTracks(withMediaType: .audio) { audioTracks, audioError in
guard let audioTrack = audioTracks?.first, audioError == nil else {
completion(.failure(audioError ?? NSError(domain: "VideoUploader", code: -2, userInfo: [NSLocalizedDescriptionKey: "Failed to load audio track"])))
return
}
let outputUrl = sourceUrl.deletingLastPathComponent().appendingPathComponent(UUID().uuidString).appendingPathExtension("mov")
guard let assetReader = try? AVAssetReader(asset: asset),
let assetWriter = try? AVAssetWriter(outputURL: outputUrl, fileType: .mov) else {
completion(.failure(NSError(domain: "VideoUploader", code: -3, userInfo: [NSLocalizedDescriptionKey: "AssetReader/Writer initialization failed"])))
return
}
let videoReaderSettings: [String: Any] = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32ARGB]
let videoSettings: [String: Any] = [
AVVideoCompressionPropertiesKey: [AVVideoAverageBitRateKey: bitrate],
AVVideoCodecKey: AVVideoCodecType.hevc,
AVVideoHeightKey: videoTrack.naturalSize.height,
AVVideoWidthKey: videoTrack.naturalSize.width
]
let assetReaderVideoOutput = AVAssetReaderTrackOutput(track: videoTrack, outputSettings: videoReaderSettings)
let assetReaderAudioOutput = AVAssetReaderTrackOutput(track: audioTrack, outputSettings: nil)
if assetReader.canAdd(assetReaderVideoOutput) {
assetReader.add(assetReaderVideoOutput)
} else {
completion(.failure(NSError(domain: "VideoUploader", code: -4, userInfo: [NSLocalizedDescriptionKey: "Couldn't add video output reader"])))
return
}
if assetReader.canAdd(assetReaderAudioOutput) {
assetReader.add(assetReaderAudioOutput)
} else {
completion(.failure(NSError(domain: "VideoUploader", code: -5, userInfo: [NSLocalizedDescriptionKey: "Couldn't add audio output reader"])))
return
}
let audioInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil)
let videoInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings)
videoInput.transform = videoTrack.preferredTransform
assetWriter.shouldOptimizeForNetworkUse = true
assetWriter.add(videoInput)
assetWriter.add(audioInput)
assetReader.startReading()
assetWriter.startWriting()
assetWriter.startSession(atSourceTime: CMTime.zero)
let videoInputQueue = DispatchQueue(label: "videoQueue")
let audioInputQueue = DispatchQueue(label: "audioQueue")
videoInput.requestMediaDataWhenReady(on: videoInputQueue) {
while videoInput.isReadyForMoreMediaData {
if let sample = assetReaderVideoOutput.copyNextSampleBuffer() {
videoInput.append(sample)
} else {
videoInput.markAsFinished()
if assetReader.status == .completed {
assetWriter.finishWriting {
completion(.success(outputUrl))
}
}
break
}
}
}
audioInput.requestMediaDataWhenReady(on: audioInputQueue) {
while audioInput.isReadyForMoreMediaData {
if let sample = assetReaderAudioOutput.copyNextSampleBuffer() {
audioInput.append(sample)
} else {
audioInput.markAsFinished()
break
}
}
}
}
}
}