Skip to content

merged file writing to use single avssetwriter for both audio and video #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sihekuang wants to merge 1 commit into
base: mic-track
Choose a base branch
from
Open

merged file writing to use single avssetwriter for both audio and video #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 52 additions & 31 deletions Azayaka/Processing.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,13 @@ import AVFAudio
import ScreenCaptureKit

extension AppDelegate {
func initVideo(conf: SCStreamConfiguration) {
func initMedia(conf: SCStreamConfiguration, audioOnly: Bool = false) {
startTime = nil

let filePathAndAssetWriter = getFilePathAndAssetWriter(audioOnly: audioOnly)

let fileEnding = ud.string(forKey: "videoFormat") ?? ""
var fileType: AVFileType?
switch fileEnding {
case VideoFormat.mov.rawValue: fileType = AVFileType.mov
case VideoFormat.mp4.rawValue: fileType = AVFileType.mp4
default: assertionFailure("loaded unknown video format")
}

filePath = "\(getFilePath()).\(fileEnding)"
vW = try? AVAssetWriter.init(outputURL: URL(fileURLWithPath: filePath), fileType: fileType!)
filePath = filePathAndAssetWriter.0
vW = filePathAndAssetWriter.1
let encoderIsH265 = ud.string(forKey: "encoder") == Encoder.h265.rawValue
let fpsMultiplier: Double = Double(ud.integer(forKey: "frameRate"))/8
let encoderMultiplier: Double = encoderIsH265 ? 0.5 : 0.9
Expand All @@ -38,14 +32,14 @@ extension AppDelegate {
] as [String : Any]
]
recordMic = ud.bool(forKey: "recordMic")
vwInput = AVAssetWriterInput(mediaType: AVMediaType.video, outputSettings: videoSettings)
vwInput = audioOnly ? nil : AVAssetWriterInput(mediaType: AVMediaType.video, outputSettings: videoSettings)
awInput = AVAssetWriterInput(mediaType: AVMediaType.audio, outputSettings: audioSettings)
micInput = AVAssetWriterInput(mediaType: AVMediaType.audio, outputSettings: audioSettings)
vwInput.expectsMediaDataInRealTime = true
vwInput?.expectsMediaDataInRealTime = true
awInput.expectsMediaDataInRealTime = true
micInput.expectsMediaDataInRealTime = true

if vW.canAdd(vwInput) {
if !audioOnly, vW.canAdd(vwInput) {
vW.add(vwInput)
}

Expand All @@ -69,10 +63,10 @@ extension AppDelegate {
vW.startWriting()
}

func closeVideo() {
func closeMedia() {
let dispatchGroup = DispatchGroup()
dispatchGroup.enter()
vwInput.markAsFinished()
vwInput?.markAsFinished()
awInput.markAsFinished()
if recordMic {
micInput.markAsFinished()
Expand All @@ -89,6 +83,11 @@ extension AppDelegate {
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of outputType: SCStreamOutputType) {
guard sampleBuffer.isValid else { return }

if vW != nil && vW?.status == .writing, startTime == nil {
startTime = Date.now
vW.startSession(atSourceTime: CMSampleBufferGetPresentationTimeStamp(sampleBuffer))
}

switch outputType {
case .screen:
if screen == nil && window == nil { break }
Expand All @@ -97,26 +96,14 @@ extension AppDelegate {
guard let statusRawValue = attachments[SCStreamFrameInfo.status] as? Int,
let status = SCFrameStatus(rawValue: statusRawValue),
status == .complete else { return }

if vW != nil && vW?.status == .writing, startTime == nil {
startTime = Date.now
vW.startSession(atSourceTime: CMSampleBufferGetPresentationTimeStamp(sampleBuffer))
}

if vwInput.isReadyForMoreMediaData {
vwInput.append(sampleBuffer)
}
break
case .audio:
if streamType == .systemaudio { // write directly to file if not video recording
guard let samples = sampleBuffer.asPCMBuffer else { return }
do {
try audioFile?.write(from: samples)
}
catch { assertionFailure("audio file writing issue") }
} else { // otherwise send the audio data to AVAssetWriter
if awInput.isReadyForMoreMediaData {
awInput.append(sampleBuffer)
}
if awInput.isReadyForMoreMediaData {
awInput.append(sampleBuffer)
}
@unknown default:
assertionFailure("unknown stream type")
Expand All @@ -131,6 +118,40 @@ extension AppDelegate {
self.stopRecording()
}
}

/**
Get the filepath of where the asset writer is writing to and asset writer itself
*/
private func getFilePathAndAssetWriter(audioOnly: Bool)-> (String, AVAssetWriter?){
var assetWriter: AVAssetWriter? = nil

var fileEnding: String!
var fileType: AVFileType!

if audioOnly{
fileEnding = ud.string(forKey: "audioFormat") ?? "wat"
fileType = .m4a // it looks like file type m4a works for all tyle extensions. but maybe need to revise this
Copy link
Owner

@Mnpn Mnpn Dec 15, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This actually doesn't work! While the file ending appears correct, the actual format in the file will be wrong. (So, for example, when recording Opus, you'll get a file with an AAC stream in a file named .ogg)

Copy link
Author

@sihekuang sihekuang Jan 4, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks for the review! Sorry has been out for a while for the Christmas + New Year holidays. Let me get back and make this right. Thanks again and happy new year!

Copy link
Owner

@Mnpn Mnpn Jan 4, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No worries at all! Likewise, happy new year :)

switch fileEnding { // todo: I'd like to store format info differently
case AudioFormat.aac.rawValue: fallthrough
case AudioFormat.alac.rawValue: fileEnding = "m4a"
case AudioFormat.flac.rawValue: fileEnding = "flac"
case AudioFormat.opus.rawValue: fileEnding = "ogg"
default: assertionFailure("loaded unknown audio format: " + fileEnding)
}
}else{
fileEnding = ud.string(forKey: "videoFormat") ?? ""
switch fileEnding {
case VideoFormat.mov.rawValue: fileType = AVFileType.mov
case VideoFormat.mp4.rawValue: fileType = AVFileType.mp4
default: assertionFailure("loaded unknown video format")
}
}

filePath = "\(getFilePath()).\(fileEnding!)"
assetWriter = try? AVAssetWriter(outputURL: URL(fileURLWithPath: filePath), fileType: fileType)

return (filePath, assetWriter)
}
}

// https://developer.apple.com/documentation/screencapturekit/capturing_screen_content_in_macos
Expand Down
26 changes: 2 additions & 24 deletions Azayaka/Recording.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,6 @@ extension AppDelegate {
}
filter = SCContentFilter(display: screen ?? availableContent!.displays.first!, excludingApplications: excluded ?? [], exceptingWindows: [])
}
if streamType == .systemaudio {
prepareAudioRecording()
}
Task { await record(audioOnly: streamType == .systemaudio, filter: filter!) }

// while recording, keep a timer which updates the menu's stats
Expand Down Expand Up @@ -63,11 +60,7 @@ extension AppDelegate {
do {
try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: .global())
try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: .global())
if !audioOnly {
initVideo(conf: conf)
} else {
startTime = Date.now
}
initMedia(conf: conf, audioOnly: audioOnly)
try await stream.startCapture()
} catch {
assertionFailure("capture failed")
Expand All @@ -87,9 +80,7 @@ extension AppDelegate {
stream.stopCapture()
}
stream = nil
if streamType != .systemaudio {
closeVideo()
}
closeMedia()
streamType = nil
audioFile = nil // close audio file
window = nil
Expand Down Expand Up @@ -121,19 +112,6 @@ extension AppDelegate {
}
}

func prepareAudioRecording() {
var fileEnding = ud.string(forKey: "audioFormat") ?? "wat"
switch fileEnding { // todo: I'd like to store format info differently
case AudioFormat.aac.rawValue: fallthrough
case AudioFormat.alac.rawValue: fileEnding = "m4a"
case AudioFormat.flac.rawValue: fileEnding = "flac"
case AudioFormat.opus.rawValue: fileEnding = "ogg"
default: assertionFailure("loaded unknown audio format: " + fileEnding)
}
filePath = "\(getFilePath()).\(fileEnding)"
audioFile = try! AVAudioFile(forWriting: URL(fileURLWithPath: filePath), settings: audioSettings, commonFormat: .pcmFormatFloat32, interleaved: false)
}

func getFilePath() -> String {
let dateFormatter = DateFormatter()
dateFormatter.dateFormat = "y-MM-dd HH.mm.ss"
Expand Down