diff --git a/Package.resolved b/Package.resolved new file mode 100644 index 0000000..31b1ef6 --- /dev/null +++ b/Package.resolved @@ -0,0 +1,141 @@ +{ + "originHash" : "f04b8fe9e82bcfef54bede3355d7647310db02857aade54807d042599ab5e7e8", + "pins" : [ + { + "identity" : "ai", + "kind" : "remoteSourceControl", + "location" : "https://github.com/PreternaturalAI/AI.git", + "state" : { + "branch" : "main", + "revision" : "4021a8f960476cebf51f5daf31483fd9d6f0452a" + } + }, + { + "identity" : "cataphyl", + "kind" : "remoteSourceControl", + "location" : "https://github.com/PreternaturalAI/Cataphyl.git", + "state" : { + "branch" : "main", + "revision" : "e53561c546fe4a0c5b50a5f3785c2bf5e65ee783" + } + }, + { + "identity" : "chatkit", + "kind" : "remoteSourceControl", + "location" : "https://github.com/PreternaturalAI/ChatKit.git", + "state" : { + "branch" : "main", + "revision" : "055ad6219fc87355b25ecdcb5eb4cbb1c6f51177" + } + }, + { + "identity" : "corepersistence", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vmanot/CorePersistence.git", + "state" : { + "branch" : "main", + "revision" : "3de61dda6b7153bde07aa7c1fbfd44c01c82e3a8" + } + }, + { + "identity" : "media", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vmanot/Media.git", + "state" : { + "branch" : "main", + "revision" : "0ba2baaebeb58667955daef68d3535ba1b217a12" + } + }, + { + "identity" : "merge", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vmanot/Merge.git", + "state" : { + "branch" : "master", + "revision" : "17e267f961c5ea9a3375c3a49807e66795a004e2" + } + }, + { + "identity" : "networkkit", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vmanot/NetworkKit.git", + "state" : { + "branch" : "master", + "revision" : "470af0276c2aa6e61acff22acc14c7ca30b99cbc" + } + }, + { + "identity" : "swallow", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vmanot/Swallow.git", + "state" : { + "branch" : "master", + "revision" : "553de76697a15c8ad0f3ff4b62ac90935b1e3a87" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "cd142fd2f64be2100422d658e7411e39489da985", + "version" : "1.2.0" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections", + "state" : { + "revision" : "671108c96644956dddcd89dd59c203dcdb36cec7", + "version" : "1.1.4" + } + }, + { + "identity" : "swift-syntax", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-syntax.git", + "state" : { + "revision" : "0687f71944021d616d34d922343dcef086855920", + "version" : "600.0.1" + } + }, + { + "identity" : "swiftapi", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vmanot/SwiftAPI.git", + "state" : { + "branch" : "master", + "revision" : "3e47cc5f9b0cefe9ed1d0971aff22583bd9ac7b0" + } + }, + { + "identity" : "swiftui-introspect", + "kind" : "remoteSourceControl", + "location" : "https://github.com/siteline/SwiftUI-Introspect.git", + "state" : { + "revision" : "807f73ce09a9b9723f12385e592b4e0aaebd3336", + "version" : "1.3.0" + } + }, + { + "identity" : "swiftuix", + "kind" : "remoteSourceControl", + "location" : "https://github.com/SwiftUIX/SwiftUIX.git", + "state" : { + "branch" : "master", + "revision" : "264cb593d0a7cbff0d95dcf715cdf5328ceb5e11" + } + }, + { + "identity" : "swiftuiz", + "kind" : "remoteSourceControl", + "location" : "https://github.com/SwiftUIX/SwiftUIZ.git", + "state" : { + "branch" : "main", + "revision" : "6f9cecdf80139854e21c43e25d26bc3a9c582969" + } + } + ], + "version" : 3 +} diff --git a/Package.swift b/Package.swift index 36218af..612e09d 100644 --- a/Package.swift +++ b/Package.swift @@ -29,6 +29,7 @@ let package = Package( .package(url: "https://github.com/SwiftUIX/SwiftUIX.git", branch: "master"), .package(url: "https://github.com/SwiftUIX/SwiftUIZ.git", branch: "main"), .package(url: "https://github.com/vmanot/CorePersistence.git", branch: "main"), + .package(url: "https://github.com/vmanot/Media.git", branch: "main"), .package(url: "https://github.com/vmanot/Merge.git", branch: "master"), .package(url: "https://github.com/vmanot/NetworkKit.git", branch: "master"), .package(url: "https://github.com/vmanot/Swallow.git", branch: "master"), @@ -98,6 +99,7 @@ let package = Package( "Cataphyl", "ChatKit", "CorePersistence", + "Media", "Merge", "NetworkKit", "SideprojectCore", @@ -119,6 +121,5 @@ let package = Package( ], path: "Tests/Sideproject" ), - ]/*, - cxxLanguageStandard: CXXLanguageStandard.cxx11*/ + ] ) diff --git a/SideprojectExample/SideprojectExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/SideprojectExample/SideprojectExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved new file mode 100644 index 0000000..d9f31a3 --- /dev/null +++ b/SideprojectExample/SideprojectExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -0,0 +1,168 @@ +{ + "originHash" : "fd6b49f16164d68a6c496e7b16ae556a43a809ab0ab84de151c20af8cd2f9bfe", + "pins" : [ + { + "identity" : "ai", + "kind" : "remoteSourceControl", + "location" : "https://github.com/PreternaturalAI/AI.git", + "state" : { + "branch" : "main", + "revision" : "4021a8f960476cebf51f5daf31483fd9d6f0452a" + } + }, + { + "identity" : "cataphyl", + "kind" : "remoteSourceControl", + "location" : "https://github.com/PreternaturalAI/Cataphyl.git", + "state" : { + "branch" : "main", + "revision" : "e53561c546fe4a0c5b50a5f3785c2bf5e65ee783" + } + }, + { + "identity" : "chatkit", + "kind" : "remoteSourceControl", + "location" : "https://github.com/PreternaturalAI/ChatKit.git", + "state" : { + "branch" : "main", + "revision" : "055ad6219fc87355b25ecdcb5eb4cbb1c6f51177" + } + }, + { + "identity" : "corepersistence", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vmanot/CorePersistence.git", + "state" : { + "branch" : "main", + "revision" : "3de61dda6b7153bde07aa7c1fbfd44c01c82e3a8" + } + }, + { + "identity" : "media", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vmanot/Media.git", + "state" : { + "branch" : "main", + "revision" : "0ba2baaebeb58667955daef68d3535ba1b217a12" + } + }, + { + "identity" : "merge", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vmanot/Merge.git", + "state" : { + "branch" : "master", + "revision" : "17e267f961c5ea9a3375c3a49807e66795a004e2" + } + }, + { + "identity" : "networkkit", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vmanot/NetworkKit.git", + "state" : { + "branch" : "master", + "revision" : "470af0276c2aa6e61acff22acc14c7ca30b99cbc" + } + }, + { + "identity" : "pow", + "kind" : "remoteSourceControl", + "location" : "https://github.com/PreternaturalAI/Pow.git", + "state" : { + "branch" : "main", + "revision" : "dc5839ef7cbb6c8b34698cda691159b21c68176d" + } + }, + { + "identity" : "swallow", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vmanot/Swallow.git", + "state" : { + "branch" : "master", + "revision" : "553de76697a15c8ad0f3ff4b62ac90935b1e3a87" + } + }, + { + "identity" : "swift-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-algorithms", + "state" : { + "revision" : "87e50f483c54e6efd60e885f7f5aa946cee68023", + "version" : "1.2.1" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "cd142fd2f64be2100422d658e7411e39489da985", + "version" : "1.2.0" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections", + "state" : { + "revision" : "671108c96644956dddcd89dd59c203dcdb36cec7", + "version" : "1.1.4" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics.git", + "state" : { + "revision" : "e0ec0f5f3af6f3e4d5e7a19d2af26b481acb6ba8", + "version" : "1.0.3" + } + }, + { + "identity" : "swift-syntax", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-syntax.git", + "state" : { + "revision" : "0687f71944021d616d34d922343dcef086855920", + "version" : "600.0.1" + } + }, + { + "identity" : "swiftapi", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vmanot/SwiftAPI.git", + "state" : { + "branch" : "master", + "revision" : "3e47cc5f9b0cefe9ed1d0971aff22583bd9ac7b0" + } + }, + { + "identity" : "swiftui-introspect", + "kind" : "remoteSourceControl", + "location" : "https://github.com/siteline/SwiftUI-Introspect.git", + "state" : { + "revision" : "807f73ce09a9b9723f12385e592b4e0aaebd3336", + "version" : "1.3.0" + } + }, + { + "identity" : "swiftuix", + "kind" : "remoteSourceControl", + "location" : "https://github.com/SwiftUIX/SwiftUIX.git", + "state" : { + "branch" : "master", + "revision" : "264cb593d0a7cbff0d95dcf715cdf5328ceb5e11" + } + }, + { + "identity" : "swiftuiz", + "kind" : "remoteSourceControl", + "location" : "https://github.com/SwiftUIX/SwiftUIZ.git", + "state" : { + "branch" : "main", + "revision" : "6f9cecdf80139854e21c43e25d26bc3a9c582969" + } + } + ], + "version" : 3 +} diff --git a/Sources/Sideproject/WIP/Media Generation (WIP)/MediaGenerationViewActor.swift b/Sources/Sideproject/WIP/Media Generation (WIP)/MediaGenerationViewActor.swift new file mode 100644 index 0000000..304c3ef --- /dev/null +++ b/Sources/Sideproject/WIP/Media Generation (WIP)/MediaGenerationViewActor.swift @@ -0,0 +1,221 @@ +// +// MediaGenerationViewActor.swift +// Sideproject +// +// Created by Jared Davidson on 1/10/25. +// + +import ElevenLabs +import AI +import Media +import SwiftUI +import SwiftUIX + +final class MediaGenerationViewActor: ObservableObject { + @Published var availableVoices: [AbstractVoice] = [] + @Published var availableModels: [VideoModel] = [] + @Published var currentInput: Any? + @Published var isLoading = false + @Published var selectedVoice: AbstractVoice.ID? + @Published var generatedFile: AnyMediaFile? + @Published var selectedVideoModel: VideoModel.ID? + @Published var speechClient: AnySpeechSynthesisRequestHandling? + @Published var videoClient: AnyVideoGenerationRequestHandling? + @Published var availableSpeechClients: [AnySpeechSynthesisRequestHandling] = [] + @Published var availableVideoClients: [AnyVideoGenerationRequestHandling] = [] + + internal let mediaType: MediaType + internal let inputModality: AnyInputModality + internal var configuration: MediaGenerationView.Configuration + internal let onComplete: ((AnyMediaFile) -> Void)? + + init( + mediaType: MediaType, + inputModality: AnyInputModality, + configuration: MediaGenerationView.Configuration, + onComplete: ((AnyMediaFile) -> Void)? + ) { + self.mediaType = mediaType + self.inputModality = inputModality + self.configuration = configuration + self.onComplete = onComplete + } + + @MainActor + internal func loadResources( + _ speechClient: (any SpeechSynthesisRequestHandling)?, + _ videoClient: (any VideoGenerationRequestHandling)? + ) async throws { + switch mediaType { + case .speech: + availableVoices = try await speechClient?.availableVoices() ?? [] + configuration.voiceSettings = .init() + + case .video: + availableModels = try await videoClient?.availableModels() ?? [] + configuration.videoSettings = .init() + } + } + + @MainActor + internal func generate( + _ speechClient: (any SpeechSynthesisRequestHandling)?, + _ videoClient: (any VideoGenerationRequestHandling)? + ) async { + isLoading = true + defer { isLoading = false } + + do { + switch mediaType { + case .speech: + try await generateSpeech(speechClient) + case .video: + try await generateVideo(videoClient) + } + } catch { + print("Error generating media: \(error)") + } + } + + @MainActor + private func generateSpeech( + _ speechClient: (any SpeechSynthesisRequestHandling)? + ) async throws { + guard let speechClient = speechClient else { + throw GenerationError.clientNotAvailable + } + + let audioData: Data? + + switch inputModality.inputType { + case is AudioFile.Type: + guard let audioFile = currentInput as? AudioFile else { return } + audioData = try await speechClient.speechToSpeech( + inputAudioURL: audioFile.url, + voiceID: selectedVoice?.id.rawValue ?? "", + voiceSettings: configuration.voiceSettings, + model: configuration.speechToSpeechModel + ) + + case is URL.Type: + guard let audioURL = currentInput as? URL else { return } + audioData = try await speechClient.speechToSpeech( + inputAudioURL: audioURL, + voiceID: selectedVoice?.id.rawValue ?? "", + voiceSettings: configuration.voiceSettings, + model: configuration.speechToSpeechModel + ) + + case is String.Type: + guard let text = currentInput as? String else { return } + audioData = try await speechClient.speech( + for: text, + voiceID: selectedVoice?.id.rawValue ?? "", + voiceSettings: configuration.voiceSettings, + model: configuration.textToSpeechModel + ) + + default: + fatalError(.unimplemented) + } + + guard let audioData = audioData else { return } + + let audioFile = try await AudioFile( + data: audioData, + name: UUID().uuidString, + id: .random() + ) + + generatedFile = .init(audioFile) + + if let onComplete = onComplete { + onComplete(AnyMediaFile(audioFile)) + } + } + + @MainActor + private func generateVideo( + _ videoClient: (any VideoGenerationRequestHandling)? + ) async throws { + guard let videoClient = videoClient else { + throw GenerationError.clientNotAvailable + } + + guard let modelID = selectedVideoModel, + let model = availableModels.first(where: { $0.id == modelID }) else { + throw GenerationError.modelNotSelected + } + + let videoData: Data? + + switch inputModality.inputType { + case is String.Type: + guard let text = currentInput as? String else { return } + videoData = try await videoClient.textToVideo( + text: text, + model: model, + settings: configuration.videoSettings + ) + + case is AppKitOrUIKitImage.Type: + guard let image = currentInput as? AppKitOrUIKitImage else { return } + let imageURL = try await saveImageTemporarily(image) + videoData = try await videoClient.imageToVideo( + imageURL: imageURL, + model: model, + settings: configuration.videoSettings + ) + + case is URL.Type: + guard let videoURL = currentInput as? URL else { return } + videoData = try await videoClient.videoToVideo( + videoURL: videoURL, + prompt: "", // Note: Would need to add prompt handling + model: model, + settings: configuration.videoSettings + ) + + default: + return + } + + guard let videoData = videoData else { + throw GenerationError.invalidVideoData + } + + let temporaryURL = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString) + .appendingPathExtension("mp4") + + try videoData.write(to: temporaryURL) + + let videoFile = try await VideoFile(url: temporaryURL) + generatedFile = .init(videoFile) + + if let onComplete = onComplete { + onComplete(AnyMediaFile(videoFile)) + } + } + + @MainActor + private func saveImageTemporarily(_ image: AppKitOrUIKitImage) async throws -> URL { + let temporaryURL = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString) + .appendingPathExtension("png") + + guard let imageData = image.pngData() else { + throw GenerationError.invalidVideoData + } + + try imageData.write(to: temporaryURL) + return temporaryURL + } +} + +fileprivate enum GenerationError: Error { + case invalidVideoData + case clientNotAvailable + case modelNotSelected + case resourceLoadingFailed +} diff --git a/Sources/Sideproject/WIP/Media Generation (WIP)/Models/AudioVariant.swift b/Sources/Sideproject/WIP/Media Generation (WIP)/Models/AudioVariant.swift new file mode 100644 index 0000000..d066135 --- /dev/null +++ b/Sources/Sideproject/WIP/Media Generation (WIP)/Models/AudioVariant.swift @@ -0,0 +1,22 @@ +// +// AudioVariant.swift +// Sideproject +// +// Created by Jared Davidson on 4/12/25. +// + +import SwiftUI + +public struct AudioVariant: OptionSet { + public let rawValue: Int + + public init(rawValue: Int) { + self.rawValue = rawValue + } + + public static let fileDrop = AudioVariant(rawValue: 1 << 0) + public static let recorder = AudioVariant(rawValue: 1 << 1) + public static let recorderWithTranscription = AudioVariant(rawValue: 1 << 2) + + public static let all: AudioVariant = [.fileDrop, .recorder, .recorderWithTranscription] +} diff --git a/Sources/Sideproject/WIP/Media Generation (WIP)/Models/Input Modalities/AnyInputModality.swift b/Sources/Sideproject/WIP/Media Generation (WIP)/Models/Input Modalities/AnyInputModality.swift new file mode 100644 index 0000000..08a41ce --- /dev/null +++ b/Sources/Sideproject/WIP/Media Generation (WIP)/Models/Input Modalities/AnyInputModality.swift @@ -0,0 +1,61 @@ +// +// InputModalityProtocol.swift +// Sideproject +// +// Created by Jared Davidson on 1/10/25. +// + +import SwiftUI +import AVFoundation +import Media + +public enum InputModality { + public static func audio(variants: AudioVariant = .all) -> AnyInputModality { + AnyInputModality(AudioInputModality(description: "Audio", variants: variants)) + } + + public static let text = AnyInputModality(TextInputModality()) + public static let image = AnyInputModality(FileInputModality(description: "Image")) + public static let video = AnyInputModality(FileInputModality(description: "Video")) +} + +extension AnyInputModality { + public static var text: Self { InputModality.text } + public static func audio(variants: AudioVariant = .all) -> Self { InputModality.audio(variants: variants) } + public static var image: Self { InputModality.image } + public static var video: Self { InputModality.video } +} + +public struct AnyInputModality { + private let _description: String + private let _makeInputView: (Binding, String) -> AnyView + private let _validate: (Any?) -> Bool + private let _type: Any.Type + + public var description: String { _description } + public var inputType: Any.Type { _type } + + public init(_ modality: T) { + self._description = modality.description + self._type = T.InputType.self + self._makeInputView = { binding, placeholder in + let typedBinding = Binding( + get: { binding.wrappedValue as? T.InputType }, + set: { binding.wrappedValue = $0 } + ) + return modality.makeInputView(inputBinding: typedBinding, placeholderText: placeholder) + } + self._validate = { input in + guard let typedInput = input as? T.InputType else { return false } + return modality.validate(typedInput) + } + } + + public func makeInputView(binding: Binding, placeholderText: String = "") -> AnyView { + _makeInputView(binding, placeholderText) + } + + public func validate(_ input: Any?) -> Bool { + _validate(input) + } +} diff --git a/Sources/Sideproject/WIP/Media Generation (WIP)/Models/Input Modalities/AudioInputModality.swift b/Sources/Sideproject/WIP/Media Generation (WIP)/Models/Input Modalities/AudioInputModality.swift new file mode 100644 index 0000000..86b1df3 --- /dev/null +++ b/Sources/Sideproject/WIP/Media Generation (WIP)/Models/Input Modalities/AudioInputModality.swift @@ -0,0 +1,37 @@ +// +// AudioInputModality.swift +// Sideproject +// +// Created by Jared Davidson on 1/14/25. +// + +import Media +import SwiftUI + +public struct AudioInputModality: InputModalityConfiguration { + public typealias InputType = AudioFile + + public var description: String + public var variants: AudioVariant + + public init( + description: String, + variants: AudioVariant = .all + ) { + self.description = description + self.variants = variants + } + + public func makeInputView(inputBinding: Binding, placeholderText: String) -> AnyView { + AnyView( + AudioInputView( + audioFile: inputBinding, + variants: variants + ) + ) + } + + public func validate(_ input: AudioFile?) -> Bool { + input != nil + } +} diff --git a/Sources/Sideproject/WIP/Media Generation (WIP)/Models/Input Modalities/FileInputModality.swift b/Sources/Sideproject/WIP/Media Generation (WIP)/Models/Input Modalities/FileInputModality.swift new file mode 100644 index 0000000..b0ac9e1 --- /dev/null +++ b/Sources/Sideproject/WIP/Media Generation (WIP)/Models/Input Modalities/FileInputModality.swift @@ -0,0 +1,35 @@ +// +// FileInputModality.swift +// Sideproject +// +// Created by Jared Davidson on 1/10/25. +// + +import Media +import SwiftUI + +public struct FileInputModality: InputModalityConfiguration { + public typealias InputType = T + + public var description: String + + public init(description: String) { + self.description = description + } + + public func makeInputView(inputBinding: Binding, placeholderText: String) -> AnyView { + AnyView( + FileDropView { files in + inputBinding.wrappedValue = files.first?.cast(to: T.self) + } content: { files in + if !files.isEmpty { + MediaFileListView(files) + } + } + ) + } + + public func validate(_ input: T?) -> Bool { + input != nil + } +} diff --git a/Sources/Sideproject/WIP/Media Generation (WIP)/Models/Input Modalities/TextInputModality.swift b/Sources/Sideproject/WIP/Media Generation (WIP)/Models/Input Modalities/TextInputModality.swift new file mode 100644 index 0000000..e699389 --- /dev/null +++ b/Sources/Sideproject/WIP/Media Generation (WIP)/Models/Input Modalities/TextInputModality.swift @@ -0,0 +1,42 @@ +// +// TextInputModality.swift +// Sideproject +// +// Created by Jared Davidson on 1/10/25. +// + +import SwiftUI + +public struct TextInputModality: InputModalityConfiguration { + public typealias InputType = String + + public var description: String { "Text" } + + public func makeInputView(inputBinding: Binding, placeholderText: String) -> AnyView { + AnyView( + TextEditor(text: Binding( + get: { inputBinding.wrappedValue ?? "" }, + set: { inputBinding.wrappedValue = $0.isEmpty ? nil : $0 } + )) + .frame(height: 100) + .overlay( + RoundedRectangle(cornerRadius: 8) + .stroke(Color.gray.opacity(0.2)) + ) + .overlay( + Group { + if inputBinding.wrappedValue?.isEmpty ?? true { + Text(placeholderText) + .foregroundColor(.gray) + .padding(.leading, 4) + } + }, + alignment: .topLeading + ) + ) + } + + public func validate(_ input: String?) -> Bool { + !(input ?? "").isEmpty + } +} diff --git a/Sources/Sideproject/WIP/Media Generation (WIP)/Protocols/InputModalityConfiguration.swift b/Sources/Sideproject/WIP/Media Generation (WIP)/Protocols/InputModalityConfiguration.swift new file mode 100644 index 0000000..a6eb056 --- /dev/null +++ b/Sources/Sideproject/WIP/Media Generation (WIP)/Protocols/InputModalityConfiguration.swift @@ -0,0 +1,15 @@ +// +// InputModalityConfiguration.swift +// Sideproject +// +// Created by Jared Davidson on 4/12/25. +// + +import SwiftUI + +public protocol InputModalityConfiguration { + associatedtype InputType + var description: String { get } + func makeInputView(inputBinding: Binding, placeholderText: String) -> AnyView + func validate(_ input: InputType?) -> Bool +} diff --git a/Sources/Sideproject/WIP/Media Generation (WIP)/Views/AudioInputView.swift b/Sources/Sideproject/WIP/Media Generation (WIP)/Views/AudioInputView.swift new file mode 100644 index 0000000..7b31a4a --- /dev/null +++ b/Sources/Sideproject/WIP/Media Generation (WIP)/Views/AudioInputView.swift @@ -0,0 +1,56 @@ +// +// AudioInputView.swift +// Sideproject +// +// Created by Jared Davidson on 1/14/25. +// + +import SwiftUI +import Media +import LargeLanguageModels + +public struct AudioInputView: View { + @Binding var audioFile: AudioFile? + let variants: AudioVariant + + public var body: some View { + VStack(spacing: 16) { + if let audioFile = audioFile { + HStack { + Button { + self.audioFile = nil + } label: { + Image(systemName: .arrowCounterclockwiseCircle) + } + } + + MediaFileView(file: audioFile) + } else { + if variants.contains(.fileDrop) { + FileDropView { files in + audioFile = files.first?.cast(to: AudioFile.self) + } content: { files in + EmptyView() + } + + if variants.contains(.recorder) { + Text("or") + .foregroundStyle(.secondary) + } + } + + if variants.contains(.recorder) || variants.contains(.recorderWithTranscription) { + AudioRecorderView(configuration: AudioRecorderViewConfiguration( + enableSpeechRecognition: variants.contains(.recorderWithTranscription) + )) { recordedAudio in + audioFile = recordedAudio + } content: { media in + if let media { + AudioFileView(file: media) + } + } + } + } + } + } +} diff --git a/Sources/Sideproject/WIP/Media Generation (WIP)/Views/MediaGenerationView+AnyInputModality.swift b/Sources/Sideproject/WIP/Media Generation (WIP)/Views/MediaGenerationView+AnyInputModality.swift new file mode 100644 index 0000000..91f2bed --- /dev/null +++ b/Sources/Sideproject/WIP/Media Generation (WIP)/Views/MediaGenerationView+AnyInputModality.swift @@ -0,0 +1,19 @@ +// +// MediaGenerationView+AnyInputModality.swift +// Sideproject +// +// Created by Jared Davidson on 4/12/25. +// + +import SwiftUI + +extension MediaGenerationView { + public func inputModality(_ modality: AnyInputModality) -> Self { + MediaGenerationView( + mediaType: self.mediaType, + inputModality: modality, + configuration: self.configuration, + onComplete: self.onComplete + ) + } +} diff --git a/Sources/Sideproject/WIP/Media Generation (WIP)/Views/MediaGenerationView+Views.swift b/Sources/Sideproject/WIP/Media Generation (WIP)/Views/MediaGenerationView+Views.swift new file mode 100644 index 0000000..b4aa89f --- /dev/null +++ b/Sources/Sideproject/WIP/Media Generation (WIP)/Views/MediaGenerationView+Views.swift @@ -0,0 +1,120 @@ +// +// MediaGenerationView+Views.swift +// Sideproject +// +// Created by Jared Davidson on 1/10/25. +// + +import SwiftUI +import ElevenLabs +import AI +import Media + +extension MediaGenerationView { + var inputView: some View { + inputModality.makeInputView( + binding: $viewActor.currentInput, + placeholderText: getPlaceholderText() + ) + } + + private func getPlaceholderText() -> String { + if case .video = mediaType, inputModality.inputType == URL.self { + return "Describe how you want to transform the video..." + } + switch inputModality.inputType { + case is String.Type: + return "Enter your text here..." + case is URL.Type: + return "Drop files here" + default: + return "" + } + } + + var clientSelectionView: some View { + VStack(alignment: .leading, spacing: 20) { + if mediaType == .speech { + Picker("Select Speech Client", selection: $viewActor.speechClient) { + ForEach(viewActor.availableSpeechClients, id: \.self) { client in + Text(client.displayName) + .tag(client as AnySpeechSynthesisRequestHandling?) + } + } + .pickerStyle(MenuPickerStyle()) + } else if mediaType == .video { + Picker("Select Video Client", selection: $viewActor.videoClient) { + ForEach(viewActor.availableVideoClients, id: \.self) { client in + Text("Video Client") + .tag(client as AnyVideoGenerationRequestHandling?) + } + } + .pickerStyle(MenuPickerStyle()) + } + } + } + + var modelSelectionView: some View { + VStack(alignment: .leading, spacing: 8) { + switch mediaType { + case .speech: + if !viewActor.availableVoices.isEmpty { + Picker("Voice", selection: $viewActor.selectedVoice) { + Text("Select a voice") + .tag(AbstractVoice.ID?.none) + ForEach(viewActor.availableVoices, id: \.id) { voice in + Text(voice.name) + .tag(Optional(AbstractVoice.ID(rawValue: voice.voiceID))) + } + } + } + + case .video: + if !viewActor.availableModels.isEmpty { + Picker("Model", selection: $viewActor.selectedVideoModel) { + Text("Select a model").tag(Optional.none) + ForEach(viewActor.availableModels, id: \.id) { model in + Text(model.name) + .tag(Optional(model.id)) + } + } + } + } + } + } + + var promptInputView: some View { + Group { + if case .video = mediaType, inputModality.inputType == URL.self { + inputModality.makeInputView( + binding: $viewActor.currentInput, + placeholderText: "Describe how you want to transform the video..." + ) + } else { + EmptyView() + } + } + } + + var controlsView: some View { + VStack(spacing: 12) { + Button { + Task { + await viewActor.generate( + viewActor.speechClient?.base, + viewActor.videoClient?.base + ) + } + } label: { + if viewActor.isLoading { + ProgressView() + .progressViewStyle(.circular) + } else { + Text("Generate") + } + } + .buttonStyle(.borderedProminent) + .disabled(viewActor.isLoading || !isGenerateEnabled) + } + } +} diff --git a/Sources/Sideproject/WIP/Media Generation (WIP)/Views/MediaGenerationView.swift b/Sources/Sideproject/WIP/Media Generation (WIP)/Views/MediaGenerationView.swift new file mode 100644 index 0000000..6106828 --- /dev/null +++ b/Sources/Sideproject/WIP/Media Generation (WIP)/Views/MediaGenerationView.swift @@ -0,0 +1,156 @@ +// +// MediaGenerationView.swift +// Sideproject +// +// Created by Jared Davidson on 1/10/25. +// + +import SwiftUI +import ElevenLabs +import Media +import AVFoundation +import SideprojectCore +import AI +import LargeLanguageModels +import Runtime + +public enum MediaType { + case speech + case video +} + +/// A simple input view to generate media of any format. +public struct MediaGenerationView: View { + public struct Configuration: Equatable { + public var textToSpeechModel: String + public var speechToSpeechModel: String + public var voiceSettings: AbstractVoiceSettings + public var videoSettings: VideoGenerationSettings + + // FIXME: - This should not be defaulting to ElevenLabs. Should be detached to an AbstractModel instead. + public init( + textToSpeechModel: String = ElevenLabs.Model.EnglishV1.rawValue, + speechToSpeechModel: String = ElevenLabs.Model.EnglishSTSV2.rawValue, + voiceSettings: AbstractVoiceSettings = .init(), + videoSettings: VideoGenerationSettings = .init() + ) { + self.textToSpeechModel = textToSpeechModel + self.speechToSpeechModel = speechToSpeechModel + self.voiceSettings = voiceSettings + self.videoSettings = videoSettings + } + } + + internal let mediaType: MediaType + internal let inputModality: AnyInputModality + internal var configuration: Configuration + internal let onComplete: ((AnyMediaFile) -> Void)? + + @StateObject internal var viewActor: MediaGenerationViewActor + + public init( + mediaType: MediaType, + configuration: Configuration = .init(), + onComplete: ((AnyMediaFile) -> Void)? = nil + ) { + // Default to text modality + self.init( + mediaType: mediaType, + inputModality: .text, + configuration: configuration, + onComplete: onComplete + ) + } + + internal init( + mediaType: MediaType, + inputModality: AnyInputModality, + configuration: Configuration = .init(), + onComplete: ((AnyMediaFile) -> Void)? = nil + ) { + self.mediaType = mediaType + self.inputModality = inputModality + self.configuration = configuration + self.onComplete = onComplete + + let viewActor = MediaGenerationViewActor( + mediaType: mediaType, + inputModality: inputModality, + configuration: configuration, + onComplete: onComplete + ) + _viewActor = StateObject(wrappedValue: viewActor) + } + + public var body: some View { + VStack(alignment: .leading, spacing: 20) { + if let mediaFile = viewActor.generatedFile { + MediaFileView(file: mediaFile.file) + } + + inputModality.makeInputView(binding: $viewActor.currentInput) + clientSelectionView + modelSelectionView + + if case .video = mediaType, inputModality.inputType == URL.self { + promptInputView + } + + controlsView + } + .padding() + .task { + await loadClients() + try? await viewActor.loadResources( + viewActor.speechClient?.base, + viewActor.videoClient?.base + ) + } + .onChange(of: viewActor.speechClient) { + oldValue, + newValue in + Task { + try? await viewActor.loadResources( + viewActor.speechClient?.base, + viewActor.videoClient?.base + ) + } + } + } + + private func loadClients() async { + do { + let services = try await Sideproject.shared.services + + self.viewActor.availableSpeechClients = services.compactMap { service in + if let service = service as? (any CoreMI._ServiceClientProtocol & SpeechSynthesisRequestHandling) { + return AnySpeechSynthesisRequestHandling(service) + } + return nil + } + + self.viewActor.availableVideoClients = services.compactMap { service in + if let service = service as? (any CoreMI._ServiceClientProtocol & VideoGenerationRequestHandling) { + return AnyVideoGenerationRequestHandling(service) + } + return nil + } + + self.viewActor.speechClient = self.viewActor.availableSpeechClients.first + self.viewActor.videoClient = self.viewActor.availableVideoClients.first + } catch { + print("Error loading clients: \(error)") + } + } + + internal var isGenerateEnabled: Bool { + let isInputValid = inputModality.validate(viewActor.currentInput) + + let isModelSelected = switch mediaType { + case .speech: viewActor.selectedVoice != nil + case .video: viewActor.selectedVideoModel != nil + } + + return isInputValid && isModelSelected + } +} diff --git a/Sources/Sideproject/WIP/Views/ModelSearchVIew.Tab.swift b/Sources/Sideproject/WIP/Views/ModelSearchVIew.Tab.swift new file mode 100644 index 0000000..d1c0bf6 --- /dev/null +++ b/Sources/Sideproject/WIP/Views/ModelSearchVIew.Tab.swift @@ -0,0 +1,34 @@ +// +// ModelSearchVIew.Tab.swift +// Sideproject +// +// Created by Jared Davidson on 4/12/25. +// + +import SwiftUI + +extension ModelSearchView { + public enum Tab { + case discover + case downloaded + + var keyPath: KeyPath { + switch self { + case .discover: return \ModelStore.models + case .downloaded: return \ModelStore.downloadedModels + } + } + } +} + + +// MARK: - Conformances + +extension ModelSearchView.Tab: CustomStringConvertible, CaseIterable { + public var description: String { + switch self { + case .discover: "Discover" + case .downloaded: "Downloaded" + } + } +} diff --git a/Sources/Sideproject/WIP/Views/ModelSearchView.swift b/Sources/Sideproject/WIP/Views/ModelSearchView.swift index f80b5a5..25f57d1 100644 --- a/Sources/Sideproject/WIP/Views/ModelSearchView.swift +++ b/Sources/Sideproject/WIP/Views/ModelSearchView.swift @@ -26,30 +26,3 @@ public struct ModelSearchView: View { .environmentObject(Sideproject.ExternalAccountStore.shared) } } - - -extension ModelSearchView { - public enum Tab { - case discover - case downloaded - - var keyPath: KeyPath { - switch self { - case .discover: return \ModelStore.models - case .downloaded: return \ModelStore.downloadedModels - } - } - } -} - - -// MARK: - Conformances - -extension ModelSearchView.Tab: CustomStringConvertible, CaseIterable { - public var description: String { - switch self { - case .discover: "Discover" - case .downloaded: "Downloaded" - } - } -} diff --git a/Sources/SideprojectCore/Intramodular/Accounts/Sideproject.ExternalAccountStore.swift b/Sources/SideprojectCore/Intramodular/Accounts/Sideproject.ExternalAccountStore.swift index 8fce2ce..66fb4b6 100644 --- a/Sources/SideprojectCore/Intramodular/Accounts/Sideproject.ExternalAccountStore.swift +++ b/Sources/SideprojectCore/Intramodular/Accounts/Sideproject.ExternalAccountStore.swift @@ -81,6 +81,13 @@ extension Sideproject.ExternalAccountStore { .compactMapValues({ $0.credential }) } + public func firstCredentialIfAvailable( + ofType type: Sideproject.ExternalAccountCredentialTypeName, + for accountType: any Sideproject.ExternalAccountTypeDescriptor + ) throws -> T? { + try credentials(for: accountType).firstAndOnly(byUnwrapping: { $0.value as? T }) + } + /// Returns all available credentials for a given account type, keyed by account IDs. /// /// For example `Sideproject.ExternalAccountStore.shared.credentials(ofType: .apiKey, for: .groq)` @@ -88,7 +95,7 @@ extension Sideproject.ExternalAccountStore { ofType type: Sideproject.ExternalAccountCredentialTypeName, for accountType: any Sideproject.ExternalAccountTypeDescriptor ) throws -> T { - try credentials(for: accountType).firstAndOnly(byUnwrapping: { $0.value as? T }).unwrap() + try firstCredentialIfAvailable(ofType: type, for: accountType).unwrap() } public func hasCredentials( diff --git a/Sources/SideprojectCore/Intramodular/Accounts/Sideproject.ExternalAccountTypeDescriptor.swift b/Sources/SideprojectCore/Intramodular/Accounts/Sideproject.ExternalAccountTypeDescriptor.swift index db81ff8..95a2747 100644 --- a/Sources/SideprojectCore/Intramodular/Accounts/Sideproject.ExternalAccountTypeDescriptor.swift +++ b/Sources/SideprojectCore/Intramodular/Accounts/Sideproject.ExternalAccountTypeDescriptor.swift @@ -234,7 +234,7 @@ extension Sideproject.ExternalAccountTypeDescriptors { } } - @HadeanIdentifier("fisul-tapos-hotak-nonov") + @HadeanIdentifier("jatap-jogaz-ritiz-vibok") public struct ElevenLabs: Sideproject.ExternalAccountTypeDescriptor, _StaticInstance { public var accountType: Sideproject.ExternalAccountTypeIdentifier { "com.vmanot.elevenlabs" diff --git a/Sources/SideprojectCore/Intramodular/Accounts/SideprojectAccountsView.swift b/Sources/SideprojectCore/Intramodular/Accounts/SideprojectAccountsView.swift index 0089e5b..9ae6ff8 100644 --- a/Sources/SideprojectCore/Intramodular/Accounts/SideprojectAccountsView.swift +++ b/Sources/SideprojectCore/Intramodular/Accounts/SideprojectAccountsView.swift @@ -54,9 +54,9 @@ public struct SideprojectAccountsView: View { cellGrid .frame(minWidth: 126) - #if os(macOS) +#if os(macOS) PathControl(url: try! store.$accounts.url) - #endif +#endif } .padding() .environmentObject(store) @@ -85,9 +85,11 @@ public struct SideprojectAccountsView: View { presentationManager.dismiss() } + #if os(macOS) ._overrideOnExitCommand { presentationManager.dismiss() } + #endif } } label: { Cell(account: $account._assigningLogicalParent(store, to: \.$store)) @@ -138,9 +140,11 @@ public struct SideprojectAccountsView: View { // ???: (@vmanot) this is a temporary hack to fix saving on first try // store.accounts = store.accounts } +#if os(macOS) ._overrideOnExitCommand { presentationManager.dismiss() } +#endif } } label: { Image(systemName: .plus) diff --git a/Sources/SideprojectCore/Intramodular/Sideproject.swift b/Sources/SideprojectCore/Intramodular/Sideproject.swift index 4464151..2eda4ec 100644 --- a/Sources/SideprojectCore/Intramodular/Sideproject.swift +++ b/Sources/SideprojectCore/Intramodular/Sideproject.swift @@ -29,8 +29,8 @@ public final class Sideproject: _CancellablesProviding, Logging, ObservableObjec #metatype((any CoreMI._ServiceClientProtocol).self), .nonAppleFramework ) - internal static var serviceTypes: [any CoreMI._ServiceClientProtocol.Type] - + private static var serviceTypes: [any CoreMI._ServiceClientProtocol.Type] + @_StaticMirrorQuery( #metatype((any Sideproject.ExternalAccountTypeDescriptor).self), .nonAppleFramework @@ -46,13 +46,14 @@ public final class Sideproject: _CancellablesProviding, Logging, ObservableObjec @Published private var autoinitializedServices: [any CoreMI._ServiceClientProtocol]? = nil { didSet { if let newValue = autoinitializedServices { + logger.info(newValue.description) logger.info("Auto-initialized \(newValue.count) service(s).") } } } @MainActor - @Published private var manuallyAddedServices: [any CoreMI._ServiceClientProtocol] = [] + @Published public var manuallyAddedServices: [any CoreMI._ServiceClientProtocol] = [] // @Published public var modelIdentifierScope: ModelIdentifierScope? @@ -156,12 +157,15 @@ extension Sideproject { let oldAccounts: [CoreMI._AnyServiceAccount] = self.autodiscoveredServiceAccounts let newAccounts = try self._serviceAccounts() + print(newAccounts) + guard oldAccounts != newAccounts else { return } let services: [any CoreMI._ServiceClientProtocol] = try await self._makeServices(forAccounts: newAccounts) + print(services) self.autodiscoveredServiceAccounts = newAccounts self.autoinitializedServices = services @@ -202,20 +206,23 @@ extension Sideproject { let serviceTypes = Sideproject.serviceTypes var result: [any CoreMI._ServiceClientProtocol] = await serviceAccounts - .asyncMap { (account: CoreMI._AnyServiceAccount) in - await serviceTypes.first(byUnwrapping: { type -> (any CoreMI._ServiceClientProtocol)? in - do { - return try await type.init(account: account) - } catch { + .asyncMap { account in + await serviceTypes + .asyncCompactMap { type in do { - return try await type.init(account: nil) - } catch(_) { - return nil + let service = try await type.init(account: account) + return service + } catch { + do { + let service = try await type.init(account: nil) + return service + } catch { + return nil + } } } - }) } - .compactMap({ $0 }) + .flatMap { $0 } result += await serviceTypes .concurrentMap({ try? await $0.init(account: nil) }) diff --git a/Sources/SideprojectCore/Intramodular/UI/_AccountPicker.swift b/Sources/SideprojectCore/Intramodular/UI/_AccountPicker.swift index fee3d4a..2701c2a 100644 --- a/Sources/SideprojectCore/Intramodular/UI/_AccountPicker.swift +++ b/Sources/SideprojectCore/Intramodular/UI/_AccountPicker.swift @@ -39,9 +39,11 @@ public struct _AccountPicker: View { } .environmentObject(store) } + #if os(macOS) ._overrideOnExitCommand { presentationMode.dismiss() } + #endif } .frame(idealWidth: 448, idealHeight: 560) .background(Color.accountModalBackgroundColor.ignoresSafeArea())