Skip to content

Commit

Permalink
Vision support for GPT4o (#44)
Browse files Browse the repository at this point in the history
* Vector store

* improving examples

* Fixing vision
  • Loading branch information
jamesrochabrun committed Jun 1, 2024
1 parent 4d90fcb commit 3be54cb
Show file tree
Hide file tree
Showing 12 changed files with 524 additions and 30 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@
7B7239AB2AF6294C00646679 /* URLImageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B7239AA2AF6294C00646679 /* URLImageView.swift */; };
7B7239AE2AF9FF0000646679 /* ChatFunctionsCallStreamProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B7239AD2AF9FF0000646679 /* ChatFunctionsCallStreamProvider.swift */; };
7B7239B12AF9FF3C00646679 /* ChatFunctionsCalllStreamDemoView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B7239B02AF9FF3C00646679 /* ChatFunctionsCalllStreamDemoView.swift */; };
7B99C2E72C0718DE00E701B3 /* FilesPicker.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B99C2E62C0718DE00E701B3 /* FilesPicker.swift */; };
7B99C2E92C0718FF00E701B3 /* FileAttachmentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B99C2E82C0718FF00E701B3 /* FileAttachmentView.swift */; };
7B99C2EB2C07191200E701B3 /* AttachmentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B99C2EA2C07191200E701B3 /* AttachmentView.swift */; };
7B99C2ED2C071B1600E701B3 /* FilesPickerProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B99C2EC2C071B1600E701B3 /* FilesPickerProvider.swift */; };
7BA788CD2AE23A48008825D5 /* SwiftOpenAIExampleApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7BA788CC2AE23A48008825D5 /* SwiftOpenAIExampleApp.swift */; };
7BA788CF2AE23A48008825D5 /* ApiKeyIntroView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7BA788CE2AE23A48008825D5 /* ApiKeyIntroView.swift */; };
7BA788D12AE23A49008825D5 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 7BA788D02AE23A49008825D5 /* Assets.xcassets */; };
Expand Down Expand Up @@ -115,6 +119,10 @@
7B7239AA2AF6294C00646679 /* URLImageView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = URLImageView.swift; sourceTree = "<group>"; };
7B7239AD2AF9FF0000646679 /* ChatFunctionsCallStreamProvider.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatFunctionsCallStreamProvider.swift; sourceTree = "<group>"; };
7B7239B02AF9FF3C00646679 /* ChatFunctionsCalllStreamDemoView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatFunctionsCalllStreamDemoView.swift; sourceTree = "<group>"; };
7B99C2E62C0718DE00E701B3 /* FilesPicker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FilesPicker.swift; sourceTree = "<group>"; };
7B99C2E82C0718FF00E701B3 /* FileAttachmentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileAttachmentView.swift; sourceTree = "<group>"; };
7B99C2EA2C07191200E701B3 /* AttachmentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AttachmentView.swift; sourceTree = "<group>"; };
7B99C2EC2C071B1600E701B3 /* FilesPickerProvider.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FilesPickerProvider.swift; sourceTree = "<group>"; };
7BA788C92AE23A48008825D5 /* SwiftOpenAIExample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = SwiftOpenAIExample.app; sourceTree = BUILT_PRODUCTS_DIR; };
7BA788CC2AE23A48008825D5 /* SwiftOpenAIExampleApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SwiftOpenAIExampleApp.swift; sourceTree = "<group>"; };
7BA788CE2AE23A48008825D5 /* ApiKeyIntroView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ApiKeyIntroView.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -297,6 +305,17 @@
path = SharedModels;
sourceTree = "<group>";
};
7B99C2E52C0718CD00E701B3 /* Files */ = {
isa = PBXGroup;
children = (
7B99C2E62C0718DE00E701B3 /* FilesPicker.swift */,
7B99C2E82C0718FF00E701B3 /* FileAttachmentView.swift */,
7B99C2EA2C07191200E701B3 /* AttachmentView.swift */,
7B99C2EC2C071B1600E701B3 /* FilesPickerProvider.swift */,
);
path = Files;
sourceTree = "<group>";
};
7BA788C02AE23A48008825D5 = {
isa = PBXGroup;
children = (
Expand All @@ -321,6 +340,7 @@
isa = PBXGroup;
children = (
7BA788CC2AE23A48008825D5 /* SwiftOpenAIExampleApp.swift */,
7B99C2E52C0718CD00E701B3 /* Files */,
7B7239AF2AF9FF1D00646679 /* SharedModels */,
7B7239A92AF6294200646679 /* SharedUI */,
7B1268032B08241200400694 /* Assistants */,
Expand Down Expand Up @@ -564,6 +584,7 @@
7B7239AB2AF6294C00646679 /* URLImageView.swift in Sources */,
7B7239B12AF9FF3C00646679 /* ChatFunctionsCalllStreamDemoView.swift in Sources */,
7BBE7EAB2B02E8FC0096A693 /* ChatMessageDisplayModel.swift in Sources */,
7B99C2E92C0718FF00E701B3 /* FileAttachmentView.swift in Sources */,
7BBE7EA52B02E8A70096A693 /* Sizes.swift in Sources */,
7B7239A22AF6260D00646679 /* ChatDisplayMessage.swift in Sources */,
0DF957862BB543F100DD2013 /* AIProxyIntroView.swift in Sources */,
Expand All @@ -572,6 +593,7 @@
7B436B962AE24A04003CE281 /* OptionsListView.swift in Sources */,
7BBE7EDE2B03718E0096A693 /* ChatFunctionCallProvider.swift in Sources */,
7B7239A62AF628F800646679 /* ChatDisplayMessageView.swift in Sources */,
7B99C2ED2C071B1600E701B3 /* FilesPickerProvider.swift in Sources */,
7B7239A02AF625F200646679 /* ChatFluidConversationProvider.swift in Sources */,
7BA788CF2AE23A48008825D5 /* ApiKeyIntroView.swift in Sources */,
7BA788CD2AE23A48008825D5 /* SwiftOpenAIExampleApp.swift in Sources */,
Expand All @@ -586,9 +608,11 @@
7B436BAD2AE788FB003CE281 /* FineTuningJobDemoView.swift in Sources */,
7B436BB02AE79369003CE281 /* FilesDemoView.swift in Sources */,
7BBE7E912AFCA52A0096A693 /* ChatVisionDemoView.swift in Sources */,
7B99C2EB2C07191200E701B3 /* AttachmentView.swift in Sources */,
7B436BAB2AE788F1003CE281 /* FineTuningJobProvider.swift in Sources */,
7B7239A42AF6289900646679 /* ChatStreamFluidConversationDemoView.swift in Sources */,
7BA788FC2AE23B42008825D5 /* AudioDemoView.swift in Sources */,
7B99C2E72C0718DE00E701B3 /* FilesPicker.swift in Sources */,
7B1268072B08247C00400694 /* AssistantConfigurationProvider.swift in Sources */,
7B436BBE2AE7ABDA003CE281 /* ModelsDemoView.swift in Sources */,
7B436BA32AE25962003CE281 /* ChatDemoView.swift in Sources */,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@ struct AssistantConfigurationDemoView: View {
@State private var isAvatarLoading = false
@State private var showAvatarFlow = false
private let service: OpenAIService

@State private var fileIDS: [String] = []
/// Used mostly to display already uploaded files if any.
@State private var filePickerInitialActions: [FilePickerAction] = []

var isCodeInterpreterOn: Binding<Bool> {
Binding(
get: {
Expand Down Expand Up @@ -72,6 +75,23 @@ struct AssistantConfigurationDemoView: View {
)
}

var isFileSearchOn: Binding<Bool> {
Binding(
get: {
let contains =
self.parameters.tools.contains { $0.displayToolType == .fileSearch } == true
return contains
},
set: { newValue in
if newValue {
self.parameters.tools.append(AssistantObject.Tool(type: .fileSearch))
} else {
self.parameters.tools.removeAll { $0.displayToolType == .fileSearch }
}
}
)
}

init(service: OpenAIService) {
self.service = service
_provider = State(initialValue: AssistantConfigurationProvider(service: service))
Expand All @@ -84,6 +104,7 @@ struct AssistantConfigurationDemoView: View {
inputViews
capabilities
footerActions
knowledge
}
.padding()
}.sheet(isPresented: $showAvatarFlow) {
Expand Down Expand Up @@ -186,11 +207,22 @@ struct AssistantConfigurationDemoView: View {
InputView(title: "Capabilities") {
VStack(spacing: 16) {
CheckboxRow(title: "Code interpreter", isChecked: isCodeInterpreterOn)
CheckboxRow(title: "File Search", isChecked: isFileSearchOn)
CheckboxRow(title: "DALL·E Image Generation", isChecked: isDalleToolOn)
}
}
.inputViewStyle(.init(verticalPadding: 16.0))
}

// TODO: Add a demo to create a vector store and add files in to it.
var knowledge: some View {
FilesPicker(
service: service,
sectionTitle: "Knowledge",
actionTitle: "Upload files",
fileIDS: $fileIDS,
actions: $filePickerInitialActions)
}
}

extension Binding where Value == String? {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,4 +67,9 @@ import SwiftOpenAI
debugPrint("\(error)")
}
}

// TODO: Create demo for this.
func createVStore ()async throws {
let _ = try await service.createVectorStore(parameters: .init(name: "Personal Data"))
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
//
// AttachmentView.swift
// SwiftOpenAIExample
//
// Created by James Rochabrun on 5/29/24.
//

import SwiftUI

struct AttachmentView: View {

let fileName: String
@Binding var actionTrigger: Bool
let isLoading: Bool

var body: some View {
HStack(spacing: Sizes.spacingExtraSmall) {
HStack {
if isLoading == true {
ProgressView()
.frame(width: 10, height: 10)
.padding(.horizontal, Sizes.spacingExtraSmall)
} else {
Image(systemName: "doc")
.resizable()
.aspectRatio(contentMode: .fit)
.frame(width: 10)
.foregroundColor(.secondary)
}
Text(fileName)
.font(.caption2)
}
Button {
actionTrigger = true

} label: {
Image(systemName: "xmark.circle.fill")
}
.disabled(isLoading)
}
.padding(.leading, Sizes.spacingMedium)
.background(
RoundedRectangle(cornerRadius: 8)
.stroke(.gray.opacity(0.5), lineWidth: 0.5)
)
}
}

#Preview {
AttachmentView(fileName: "Mydocument.pdf", actionTrigger: .constant(true), isLoading: true)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
//
// FileAttachmentView.swift
// SwiftOpenAIExample
//
// Created by James Rochabrun on 5/29/24.
//

import SwiftUI
import SwiftOpenAI

struct FileAttachmentView: View {

init(
service: OpenAIService,
action: FilePickerAction,
fileUploadedCompletion: @escaping (_ file: FileObject) -> Void,
fileDeletedCompletion: @escaping (_ parameters: FilePickerAction, _ id: String) -> Void)
{
self.fileProvider = FilesPickerProvider(service: service)
self.action = action
self.fileUploadedCompletion = fileUploadedCompletion
self.fileDeletedCompletion = fileDeletedCompletion
}

func newUploadedFileView(
parameters: FileParameters)
-> some View
{
AttachmentView(fileName: fileObject?.filename ?? parameters.fileName, actionTrigger: $deleted, isLoading: fileObject == nil || deleted)
.disabled(fileObject == nil)
.opacity(fileObject == nil ? 0.3 : 1)
.onFirstAppear {
Task {
fileObject = try await fileProvider.uploadFile(parameters: parameters)
}
}
.onChange(of: fileObject) { oldValue, newValue in
if oldValue != newValue, let newValue {
fileUploadedCompletion(newValue)
}
}
}

func previousUploadedFileView(
id: String)
-> some View
{
AttachmentView(fileName: fileObject?.filename ?? "Document", actionTrigger: $deleted, isLoading: fileObject == nil || deleted)
.onFirstAppear {
Task {
fileObject = try await fileProvider.retrieveFileWith(id: id)
}
}
}

var body: some View {
Group {
switch action {
case .request(let parameters):
newUploadedFileView(parameters: parameters)
case .retrieveAndDisplay(let id):
previousUploadedFileView(id: id)
}
}
.onChange(of: deleted) { oldValue, newValue in
if oldValue != newValue, newValue {
Task {
if let fileObject {
fileDeleteStatus = try await fileProvider.deleteFileWith(id: fileObject.id)
}
}
}
}
.onChange(of: fileDeleteStatus) { oldValue, newValue in
if oldValue != newValue, let newValue, newValue.deleted {
fileDeletedCompletion(action, newValue.id)
}
}
}

// MARK: Private

private let fileProvider: FilesPickerProvider
private let fileUploadedCompletion: (_ file: FileObject) -> Void
private let fileDeletedCompletion: (_ action: FilePickerAction, _ id: String) -> Void
private let action: FilePickerAction
@State private var fileObject: FileObject?
@State private var fileDeleteStatus: DeletionStatus?
@State private var deleted: Bool = false
}


private struct OnFirstAppear: ViewModifier {
let perform: () -> Void

@State private var firstTime = true

func body(content: Content) -> some View {
content.onAppear {
if firstTime {
firstTime = false
perform()
}
}
}
}

extension View {
func onFirstAppear(perform: @escaping () -> Void) -> some View {
modifier(OnFirstAppear(perform: perform))
}
}

extension DeletionStatus: Equatable {
public static func == (lhs: DeletionStatus, rhs: DeletionStatus) -> Bool {
lhs.id == rhs.id
}
}
Loading

0 comments on commit 3be54cb

Please sign in to comment.