From b3399110ec3d0f0586da4dbe104a3f59a8ea8ec0 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 9 Jul 2024 14:20:04 +0800 Subject: [PATCH 01/59] Init objc bindings --- objectivec/cxx_api.h | 17 +++++++++++++++ objectivec/error_utils.h | 29 +++++++++++++++++++++++++ objectivec/error_utils.mm | 33 +++++++++++++++++++++++++++++ objectivec/include/ort_genai_objc.h | 12 +++++++++++ objectivec/oga_model.mm | 25 ++++++++++++++++++++++ onnxruntime-genai-objc.podspec | 27 +++++++++++++++++++++++ 6 files changed, 143 insertions(+) create mode 100644 objectivec/cxx_api.h create mode 100644 objectivec/error_utils.h create mode 100644 objectivec/error_utils.mm create mode 100644 objectivec/include/ort_genai_objc.h create mode 100644 objectivec/oga_model.mm create mode 100644 onnxruntime-genai-objc.podspec diff --git a/objectivec/cxx_api.h b/objectivec/cxx_api.h new file mode 100644 index 000000000..66b0087b0 --- /dev/null +++ b/objectivec/cxx_api.h @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// wrapper for ORT C/C++ API headers + +#if defined(__clang__) +#pragma clang diagnostic push +// ignore clang documentation-related warnings +// instead, we will rely on Doxygen warnings for the C/C++ API headers +#pragma clang diagnostic ignored "-Wdocumentation" +#endif // defined(__clang__) + +#import "ort_genai.h" + +#if defined(__clang__) +#pragma clang diagnostic pop +#endif // defined(__clang__) \ No newline at end of file diff --git a/objectivec/error_utils.h b/objectivec/error_utils.h new file mode 100644 index 000000000..4e213ac6c --- /dev/null +++ b/objectivec/error_utils.h @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import + +#include + +#import "cxx_api.h" + +NS_ASSUME_NONNULL_BEGIN + +void OGASaveCodeAndDescriptionToError(int code, const char* description, NSError** error); +void OGASaveCodeAndDescriptionToError(int code, NSString* description, NSError** error); +void OGASaveExceptionToError(const std::exception& e, NSError** error); + +// helper macros to catch and handle C++ exceptions +#define OGA_OBJC_API_IMPL_CATCH(error, failure_return_value) \ + catch (const std::exception& e) { \ + OGASaveExceptionToError(e, (error)); \ + return (failure_return_value); \ + } + +#define OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) \ + OGA_OBJC_API_IMPL_CATCH(error, NO) + +#define OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) \ + OGA_OBJC_API_IMPL_CATCH(error, nil) + +NS_ASSUME_NONNULL_END diff --git a/objectivec/error_utils.mm b/objectivec/error_utils.mm new file mode 100644 index 000000000..cbfdc0a9e --- /dev/null +++ b/objectivec/error_utils.mm @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import "error_utils.h" + +NS_ASSUME_NONNULL_BEGIN + +static NSString* const kOgaErrorDomain = @"onnxruntime-genai"; + +void OGASaveCodeAndDescriptionToError(int code, const char* descriptionCstr, NSError** error) { + if (!error) return; + + NSString* description = [NSString stringWithCString:descriptionCstr + encoding:NSASCIIStringEncoding]; + + *error = [NSError errorWithDomain:kOgaErrorDomain + code:code + userInfo:@{NSLocalizedDescriptionKey : description}]; +} + +void OGASaveCodeAndDescriptionToError(int code, NSString* description, NSError** error) { + if (!error) return; + + *error = [NSError errorWithDomain:kOgaErrorDomain + code:code + userInfo:@{NSLocalizedDescriptionKey : description}]; +} + +void OGASaveExceptionToError(const std::exception& e, NSError** error) { + OGASaveCodeAndDescriptionToError(0x0A, e.what(), error); +} + +NS_ASSUME_NONNULL_END diff --git a/objectivec/include/ort_genai_objc.h b/objectivec/include/ort_genai_objc.h new file mode 100644 index 000000000..aea001803 --- /dev/null +++ b/objectivec/include/ort_genai_objc.h @@ -0,0 +1,12 @@ +NS_ASSUME_NONNULL_BEGIN + +@interface OGAModel : NSObject + +- (instancetype)init NS_UNAVAILABLE; +- (nullable)initWithConfigPath:(NSString *)path + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + + +@end + +NS_ASSUME_NONNULL_END diff --git a/objectivec/oga_model.mm b/objectivec/oga_model.mm new file mode 100644 index 000000000..6e2532506 --- /dev/null +++ b/objectivec/oga_model.mm @@ -0,0 +1,25 @@ +#import "ort_genai_objc.h" +#import "cxx_api.h" +#import "error_utils.h" + +@implementation OGAModel { + std::unique_ptr _model; +} + + +- (nullable)initWithConfigPath:(NSString *)path + error:(NSError **)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _model = OgaModel::Create("phi-2"); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) + +} + + +@end diff --git a/onnxruntime-genai-objc.podspec b/onnxruntime-genai-objc.podspec new file mode 100644 index 000000000..4eb2dd202 --- /dev/null +++ b/onnxruntime-genai-objc.podspec @@ -0,0 +1,27 @@ + +Pod::Spec.new do |s| + s.name = "onnxruntime-genai-objc" + s.version = "0.4.0" + s.summary = "SUMMARY" + s.homepage = "https://github.com/microsoft/onnxruntime-genai" + s.license = { :type => "MIT" } + s.author = {"ONNX Runtime" => "onnxruntime@microsoft.com"} + s.source = { :path => './objectivec'} + + s.static_framework = true + s.public_header_files = [ + "objectivec/include/ort_genai_objc.h" + ] + + s.source_files = [ + "objectivec/include/ort_genai_objc.h", + "objectivec/cxx_api.h", + "objectivec/oga_model.mm", + "objectivec/error_utils.h", + "objectivec/error_utils.mm", + "src/ort_genai.h" + ] + + s.dependency 'onnxruntime-objc', '~> 1.18.0' +end + From 1e14fa76eb21e626cb48745f80665520e4acf3f0 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 16 Jul 2024 14:37:57 +0800 Subject: [PATCH 02/59] More objc --- objectivec/cxx_api.h | 2 +- objectivec/include/ort_genai_objc.h | 49 +++++++++++++++++++++++++ objectivec/oga_generator_params.mm | 57 +++++++++++++++++++++++++++++ objectivec/oga_internal.h | 34 +++++++++++++++++ objectivec/oga_model.mm | 17 +++++++-- objectivec/oga_sequences.mm | 43 ++++++++++++++++++++++ objectivec/oga_tokenizer.mm | 47 ++++++++++++++++++++++++ 7 files changed, 245 insertions(+), 4 deletions(-) create mode 100644 objectivec/oga_generator_params.mm create mode 100644 objectivec/oga_internal.h create mode 100644 objectivec/oga_sequences.mm create mode 100644 objectivec/oga_tokenizer.mm diff --git a/objectivec/cxx_api.h b/objectivec/cxx_api.h index 66b0087b0..d1f5fa06f 100644 --- a/objectivec/cxx_api.h +++ b/objectivec/cxx_api.h @@ -14,4 +14,4 @@ #if defined(__clang__) #pragma clang diagnostic pop -#endif // defined(__clang__) \ No newline at end of file +#endif // defined(__clang__) diff --git a/objectivec/include/ort_genai_objc.h b/objectivec/include/ort_genai_objc.h index aea001803..2b2a52126 100644 --- a/objectivec/include/ort_genai_objc.h +++ b/objectivec/include/ort_genai_objc.h @@ -1,5 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + NS_ASSUME_NONNULL_BEGIN +@class OGASequences; +@class OGAGeneratorParams; + @interface OGAModel : NSObject - (instancetype)init NS_UNAVAILABLE; @@ -7,6 +13,49 @@ NS_ASSUME_NONNULL_BEGIN error:(NSError **)error NS_DESIGNATED_INITIALIZER; +- (nullable OGASequences *)generate:(OGAGeneratorParams *)params + error:(NSError **)error; + +@end + + +@interface OGATokenizer : NSObject +- (instancetype)init NS_UNAVAILABLE; +- (nullable)initWithModel:(OGAModel *)model + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +- (nullable OGASequences *)encode:(NSString *)str + error:(NSError **)error; + +- (nullable NSString *)decode:(NSData *)data + error:(NSError **)error; +@end + +@interface OGASequences : NSObject + +- (instancetype)init NS_UNAVAILABLE; +- (nullable)initWithError:(NSError **)error NS_DESIGNATED_INITIALIZER; + +- (size_t)count; +- (nullable NSData *)sequenceAtIndex:(size_t)index; + +@end + +@interface OGAGeneratorParams : NSObject +- (instancetype)init NS_UNAVAILABLE; +- (nullable)initWithModel:(OGAModel *)model + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +- (BOOL)setInputSequences:(OGASequences *)sequences + error:(NSError **)error; + +- (BOOL)setSearchOption:(NSString *)key + doubleValue:(double)value + error:(NSError **)error; + +- (BOOL)setSearchOption:(NSString *)key + boolValue:(BOOL)value + error:(NSError **)error; @end NS_ASSUME_NONNULL_END diff --git a/objectivec/oga_generator_params.mm b/objectivec/oga_generator_params.mm new file mode 100644 index 000000000..d71eaa49c --- /dev/null +++ b/objectivec/oga_generator_params.mm @@ -0,0 +1,57 @@ +#import "ort_genai_objc.h" +#import "error_utils.h" +#import "oga_internal.h" + +@implementation OGAGeneratorParams { + std::unique_ptr _generatorParams; +} + + +- (nullable)initWithModel:(OGAModel *)model + error:(NSError **)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _generatorParams = OgaGeneratorParams::Create([model CXXAPIOgaModel]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (BOOL)setInputSequences:(OGASequences *)sequences + error:(NSError **)error { + try { + _generatorParams->SetInputSequences([sequences CXXAPIOgaSequences]); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + +- (BOOL)setSearchOption:(NSString *)key + doubleValue:(double)value + error:(NSError **)error { + try { + _generatorParams->SetSearchOption([key UTF8String], value); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + +- (BOOL)setSearchOption:(NSString *)key + boolValue:(BOOL)value + error:(NSError **)error { + try { + _generatorParams->SetSearchOptionBool([key UTF8String], value); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + + +- (OgaGeneratorParams&)CXXAPIOgaGeneratorParams { + return *(_generatorParams.get()); +} + +@end diff --git a/objectivec/oga_internal.h b/objectivec/oga_internal.h new file mode 100644 index 000000000..86e084ed6 --- /dev/null +++ b/objectivec/oga_internal.h @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#import "cxx_api.h" + + +NS_ASSUME_NONNULL_BEGIN + +@interface OGAModel () + +- (const OgaModel&)CXXAPIOgaModel; + +@end + +@interface OGATokenizer () + +- (const OgaTokenizer&)CXXAPIOgaTokenizer; + +@end + +@interface OGASequences () + +- (instancetype)initWithNativeSeqquences:(std::unique_ptr)ptr; + +- (OgaSequences&)CXXAPIOgaSequences; + +@end + +@interface OGAGeneratorParams () + +- (OgaGeneratorParams&)CXXAPIOgaGeneratorParams; + +@end + +NS_ASSUME_NONNULL_END diff --git a/objectivec/oga_model.mm b/objectivec/oga_model.mm index 6e2532506..b78d2d8da 100644 --- a/objectivec/oga_model.mm +++ b/objectivec/oga_model.mm @@ -1,6 +1,6 @@ #import "ort_genai_objc.h" -#import "cxx_api.h" #import "error_utils.h" +#import "oga_internal.h" @implementation OGAModel { std::unique_ptr _model; @@ -14,12 +14,23 @@ - (nullable)initWithConfigPath:(NSString *)path } try { - _model = OgaModel::Create("phi-2"); + _model = OgaModel::Create(path.UTF8String); return self; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) - } +- (nullable OGASequences *)generate:(OGAGeneratorParams *)params + error:(NSError **)error { + try { + std::unique_ptr output_sequences = _model->Generate([params CXXAPIOgaGeneratorParams]); + return [[OGASequences alloc] initWithNativeSeqquences:std::move(output_sequences)]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (const OgaModel&)CXXAPIOgaModel { + return *(_model.get()); +} @end diff --git a/objectivec/oga_sequences.mm b/objectivec/oga_sequences.mm new file mode 100644 index 000000000..a5cce4005 --- /dev/null +++ b/objectivec/oga_sequences.mm @@ -0,0 +1,43 @@ +#import "ort_genai_objc.h" +#import "error_utils.h" +#import "oga_internal.h" + +@implementation OGASequences { + std::unique_ptr _sequences; +} + +- (instancetype)initWithNativeSeqquences:(std::unique_ptr)ptr { + _sequences = std::move(ptr); + return self; +} + +- (nullable)initWithError:(NSError **)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _sequences = OgaSequences::Create(); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (size_t)count { + return _sequences->Count(); +} + +- (nullable NSData *)sequenceAtIndex:(size_t) index { + if (index >= [self count]) { + return nil; + } + size_t sequenceLength = _sequences->SequenceCount(index); + const int32_t* data = _sequences->SequenceData(index); + return [[NSData alloc] initWithBytes:data length:sequenceLength]; +} + +- (OgaSequences&) CXXAPIOgaSequences { + return *(_sequences.get()); +} + +@end diff --git a/objectivec/oga_tokenizer.mm b/objectivec/oga_tokenizer.mm new file mode 100644 index 000000000..0e3474e2a --- /dev/null +++ b/objectivec/oga_tokenizer.mm @@ -0,0 +1,47 @@ + +#import "ort_genai_objc.h" +#import "error_utils.h" +#import "oga_internal.h" + +@implementation OGATokenizer { + std::unique_ptr _tokenizer; +} + + +- (nullable)initWithModel:(OGAModel *)model + error:(NSError **)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _tokenizer = OgaTokenizer::Create([model CXXAPIOgaModel]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (nullable OGASequences *)encode:(NSString *)str + error:(NSError **)error { + OGASequences *sequences = [[OGASequences alloc] initWithError:error]; + if (error) { + return nil; + } + _tokenizer->Encode([str UTF8String], [sequences CXXAPIOgaSequences]); + return sequences; +} + +- (nullable NSString *)decode:(NSData *)data + error:(NSError **)error { + try { + OgaString result = _tokenizer->Decode((const int32_t *)[data bytes], [data length]); + return [NSString stringWithUTF8String:result]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (const OgaTokenizer&)CXXAPIOgaTokenizer { + return *(_tokenizer.get()); +} + +@end From 683daf55f433f1c6c9daf77a43002d3c91eda0f5 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 16 Jul 2024 14:48:06 +0800 Subject: [PATCH 03/59] Fix --- objectivec/include/ort_genai_objc.h | 1 - objectivec/oga_internal.h | 1 + objectivec/oga_tokenizer.mm | 9 ++++++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/objectivec/include/ort_genai_objc.h b/objectivec/include/ort_genai_objc.h index 2b2a52126..002c1ae83 100644 --- a/objectivec/include/ort_genai_objc.h +++ b/objectivec/include/ort_genai_objc.h @@ -34,7 +34,6 @@ NS_ASSUME_NONNULL_BEGIN @interface OGASequences : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithError:(NSError **)error NS_DESIGNATED_INITIALIZER; - (size_t)count; - (nullable NSData *)sequenceAtIndex:(size_t)index; diff --git a/objectivec/oga_internal.h b/objectivec/oga_internal.h index 86e084ed6..8d29d3ed1 100644 --- a/objectivec/oga_internal.h +++ b/objectivec/oga_internal.h @@ -19,6 +19,7 @@ NS_ASSUME_NONNULL_BEGIN @interface OGASequences () +- (nullable)initWithError:(NSError **)error; - (instancetype)initWithNativeSeqquences:(std::unique_ptr)ptr; - (OgaSequences&)CXXAPIOgaSequences; diff --git a/objectivec/oga_tokenizer.mm b/objectivec/oga_tokenizer.mm index 0e3474e2a..17c50808c 100644 --- a/objectivec/oga_tokenizer.mm +++ b/objectivec/oga_tokenizer.mm @@ -24,11 +24,14 @@ - (nullable)initWithModel:(OGAModel *)model - (nullable OGASequences *)encode:(NSString *)str error:(NSError **)error { OGASequences *sequences = [[OGASequences alloc] initWithError:error]; - if (error) { + if (*error) { return nil; } - _tokenizer->Encode([str UTF8String], [sequences CXXAPIOgaSequences]); - return sequences; + try { + _tokenizer->Encode([str UTF8String], [sequences CXXAPIOgaSequences]); + return sequences; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } - (nullable NSString *)decode:(NSData *)data From e77fb609d93e919b4f0bb90db5dfebc068bc83f6 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 16 Jul 2024 16:58:56 +0800 Subject: [PATCH 04/59] OGASpan --- objectivec/include/ort_genai_objc.h | 40 +++++++++++++++++++++++++++-- objectivec/oga_generator.mm | 39 ++++++++++++++++++++++++++++ objectivec/oga_internal.h | 10 ++++++++ objectivec/oga_sequences.mm | 30 ++++++++++++++++++++-- objectivec/oga_tokenizer.mm | 31 ++++++++++++++++++++-- 5 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 objectivec/oga_generator.mm diff --git a/objectivec/include/ort_genai_objc.h b/objectivec/include/ort_genai_objc.h index 002c1ae83..4de56e070 100644 --- a/objectivec/include/ort_genai_objc.h +++ b/objectivec/include/ort_genai_objc.h @@ -3,8 +3,12 @@ NS_ASSUME_NONNULL_BEGIN + +@class OGASpan; @class OGASequences; @class OGAGeneratorParams; +@class OGATokenizerStream; + @interface OGAModel : NSObject @@ -27,8 +31,27 @@ NS_ASSUME_NONNULL_BEGIN - (nullable OGASequences *)encode:(NSString *)str error:(NSError **)error; -- (nullable NSString *)decode:(NSData *)data +- (nullable NSString *)decode:(OGASpan *)data error:(NSError **)error; + +@end + +@interface OGATokenizerStream: NSObject +- (instancetype)init NS_UNAVAILABLE; +- (nullable)initWithTokenizer:(OGATokenizer *)tokenizer + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +- (nullable NSString *)decode:(int32_t)token + error:(NSError **)error; +@end; + + +@interface OGASpan : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +- (int32_t)last; + @end @interface OGASequences : NSObject @@ -36,7 +59,7 @@ NS_ASSUME_NONNULL_BEGIN - (instancetype)init NS_UNAVAILABLE; - (size_t)count; -- (nullable NSData *)sequenceAtIndex:(size_t)index; +- (nullable OGASpan *)sequenceAtIndex:(size_t)index; @end @@ -57,4 +80,17 @@ NS_ASSUME_NONNULL_BEGIN error:(NSError **)error; @end +@interface OGAGenerator : NSObject +- (instancetype)init NS_UNAVAILABLE; +- (nullable)initWithModel:(OGAModel *)model + params:(OGAGeneratorParams *)params + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +- (BOOL)isDone; +- (void)ComputeLogits; +- (void)GenerateNextToken; + +- (nullable OGASpan *)sequenceAtIndex:(size_t) index; +@end + NS_ASSUME_NONNULL_END diff --git a/objectivec/oga_generator.mm b/objectivec/oga_generator.mm new file mode 100644 index 000000000..b6177bc4f --- /dev/null +++ b/objectivec/oga_generator.mm @@ -0,0 +1,39 @@ +#import "ort_genai_objc.h" +#import "error_utils.h" +#import "oga_internal.h" + +@implementation OGAGenerator { + std::unique_ptr _generator; +} + +- (nullable)initWithModel:(OGAModel *)model + params:(OGAGeneratorParams *)params + error:(NSError **)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _generator = OgaGenerator::Create([model CXXAPIOgaModel], [params CXXAPIOgaGeneratorParams]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (BOOL)isDone { + return _generator->IsDone(); +} +- (void)ComputeLogits { + _generator->ComputeLogits(); +} +- (void)GenerateNextToken { + _generator->GenerateNextToken(); +} + +- (nullable OGASpan *)sequenceAtIndex:(size_t) index { + size_t sequenceLength = _generator->GetSequenceCount(index); + const int32_t* data = _generator->GetSequenceData(index); + return [[OGASpan alloc] initWithRawPointer:data size:sequenceLength]; +} + +@end diff --git a/objectivec/oga_internal.h b/objectivec/oga_internal.h index 8d29d3ed1..e8350952e 100644 --- a/objectivec/oga_internal.h +++ b/objectivec/oga_internal.h @@ -17,6 +17,16 @@ NS_ASSUME_NONNULL_BEGIN @end +@interface OGASpan () + +- (nullable)initWithRawPointer:(const int32_t *) pointer + size:(size_t)size; + +- (const int32_t *)pointer; +- (size_t)size; + +@end + @interface OGASequences () - (nullable)initWithError:(NSError **)error; diff --git a/objectivec/oga_sequences.mm b/objectivec/oga_sequences.mm index a5cce4005..0adb40baf 100644 --- a/objectivec/oga_sequences.mm +++ b/objectivec/oga_sequences.mm @@ -2,6 +2,32 @@ #import "error_utils.h" #import "oga_internal.h" +@implementation OGASpan { + const int32_t * _ptr; + size_t _size; +} + +- (nullable)initWithRawPointer:(const int32_t * )pointer + size:(size_t)size { + _ptr = pointer; + _size = size; + return [self init]; +} + +- (const int32_t * )pointer { + return _ptr; +} + +- (size_t)size { + return _size; +} + +- (int32_t)last { + return *(_ptr + (_size - 1)); +} + +@end + @implementation OGASequences { std::unique_ptr _sequences; } @@ -27,13 +53,13 @@ - (size_t)count { return _sequences->Count(); } -- (nullable NSData *)sequenceAtIndex:(size_t) index { +- (nullable OGASpan *)sequenceAtIndex:(size_t) index { if (index >= [self count]) { return nil; } size_t sequenceLength = _sequences->SequenceCount(index); const int32_t* data = _sequences->SequenceData(index); - return [[NSData alloc] initWithBytes:data length:sequenceLength]; + return [[OGASpan alloc] initWithRawPointer:data size: sequenceLength]; } - (OgaSequences&) CXXAPIOgaSequences { diff --git a/objectivec/oga_tokenizer.mm b/objectivec/oga_tokenizer.mm index 17c50808c..a13d61a81 100644 --- a/objectivec/oga_tokenizer.mm +++ b/objectivec/oga_tokenizer.mm @@ -34,10 +34,10 @@ - (nullable OGASequences *)encode:(NSString *)str OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable NSString *)decode:(NSData *)data +- (nullable NSString *)decode:(OGASpan *) data error:(NSError **)error { try { - OgaString result = _tokenizer->Decode((const int32_t *)[data bytes], [data length]); + OgaString result = _tokenizer->Decode(data.pointer, data.size); return [NSString stringWithUTF8String:result]; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) @@ -48,3 +48,30 @@ - (nullable NSString *)decode:(NSData *)data } @end + +@implementation OGATokenizerStream { + std::unique_ptr _stream; +} + +- (nullable)initWithTokenizer:(OGATokenizer *)tokenizer + error:(NSError **)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _stream = OgaTokenizerStream::Create([tokenizer CXXAPIOgaTokenizer]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (nullable NSString *)decode:(int32_t)token + error:(NSError **)error { + try { + return [NSString stringWithUTF8String:_stream->Decode(token)]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +@end From ad1c9e8d349756a89a107ea500c0bc7eda81f89d Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Wed, 17 Jul 2024 13:56:44 +0800 Subject: [PATCH 05/59] More objc bindings --- objectivec/include/ort_genai_objc.h | 43 ++++++++++++++++++-- objectivec/oga_generator.mm | 6 ++- objectivec/oga_generator_params.mm | 19 +++++++++ objectivec/oga_images.mm | 27 +++++++++++++ objectivec/oga_internal.h | 53 ++++++++++++++++++++++++- objectivec/oga_model.mm | 6 +-- objectivec/oga_multi_modal_processor.mm | 45 +++++++++++++++++++++ objectivec/oga_named_tensors.mm | 19 +++++++++ objectivec/oga_sequences.mm | 26 ------------ objectivec/oga_tensor.mm | 37 +++++++++++++++++ 10 files changed, 245 insertions(+), 36 deletions(-) create mode 100644 objectivec/oga_images.mm create mode 100644 objectivec/oga_multi_modal_processor.mm create mode 100644 objectivec/oga_named_tensors.mm create mode 100644 objectivec/oga_tensor.mm diff --git a/objectivec/include/ort_genai_objc.h b/objectivec/include/ort_genai_objc.h index 4de56e070..f90701734 100644 --- a/objectivec/include/ort_genai_objc.h +++ b/objectivec/include/ort_genai_objc.h @@ -5,16 +5,34 @@ NS_ASSUME_NONNULL_BEGIN @class OGASpan; +@class OGATensor; @class OGASequences; @class OGAGeneratorParams; @class OGATokenizerStream; +typedef NS_ENUM(NSInteger, OGAElementType) { + OGAElementTypeUndefined, + OGAElementTypeFloat32, // maps to c type float + OGAElementTypeUint8, // maps to c type uint8_t + OGAElementTypeInt8, // maps to c type int8_t + OGAElementTypeUint16, // maps to c type uint16_t + OGAElementTypeInt16, // maps to c type int16_t + OGAElementTypeInt32, // maps to c type int32_t + OGAElementTypeInt64, // maps to c type int64_t + OGAElementTypeString, // string type (not currently supported by Oga) + OGAElementTypeBool, // maps to c type bool + OGAElementTypeFloat16, // IEEE 752-2008 binary16 format, 1 sign bit, 5 bit exponent, 10 bit fraction + OGAElementTypeFloat64, // maps to c type double + OGAElementTypeUint32, // maps to c type uint32_t + OGAElementTypeUint64, // maps to c type uint64_t +}; + @interface OGAModel : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithConfigPath:(NSString *)path - error:(NSError **)error NS_DESIGNATED_INITIALIZER; +- (nullable)initWithPath:(NSString *)path + error:(NSError **)error NS_DESIGNATED_INITIALIZER; - (nullable OGASequences *)generate:(OGAGeneratorParams *)params @@ -68,9 +86,16 @@ NS_ASSUME_NONNULL_BEGIN - (nullable)initWithModel:(OGAModel *)model error:(NSError **)error NS_DESIGNATED_INITIALIZER; +- (BOOL)setInputs:(OGANamedTensors *)namedTensors + error:(NSError **)error; + - (BOOL)setInputSequences:(OGASequences *)sequences error:(NSError **)error; +- (BOOL)setModelInput:(NSString *)name + tensor:(OGATensor* tensor) + error:(NSError **)error; + - (BOOL)setSearchOption:(NSString *)key doubleValue:(double)value error:(NSError **)error; @@ -81,16 +106,26 @@ NS_ASSUME_NONNULL_BEGIN @end @interface OGAGenerator : NSObject + - (instancetype)init NS_UNAVAILABLE; - (nullable)initWithModel:(OGAModel *)model params:(OGAGeneratorParams *)params error:(NSError **)error NS_DESIGNATED_INITIALIZER; - (BOOL)isDone; -- (void)ComputeLogits; -- (void)GenerateNextToken; +- (void)computeLogits; +- (void)generateNextToken; - (nullable OGASpan *)sequenceAtIndex:(size_t) index; + +@end + +@interface OGAImages : NSObject + +- (instancetype)init NS_UNAVAILABLE; +- (nullable)initWithPath:(NSString *)path + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + @end NS_ASSUME_NONNULL_END diff --git a/objectivec/oga_generator.mm b/objectivec/oga_generator.mm index b6177bc4f..992f8c808 100644 --- a/objectivec/oga_generator.mm +++ b/objectivec/oga_generator.mm @@ -23,10 +23,12 @@ - (nullable)initWithModel:(OGAModel *)model - (BOOL)isDone { return _generator->IsDone(); } -- (void)ComputeLogits { + +- (void)computeLogits { _generator->ComputeLogits(); } -- (void)GenerateNextToken { + +- (void)generateNextToken { _generator->GenerateNextToken(); } diff --git a/objectivec/oga_generator_params.mm b/objectivec/oga_generator_params.mm index d71eaa49c..16b79a440 100644 --- a/objectivec/oga_generator_params.mm +++ b/objectivec/oga_generator_params.mm @@ -20,6 +20,15 @@ - (nullable)initWithModel:(OGAModel *)model OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } +- (BOOL)setInputs:(OGANamedTensors *)namedTensors + error:(NSError **)error { + try { + _generatorParams->SetInputs([namedTensors CXXAPIOgaNamedTensors]); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + - (BOOL)setInputSequences:(OGASequences *)sequences error:(NSError **)error { try { @@ -29,6 +38,16 @@ - (BOOL)setInputSequences:(OGASequences *)sequences OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } +- (BOOL)setModelInput:(NSString *)name + tensor:(OGATensor* tensor) + error:(NSError **)error { + try { + _generatorParams->SetModelInput(name.UTF8String, [tensor CXXAPIOgaTensor]); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + - (BOOL)setSearchOption:(NSString *)key doubleValue:(double)value error:(NSError **)error { diff --git a/objectivec/oga_images.mm b/objectivec/oga_images.mm new file mode 100644 index 000000000..35c9432c8 --- /dev/null +++ b/objectivec/oga_images.mm @@ -0,0 +1,27 @@ +#import "ort_genai_objc.h" +#import "error_utils.h" +#import "oga_internal.h" + +@implementation OGAImages { + std::unique_ptr _images; +} + + +- (nullable)initWithPath:(NSString *)path + error:(NSError **)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _images = OgaImages::Load(path.UTF8String); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (OgaImages *)CXXAPIOgaImages { + return _images.get(); +} + +@end \ No newline at end of file diff --git a/objectivec/oga_internal.h b/objectivec/oga_internal.h index e8350952e..564fe2da5 100644 --- a/objectivec/oga_internal.h +++ b/objectivec/oga_internal.h @@ -27,10 +27,37 @@ NS_ASSUME_NONNULL_BEGIN @end +@implementation OGASpan { + const int32_t * _ptr; + size_t _size; +} + +- (nullable)initWithRawPointer:(const int32_t * )pointer + size:(size_t)size { + _ptr = pointer; + _size = size; + return [self init]; +} + +- (const int32_t * )pointer { + return _ptr; +} + +- (size_t)size { + return _size; +} + +- (int32_t)last { + return *(_ptr + (_size - 1)); +} + +@end + + @interface OGASequences () - (nullable)initWithError:(NSError **)error; -- (instancetype)initWithNativeSeqquences:(std::unique_ptr)ptr; +- (instancetype)initWithNativePointer:(std::unique_ptr)ptr; - (OgaSequences&)CXXAPIOgaSequences; @@ -42,4 +69,28 @@ NS_ASSUME_NONNULL_BEGIN @end +@interface OGAImages () + +- (OgaImages *)CXXAPIOgaImages; + +@end + +@interface OGATensor () + +- (OGATensor&)CXXAPIOgaTensor; + +@end + +@interface OGANamedTensors () + +- (OgaNamedTensor&)CXXAPIOgaNamedTensors; + +@end + +@interface OGAMultiModalProcessor () + +- (const OgaMultiModalProcessor&)CXXAPIOgaMultiModalProcessor; + +@end + NS_ASSUME_NONNULL_END diff --git a/objectivec/oga_model.mm b/objectivec/oga_model.mm index b78d2d8da..72812cfa3 100644 --- a/objectivec/oga_model.mm +++ b/objectivec/oga_model.mm @@ -7,12 +7,12 @@ @implementation OGAModel { } -- (nullable)initWithConfigPath:(NSString *)path - error:(NSError **)error { +- (nullable)initWithPath:(NSString *)path + error:(NSError **)error { if ((self = [super init]) == nil) { return nil; } - + try { _model = OgaModel::Create(path.UTF8String); return self; diff --git a/objectivec/oga_multi_modal_processor.mm b/objectivec/oga_multi_modal_processor.mm new file mode 100644 index 000000000..364677580 --- /dev/null +++ b/objectivec/oga_multi_modal_processor.mm @@ -0,0 +1,45 @@ + +#import "ort_genai_objc.h" +#import "error_utils.h" +#import "oga_internal.h" + +@implementation OgaMultiModalProcessor { + std::unique_ptr _processor; +} + + +- (nullable)initWithModel:(OGAModel *)model + error:(NSError **)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _processor = OgaMultiModalProcessor::Create([model CXXAPIOgaModel]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (nullable OGANamedTensor *)processImages:(NSString *)prompt + images:(OGAImages *)images + error:(NSError **)error { + try { + OGANamedTensor *result = [OGANamedTensor alloc] initWithNativePointer:_processor->ProcessImages([prompt UTF8String], [images CXXAPIOgaImages])]; + return result; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (nullable NSString *)decode:(OGASpan *) data + error:(NSError **)error { + try { + OgaString result = _processor->Decode(data.pointer, data.size); + return [NSString stringWithUTF8String:result]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (const OgaMultiModalProcessor&)CXXAPIOgaMultiModalProcessor { + return *(_processor.get()); +} diff --git a/objectivec/oga_named_tensors.mm b/objectivec/oga_named_tensors.mm new file mode 100644 index 000000000..fcebd42a5 --- /dev/null +++ b/objectivec/oga_named_tensors.mm @@ -0,0 +1,19 @@ +#import "ort_genai_objc.h" +#import "error_utils.h" +#import "oga_internal.h" + +@implementation OGANamedTensors { + std::unique_ptr _tensor; +} + + +- (instancetype)initWithNativePointer:(std::unique_ptr)ptr; { + _tensor = std::move(ptr); + return self; +} + +- (OgaNamedTensors&)CXXAPIOgaNamedTensors { + return *(_tensor.get()); +} + +@end diff --git a/objectivec/oga_sequences.mm b/objectivec/oga_sequences.mm index 0adb40baf..7179e17d9 100644 --- a/objectivec/oga_sequences.mm +++ b/objectivec/oga_sequences.mm @@ -2,32 +2,6 @@ #import "error_utils.h" #import "oga_internal.h" -@implementation OGASpan { - const int32_t * _ptr; - size_t _size; -} - -- (nullable)initWithRawPointer:(const int32_t * )pointer - size:(size_t)size { - _ptr = pointer; - _size = size; - return [self init]; -} - -- (const int32_t * )pointer { - return _ptr; -} - -- (size_t)size { - return _size; -} - -- (int32_t)last { - return *(_ptr + (_size - 1)); -} - -@end - @implementation OGASequences { std::unique_ptr _sequences; } diff --git a/objectivec/oga_tensor.mm b/objectivec/oga_tensor.mm new file mode 100644 index 000000000..5e998a96b --- /dev/null +++ b/objectivec/oga_tensor.mm @@ -0,0 +1,37 @@ +#import "ort_genai_objc.h" +#import "error_utils.h" +#import "oga_internal.h" + +@implementation OGATensor { + std::unique_ptr _tensor; +} + + +- (nullable)initWithDataPointer:(void *)data + shape:(OGASpan)shape + type:(OGAElementType)elementType + error:(NSError **)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _model = OgaTensor::Create(data, shape.pointer, shape.size, elementType); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (OGAElementType)type { + return _tensor->Type(); +} + +- (void *)data { + return _tensor->Data(); +} + +- (OgaModel&)CXXAPIOgaTensor { + return *(_tensor.get()); +} + +@end From 83d127322a05d0616d86e2dfa59f4be98b11dbe0 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Wed, 17 Jul 2024 14:01:34 +0800 Subject: [PATCH 06/59] license header --- objectivec/oga_generator.mm | 3 +++ objectivec/oga_generator_params.mm | 3 +++ objectivec/oga_images.mm | 3 +++ objectivec/oga_model.mm | 3 +++ objectivec/oga_multi_modal_processor.mm | 2 ++ objectivec/oga_named_tensors.mm | 3 +++ objectivec/oga_sequences.mm | 3 +++ objectivec/oga_tensor.mm | 3 +++ objectivec/oga_tokenizer.mm | 2 ++ 9 files changed, 25 insertions(+) diff --git a/objectivec/oga_generator.mm b/objectivec/oga_generator.mm index 992f8c808..f29725f82 100644 --- a/objectivec/oga_generator.mm +++ b/objectivec/oga_generator.mm @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + #import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" diff --git a/objectivec/oga_generator_params.mm b/objectivec/oga_generator_params.mm index 16b79a440..988056292 100644 --- a/objectivec/oga_generator_params.mm +++ b/objectivec/oga_generator_params.mm @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + #import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" diff --git a/objectivec/oga_images.mm b/objectivec/oga_images.mm index 35c9432c8..d2b7ba499 100644 --- a/objectivec/oga_images.mm +++ b/objectivec/oga_images.mm @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + #import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" diff --git a/objectivec/oga_model.mm b/objectivec/oga_model.mm index 72812cfa3..f060190a5 100644 --- a/objectivec/oga_model.mm +++ b/objectivec/oga_model.mm @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + #import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" diff --git a/objectivec/oga_multi_modal_processor.mm b/objectivec/oga_multi_modal_processor.mm index 364677580..9ef474f3c 100644 --- a/objectivec/oga_multi_modal_processor.mm +++ b/objectivec/oga_multi_modal_processor.mm @@ -1,3 +1,5 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. #import "ort_genai_objc.h" #import "error_utils.h" diff --git a/objectivec/oga_named_tensors.mm b/objectivec/oga_named_tensors.mm index fcebd42a5..b65526fa5 100644 --- a/objectivec/oga_named_tensors.mm +++ b/objectivec/oga_named_tensors.mm @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + #import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" diff --git a/objectivec/oga_sequences.mm b/objectivec/oga_sequences.mm index 7179e17d9..2d5f74798 100644 --- a/objectivec/oga_sequences.mm +++ b/objectivec/oga_sequences.mm @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + #import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" diff --git a/objectivec/oga_tensor.mm b/objectivec/oga_tensor.mm index 5e998a96b..05b7c4e71 100644 --- a/objectivec/oga_tensor.mm +++ b/objectivec/oga_tensor.mm @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + #import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" diff --git a/objectivec/oga_tokenizer.mm b/objectivec/oga_tokenizer.mm index a13d61a81..caf308a9d 100644 --- a/objectivec/oga_tokenizer.mm +++ b/objectivec/oga_tokenizer.mm @@ -1,3 +1,5 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. #import "ort_genai_objc.h" #import "error_utils.h" From d635d63b2be06ed0309d694c19565f722b118616 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:59:26 +0800 Subject: [PATCH 07/59] Update --- objectivec/include/ort_genai_objc.h | 3 +++ objectivec/oga_generator_params.mm | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/objectivec/include/ort_genai_objc.h b/objectivec/include/ort_genai_objc.h index f90701734..3f96e37f2 100644 --- a/objectivec/include/ort_genai_objc.h +++ b/objectivec/include/ort_genai_objc.h @@ -59,6 +59,9 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (nullable)initWithTokenizer:(OGATokenizer *)tokenizer error:(NSError **)error NS_DESIGNATED_INITIALIZER; +- (nullable)initWithTokenizer:(OGAMultiModalProcessor *)processor + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + - (nullable NSString *)decode:(int32_t)token error:(NSError **)error; @end; diff --git a/objectivec/oga_generator_params.mm b/objectivec/oga_generator_params.mm index 988056292..ad6cda963 100644 --- a/objectivec/oga_generator_params.mm +++ b/objectivec/oga_generator_params.mm @@ -23,6 +23,20 @@ - (nullable)initWithModel:(OGAModel *)model OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } + +- (nullable)initWithTokenizer:(OGAMultiModalProcessor *)processor + error:(NSError **) { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _generatorParams = OgaGeneratorParams::Create([processor CXXAPIOgaMultiModalProcessor]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + - (BOOL)setInputs:(OGANamedTensors *)namedTensors error:(NSError **)error { try { From 60992363a7258c4d5bd964a0261f619175169ae4 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:35:05 +0800 Subject: [PATCH 08/59] Update folder structure --- {objectivec => src/objectivec}/cxx_api.h | 2 +- {objectivec => src/objectivec}/error_utils.h | 0 {objectivec => src/objectivec}/error_utils.mm | 0 .../objectivec}/include/ort_genai_objc.h | 40 ++++++++++--- .../objectivec}/oga_generator.mm | 4 +- .../objectivec}/oga_generator_params.mm | 16 +---- {objectivec => src/objectivec}/oga_images.mm | 0 {objectivec => src/objectivec}/oga_internal.h | 33 +++------- src/objectivec/oga_internal.mm | 60 +++++++++++++++++++ {objectivec => src/objectivec}/oga_model.mm | 2 +- .../objectivec}/oga_multi_modal_processor.mm | 10 ++-- .../objectivec}/oga_named_tensors.mm | 0 .../objectivec}/oga_sequences.mm | 4 +- {objectivec => src/objectivec}/oga_tensor.mm | 8 +-- .../objectivec}/oga_tokenizer.mm | 15 ++++- 15 files changed, 132 insertions(+), 62 deletions(-) rename {objectivec => src/objectivec}/cxx_api.h (94%) rename {objectivec => src/objectivec}/error_utils.h (100%) rename {objectivec => src/objectivec}/error_utils.mm (100%) rename {objectivec => src/objectivec}/include/ort_genai_objc.h (81%) rename {objectivec => src/objectivec}/oga_generator.mm (87%) rename {objectivec => src/objectivec}/oga_generator_params.mm (83%) rename {objectivec => src/objectivec}/oga_images.mm (100%) rename {objectivec => src/objectivec}/oga_internal.h (67%) create mode 100644 src/objectivec/oga_internal.mm rename {objectivec => src/objectivec}/oga_model.mm (90%) rename {objectivec => src/objectivec}/oga_multi_modal_processor.mm (78%) rename {objectivec => src/objectivec}/oga_named_tensors.mm (100%) rename {objectivec => src/objectivec}/oga_sequences.mm (87%) rename {objectivec => src/objectivec}/oga_tensor.mm (73%) rename {objectivec => src/objectivec}/oga_tokenizer.mm (81%) diff --git a/objectivec/cxx_api.h b/src/objectivec/cxx_api.h similarity index 94% rename from objectivec/cxx_api.h rename to src/objectivec/cxx_api.h index d1f5fa06f..b452a490f 100644 --- a/objectivec/cxx_api.h +++ b/src/objectivec/cxx_api.h @@ -10,7 +10,7 @@ #pragma clang diagnostic ignored "-Wdocumentation" #endif // defined(__clang__) -#import "ort_genai.h" +#import "../ort_genai.h" #if defined(__clang__) #pragma clang diagnostic pop diff --git a/objectivec/error_utils.h b/src/objectivec/error_utils.h similarity index 100% rename from objectivec/error_utils.h rename to src/objectivec/error_utils.h diff --git a/objectivec/error_utils.mm b/src/objectivec/error_utils.mm similarity index 100% rename from objectivec/error_utils.mm rename to src/objectivec/error_utils.mm diff --git a/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h similarity index 81% rename from objectivec/include/ort_genai_objc.h rename to src/objectivec/include/ort_genai_objc.h index 3f96e37f2..99dab562e 100644 --- a/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -4,11 +4,13 @@ NS_ASSUME_NONNULL_BEGIN -@class OGASpan; +@class OGAInt32Span; @class OGATensor; @class OGASequences; +@class OGANamedTensors; @class OGAGeneratorParams; @class OGATokenizerStream; +@class OGAMultiModalProcessor; typedef NS_ENUM(NSInteger, OGAElementType) { OGAElementTypeUndefined, @@ -49,7 +51,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (nullable OGASequences *)encode:(NSString *)str error:(NSError **)error; -- (nullable NSString *)decode:(OGASpan *)data +- (nullable NSString *)decode:(OGAInt32Span *)data error:(NSError **)error; @end @@ -59,15 +61,15 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (nullable)initWithTokenizer:(OGATokenizer *)tokenizer error:(NSError **)error NS_DESIGNATED_INITIALIZER; -- (nullable)initWithTokenizer:(OGAMultiModalProcessor *)processor - error:(NSError **)error NS_DESIGNATED_INITIALIZER; +- (nullable)initWithMultiModalProcessor:(OGAMultiModalProcessor *)processor + error:(NSError **)error NS_DESIGNATED_INITIALIZER; - (nullable NSString *)decode:(int32_t)token error:(NSError **)error; @end; -@interface OGASpan : NSObject +@interface OGAInt32Span : NSObject - (instancetype)init NS_UNAVAILABLE; @@ -75,12 +77,20 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @end +@interface OGAInt64Span : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +- (int64_t)last; + +@end + @interface OGASequences : NSObject - (instancetype)init NS_UNAVAILABLE; - (size_t)count; -- (nullable OGASpan *)sequenceAtIndex:(size_t)index; +- (nullable OGAInt32Span *)sequenceAtIndex:(size_t)index; @end @@ -96,7 +106,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { error:(NSError **)error; - (BOOL)setModelInput:(NSString *)name - tensor:(OGATensor* tensor) + tensor:(OGATensor*)tensor error:(NSError **)error; - (BOOL)setSearchOption:(NSString *)key @@ -119,7 +129,17 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (void)computeLogits; - (void)generateNextToken; -- (nullable OGASpan *)sequenceAtIndex:(size_t) index; +- (nullable OGAInt32Span *)sequenceAtIndex:(size_t) index; + +@end + +@interface OGATensor : NSObject + +@end + +@interface OGANamedTensors : NSObject + +- (instancetype)init NS_UNAVAILABLE; @end @@ -131,4 +151,8 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @end +@interface OGAMultiModalProcessor : NSObject + +@end + NS_ASSUME_NONNULL_END diff --git a/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm similarity index 87% rename from objectivec/oga_generator.mm rename to src/objectivec/oga_generator.mm index f29725f82..a7e6bb5a4 100644 --- a/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -35,10 +35,10 @@ - (void)generateNextToken { _generator->GenerateNextToken(); } -- (nullable OGASpan *)sequenceAtIndex:(size_t) index { +- (nullable OGAInt32Span *)sequenceAtIndex:(size_t) index { size_t sequenceLength = _generator->GetSequenceCount(index); const int32_t* data = _generator->GetSequenceData(index); - return [[OGASpan alloc] initWithRawPointer:data size:sequenceLength]; + return [[OGAInt32Span alloc] initWithRawPointer:data size:sequenceLength]; } @end diff --git a/objectivec/oga_generator_params.mm b/src/objectivec/oga_generator_params.mm similarity index 83% rename from objectivec/oga_generator_params.mm rename to src/objectivec/oga_generator_params.mm index ad6cda963..de9f9a8a7 100644 --- a/objectivec/oga_generator_params.mm +++ b/src/objectivec/oga_generator_params.mm @@ -23,20 +23,6 @@ - (nullable)initWithModel:(OGAModel *)model OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } - -- (nullable)initWithTokenizer:(OGAMultiModalProcessor *)processor - error:(NSError **) { - if ((self = [super init]) == nil) { - return nil; - } - - try { - _generatorParams = OgaGeneratorParams::Create([processor CXXAPIOgaMultiModalProcessor]); - return self; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) -} - - (BOOL)setInputs:(OGANamedTensors *)namedTensors error:(NSError **)error { try { @@ -56,7 +42,7 @@ - (BOOL)setInputSequences:(OGASequences *)sequences } - (BOOL)setModelInput:(NSString *)name - tensor:(OGATensor* tensor) + tensor:(OGATensor*)tensor error:(NSError **)error { try { _generatorParams->SetModelInput(name.UTF8String, [tensor CXXAPIOgaTensor]); diff --git a/objectivec/oga_images.mm b/src/objectivec/oga_images.mm similarity index 100% rename from objectivec/oga_images.mm rename to src/objectivec/oga_images.mm diff --git a/objectivec/oga_internal.h b/src/objectivec/oga_internal.h similarity index 67% rename from objectivec/oga_internal.h rename to src/objectivec/oga_internal.h index 564fe2da5..e5e685cae 100644 --- a/objectivec/oga_internal.h +++ b/src/objectivec/oga_internal.h @@ -17,7 +17,7 @@ NS_ASSUME_NONNULL_BEGIN @end -@interface OGASpan () +@interface OGAInt32Span () - (nullable)initWithRawPointer:(const int32_t *) pointer size:(size_t)size; @@ -27,29 +27,13 @@ NS_ASSUME_NONNULL_BEGIN @end -@implementation OGASpan { - const int32_t * _ptr; - size_t _size; -} +@interface OGAInt64Span () -- (nullable)initWithRawPointer:(const int32_t * )pointer - size:(size_t)size { - _ptr = pointer; - _size = size; - return [self init]; -} - -- (const int32_t * )pointer { - return _ptr; -} - -- (size_t)size { - return _size; -} +- (nullable)initWithRawPointer:(const int64_t *) pointer + size:(size_t)size; -- (int32_t)last { - return *(_ptr + (_size - 1)); -} +- (const int64_t *)pointer; +- (size_t)size; @end @@ -77,13 +61,14 @@ NS_ASSUME_NONNULL_BEGIN @interface OGATensor () -- (OGATensor&)CXXAPIOgaTensor; +- (OgaTensor&)CXXAPIOgaTensor; @end @interface OGANamedTensors () -- (OgaNamedTensor&)CXXAPIOgaNamedTensors; +- (instancetype)initWithNativePointer:(std::unique_ptr)ptr NS_DESIGNATED_INITIALIZER; +- (OgaNamedTensors&)CXXAPIOgaNamedTensors; @end diff --git a/src/objectivec/oga_internal.mm b/src/objectivec/oga_internal.mm new file mode 100644 index 000000000..3f69e4dfc --- /dev/null +++ b/src/objectivec/oga_internal.mm @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import "ort_genai_objc.h" +#import "error_utils.h" +#import "oga_internal.h" + + +@implementation OGAInt32Span { + const int32_t * _ptr; + size_t _size; +} + +- (nullable)initWithRawPointer:(const int32_t * )pointer + size:(size_t)size { + _ptr = pointer; + _size = size; + return [self init]; +} + +- (const int32_t * )pointer { + return _ptr; +} + +- (size_t)size { + return _size; +} + +- (int32_t)last { + return *(_ptr + (_size - 1)); +} + +@end + +@implementation OGAInt64Span { + const int64_t * _ptr; + size_t _size; +} + +- (nullable)initWithRawPointer:(const int64_t * )pointer + size:(size_t)size { + _ptr = pointer; + _size = size; + return [self init]; +} + +- (const int64_t * )pointer { + return _ptr; +} + +- (size_t)size { + return _size; +} + +- (int64_t)last { + return *(_ptr + (_size - 1)); +} + +@end + diff --git a/objectivec/oga_model.mm b/src/objectivec/oga_model.mm similarity index 90% rename from objectivec/oga_model.mm rename to src/objectivec/oga_model.mm index f060190a5..6b187132e 100644 --- a/objectivec/oga_model.mm +++ b/src/objectivec/oga_model.mm @@ -27,7 +27,7 @@ - (nullable OGASequences *)generate:(OGAGeneratorParams *)params error:(NSError **)error { try { std::unique_ptr output_sequences = _model->Generate([params CXXAPIOgaGeneratorParams]); - return [[OGASequences alloc] initWithNativeSeqquences:std::move(output_sequences)]; + return [[OGASequences alloc] initWithNativePointer:std::move(output_sequences)]; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } diff --git a/objectivec/oga_multi_modal_processor.mm b/src/objectivec/oga_multi_modal_processor.mm similarity index 78% rename from objectivec/oga_multi_modal_processor.mm rename to src/objectivec/oga_multi_modal_processor.mm index 9ef474f3c..a9fe53860 100644 --- a/objectivec/oga_multi_modal_processor.mm +++ b/src/objectivec/oga_multi_modal_processor.mm @@ -5,7 +5,7 @@ #import "error_utils.h" #import "oga_internal.h" -@implementation OgaMultiModalProcessor { +@implementation OGAMultiModalProcessor { std::unique_ptr _processor; } @@ -23,17 +23,17 @@ - (nullable)initWithModel:(OGAModel *)model OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable OGANamedTensor *)processImages:(NSString *)prompt +- (nullable OGANamedTensors *)processImages:(NSString *)prompt images:(OGAImages *)images error:(NSError **)error { try { - OGANamedTensor *result = [OGANamedTensor alloc] initWithNativePointer:_processor->ProcessImages([prompt UTF8String], [images CXXAPIOgaImages])]; + OGANamedTensors *result = [[OGANamedTensors alloc] initWithNativePointer:_processor->ProcessImages([prompt UTF8String], [images CXXAPIOgaImages])]; return result; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable NSString *)decode:(OGASpan *) data +- (nullable NSString *)decode:(OGAInt32Span *) data error:(NSError **)error { try { OgaString result = _processor->Decode(data.pointer, data.size); @@ -45,3 +45,5 @@ - (nullable NSString *)decode:(OGASpan *) data - (const OgaMultiModalProcessor&)CXXAPIOgaMultiModalProcessor { return *(_processor.get()); } + +@end diff --git a/objectivec/oga_named_tensors.mm b/src/objectivec/oga_named_tensors.mm similarity index 100% rename from objectivec/oga_named_tensors.mm rename to src/objectivec/oga_named_tensors.mm diff --git a/objectivec/oga_sequences.mm b/src/objectivec/oga_sequences.mm similarity index 87% rename from objectivec/oga_sequences.mm rename to src/objectivec/oga_sequences.mm index 2d5f74798..cede09a89 100644 --- a/objectivec/oga_sequences.mm +++ b/src/objectivec/oga_sequences.mm @@ -30,13 +30,13 @@ - (size_t)count { return _sequences->Count(); } -- (nullable OGASpan *)sequenceAtIndex:(size_t) index { +- (nullable OGAInt32Span *)sequenceAtIndex:(size_t) index { if (index >= [self count]) { return nil; } size_t sequenceLength = _sequences->SequenceCount(index); const int32_t* data = _sequences->SequenceData(index); - return [[OGASpan alloc] initWithRawPointer:data size: sequenceLength]; + return [[OGAInt32Span alloc] initWithRawPointer:data size: sequenceLength]; } - (OgaSequences&) CXXAPIOgaSequences { diff --git a/objectivec/oga_tensor.mm b/src/objectivec/oga_tensor.mm similarity index 73% rename from objectivec/oga_tensor.mm rename to src/objectivec/oga_tensor.mm index 05b7c4e71..25ebe6d99 100644 --- a/objectivec/oga_tensor.mm +++ b/src/objectivec/oga_tensor.mm @@ -11,7 +11,7 @@ @implementation OGATensor { - (nullable)initWithDataPointer:(void *)data - shape:(OGASpan)shape + shape:(OGAInt64Span *)shape type:(OGAElementType)elementType error:(NSError **)error { if ((self = [super init]) == nil) { @@ -19,21 +19,21 @@ - (nullable)initWithDataPointer:(void *)data } try { - _model = OgaTensor::Create(data, shape.pointer, shape.size, elementType); + _tensor = OgaTensor::Create(data, shape.pointer, shape.size, static_cast(elementType)); return self; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } - (OGAElementType)type { - return _tensor->Type(); + return OGAElementType(_tensor->Type()); } - (void *)data { return _tensor->Data(); } -- (OgaModel&)CXXAPIOgaTensor { +- (OgaTensor&)CXXAPIOgaTensor { return *(_tensor.get()); } diff --git a/objectivec/oga_tokenizer.mm b/src/objectivec/oga_tokenizer.mm similarity index 81% rename from objectivec/oga_tokenizer.mm rename to src/objectivec/oga_tokenizer.mm index caf308a9d..f90ad99e9 100644 --- a/objectivec/oga_tokenizer.mm +++ b/src/objectivec/oga_tokenizer.mm @@ -36,7 +36,7 @@ - (nullable OGASequences *)encode:(NSString *)str OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable NSString *)decode:(OGASpan *) data +- (nullable NSString *)decode:(OGAInt32Span *) data error:(NSError **)error { try { OgaString result = _tokenizer->Decode(data.pointer, data.size); @@ -68,6 +68,19 @@ - (nullable)initWithTokenizer:(OGATokenizer *)tokenizer OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } +- (nullable)initWithMultiModalProcessor:(OGAMultiModalProcessor *)processor + error:(NSError **)error{ + if ((self = [super init]) == nil) { + return nil; + } + + try { + _stream = OgaTokenizerStream::Create([processor CXXAPIOgaMultiModalProcessor]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + - (nullable NSString *)decode:(int32_t)token error:(NSError **)error { try { From fda89e80014d7cfbd352fffc793b65e3ee002c33 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:48:13 +0800 Subject: [PATCH 09/59] format --- src/objectivec/error_utils.h | 6 +- src/objectivec/oga_generator.mm | 42 ++++---- src/objectivec/oga_generator_params.mm | 95 ++++++++--------- src/objectivec/oga_images.mm | 28 +++-- src/objectivec/oga_internal.h | 22 ++-- src/objectivec/oga_internal.mm | 46 ++++---- src/objectivec/oga_model.mm | 40 ++++--- src/objectivec/oga_multi_modal_processor.mm | 57 +++++----- src/objectivec/oga_named_tensors.mm | 14 +-- src/objectivec/oga_sequences.mm | 48 ++++----- src/objectivec/oga_tensor.mm | 38 +++---- src/objectivec/oga_tokenizer.mm | 111 +++++++++----------- 12 files changed, 258 insertions(+), 289 deletions(-) diff --git a/src/objectivec/error_utils.h b/src/objectivec/error_utils.h index 4e213ac6c..a9be5b7fc 100644 --- a/src/objectivec/error_utils.h +++ b/src/objectivec/error_utils.h @@ -20,10 +20,8 @@ void OGASaveExceptionToError(const std::exception& e, NSError** error); return (failure_return_value); \ } -#define OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) \ - OGA_OBJC_API_IMPL_CATCH(error, NO) +#define OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) OGA_OBJC_API_IMPL_CATCH(error, NO) -#define OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) \ - OGA_OBJC_API_IMPL_CATCH(error, nil) +#define OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) OGA_OBJC_API_IMPL_CATCH(error, nil) NS_ASSUME_NONNULL_END diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm index a7e6bb5a4..424f40229 100644 --- a/src/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -1,44 +1,44 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" +#import "ort_genai_objc.h" @implementation OGAGenerator { - std::unique_ptr _generator; + std::unique_ptr _generator; } -- (nullable)initWithModel:(OGAModel *)model - params:(OGAGeneratorParams *)params - error:(NSError **)error { - if ((self = [super init]) == nil) { - return nil; - } - - try { - _generator = OgaGenerator::Create([model CXXAPIOgaModel], [params CXXAPIOgaGeneratorParams]); - return self; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +- (nullable)initWithModel:(OGAModel*)model + params:(OGAGeneratorParams*)params + error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _generator = OgaGenerator::Create([model CXXAPIOgaModel], [params CXXAPIOgaGeneratorParams]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } - (BOOL)isDone { - return _generator->IsDone(); + return _generator->IsDone(); } - (void)computeLogits { - _generator->ComputeLogits(); + _generator->ComputeLogits(); } - (void)generateNextToken { - _generator->GenerateNextToken(); + _generator->GenerateNextToken(); } -- (nullable OGAInt32Span *)sequenceAtIndex:(size_t) index { - size_t sequenceLength = _generator->GetSequenceCount(index); - const int32_t* data = _generator->GetSequenceData(index); - return [[OGAInt32Span alloc] initWithRawPointer:data size:sequenceLength]; +- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index { + size_t sequenceLength = _generator->GetSequenceCount(index); + const int32_t* data = _generator->GetSequenceData(index); + return [[OGAInt32Span alloc] initWithRawPointer:data size:sequenceLength]; } @end diff --git a/src/objectivec/oga_generator_params.mm b/src/objectivec/oga_generator_params.mm index de9f9a8a7..f0848cc62 100644 --- a/src/objectivec/oga_generator_params.mm +++ b/src/objectivec/oga_generator_params.mm @@ -1,79 +1,68 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" +#import "ort_genai_objc.h" @implementation OGAGeneratorParams { - std::unique_ptr _generatorParams; + std::unique_ptr _generatorParams; } +- (nullable)initWithModel:(OGAModel*)model error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } -- (nullable)initWithModel:(OGAModel *)model - error:(NSError **)error { - if ((self = [super init]) == nil) { - return nil; - } - - try { - _generatorParams = OgaGeneratorParams::Create([model CXXAPIOgaModel]); - return self; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) + try { + _generatorParams = OgaGeneratorParams::Create([model CXXAPIOgaModel]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (BOOL)setInputs:(OGANamedTensors *)namedTensors - error:(NSError **)error { - try { - _generatorParams->SetInputs([namedTensors CXXAPIOgaNamedTensors]); - return YES; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +- (BOOL)setInputs:(OGANamedTensors*)namedTensors error:(NSError**)error { + try { + _generatorParams->SetInputs([namedTensors CXXAPIOgaNamedTensors]); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } -- (BOOL)setInputSequences:(OGASequences *)sequences - error:(NSError **)error { - try { - _generatorParams->SetInputSequences([sequences CXXAPIOgaSequences]); - return YES; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +- (BOOL)setInputSequences:(OGASequences*)sequences error:(NSError**)error { + try { + _generatorParams->SetInputSequences([sequences CXXAPIOgaSequences]); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } -- (BOOL)setModelInput:(NSString *)name - tensor:(OGATensor*)tensor - error:(NSError **)error { - try { - _generatorParams->SetModelInput(name.UTF8String, [tensor CXXAPIOgaTensor]); - return YES; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +- (BOOL)setModelInput:(NSString*)name tensor:(OGATensor*)tensor error:(NSError**)error { + try { + _generatorParams->SetModelInput(name.UTF8String, [tensor CXXAPIOgaTensor]); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } -- (BOOL)setSearchOption:(NSString *)key - doubleValue:(double)value - error:(NSError **)error { - try { - _generatorParams->SetSearchOption([key UTF8String], value); - return YES; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +- (BOOL)setSearchOption:(NSString*)key doubleValue:(double)value error:(NSError**)error { + try { + _generatorParams->SetSearchOption([key UTF8String], value); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } -- (BOOL)setSearchOption:(NSString *)key - boolValue:(BOOL)value - error:(NSError **)error { - try { - _generatorParams->SetSearchOptionBool([key UTF8String], value); - return YES; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +- (BOOL)setSearchOption:(NSString*)key boolValue:(BOOL)value error:(NSError**)error { + try { + _generatorParams->SetSearchOptionBool([key UTF8String], value); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } - - (OgaGeneratorParams&)CXXAPIOgaGeneratorParams { - return *(_generatorParams.get()); + return *(_generatorParams.get()); } @end diff --git a/src/objectivec/oga_images.mm b/src/objectivec/oga_images.mm index d2b7ba499..a984190a4 100644 --- a/src/objectivec/oga_images.mm +++ b/src/objectivec/oga_images.mm @@ -1,30 +1,28 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" +#import "ort_genai_objc.h" @implementation OGAImages { - std::unique_ptr _images; + std::unique_ptr _images; } +- (nullable)initWithPath:(NSString*)path error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } -- (nullable)initWithPath:(NSString *)path - error:(NSError **)error { - if ((self = [super init]) == nil) { - return nil; - } - - try { - _images = OgaImages::Load(path.UTF8String); - return self; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) + try { + _images = OgaImages::Load(path.UTF8String); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (OgaImages *)CXXAPIOgaImages { - return _images.get(); +- (OgaImages*)CXXAPIOgaImages { + return _images.get(); } @end \ No newline at end of file diff --git a/src/objectivec/oga_internal.h b/src/objectivec/oga_internal.h index e5e685cae..aa9948f61 100644 --- a/src/objectivec/oga_internal.h +++ b/src/objectivec/oga_internal.h @@ -1,7 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#import "cxx_api.h" +#import "cxx_api.h" +#import "error_utils.h" +#import "ort_genai_objc.h" NS_ASSUME_NONNULL_BEGIN @@ -19,28 +21,25 @@ NS_ASSUME_NONNULL_BEGIN @interface OGAInt32Span () -- (nullable)initWithRawPointer:(const int32_t *) pointer - size:(size_t)size; +- (nullable)initWithRawPointer:(const int32_t*)pointer size:(size_t)size; -- (const int32_t *)pointer; +- (const int32_t*)pointer; - (size_t)size; @end @interface OGAInt64Span () -- (nullable)initWithRawPointer:(const int64_t *) pointer - size:(size_t)size; +- (nullable)initWithRawPointer:(const int64_t*)pointer size:(size_t)size; -- (const int64_t *)pointer; +- (const int64_t*)pointer; - (size_t)size; @end - @interface OGASequences () -- (nullable)initWithError:(NSError **)error; +- (nullable)initWithError:(NSError**)error; - (instancetype)initWithNativePointer:(std::unique_ptr)ptr; - (OgaSequences&)CXXAPIOgaSequences; @@ -55,7 +54,7 @@ NS_ASSUME_NONNULL_BEGIN @interface OGAImages () -- (OgaImages *)CXXAPIOgaImages; +- (OgaImages*)CXXAPIOgaImages; @end @@ -67,7 +66,8 @@ NS_ASSUME_NONNULL_BEGIN @interface OGANamedTensors () -- (instancetype)initWithNativePointer:(std::unique_ptr)ptr NS_DESIGNATED_INITIALIZER; +- (instancetype)initWithNativePointer:(std::unique_ptr)ptr + NS_DESIGNATED_INITIALIZER; - (OgaNamedTensors&)CXXAPIOgaNamedTensors; @end diff --git a/src/objectivec/oga_internal.mm b/src/objectivec/oga_internal.mm index 3f69e4dfc..aafb1efd3 100644 --- a/src/objectivec/oga_internal.mm +++ b/src/objectivec/oga_internal.mm @@ -1,60 +1,54 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#import "ort_genai_objc.h" -#import "error_utils.h" #import "oga_internal.h" - @implementation OGAInt32Span { - const int32_t * _ptr; - size_t _size; + const int32_t* _ptr; + size_t _size; } -- (nullable)initWithRawPointer:(const int32_t * )pointer - size:(size_t)size { - _ptr = pointer; - _size = size; - return [self init]; +- (nullable)initWithRawPointer:(const int32_t*)pointer size:(size_t)size { + _ptr = pointer; + _size = size; + return [self init]; } -- (const int32_t * )pointer { - return _ptr; +- (const int32_t*)pointer { + return _ptr; } - (size_t)size { - return _size; + return _size; } - (int32_t)last { - return *(_ptr + (_size - 1)); + return *(_ptr + (_size - 1)); } @end @implementation OGAInt64Span { - const int64_t * _ptr; - size_t _size; + const int64_t* _ptr; + size_t _size; } -- (nullable)initWithRawPointer:(const int64_t * )pointer - size:(size_t)size { - _ptr = pointer; - _size = size; - return [self init]; +- (nullable)initWithRawPointer:(const int64_t*)pointer size:(size_t)size { + _ptr = pointer; + _size = size; + return [self init]; } -- (const int64_t * )pointer { - return _ptr; +- (const int64_t*)pointer { + return _ptr; } - (size_t)size { - return _size; + return _size; } - (int64_t)last { - return *(_ptr + (_size - 1)); + return *(_ptr + (_size - 1)); } @end - diff --git a/src/objectivec/oga_model.mm b/src/objectivec/oga_model.mm index 6b187132e..2a54bfac4 100644 --- a/src/objectivec/oga_model.mm +++ b/src/objectivec/oga_model.mm @@ -1,39 +1,37 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" +#import "ort_genai_objc.h" @implementation OGAModel { - std::unique_ptr _model; + std::unique_ptr _model; } +- (nullable)initWithPath:(NSString*)path error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } -- (nullable)initWithPath:(NSString *)path - error:(NSError **)error { - if ((self = [super init]) == nil) { - return nil; - } - - try { - _model = OgaModel::Create(path.UTF8String); - return self; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) + try { + _model = OgaModel::Create(path.UTF8String); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable OGASequences *)generate:(OGAGeneratorParams *)params - error:(NSError **)error { - try { - std::unique_ptr output_sequences = _model->Generate([params CXXAPIOgaGeneratorParams]); - return [[OGASequences alloc] initWithNativePointer:std::move(output_sequences)]; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +- (nullable OGASequences*)generate:(OGAGeneratorParams*)params error:(NSError**)error { + try { + std::unique_ptr output_sequences = + _model->Generate([params CXXAPIOgaGeneratorParams]); + return [[OGASequences alloc] initWithNativePointer:std::move(output_sequences)]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } - (const OgaModel&)CXXAPIOgaModel { - return *(_model.get()); + return *(_model.get()); } @end diff --git a/src/objectivec/oga_multi_modal_processor.mm b/src/objectivec/oga_multi_modal_processor.mm index a9fe53860..9790760b9 100644 --- a/src/objectivec/oga_multi_modal_processor.mm +++ b/src/objectivec/oga_multi_modal_processor.mm @@ -1,49 +1,48 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" +#import "ort_genai_objc.h" @implementation OGAMultiModalProcessor { - std::unique_ptr _processor; + std::unique_ptr _processor; } +- (nullable)initWithModel:(OGAModel*)model error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } -- (nullable)initWithModel:(OGAModel *)model - error:(NSError **)error { - if ((self = [super init]) == nil) { - return nil; - } - - try { - _processor = OgaMultiModalProcessor::Create([model CXXAPIOgaModel]); - return self; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) + try { + _processor = OgaMultiModalProcessor::Create([model CXXAPIOgaModel]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable OGANamedTensors *)processImages:(NSString *)prompt - images:(OGAImages *)images - error:(NSError **)error { - try { - OGANamedTensors *result = [[OGANamedTensors alloc] initWithNativePointer:_processor->ProcessImages([prompt UTF8String], [images CXXAPIOgaImages])]; - return result; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +- (nullable OGANamedTensors*)processImages:(NSString*)prompt + images:(OGAImages*)images + error:(NSError**)error { + try { + OGANamedTensors* result = [[OGANamedTensors alloc] + initWithNativePointer:_processor->ProcessImages([prompt UTF8String], + [images CXXAPIOgaImages])]; + return result; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable NSString *)decode:(OGAInt32Span *) data - error:(NSError **)error { - try { - OgaString result = _processor->Decode(data.pointer, data.size); - return [NSString stringWithUTF8String:result]; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +- (nullable NSString*)decode:(OGAInt32Span*)data error:(NSError**)error { + try { + OgaString result = _processor->Decode(data.pointer, data.size); + return [NSString stringWithUTF8String:result]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } - (const OgaMultiModalProcessor&)CXXAPIOgaMultiModalProcessor { - return *(_processor.get()); + return *(_processor.get()); } @end diff --git a/src/objectivec/oga_named_tensors.mm b/src/objectivec/oga_named_tensors.mm index b65526fa5..dab8e55ae 100644 --- a/src/objectivec/oga_named_tensors.mm +++ b/src/objectivec/oga_named_tensors.mm @@ -1,22 +1,22 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" +#import "ort_genai_objc.h" @implementation OGANamedTensors { - std::unique_ptr _tensor; + std::unique_ptr _tensor; } - -- (instancetype)initWithNativePointer:(std::unique_ptr)ptr; { - _tensor = std::move(ptr); - return self; +- (instancetype)initWithNativePointer:(std::unique_ptr)ptr; +{ + _tensor = std::move(ptr); + return self; } - (OgaNamedTensors&)CXXAPIOgaNamedTensors { - return *(_tensor.get()); + return *(_tensor.get()); } @end diff --git a/src/objectivec/oga_sequences.mm b/src/objectivec/oga_sequences.mm index cede09a89..154afb124 100644 --- a/src/objectivec/oga_sequences.mm +++ b/src/objectivec/oga_sequences.mm @@ -1,46 +1,46 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" +#import "ort_genai_objc.h" @implementation OGASequences { - std::unique_ptr _sequences; + std::unique_ptr _sequences; } -- (instancetype)initWithNativeSeqquences:(std::unique_ptr)ptr { - _sequences = std::move(ptr); - return self; +- (instancetype)initWithNativePointer:(std::unique_ptr)ptr { + _sequences = std::move(ptr); + return self; } -- (nullable)initWithError:(NSError **)error { - if ((self = [super init]) == nil) { - return nil; - } +- (nullable)initWithError:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } - try { - _sequences = OgaSequences::Create(); - return self; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) + try { + _sequences = OgaSequences::Create(); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } - (size_t)count { - return _sequences->Count(); + return _sequences->Count(); } -- (nullable OGAInt32Span *)sequenceAtIndex:(size_t) index { - if (index >= [self count]) { - return nil; - } - size_t sequenceLength = _sequences->SequenceCount(index); - const int32_t* data = _sequences->SequenceData(index); - return [[OGAInt32Span alloc] initWithRawPointer:data size: sequenceLength]; +- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index { + if (index >= [self count]) { + return nil; + } + size_t sequenceLength = _sequences->SequenceCount(index); + const int32_t* data = _sequences->SequenceData(index); + return [[OGAInt32Span alloc] initWithRawPointer:data size:sequenceLength]; } -- (OgaSequences&) CXXAPIOgaSequences { - return *(_sequences.get()); +- (OgaSequences&)CXXAPIOgaSequences { + return *(_sequences.get()); } @end diff --git a/src/objectivec/oga_tensor.mm b/src/objectivec/oga_tensor.mm index 25ebe6d99..617f76425 100644 --- a/src/objectivec/oga_tensor.mm +++ b/src/objectivec/oga_tensor.mm @@ -1,40 +1,40 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" +#import "ort_genai_objc.h" @implementation OGATensor { - std::unique_ptr _tensor; + std::unique_ptr _tensor; } - -- (nullable)initWithDataPointer:(void *)data - shape:(OGAInt64Span *)shape +- (nullable)initWithDataPointer:(void*)data + shape:(OGAInt64Span*)shape type:(OGAElementType)elementType - error:(NSError **)error { - if ((self = [super init]) == nil) { - return nil; - } - - try { - _tensor = OgaTensor::Create(data, shape.pointer, shape.size, static_cast(elementType)); - return self; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) + error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _tensor = OgaTensor::Create(data, shape.pointer, shape.size, + static_cast(elementType)); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } - (OGAElementType)type { - return OGAElementType(_tensor->Type()); + return OGAElementType(_tensor->Type()); } -- (void *)data { - return _tensor->Data(); +- (void*)data { + return _tensor->Data(); } - (OgaTensor&)CXXAPIOgaTensor { - return *(_tensor.get()); + return *(_tensor.get()); } @end diff --git a/src/objectivec/oga_tokenizer.mm b/src/objectivec/oga_tokenizer.mm index f90ad99e9..0cc2b2394 100644 --- a/src/objectivec/oga_tokenizer.mm +++ b/src/objectivec/oga_tokenizer.mm @@ -1,92 +1,85 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#import "ort_genai_objc.h" #import "error_utils.h" #import "oga_internal.h" +#import "ort_genai_objc.h" @implementation OGATokenizer { - std::unique_ptr _tokenizer; + std::unique_ptr _tokenizer; } +- (nullable)initWithModel:(OGAModel*)model error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } -- (nullable)initWithModel:(OGAModel *)model - error:(NSError **)error { - if ((self = [super init]) == nil) { - return nil; - } - - try { - _tokenizer = OgaTokenizer::Create([model CXXAPIOgaModel]); - return self; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) + try { + _tokenizer = OgaTokenizer::Create([model CXXAPIOgaModel]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable OGASequences *)encode:(NSString *)str - error:(NSError **)error { - OGASequences *sequences = [[OGASequences alloc] initWithError:error]; - if (*error) { - return nil; - } - try { - _tokenizer->Encode([str UTF8String], [sequences CXXAPIOgaSequences]); - return sequences; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +- (nullable OGASequences*)encode:(NSString*)str error:(NSError**)error { + OGASequences* sequences = [[OGASequences alloc] initWithError:error]; + if (*error) { + return nil; + } + try { + _tokenizer->Encode([str UTF8String], [sequences CXXAPIOgaSequences]); + return sequences; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable NSString *)decode:(OGAInt32Span *) data - error:(NSError **)error { - try { - OgaString result = _tokenizer->Decode(data.pointer, data.size); - return [NSString stringWithUTF8String:result]; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +- (nullable NSString*)decode:(OGAInt32Span*)data error:(NSError**)error { + try { + OgaString result = _tokenizer->Decode(data.pointer, data.size); + return [NSString stringWithUTF8String:result]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } - (const OgaTokenizer&)CXXAPIOgaTokenizer { - return *(_tokenizer.get()); + return *(_tokenizer.get()); } @end @implementation OGATokenizerStream { - std::unique_ptr _stream; + std::unique_ptr _stream; } -- (nullable)initWithTokenizer:(OGATokenizer *)tokenizer - error:(NSError **)error { - if ((self = [super init]) == nil) { - return nil; - } +- (nullable)initWithTokenizer:(OGATokenizer*)tokenizer error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } - try { - _stream = OgaTokenizerStream::Create([tokenizer CXXAPIOgaTokenizer]); - return self; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) + try { + _stream = OgaTokenizerStream::Create([tokenizer CXXAPIOgaTokenizer]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable)initWithMultiModalProcessor:(OGAMultiModalProcessor *)processor - error:(NSError **)error{ - if ((self = [super init]) == nil) { - return nil; - } +- (nullable)initWithMultiModalProcessor:(OGAMultiModalProcessor*)processor error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } - try { - _stream = OgaTokenizerStream::Create([processor CXXAPIOgaMultiModalProcessor]); - return self; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) + try { + _stream = OgaTokenizerStream::Create([processor CXXAPIOgaMultiModalProcessor]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable NSString *)decode:(int32_t)token - error:(NSError **)error { - try { - return [NSString stringWithUTF8String:_stream->Decode(token)]; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +- (nullable NSString*)decode:(int32_t)token error:(NSError**)error { + try { + return [NSString stringWithUTF8String:_stream->Decode(token)]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } @end From 844e7d52282a5c0619f4deffeec86009b1235952 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:53:56 +0800 Subject: [PATCH 10/59] Rm podspec for now --- onnxruntime-genai-objc.podspec | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 onnxruntime-genai-objc.podspec diff --git a/onnxruntime-genai-objc.podspec b/onnxruntime-genai-objc.podspec deleted file mode 100644 index 4eb2dd202..000000000 --- a/onnxruntime-genai-objc.podspec +++ /dev/null @@ -1,27 +0,0 @@ - -Pod::Spec.new do |s| - s.name = "onnxruntime-genai-objc" - s.version = "0.4.0" - s.summary = "SUMMARY" - s.homepage = "https://github.com/microsoft/onnxruntime-genai" - s.license = { :type => "MIT" } - s.author = {"ONNX Runtime" => "onnxruntime@microsoft.com"} - s.source = { :path => './objectivec'} - - s.static_framework = true - s.public_header_files = [ - "objectivec/include/ort_genai_objc.h" - ] - - s.source_files = [ - "objectivec/include/ort_genai_objc.h", - "objectivec/cxx_api.h", - "objectivec/oga_model.mm", - "objectivec/error_utils.h", - "objectivec/error_utils.mm", - "src/ort_genai.h" - ] - - s.dependency 'onnxruntime-objc', '~> 1.18.0' -end - From 891ec8c92d910b6a53b39a8218621fcfcf7e5c6a Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 18 Jul 2024 12:11:02 +0800 Subject: [PATCH 11/59] header --- src/objectivec/include/ort_genai_objc.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 99dab562e..08c0acb79 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -135,6 +135,11 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGATensor : NSObject +- (instancetype)init NS_UNAVAILABLE; +- (nullable)initWithDataPointer:(void*)data + shape:(OGAInt64Span*)shape + type:(OGAElementType)elementType + error:(NSError**)error; @end @interface OGANamedTensors : NSObject From a4af1b5fc6197020ad1884aff2e1ae5df3777ac9 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 18 Jul 2024 12:14:42 +0800 Subject: [PATCH 12/59] more header --- src/objectivec/include/ort_genai_objc.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 08c0acb79..9e64f7903 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -140,6 +140,8 @@ typedef NS_ENUM(NSInteger, OGAElementType) { shape:(OGAInt64Span*)shape type:(OGAElementType)elementType error:(NSError**)error; +- (OGAElementType)type; + @end @interface OGANamedTensors : NSObject @@ -158,6 +160,13 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGAMultiModalProcessor : NSObject +- (instancetype)init NS_UNAVAILABLE; +- (nullable)initWithModel:(OGAModel*)model + error:(NSError**)error NS_DESIGNATED_INITIALIZER; + +- (nullable OGANamedTensors*)processImages:(NSString*)prompt + images:(OGAImages*)images + error:(NSError**)error; @end NS_ASSUME_NONNULL_END From fef5a1a9dd751edd13e71dcf472cfc711620fec0 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 18 Jul 2024 12:19:24 +0800 Subject: [PATCH 13/59] catch --- src/objectivec/include/ort_genai_objc.h | 3 ++- src/objectivec/oga_generator.mm | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 9e64f7903..b525c99c7 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -129,7 +129,8 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (void)computeLogits; - (void)generateNextToken; -- (nullable OGAInt32Span *)sequenceAtIndex:(size_t) index; +- (nullable OGAInt32Span *)sequenceAtIndex:(size_t) index + error:(NSError **)error; @end diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm index 424f40229..e4f23f0b3 100644 --- a/src/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -35,10 +35,13 @@ - (void)generateNextToken { _generator->GenerateNextToken(); } -- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index { - size_t sequenceLength = _generator->GetSequenceCount(index); - const int32_t* data = _generator->GetSequenceData(index); - return [[OGAInt32Span alloc] initWithRawPointer:data size:sequenceLength]; +- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index error:(NSError**)error { + try { + size_t sequenceLength = _generator->GetSequenceCount(index); + const int32_t* data = _generator->GetSequenceData(index); + return [[OGAInt32Span alloc] initWithRawPointer:data size:sequenceLength]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } @end From cda7411621dade65c733897c155ed8571771f19e Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 18 Jul 2024 13:06:38 +0800 Subject: [PATCH 14/59] Lint --- src/objectivec/include/ort_genai_objc.h | 69 ++++++++++++------------- src/objectivec/oga_generator.mm | 5 +- 2 files changed, 36 insertions(+), 38 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index b525c99c7..2419bb53c 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -3,7 +3,6 @@ NS_ASSUME_NONNULL_BEGIN - @class OGAInt32Span; @class OGATensor; @class OGASequences; @@ -29,43 +28,42 @@ typedef NS_ENUM(NSInteger, OGAElementType) { OGAElementTypeUint64, // maps to c type uint64_t }; - @interface OGAModel : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithPath:(NSString *)path - error:(NSError **)error NS_DESIGNATED_INITIALIZER; +- (nullable)initWithPath:(NSString*)path + error:(NSError**)error NS_DESIGNATED_INITIALIZER; -- (nullable OGASequences *)generate:(OGAGeneratorParams *)params - error:(NSError **)error; +- (nullable OGASequences *)generate:(OGAGeneratorParams*)params + error:(NSError**)error; @end @interface OGATokenizer : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithModel:(OGAModel *)model - error:(NSError **)error NS_DESIGNATED_INITIALIZER; +- (nullable)initWithModel:(OGAModel*)model + error:(NSError**)error NS_DESIGNATED_INITIALIZER; -- (nullable OGASequences *)encode:(NSString *)str - error:(NSError **)error; +- (nullable OGASequences *)encode:(NSString*)str + error:(NSError**)error; -- (nullable NSString *)decode:(OGAInt32Span *)data - error:(NSError **)error; +- (nullable NSString *)decode:(OGAInt32Span*)data + error:(NSError**)error; @end @interface OGATokenizerStream: NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithTokenizer:(OGATokenizer *)tokenizer - error:(NSError **)error NS_DESIGNATED_INITIALIZER; +- (nullable)initWithTokenizer:(OGATokenizer*)tokenizer + error:(NSError**)error NS_DESIGNATED_INITIALIZER; -- (nullable)initWithMultiModalProcessor:(OGAMultiModalProcessor *)processor - error:(NSError **)error NS_DESIGNATED_INITIALIZER; +- (nullable)initWithMultiModalProcessor:(OGAMultiModalProcessor*)processor + error:(NSError**)error NS_DESIGNATED_INITIALIZER; - (nullable NSString *)decode:(int32_t)token - error:(NSError **)error; + error:(NSError**)error; @end; @@ -90,47 +88,46 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (instancetype)init NS_UNAVAILABLE; - (size_t)count; -- (nullable OGAInt32Span *)sequenceAtIndex:(size_t)index; +- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index; @end @interface OGAGeneratorParams : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithModel:(OGAModel *)model - error:(NSError **)error NS_DESIGNATED_INITIALIZER; +- (nullable)initWithModel:(OGAModel*)model + error:(NSError**)error NS_DESIGNATED_INITIALIZER; -- (BOOL)setInputs:(OGANamedTensors *)namedTensors - error:(NSError **)error; +- (BOOL)setInputs:(OGANamedTensors*)namedTensors + error:(NSError**)error; -- (BOOL)setInputSequences:(OGASequences *)sequences +- (BOOL)setInputSequences:(OGASequences*)sequences error:(NSError **)error; -- (BOOL)setModelInput:(NSString *)name +- (BOOL)setModelInput:(NSString*)name tensor:(OGATensor*)tensor - error:(NSError **)error; + error:(NSError**)error; -- (BOOL)setSearchOption:(NSString *)key +- (BOOL)setSearchOption:(NSString*)key doubleValue:(double)value - error:(NSError **)error; + error:(NSError**)error; -- (BOOL)setSearchOption:(NSString *)key +- (BOOL)setSearchOption:(NSString*)key boolValue:(BOOL)value - error:(NSError **)error; + error:(NSError**)error; @end @interface OGAGenerator : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithModel:(OGAModel *)model - params:(OGAGeneratorParams *)params - error:(NSError **)error NS_DESIGNATED_INITIALIZER; +- (nullable)initWithModel:(OGAModel*)model + params:(OGAGeneratorParams*)params + error:(NSError**)error NS_DESIGNATED_INITIALIZER; - (BOOL)isDone; - (void)computeLogits; - (void)generateNextToken; -- (nullable OGAInt32Span *)sequenceAtIndex:(size_t) index - error:(NSError **)error; +- (nullable OGAInt32Span*)sequenceAtIndex:(size_t) index; @end @@ -154,8 +151,8 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGAImages : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithPath:(NSString *)path - error:(NSError **)error NS_DESIGNATED_INITIALIZER; +- (nullable)initWithPath:(NSString*)path + error:(NSError**)error NS_DESIGNATED_INITIALIZER; @end diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm index e4f23f0b3..4db451dfa 100644 --- a/src/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -35,13 +35,14 @@ - (void)generateNextToken { _generator->GenerateNextToken(); } -- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index error:(NSError**)error { +- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index { try { size_t sequenceLength = _generator->GetSequenceCount(index); const int32_t* data = _generator->GetSequenceData(index); return [[OGAInt32Span alloc] initWithRawPointer:data size:sequenceLength]; + } catch (std::exception) { + return nil; } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } @end From edd86f6360fdbe89989867d31ed9850a425a453b Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 18 Jul 2024 13:11:05 +0800 Subject: [PATCH 15/59] Lint --- src/objectivec/include/ort_genai_objc.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 2419bb53c..c3236fb84 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -35,8 +35,8 @@ typedef NS_ENUM(NSInteger, OGAElementType) { error:(NSError**)error NS_DESIGNATED_INITIALIZER; -- (nullable OGASequences *)generate:(OGAGeneratorParams*)params - error:(NSError**)error; +- (nullable OGASequences*)generate:(OGAGeneratorParams*)params + error:(NSError**)error; @end @@ -49,12 +49,12 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (nullable OGASequences *)encode:(NSString*)str error:(NSError**)error; -- (nullable NSString *)decode:(OGAInt32Span*)data - error:(NSError**)error; +- (nullable NSString*)decode:(OGAInt32Span*)data + error:(NSError**)error; @end -@interface OGATokenizerStream: NSObject +@interface OGATokenizerStream : NSObject - (instancetype)init NS_UNAVAILABLE; - (nullable)initWithTokenizer:(OGATokenizer*)tokenizer error:(NSError**)error NS_DESIGNATED_INITIALIZER; @@ -62,8 +62,8 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (nullable)initWithMultiModalProcessor:(OGAMultiModalProcessor*)processor error:(NSError**)error NS_DESIGNATED_INITIALIZER; -- (nullable NSString *)decode:(int32_t)token - error:(NSError**)error; +- (nullable NSString*)decode:(int32_t)token + error:(NSError**)error; @end; @@ -101,7 +101,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { error:(NSError**)error; - (BOOL)setInputSequences:(OGASequences*)sequences - error:(NSError **)error; + error:(NSError**)error; - (BOOL)setModelInput:(NSString*)name tensor:(OGATensor*)tensor @@ -127,7 +127,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (void)computeLogits; - (void)generateNextToken; -- (nullable OGAInt32Span*)sequenceAtIndex:(size_t) index; +- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index; @end From 2eeb42c9e41fccc3a434e66e6206a1f5782c18f0 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 18 Jul 2024 13:13:27 +0800 Subject: [PATCH 16/59] Lint --- src/objectivec/include/ort_genai_objc.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index c3236fb84..a07173367 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -36,18 +36,17 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (nullable OGASequences*)generate:(OGAGeneratorParams*)params - error:(NSError**)error; + error:(NSError**)error; @end - @interface OGATokenizer : NSObject - (instancetype)init NS_UNAVAILABLE; - (nullable)initWithModel:(OGAModel*)model error:(NSError**)error NS_DESIGNATED_INITIALIZER; -- (nullable OGASequences *)encode:(NSString*)str - error:(NSError**)error; +- (nullable OGASequences*)encode:(NSString*)str + error:(NSError**)error; - (nullable NSString*)decode:(OGAInt32Span*)data error:(NSError**)error; @@ -66,7 +65,6 @@ typedef NS_ENUM(NSInteger, OGAElementType) { error:(NSError**)error; @end; - @interface OGAInt32Span : NSObject - (instancetype)init NS_UNAVAILABLE; From 27d5494f1968561688024e65bec4704520e8add4 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 18 Jul 2024 13:14:34 +0800 Subject: [PATCH 17/59] Lint --- src/objectivec/include/ort_genai_objc.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index a07173367..722831e88 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -34,7 +34,6 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (nullable)initWithPath:(NSString*)path error:(NSError**)error NS_DESIGNATED_INITIALIZER; - - (nullable OGASequences*)generate:(OGAGeneratorParams*)params error:(NSError**)error; @@ -63,7 +62,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (nullable NSString*)decode:(int32_t)token error:(NSError**)error; -@end; +@end @interface OGAInt32Span : NSObject From 808398a97211b5f5d6ebf0a4b4af2120f7422bad Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 5 Sep 2024 13:34:46 +0800 Subject: [PATCH 18/59] save --- onnxruntime-genai-c.podspec | 21 ++++++++++++++++++ onnxruntime-genai-objc.podspec | 40 ++++++++++++++++++++++++++++++++++ src/objectivec/cxx_api.h | 2 +- 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 onnxruntime-genai-c.podspec create mode 100644 onnxruntime-genai-objc.podspec diff --git a/onnxruntime-genai-c.podspec b/onnxruntime-genai-c.podspec new file mode 100644 index 000000000..15b7cb15f --- /dev/null +++ b/onnxruntime-genai-c.podspec @@ -0,0 +1,21 @@ +Pod::Spec.new do |s| + s.name = "onnxruntime-genai-c" + s.version = "0.5.0" + s.summary = "SUMMARY" + s.homepage = "https://github.com/microsoft/onnxruntime-genai" + s.license = { :type => "MIT" } + s.author = {"ONNX Runtime" => "onnxruntime@microsoft.com"} + s.source = { :http => 'file:///Users/chester/Projects/onnxruntime-genai/build/apple_framework/framework_out/archive.zip'} + s.platform = :ios, '13.0', :osx, '11.0' + + s.vendored_frameworks = "onnxruntime-genai.xcframework" + s.static_framework = true + + s.source_files = [ + "Headers/*.h" + ] + + s.libraries = "c++" + s.dependency 'onnxruntime-c', '~> 1.19.0' +end + diff --git a/onnxruntime-genai-objc.podspec b/onnxruntime-genai-objc.podspec new file mode 100644 index 000000000..3e3b15608 --- /dev/null +++ b/onnxruntime-genai-objc.podspec @@ -0,0 +1,40 @@ +Pod::Spec.new do |s| + s.name = "onnxruntime-genai-objc" + s.version = "0.5.0" + s.summary = "SUMMARY" + s.homepage = "https://github.com/microsoft/onnxruntime-genai" + s.license = { :type => "MIT" } + s.author = {"ONNX Runtime" => "onnxruntime@microsoft.com"} + s.source = { :http => 'file:///Users/chester/Projects/onnxruntime-genai/src/archive.zip'} + s.platform = :ios, '13.0', :osx, '11.0' + + s.default_subspec = 'Core' + s.static_framework = true + + s.subspec 'Core' do |sp| + sp.requires_arc = true + sp.public_header_files = [ + "objectivec/include/ort_genai_objc.h" + ] + sp.source_files = [ + "objectivec/include/ort_genai_objc.h", + "objectivec/cxx_api.h", + "objectivec/error_utils.h", + "objectivec/error_utils.mm", + "objectivec/oga_internal.h", + "objectivec/oga_internal.mm", + "objectivec/oga_model.mm", + "objectivec/oga_tokenizer.mm", + "objectivec/oga_sequences.mm", + "objectivec/oga_generator.mm", + "objectivec/oga_generator_params.mm", + "objectivec/oga_multi_modal_processor.mm", + "objectivec/oga_named_tensors.mm", + "objectivec/oga_tensor.mm", + "objectivec/oga_images.mm", + ] + + end + s.dependency 'onnxruntime-objc', '~> 1.19.0' +end + diff --git a/src/objectivec/cxx_api.h b/src/objectivec/cxx_api.h index b452a490f..d1f5fa06f 100644 --- a/src/objectivec/cxx_api.h +++ b/src/objectivec/cxx_api.h @@ -10,7 +10,7 @@ #pragma clang diagnostic ignored "-Wdocumentation" #endif // defined(__clang__) -#import "../ort_genai.h" +#import "ort_genai.h" #if defined(__clang__) #pragma clang diagnostic pop From cda2e395f0bf08b086743cc09ac06e5671a16d05 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:41:53 +0800 Subject: [PATCH 19/59] Update interface --- src/objectivec/include/ort_genai_objc.h | 2 +- src/objectivec/oga_generator_params.mm | 2 +- src/objectivec/oga_images.mm | 11 +++++++++-- src/objectivec/oga_model.mm | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 722831e88..8fa7b95de 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -148,7 +148,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGAImages : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithPath:(NSString*)path +- (nullable)initWithPath:(NSArray *)paths error:(NSError**)error NS_DESIGNATED_INITIALIZER; @end diff --git a/src/objectivec/oga_generator_params.mm b/src/objectivec/oga_generator_params.mm index f0848cc62..95e840944 100644 --- a/src/objectivec/oga_generator_params.mm +++ b/src/objectivec/oga_generator_params.mm @@ -39,7 +39,7 @@ - (BOOL)setInputSequences:(OGASequences*)sequences error:(NSError**)error { - (BOOL)setModelInput:(NSString*)name tensor:(OGATensor*)tensor error:(NSError**)error { try { - _generatorParams->SetModelInput(name.UTF8String, [tensor CXXAPIOgaTensor]); + _generatorParams->SetModelInput([name UTF8String], [tensor CXXAPIOgaTensor]); return YES; } OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) diff --git a/src/objectivec/oga_images.mm b/src/objectivec/oga_images.mm index a984190a4..9f4e89815 100644 --- a/src/objectivec/oga_images.mm +++ b/src/objectivec/oga_images.mm @@ -9,13 +9,20 @@ @implementation OGAImages { std::unique_ptr _images; } -- (nullable)initWithPath:(NSString*)path error:(NSError**)error { +- (nullable)initWithPath:(NSArray *)paths error:(NSError**)error { if ((self = [super init]) == nil) { return nil; } + std::vector cpp_paths; + cpp_paths.reserve([paths count]); + + for (NSString* path in paths){ + cpp_paths.push_back([path UTF8String]); + } + try { - _images = OgaImages::Load(path.UTF8String); + _images = OgaImages::Load(cpp_paths); return self; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) diff --git a/src/objectivec/oga_model.mm b/src/objectivec/oga_model.mm index 2a54bfac4..dbb20d7bb 100644 --- a/src/objectivec/oga_model.mm +++ b/src/objectivec/oga_model.mm @@ -15,7 +15,7 @@ - (nullable)initWithPath:(NSString*)path error:(NSError**)error { } try { - _model = OgaModel::Create(path.UTF8String); + _model = OgaModel::Create([path UTF8String]); return self; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) From 01c04161f84a6ce9c93f49e9d670baa28f13cdd1 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:09:23 +0800 Subject: [PATCH 20/59] init pipeline --- .pipelines/ios-cocoapods-publishing.yml | 22 +++++++++++++++++++ .../stages/ios-cocoapods-packaging-stage.yml | 15 +++++++++++++ .../jobs/ios-cocoapods-packaginb-job.yml | 0 3 files changed, 37 insertions(+) create mode 100644 .pipelines/ios-cocoapods-publishing.yml create mode 100644 .pipelines/stages/ios-cocoapods-packaging-stage.yml create mode 100644 .pipelines/stages/jobs/ios-cocoapods-packaginb-job.yml diff --git a/.pipelines/ios-cocoapods-publishing.yml b/.pipelines/ios-cocoapods-publishing.yml new file mode 100644 index 000000000..a79462249 --- /dev/null +++ b/.pipelines/ios-cocoapods-publishing.yml @@ -0,0 +1,22 @@ +parameters: +- name: ort_version + displayName: 'OnnxRuntime version' + type: string + default: '1.18.0' + +- name: build_config + displayName: 'Build Configuration' + type: string + default: 'Release' + values: + - 'Debug' + - 'RelWithDebInfo' + - 'Release' + - 'MinSizeRel' + +trigger: none +stages: +- template: stages/ios-cocoapods-packaging-stage.yml + parameters: + ort_version: ${{ parameters.ort_version }} + build_config: ${{ parameters.build_config }} diff --git a/.pipelines/stages/ios-cocoapods-packaging-stage.yml b/.pipelines/stages/ios-cocoapods-packaging-stage.yml new file mode 100644 index 000000000..da72b6b83 --- /dev/null +++ b/.pipelines/stages/ios-cocoapods-packaging-stage.yml @@ -0,0 +1,15 @@ +parameters: +- name: ort_version + type: string + +- name: build_config + type: string + default: 'Release' + +stages: +- stage: Build_iOS_CocoaPods_Archive + jobs: + - template: jobs/android-java-api-aar.yml + parameters: + ort_version: ${{ parameters.ort_version }} + build_config: ${{ parameters.build_config }} diff --git a/.pipelines/stages/jobs/ios-cocoapods-packaginb-job.yml b/.pipelines/stages/jobs/ios-cocoapods-packaginb-job.yml new file mode 100644 index 000000000..e69de29bb From 318cce1b96afa69c844cc6e1c585d3db3329b5f4 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:20:13 +0800 Subject: [PATCH 21/59] cocoapods pipeline --- .pipelines/stages/jobs/capi-packaging-job.yml | 1 + .../stages/jobs/ios-cocoapods-packaginb-job.yml | 0 .../stages/jobs/ios-cocoapods-packaging-job.yml | 17 +++++++++++++++++ .../jobs/steps/capi-appleframework-step.yml | 9 +++++++-- 4 files changed, 25 insertions(+), 2 deletions(-) delete mode 100644 .pipelines/stages/jobs/ios-cocoapods-packaginb-job.yml create mode 100644 .pipelines/stages/jobs/ios-cocoapods-packaging-job.yml diff --git a/.pipelines/stages/jobs/capi-packaging-job.yml b/.pipelines/stages/jobs/capi-packaging-job.yml index 8759d2b69..7c1efe747 100644 --- a/.pipelines/stages/jobs/capi-packaging-job.yml +++ b/.pipelines/stages/jobs/capi-packaging-job.yml @@ -148,5 +148,6 @@ jobs: - template: steps/capi-appleframework-step.yml parameters: build_config: ${{ parameters.build_config }} + framework_config_file: "tools/ci_build/github/apple/default_full_ios_framework_build_settings.json" - template: steps/compliant-and-cleanup-step.yml diff --git a/.pipelines/stages/jobs/ios-cocoapods-packaginb-job.yml b/.pipelines/stages/jobs/ios-cocoapods-packaginb-job.yml deleted file mode 100644 index e69de29bb..000000000 diff --git a/.pipelines/stages/jobs/ios-cocoapods-packaging-job.yml b/.pipelines/stages/jobs/ios-cocoapods-packaging-job.yml new file mode 100644 index 000000000..64b493549 --- /dev/null +++ b/.pipelines/stages/jobs/ios-cocoapods-packaging-job.yml @@ -0,0 +1,17 @@ +parameters: +- name: build_config + type: string + default: 'release' + +jobs: +- job: ios_cocoapods_packaging + pool: + vmImage: 'macOS-latest' + + workspace: + clean: all + steps: + - template: steps/capi-appleframework-step.yml + parameters: + build_config: ${{ parameters.build_config }} + framework_config_file: "tools/ci_build/github/apple/default_full_apple_framework_build_settings.json" diff --git a/.pipelines/stages/jobs/steps/capi-appleframework-step.yml b/.pipelines/stages/jobs/steps/capi-appleframework-step.yml index 1970812ed..130a8d2e8 100644 --- a/.pipelines/stages/jobs/steps/capi-appleframework-step.yml +++ b/.pipelines/stages/jobs/steps/capi-appleframework-step.yml @@ -2,7 +2,8 @@ parameters: - name: build_config type: string default: 'release' - +- name: framework_config_file + type: string steps: - checkout: self @@ -11,6 +12,9 @@ steps: submodules: recursive - template: utils/set-genai-version.yml +- template: steps/utils/set-cmake-build-type.yml + parameters: + build_config: ${{parameters.build_config}} - template: utils/set-nightly-build-option-variable.yml @@ -19,7 +23,8 @@ steps: python3 -m pip install requests python3 tools/ci_build/github/apple/build_apple_framework.py \ --build_dir "$(Build.BinariesDirectory)/apple_framework" \ - tools/ci_build/github/apple/default_full_ios_framework_build_settings.json + ${{ parameters.framework_config_file }} + --config $(cmake_build_type) mkdir $(Build.BinariesDirectory)/artifacts mkdir -p $(Build.BinariesDirectory)/artifacts_staging/onnxruntime-genai.xcframework From d50175026cf1a9687726006634748f9ea5f2fcc4 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 15 Oct 2024 13:42:36 +0800 Subject: [PATCH 22/59] tools and YAMLs --- .pipelines/stages/jobs/capi-packaging-job.yml | 2 +- .../jobs/ios-cocoapods-packaging-job.yml | 17 -- .../macos-ios-cocoapods-packaging-job.yml | 37 ++++ .../jobs/steps/capi-appleframework-step.yml | 4 +- ...> macos-ios-cocoapods-packaging-stage.yml} | 4 +- build.py | 1 + onnxruntime-genai-c.podspec | 21 -- .../apple/build_and_assemble_apple_pods.py | 190 ++++++++++++++++++ .../github/apple/c/assemble_c_pod_package.py | 154 ++++++++++++++ .../github/apple/c/c.podspec.template | 27 +++ .../apple/c/onnxruntime-genai-c.config.json | 5 + .../objectivec/assemble_objc_pod_package.py | 178 ++++++++++++++++ .../apple/objectivec/objc.podspec.template | 43 ++++ .../github/apple/package_assembly_utils.py | 161 +++++++++++++++ 14 files changed, 801 insertions(+), 43 deletions(-) delete mode 100644 .pipelines/stages/jobs/ios-cocoapods-packaging-job.yml create mode 100644 .pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml rename .pipelines/stages/{ios-cocoapods-packaging-stage.yml => macos-ios-cocoapods-packaging-stage.yml} (70%) delete mode 100644 onnxruntime-genai-c.podspec create mode 100755 tools/ci_build/github/apple/build_and_assemble_apple_pods.py create mode 100644 tools/ci_build/github/apple/c/assemble_c_pod_package.py create mode 100644 tools/ci_build/github/apple/c/c.podspec.template create mode 100644 tools/ci_build/github/apple/c/onnxruntime-genai-c.config.json create mode 100755 tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py create mode 100644 tools/ci_build/github/apple/objectivec/objc.podspec.template create mode 100644 tools/ci_build/github/apple/package_assembly_utils.py diff --git a/.pipelines/stages/jobs/capi-packaging-job.yml b/.pipelines/stages/jobs/capi-packaging-job.yml index 7c1efe747..c14f3092c 100644 --- a/.pipelines/stages/jobs/capi-packaging-job.yml +++ b/.pipelines/stages/jobs/capi-packaging-job.yml @@ -148,6 +148,6 @@ jobs: - template: steps/capi-appleframework-step.yml parameters: build_config: ${{ parameters.build_config }} - framework_config_file: "tools/ci_build/github/apple/default_full_ios_framework_build_settings.json" + build_settings_file: "tools/ci_build/github/apple/default_full_ios_framework_build_settings.json" - template: steps/compliant-and-cleanup-step.yml diff --git a/.pipelines/stages/jobs/ios-cocoapods-packaging-job.yml b/.pipelines/stages/jobs/ios-cocoapods-packaging-job.yml deleted file mode 100644 index 64b493549..000000000 --- a/.pipelines/stages/jobs/ios-cocoapods-packaging-job.yml +++ /dev/null @@ -1,17 +0,0 @@ -parameters: -- name: build_config - type: string - default: 'release' - -jobs: -- job: ios_cocoapods_packaging - pool: - vmImage: 'macOS-latest' - - workspace: - clean: all - steps: - - template: steps/capi-appleframework-step.yml - parameters: - build_config: ${{ parameters.build_config }} - framework_config_file: "tools/ci_build/github/apple/default_full_apple_framework_build_settings.json" diff --git a/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml b/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml new file mode 100644 index 000000000..1e32247a6 --- /dev/null +++ b/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml @@ -0,0 +1,37 @@ +parameters: +- name: build_config + type: string + default: 'release' + +jobs: +- job: macos_ios_cocoapods_packaging + pool: + vmImage: 'macOS-latest' + variables: + buildSettingsFile: "tools/ci_build/github/apple/default_full_apple_framework_build_settings.json" + + workspace: + clean: all + steps: + - checkout: self + clean: true + submodules: none + + - template: steps/utils/set-genai-version.yml + - template: steps/utils/set-cmake-build-type.yml + parameters: + build_config: ${{parameters.build_config}} + + + - task: CmdLine@2 + displayName: Build MacOS & iOS CocoaPods Packages + inputs: + script: | + set -e -x + python tools/ci_build/github/apple/build_and_assemble_apple_pods.py \ + --build-dir "$(Build.BinariesDirectory)/apple_framework" \ + --staging-dir "$(Build.BinariesDirectory)/staging" \ + --pod-version "$(genai_version)" \ + --test \ + --variant Full \ + --build-settings-file "${{ variables.buildSettingsFile }}" \ diff --git a/.pipelines/stages/jobs/steps/capi-appleframework-step.yml b/.pipelines/stages/jobs/steps/capi-appleframework-step.yml index 130a8d2e8..62a356173 100644 --- a/.pipelines/stages/jobs/steps/capi-appleframework-step.yml +++ b/.pipelines/stages/jobs/steps/capi-appleframework-step.yml @@ -2,7 +2,7 @@ parameters: - name: build_config type: string default: 'release' -- name: framework_config_file +- name: build_settings_file type: string steps: @@ -23,7 +23,7 @@ steps: python3 -m pip install requests python3 tools/ci_build/github/apple/build_apple_framework.py \ --build_dir "$(Build.BinariesDirectory)/apple_framework" \ - ${{ parameters.framework_config_file }} + ${{ parameters.build_settings_file }} --config $(cmake_build_type) mkdir $(Build.BinariesDirectory)/artifacts diff --git a/.pipelines/stages/ios-cocoapods-packaging-stage.yml b/.pipelines/stages/macos-ios-cocoapods-packaging-stage.yml similarity index 70% rename from .pipelines/stages/ios-cocoapods-packaging-stage.yml rename to .pipelines/stages/macos-ios-cocoapods-packaging-stage.yml index da72b6b83..0e3b27ac8 100644 --- a/.pipelines/stages/ios-cocoapods-packaging-stage.yml +++ b/.pipelines/stages/macos-ios-cocoapods-packaging-stage.yml @@ -7,9 +7,9 @@ parameters: default: 'Release' stages: -- stage: Build_iOS_CocoaPods_Archive +- stage: Build_MacOS_iOS_CocoaPods_Archive jobs: - - template: jobs/android-java-api-aar.yml + - template: jobs/macos-ios-cocoapods-packaging-job.yml parameters: ort_version: ${{ parameters.ort_version }} build_config: ${{ parameters.build_config }} diff --git a/build.py b/build.py index b2087a46f..62a463b5a 100644 --- a/build.py +++ b/build.py @@ -500,6 +500,7 @@ def update(args: argparse.Namespace, env: dict[str, str]): "-DENABLE_PYTHON=OFF", "-DENABLE_TESTS=OFF", "-DENABLE_MODEL_BENCHMARK=OFF", + "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + args.apple_deploy_target, f"-DBUILD_APPLE_FRAMEWORK={'ON' if args.build_apple_framework else 'OFF'}", "-DPLATFORM_NAME=" + platform_name, ] diff --git a/onnxruntime-genai-c.podspec b/onnxruntime-genai-c.podspec deleted file mode 100644 index 15b7cb15f..000000000 --- a/onnxruntime-genai-c.podspec +++ /dev/null @@ -1,21 +0,0 @@ -Pod::Spec.new do |s| - s.name = "onnxruntime-genai-c" - s.version = "0.5.0" - s.summary = "SUMMARY" - s.homepage = "https://github.com/microsoft/onnxruntime-genai" - s.license = { :type => "MIT" } - s.author = {"ONNX Runtime" => "onnxruntime@microsoft.com"} - s.source = { :http => 'file:///Users/chester/Projects/onnxruntime-genai/build/apple_framework/framework_out/archive.zip'} - s.platform = :ios, '13.0', :osx, '11.0' - - s.vendored_frameworks = "onnxruntime-genai.xcframework" - s.static_framework = true - - s.source_files = [ - "Headers/*.h" - ] - - s.libraries = "c++" - s.dependency 'onnxruntime-c', '~> 1.19.0' -end - diff --git a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py new file mode 100755 index 000000000..71aeb9e7b --- /dev/null +++ b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import logging +import pathlib +import shutil +import sys +import tempfile + +from c.assemble_c_pod_package import assemble_c_pod_package +from objectivec.assemble_objc_pod_package import assemble_objc_pod_package +from package_assembly_utils import PackageVariant, get_ort_version + +SCRIPT_PATH = pathlib.Path(__file__).resolve() +SCRIPT_DIR = SCRIPT_PATH.parent +REPO_DIR = SCRIPT_PATH.parents[4] + + +logging.basicConfig(format="%(asctime)s %(name)s [%(levelname)s] - %(message)s", level=logging.DEBUG) +log = logging.getLogger(SCRIPT_PATH.stem) + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Builds an iOS framework and uses it to assemble iOS pod package files.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "--build-dir", + type=pathlib.Path, + default=REPO_DIR / "build" / "apple_framework", + help="The build directory. This will contain the iOS framework build output.", + ) + parser.add_argument( + "--staging-dir", + type=pathlib.Path, + default=REPO_DIR / "build" / "apple_pod_staging", + help="The staging directory. This will contain the iOS pod package files. " + "The pod package files do not have dependencies on files in the build directory.", + ) + + parser.add_argument( + "--pod-version", + default=f"{get_ort_version()}-local", + help="The version string of the pod. The same version is used for all pods.", + ) + + parser.add_argument( + "--variant", + choices=PackageVariant.release_variant_names(), + default=PackageVariant.Full.name, + help="Pod package variant.", + ) + + parser.add_argument("--test", action="store_true", help="Run tests on the framework and pod package files.") + parser.add_argument( + "--skip-build", + action="store_true", + help="Use build from previous run. Useful to debug test issues or packaging changes.", + ) + + build_framework_group = parser.add_argument_group( + title="iOS framework build arguments", + description="See the corresponding arguments in build_apple_framework.py for details.", + ) + + build_framework_group.add_argument("--include-ops-by-config") + build_framework_group.add_argument( + "--build-settings-file", required=True, help="The positional argument of build_apple_framework.py." + ) + build_framework_group.add_argument( + "-b", + "--build-apple-framework-arg", + action="append", + dest="build_apple_framework_extra_args", + default=[], + help="Pass an argument through to build_apple_framework.py. This may be specified multiple times.", + ) + + args = parser.parse_args() + + return args + + +def run(arg_list, cwd=None): + import os + import shlex + import subprocess + + log.info("Running subprocess in '%s'\n %s", cwd or os.getcwd(), " ".join([shlex.quote(arg) for arg in arg_list])) + + return subprocess.run(arg_list, check=True, cwd=cwd) + + +def main(): + args = parse_args() + + build_dir = args.build_dir.resolve() + staging_dir = args.staging_dir.resolve() + + # build framework + package_variant = PackageVariant[args.variant] + framework_info_file = build_dir / "xcframework_info.json" + + log.info("Building Apple framework.") + + build_apple_framework_args = [ + sys.executable, + str(SCRIPT_DIR / "build_apple_framework.py"), + *args.build_apple_framework_extra_args, + ] + + if args.include_ops_by_config is not None: + build_apple_framework_args += ["--include_ops_by_config", args.include_ops_by_config] + + build_apple_framework_args += ["--build_dir", str(build_dir), args.build_settings_file] + + if not args.skip_build: + run(build_apple_framework_args) + + if args.test: + test_apple_packages_args = [ + sys.executable, + str(SCRIPT_DIR / "test_apple_packages.py"), + "--fail_if_cocoapods_missing", + "--framework_info_file", + str(framework_info_file), + "--c_framework_dir", + str(build_dir / "framework_out"), + "--variant", + package_variant.name, + ] + + run(test_apple_packages_args) + + # assemble pods and then move them to their target locations (staging_dir/) + staging_dir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=staging_dir) as pod_assembly_dir_name: + pod_assembly_dir = pathlib.Path(pod_assembly_dir_name) + + log.info("Assembling C/C++ pod.") + + c_pod_staging_dir = pod_assembly_dir / "c_pod" + c_pod_name, c_pod_podspec = assemble_c_pod_package( + staging_dir=c_pod_staging_dir, + pod_version=args.pod_version, + framework_info_file=framework_info_file, + framework_dir=build_dir / "framework_out" / "onnxruntime.xcframework", + public_headers_dir=build_dir / "framework_out" / "Headers", + package_variant=package_variant, + ) + + if args.test: + test_c_pod_args = ["pod", "lib", "lint", "--verbose"] + + run(test_c_pod_args, cwd=c_pod_staging_dir) + + log.info("Assembling Objective-C pod.") + + objc_pod_staging_dir = pod_assembly_dir / "objc_pod" + objc_pod_name, objc_pod_podspec = assemble_objc_pod_package( + staging_dir=objc_pod_staging_dir, + pod_version=args.pod_version, + framework_info_file=framework_info_file, + package_variant=package_variant, + ) + + if args.test: + test_objc_pod_args = ["pod", "lib", "lint", "--verbose", f"--include-podspecs={c_pod_podspec}"] + + run(test_objc_pod_args, cwd=objc_pod_staging_dir) + + def move_dir(src, dst): + if dst.is_dir(): + shutil.rmtree(dst) + shutil.copytree(src, dst, symlinks=True) + shutil.rmtree(src) + + move_dir(c_pod_staging_dir, staging_dir / c_pod_name) + move_dir(objc_pod_staging_dir, staging_dir / objc_pod_name) + + log.info(f"Successfully assembled iOS pods at '{staging_dir}'.") + + +if __name__ == "__main__": + main() diff --git a/tools/ci_build/github/apple/c/assemble_c_pod_package.py b/tools/ci_build/github/apple/c/assemble_c_pod_package.py new file mode 100644 index 000000000..a1bb6e4e9 --- /dev/null +++ b/tools/ci_build/github/apple/c/assemble_c_pod_package.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import pathlib +import shutil +import sys + +_script_dir = pathlib.Path(__file__).parent.resolve(strict=True) +sys.path.append(str(_script_dir.parent)) + + +from package_assembly_utils import ( # noqa: E402 + PackageVariant, + copy_repo_relative_to_dir, + gen_file_from_template, + get_podspec_values, + load_json_config, +) + + +def get_pod_config_file(package_variant: PackageVariant): + """ + Gets the pod configuration file path for the given package variant. + """ + if package_variant == PackageVariant.Full: + return _script_dir / "onnxruntime-genai-c.config.json" + else: + raise ValueError(f"Unhandled package variant: {package_variant}") + + +def assemble_c_pod_package( + staging_dir: pathlib.Path, + pod_version: str, + framework_info_file: pathlib.Path, + public_headers_dir: pathlib.Path, + framework_dir: pathlib.Path, + package_variant: PackageVariant, +): + """ + Assembles the files for the C/C++ pod package in a staging directory. + + :param staging_dir Path to the staging directory for the C/C++ pod files. + :param pod_version C/C++ pod version. + :param framework_info_file Path to the framework_info.json or xcframework_info.json file containing additional values for the podspec. + :param public_headers_dir Path to the public headers directory to include in the pod. + :param framework_dir Path to the onnxruntime framework directory to include in the pod. + :param package_variant The pod package variant. + :return Tuple of (package name, path to the podspec file). + """ + staging_dir = staging_dir.resolve() + framework_info_file = framework_info_file.resolve(strict=True) + public_headers_dir = public_headers_dir.resolve(strict=True) + framework_dir = framework_dir.resolve(strict=True) + + framework_info = load_json_config(framework_info_file) + pod_config = load_json_config(get_pod_config_file(package_variant)) + + pod_name = pod_config["name"] + + print(f"Assembling files in staging directory: {staging_dir}") + if staging_dir.exists(): + print("Warning: staging directory already exists", file=sys.stderr) + + # copy the necessary files to the staging directory + shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True, symlinks=True) + shutil.copytree(public_headers_dir, staging_dir / public_headers_dir.name, dirs_exist_ok=True, symlinks=True) + copy_repo_relative_to_dir(["LICENSE"], staging_dir) + + (ios_deployment_target, macos_deployment_target, weak_framework) = get_podspec_values(framework_info) + + # generate the podspec file from the template + variable_substitutions = { + "DESCRIPTION": pod_config["description"], + # By default, we build both "iphoneos" and "iphonesimulator" architectures, and the deployment target should be the same between these two. + "IOS_DEPLOYMENT_TARGET": ios_deployment_target, + "MACOSX_DEPLOYMENT_TARGET": macos_deployment_target, + "LICENSE_FILE": "LICENSE", + "NAME": pod_name, + "ORT_C_FRAMEWORK": framework_dir.name, + "ORT_C_HEADERS_DIR": public_headers_dir.name, + "SUMMARY": pod_config["summary"], + "VERSION": pod_version, + "WEAK_FRAMEWORK": weak_framework, + } + + podspec_template = _script_dir / "c.podspec.template" + podspec = staging_dir / f"{pod_name}.podspec" + + gen_file_from_template(podspec_template, podspec, variable_substitutions) + + return pod_name, podspec + + +def parse_args(): + parser = argparse.ArgumentParser( + description=""" + Assembles the files for the C/C++ pod package in a staging directory. + This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release. + """ + ) + + parser.add_argument( + "--staging-dir", + type=pathlib.Path, + default=pathlib.Path("./c-staging"), + help="Path to the staging directory for the C/C++ pod files.", + ) + parser.add_argument("--pod-version", required=True, help="C/C++ pod version.") + parser.add_argument( + "--framework-info-file", + type=pathlib.Path, + required=True, + help="Path to the framework_info.json or xcframework_info.json file containing additional values for the podspec. " + "This file should be generated by CMake in the build directory.", + ) + parser.add_argument( + "--public-headers-dir", + type=pathlib.Path, + required=True, + help="Path to the public headers directory to include in the pod.", + ) + parser.add_argument( + "--framework-dir", + type=pathlib.Path, + required=True, + help="Path to the onnxruntime framework directory to include in the pod.", + ) + parser.add_argument( + "--variant", choices=PackageVariant.all_variant_names(), required=True, help="Pod package variant." + ) + + return parser.parse_args() + + +def main(): + args = parse_args() + + assemble_c_pod_package( + staging_dir=args.staging_dir, + pod_version=args.pod_version, + framework_info_file=args.framework_info_file, + public_headers_dir=args.public_headers_dir, + framework_dir=args.framework_dir, + package_variant=PackageVariant[args.variant], + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci_build/github/apple/c/c.podspec.template b/tools/ci_build/github/apple/c/c.podspec.template new file mode 100644 index 000000000..899a2a40e --- /dev/null +++ b/tools/ci_build/github/apple/c/c.podspec.template @@ -0,0 +1,27 @@ +Pod::Spec.new do |s| + s.name = "@NAME@" + s.version = "@VERSION@" + s.summary = "@SUMMARY@" + s.homepage = "https://github.com/microsoft/onnxruntime-genai" + s.license = { :type => "MIT", :file => "@LICENSE_FILE@" } + s.author = {"ONNX Runtime" => "onnxruntime@microsoft.com"} + s.source = { :http => 'file:///Users/chester/Projects/onnxruntime-genai/build/apple_framework/framework_out/archive.zip'} + + s.platform = :ios, "@IOS_DEPLOYMENT_TARGET@", :osx, "@MACOSX_DEPLOYMENT_TARGET@" + + s.vendored_frameworks = "@ORTGENAI_C_FRAMEWORK@" + s.static_framework = true + + s.source_files = [ + "Headers/*.h" + ] + s.preserve_paths = [ "@LICENSE_FILE@" ] + s.description = "@DESCRIPTION@" + s.libraries = "c++" + s.dependency 'onnxruntime-c', '~> @ORT_VERSION@' + + s.pod_target_xcconfig = { + "OTHER_CPLUSPLUSFLAGS" => "-fvisibility=hidden -fvisibility-inlines-hidden", + } +end + diff --git a/tools/ci_build/github/apple/c/onnxruntime-genai-c.config.json b/tools/ci_build/github/apple/c/onnxruntime-genai-c.config.json new file mode 100644 index 000000000..d3eda57c1 --- /dev/null +++ b/tools/ci_build/github/apple/c/onnxruntime-genai-c.config.json @@ -0,0 +1,5 @@ +{ + "name": "onnxruntime-c", + "summary": "ONNX Runtime C/C++ Pod", + "description": "A pod for the ONNX Runtime C/C++ library." +} diff --git a/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py new file mode 100755 index 000000000..d5cd1f71c --- /dev/null +++ b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import pathlib +import sys + +_script_dir = pathlib.Path(__file__).parent.resolve(strict=True) +sys.path.append(str(_script_dir.parent)) + + +from c.assemble_c_pod_package import get_pod_config_file as get_c_pod_config_file # noqa: E402 +from package_assembly_utils import ( # noqa: E402 + PackageVariant, + copy_repo_relative_to_dir, + filter_files, + gen_file_from_template, + get_podspec_values, + load_json_config, +) + +# these variables contain paths or path patterns that are relative to the repo root + +# the license file +license_file = "LICENSE" + +# include directories for compiling the pod itself +include_dirs = [ + "objectivec", +] + +all_objc_files = { + "source_files": [ + "objectivec/include/*.h", + "objectivec/*.h", + "objectivec/*.m", + "objectivec/*.mm", + ], + "public_header_files": [ + "objectivec/include/*.h", + ], + "test_source_files": [ + "objectivec/test/*.h", + "objectivec/test/*.m", + "objectivec/test/*.mm", + ] +} + +def get_pod_files(package_variant: PackageVariant): + """ + Gets the source and header files for the given package variant. + """ + filtered_pod_files = {} + for key in all_objc_files: + filtered_pod_files[key] = filter_files(all_objc_files[key]) + return filtered_pod_files + + +def get_pod_config_file(package_variant: PackageVariant): + """ + Gets the pod configuration file path for the given package variant. + """ + if package_variant == PackageVariant.Full: + return _script_dir / "onnxruntime-objc.config.json" + else: + raise ValueError(f"Unhandled package variant: {package_variant}") + + +def assemble_objc_pod_package( + staging_dir: pathlib.Path, pod_version: str, framework_info_file: pathlib.Path, package_variant: PackageVariant +): + """ + Assembles the files for the Objective-C pod package in a staging directory. + + :param staging_dir Path to the staging directory for the Objective-C pod files. + :param pod_version Objective-C pod version. + :param framework_info_file Path to the framework_info.json or xcframework_info.json file containing additional values for the podspec. + :param package_variant The pod package variant. + :return Tuple of (package name, path to the podspec file). + """ + staging_dir = staging_dir.resolve() + framework_info_file = framework_info_file.resolve(strict=True) + + framework_info = load_json_config(framework_info_file) + pod_config = load_json_config(get_pod_config_file(package_variant)) + c_pod_config = load_json_config(get_c_pod_config_file(package_variant)) + + pod_name = pod_config["name"] + + print(f"Assembling files in staging directory: {staging_dir}") + if staging_dir.exists(): + print("Warning: staging directory already exists", file=sys.stderr) + + pod_files = get_pod_files(package_variant) + + # copy the necessary files to the staging directory + copy_repo_relative_to_dir( + [license_file, *pod_files["source_files"], *pod_files["test_source_files"], *pod_files["test_resource_files"]], + staging_dir, + ) + + # generate the podspec file from the template + + def path_patterns_as_variable_value(patterns: list[str]): + return ", ".join([f'"{pattern}"' for pattern in patterns]) + + (ios_deployment_target, macos_deployment_target, _) = get_podspec_values(framework_info) + + variable_substitutions = { + "C_POD_NAME": c_pod_config["name"], + "DESCRIPTION": pod_config["description"], + "INCLUDE_DIR_LIST": path_patterns_as_variable_value(include_dirs), + "IOS_DEPLOYMENT_TARGET": ios_deployment_target, + "MACOSX_DEPLOYMENT_TARGET": macos_deployment_target, + "LICENSE_FILE": license_file, + "NAME": pod_name, + "PUBLIC_HEADER_FILE_LIST": path_patterns_as_variable_value(pod_files["public_header_files"]), + "SOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["source_files"]), + "SUMMARY": pod_config["summary"], + "TEST_RESOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_resource_files"]), + "TEST_SOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_source_files"]), + "VERSION": pod_version, + } + + podspec_template = _script_dir / "objc.podspec.template" + podspec = staging_dir / f"{pod_name}.podspec" + + gen_file_from_template(podspec_template, podspec, variable_substitutions) + + return pod_name, podspec + + +def parse_args(): + parser = argparse.ArgumentParser( + description=""" + Assembles the files for the Objective-C pod package in a staging directory. + This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release. + """ + ) + + parser.add_argument( + "--staging-dir", + type=pathlib.Path, + default=pathlib.Path("./objc-staging"), + help="Path to the staging directory for the Objective-C pod files.", + ) + parser.add_argument("--pod-version", required=True, help="Objective-C pod version.") + parser.add_argument( + "--framework-info-file", + type=pathlib.Path, + required=True, + help="Path to the framework_info.json or xcframework_info.json file containing additional values for the podspec. " + "This file should be generated by CMake in the build directory.", + ) + parser.add_argument( + "--variant", choices=PackageVariant.release_variant_names(), required=True, help="Pod package variant." + ) + + return parser.parse_args() + + +def main(): + args = parse_args() + + assemble_objc_pod_package( + staging_dir=args.staging_dir, + pod_version=args.pod_version, + framework_info_file=args.framework_info_file, + package_variant=PackageVariant[args.variant], + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci_build/github/apple/objectivec/objc.podspec.template b/tools/ci_build/github/apple/objectivec/objc.podspec.template new file mode 100644 index 000000000..9339290d3 --- /dev/null +++ b/tools/ci_build/github/apple/objectivec/objc.podspec.template @@ -0,0 +1,43 @@ +Pod::Spec.new do |s| + s.name = "@NAME@" + s.version = "@VERSION@" + s.summary = "@SUMMARY@" + s.homepage = "https://github.com/microsoft/onnxruntime-genai" + s.license = { :type => "MIT", :file => "@LICENSE_FILE@" } + s.author = {"ONNX Runtime" => "onnxruntime@microsoft.com"} + s.source = { :http => 'file:///Users/chester/Projects/onnxruntime-genai/src/archive.zip'} + s.platform = :ios, "@IOS_DEPLOYMENT_TARGET@", :osx, "@MACOSX_DEPLOYMENT_TARGET@" + + s.default_subspec = 'Core' + s.static_framework = true + + s.subspec 'Core' do |core| + core.dependency "@C_POD_NAME@", "#{s.version}" + core.requires_arc = true + core.compiler_flags = "-std=c++17", "-fobjc-arc-exceptions", "-Wall", "-Wextra", "-Werror" + core.pod_target_xcconfig = { + "HEADER_SEARCH_PATHS" => include_dirs.join(" "), + "OTHER_CPLUSPLUSFLAGS" => "-fvisibility=hidden -fvisibility-inlines-hidden", + } + + core.public_header_files = [ + @PUBLIC_HEADER_FILE_LIST@ + ] + + core.source_files = [ + @SOURCE_FILE_LIST@ + ] + + core.test_spec "Tests" do |test| + test.source_files = [ + @TEST_SOURCE_FILE_LIST@ + ] + + test.resources = [ + @TEST_RESOURCE_FILE_LIST@ + ] + end + end + s.dependency 'onnxruntime-objc', '~> @ORT_VERSION@' +end + diff --git a/tools/ci_build/github/apple/package_assembly_utils.py b/tools/ci_build/github/apple/package_assembly_utils.py new file mode 100644 index 000000000..8f1b8d735 --- /dev/null +++ b/tools/ci_build/github/apple/package_assembly_utils.py @@ -0,0 +1,161 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import enum +import json +import os +import pathlib +import re +import shutil +from typing import Dict, List + +_script_dir = pathlib.Path(__file__).parent.resolve(strict=True) +repo_root = _script_dir.parents[3] + + +class PackageVariant(enum.Enum): + Full = 0 + + @classmethod + def release_variant_names(cls): + return [v.name for v in cls if v.value >= 0] + + +_template_variable_pattern = re.compile(r"@(\w+)@") # match "@var@" + + +def gen_file_from_template( + template_file: pathlib.Path, output_file: pathlib.Path, variable_substitutions: Dict[str, str], strict: bool = True +): + """ + Generates a file from a template file. + The template file may contain template variables that will be substituted + with the provided values in the generated output file. + In the template file, template variable names are delimited by "@"'s, + e.g., "@var@". + + :param template_file The template file path. + :param output_file The generated output file path. + :param variable_substitutions The mapping from template variable name to value. + :param strict Whether to require the set of template variable names in the file and the keys of + `variable_substitutions` to be equal. + """ + with open(template_file) as template: + content = template.read() + + variables_in_file = set() + + def replace_template_variable(match): + variable_name = match.group(1) + variables_in_file.add(variable_name) + return variable_substitutions.get(variable_name, match.group(0)) + + content = _template_variable_pattern.sub(replace_template_variable, content) + + if strict and variables_in_file != variable_substitutions.keys(): + variables_in_substitutions = set(variable_substitutions.keys()) + raise ValueError( + f"Template file variables and substitution variables do not match. " + f"Only in template file: {sorted(variables_in_file - variables_in_substitutions)}. " + f"Only in substitutions: {sorted(variables_in_substitutions - variables_in_file)}." + ) + + with open(output_file, mode="w") as output: + output.write(content) + + +def filter_files(all_file_patterns: List[str], excluded_file_patterns: List[str]): + """ + Filters file paths based on inclusion and exclusion patterns + + :param all_file_patterns The list of file paths to filter. + :param excluded_file_patterns The list of exclusion patterns. + + :return The filtered list of file paths + """ + # get all files matching the patterns in all_file_patterns + all_files = [str(path.relative_to(repo_root)) for pattern in all_file_patterns for path in repo_root.glob(pattern)] + + # get all files matching the patterns in excluded_file_patterns + exclude_files = [ + str(path.relative_to(repo_root)) for pattern in excluded_file_patterns for path in repo_root.glob(pattern) + ] + + # return the difference + return list(set(all_files) - set(exclude_files)) + + +def copy_repo_relative_to_dir(patterns: List[str], dest_dir: pathlib.Path): + """ + Copies file paths relative to the repo root to a directory. + The given paths or path patterns are relative to the repo root, and the + repo root-relative intermediate directory structure is maintained. + + :param patterns The paths or path patterns relative to the repo root. + :param dest_dir The destination directory. + """ + paths = [path for pattern in patterns for path in repo_root.glob(pattern)] + for path in paths: + repo_relative_path = path.relative_to(repo_root) + dst_path = dest_dir / repo_relative_path + os.makedirs(dst_path.parent, exist_ok=True) + shutil.copy(path, dst_path) + + +def load_json_config(json_config_file: pathlib.Path): + """ + Loads configuration info from a JSON file. + + :param json_config_file The JSON configuration file path. + :return The configuration info values. + """ + with open(json_config_file) as config: + return json.load(config) + + +def get_podspec_values(framework_info): + """ + Get the podspec deployement targets and weak framework info from the dictionary that load_json_config returned. + Looks for iphonesimulator, iphoneos and macos settings. + Handles missing platforms and checks consistency. + Returns empty string for deployment target if that platofrm is not enabled. + + :return (ios_deployment_target, macos_deployment_target, weak_framework) + """ + ios_deployment_target = "" + macos_deployment_target = "" + weak_framework = "" # should be the same for all platforms + # get info, allowing for a subset of platforms to be specified + for framework in ("iphonesimulator", "iphoneos", "macosx"): + if framework not in framework_info: + continue + + target = framework_info[framework]["APPLE_DEPLOYMENT_TARGET"] + weak = framework_info[framework]["WEAK_FRAMEWORK"] + + if not weak_framework: + weak_framework = weak + else: + # should be consistent + assert weak == weak_framework + + if framework == "macosx": + macos_deployment_target = target + else: + if not ios_deployment_target: + ios_deployment_target = target + else: + # should be consistent + assert ios_deployment_target == target + + return (ios_deployment_target, macos_deployment_target, weak_framework) + + +def get_ort_version(): + """ + Gets the version string from the repo. + + :return The version string. + """ + with open(repo_root / "VERSION_INFO") as version_file: + return version_file.read().strip() From bb448541d7d73f01b3679c025891ae04e5edd459 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 15 Oct 2024 14:23:38 +0800 Subject: [PATCH 23/59] [skip ci] assemble pod --- onnxruntime-genai-objc.podspec | 40 ------------------- .../apple/build_and_assemble_apple_pods.py | 8 +++- .../github/apple/c/assemble_c_pod_package.py | 10 ++++- .../github/apple/c/c.podspec.template | 8 ++-- .../apple/c/onnxruntime-genai-c.config.json | 6 +-- .../objectivec/assemble_objc_pod_package.py | 21 +++++----- .../apple/objectivec/objc.podspec.template | 16 +++++--- .../onnxruntime-genai-objc.config.json | 5 +++ 8 files changed, 49 insertions(+), 65 deletions(-) delete mode 100644 onnxruntime-genai-objc.podspec create mode 100644 tools/ci_build/github/apple/objectivec/onnxruntime-genai-objc.config.json diff --git a/onnxruntime-genai-objc.podspec b/onnxruntime-genai-objc.podspec deleted file mode 100644 index 3e3b15608..000000000 --- a/onnxruntime-genai-objc.podspec +++ /dev/null @@ -1,40 +0,0 @@ -Pod::Spec.new do |s| - s.name = "onnxruntime-genai-objc" - s.version = "0.5.0" - s.summary = "SUMMARY" - s.homepage = "https://github.com/microsoft/onnxruntime-genai" - s.license = { :type => "MIT" } - s.author = {"ONNX Runtime" => "onnxruntime@microsoft.com"} - s.source = { :http => 'file:///Users/chester/Projects/onnxruntime-genai/src/archive.zip'} - s.platform = :ios, '13.0', :osx, '11.0' - - s.default_subspec = 'Core' - s.static_framework = true - - s.subspec 'Core' do |sp| - sp.requires_arc = true - sp.public_header_files = [ - "objectivec/include/ort_genai_objc.h" - ] - sp.source_files = [ - "objectivec/include/ort_genai_objc.h", - "objectivec/cxx_api.h", - "objectivec/error_utils.h", - "objectivec/error_utils.mm", - "objectivec/oga_internal.h", - "objectivec/oga_internal.mm", - "objectivec/oga_model.mm", - "objectivec/oga_tokenizer.mm", - "objectivec/oga_sequences.mm", - "objectivec/oga_generator.mm", - "objectivec/oga_generator_params.mm", - "objectivec/oga_multi_modal_processor.mm", - "objectivec/oga_named_tensors.mm", - "objectivec/oga_tensor.mm", - "objectivec/oga_images.mm", - ] - - end - s.dependency 'onnxruntime-objc', '~> 1.19.0' -end - diff --git a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py index 71aeb9e7b..795076d0d 100755 --- a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py +++ b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py @@ -81,6 +81,10 @@ def parse_args(): help="Pass an argument through to build_apple_framework.py. This may be specified multiple times.", ) + parser.add_argument( + "--ort_version", required=True, help="The ORT version to depend on." + ) + args = parser.parse_args() return args @@ -149,9 +153,10 @@ def main(): staging_dir=c_pod_staging_dir, pod_version=args.pod_version, framework_info_file=framework_info_file, - framework_dir=build_dir / "framework_out" / "onnxruntime.xcframework", + framework_dir=build_dir / "framework_out" / "onnxruntime-genai.xcframework", public_headers_dir=build_dir / "framework_out" / "Headers", package_variant=package_variant, + ort_version=args.ort_version ) if args.test: @@ -167,6 +172,7 @@ def main(): pod_version=args.pod_version, framework_info_file=framework_info_file, package_variant=package_variant, + ort_version=args.ort_version ) if args.test: diff --git a/tools/ci_build/github/apple/c/assemble_c_pod_package.py b/tools/ci_build/github/apple/c/assemble_c_pod_package.py index a1bb6e4e9..a94fb6195 100644 --- a/tools/ci_build/github/apple/c/assemble_c_pod_package.py +++ b/tools/ci_build/github/apple/c/assemble_c_pod_package.py @@ -38,6 +38,7 @@ def assemble_c_pod_package( public_headers_dir: pathlib.Path, framework_dir: pathlib.Path, package_variant: PackageVariant, + ort_version: str ): """ Assembles the files for the C/C++ pod package in a staging directory. @@ -79,10 +80,11 @@ def assemble_c_pod_package( "MACOSX_DEPLOYMENT_TARGET": macos_deployment_target, "LICENSE_FILE": "LICENSE", "NAME": pod_name, - "ORT_C_FRAMEWORK": framework_dir.name, - "ORT_C_HEADERS_DIR": public_headers_dir.name, + "ORTGENAI_C_FRAMEWORK": framework_dir.name, + "ORTGENAI_C_HEADERS_DIR": public_headers_dir.name, "SUMMARY": pod_config["summary"], "VERSION": pod_version, + "ORT_VERSION": ort_version, "WEAK_FRAMEWORK": weak_framework, } @@ -132,6 +134,9 @@ def parse_args(): "--variant", choices=PackageVariant.all_variant_names(), required=True, help="Pod package variant." ) + parser.add_argument( + "--ort_version", required=True, help="The ORT version to depend on." + ) return parser.parse_args() @@ -145,6 +150,7 @@ def main(): public_headers_dir=args.public_headers_dir, framework_dir=args.framework_dir, package_variant=PackageVariant[args.variant], + ort_version=args.ort_version ) return 0 diff --git a/tools/ci_build/github/apple/c/c.podspec.template b/tools/ci_build/github/apple/c/c.podspec.template index 899a2a40e..0000b5014 100644 --- a/tools/ci_build/github/apple/c/c.podspec.template +++ b/tools/ci_build/github/apple/c/c.podspec.template @@ -11,10 +11,10 @@ Pod::Spec.new do |s| s.vendored_frameworks = "@ORTGENAI_C_FRAMEWORK@" s.static_framework = true + s.weak_framework = [ @WEAK_FRAMEWORK@ ] + + s.source_files = "@ORTGENAI_C_HEADERS_DIR@/*.h" - s.source_files = [ - "Headers/*.h" - ] s.preserve_paths = [ "@LICENSE_FILE@" ] s.description = "@DESCRIPTION@" s.libraries = "c++" @@ -22,6 +22,6 @@ Pod::Spec.new do |s| s.pod_target_xcconfig = { "OTHER_CPLUSPLUSFLAGS" => "-fvisibility=hidden -fvisibility-inlines-hidden", - } + } end diff --git a/tools/ci_build/github/apple/c/onnxruntime-genai-c.config.json b/tools/ci_build/github/apple/c/onnxruntime-genai-c.config.json index d3eda57c1..5129808f6 100644 --- a/tools/ci_build/github/apple/c/onnxruntime-genai-c.config.json +++ b/tools/ci_build/github/apple/c/onnxruntime-genai-c.config.json @@ -1,5 +1,5 @@ { - "name": "onnxruntime-c", - "summary": "ONNX Runtime C/C++ Pod", - "description": "A pod for the ONNX Runtime C/C++ library." + "name": "onnxruntime-genai-c", + "summary": "ONNX Runtime GenAI C/C++ Pod", + "description": "A pod for the ONNX Runtime GenAI C/C++ library." } diff --git a/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py index d5cd1f71c..895f2f6ee 100755 --- a/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py +++ b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py @@ -54,7 +54,7 @@ def get_pod_files(package_variant: PackageVariant): """ filtered_pod_files = {} for key in all_objc_files: - filtered_pod_files[key] = filter_files(all_objc_files[key]) + filtered_pod_files[key] = filter_files(all_objc_files[key], []) return filtered_pod_files @@ -63,13 +63,13 @@ def get_pod_config_file(package_variant: PackageVariant): Gets the pod configuration file path for the given package variant. """ if package_variant == PackageVariant.Full: - return _script_dir / "onnxruntime-objc.config.json" + return _script_dir / "onnxruntime-genai-objc.config.json" else: raise ValueError(f"Unhandled package variant: {package_variant}") def assemble_objc_pod_package( - staging_dir: pathlib.Path, pod_version: str, framework_info_file: pathlib.Path, package_variant: PackageVariant + staging_dir: pathlib.Path, pod_version: str, framework_info_file: pathlib.Path, package_variant: PackageVariant, ort_version: str ): """ Assembles the files for the Objective-C pod package in a staging directory. @@ -96,10 +96,10 @@ def assemble_objc_pod_package( pod_files = get_pod_files(package_variant) # copy the necessary files to the staging directory - copy_repo_relative_to_dir( - [license_file, *pod_files["source_files"], *pod_files["test_source_files"], *pod_files["test_resource_files"]], - staging_dir, - ) + need_copy = [license_file, *pod_files["source_files"], *pod_files["test_source_files"]] + if "test_resource_files" in pod_files: + need_copy.append(*pod_files["test_resource_files"]) + copy_repo_relative_to_dir(need_copy, staging_dir,) # generate the podspec file from the template @@ -119,9 +119,10 @@ def path_patterns_as_variable_value(patterns: list[str]): "PUBLIC_HEADER_FILE_LIST": path_patterns_as_variable_value(pod_files["public_header_files"]), "SOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["source_files"]), "SUMMARY": pod_config["summary"], - "TEST_RESOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_resource_files"]), + # "TEST_RESOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_resource_files"]), "TEST_SOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_source_files"]), "VERSION": pod_version, + "ORT_VERSION": ort_version } podspec_template = _script_dir / "objc.podspec.template" @@ -157,7 +158,9 @@ def parse_args(): parser.add_argument( "--variant", choices=PackageVariant.release_variant_names(), required=True, help="Pod package variant." ) - + parser.add_argument( + "--ort_version", required=True, help="The ORT version to depend on." + ) return parser.parse_args() diff --git a/tools/ci_build/github/apple/objectivec/objc.podspec.template b/tools/ci_build/github/apple/objectivec/objc.podspec.template index 9339290d3..04b6e7661 100644 --- a/tools/ci_build/github/apple/objectivec/objc.podspec.template +++ b/tools/ci_build/github/apple/objectivec/objc.podspec.template @@ -1,7 +1,9 @@ Pod::Spec.new do |s| - s.name = "@NAME@" + s.name = "@NAME@" s.version = "@VERSION@" - s.summary = "@SUMMARY@" + s.summary = "@SUMMARY@ + s.description = "@DESCRIPTION@" + s.homepage = "https://github.com/microsoft/onnxruntime-genai" s.license = { :type => "MIT", :file => "@LICENSE_FILE@" } s.author = {"ONNX Runtime" => "onnxruntime@microsoft.com"} @@ -20,6 +22,12 @@ Pod::Spec.new do |s| "OTHER_CPLUSPLUSFLAGS" => "-fvisibility=hidden -fvisibility-inlines-hidden", } + include_dirs = [ + @INCLUDE_DIR_LIST@ + ].map { |relative_include_dir| + '"${PODS_TARGET_SRCROOT}/' + relative_include_dir + '"' + } + core.public_header_files = [ @PUBLIC_HEADER_FILE_LIST@ ] @@ -32,10 +40,6 @@ Pod::Spec.new do |s| test.source_files = [ @TEST_SOURCE_FILE_LIST@ ] - - test.resources = [ - @TEST_RESOURCE_FILE_LIST@ - ] end end s.dependency 'onnxruntime-objc', '~> @ORT_VERSION@' diff --git a/tools/ci_build/github/apple/objectivec/onnxruntime-genai-objc.config.json b/tools/ci_build/github/apple/objectivec/onnxruntime-genai-objc.config.json new file mode 100644 index 000000000..017564db7 --- /dev/null +++ b/tools/ci_build/github/apple/objectivec/onnxruntime-genai-objc.config.json @@ -0,0 +1,5 @@ +{ + "name": "onnxruntime-genai-objc", + "summary": "ONNX Runtime GenAI Objective-C Pod", + "description": "A pod for the ONNX Runtime GenAI Objective-C API." +} From 65d824042b4c9c426ed5a55446d3bea2ceeb2207 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 15 Oct 2024 15:18:48 +0800 Subject: [PATCH 24/59] [skip ci] podspec templat --- tools/ci_build/github/apple/c/c.podspec.template | 4 ++-- .../github/apple/objectivec/objc.podspec.template | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tools/ci_build/github/apple/c/c.podspec.template b/tools/ci_build/github/apple/c/c.podspec.template index 0000b5014..c0780425d 100644 --- a/tools/ci_build/github/apple/c/c.podspec.template +++ b/tools/ci_build/github/apple/c/c.podspec.template @@ -1,6 +1,6 @@ Pod::Spec.new do |s| - s.name = "@NAME@" - s.version = "@VERSION@" + s.name = "@NAME@" + s.version = "@VERSION@" s.summary = "@SUMMARY@" s.homepage = "https://github.com/microsoft/onnxruntime-genai" s.license = { :type => "MIT", :file => "@LICENSE_FILE@" } diff --git a/tools/ci_build/github/apple/objectivec/objc.podspec.template b/tools/ci_build/github/apple/objectivec/objc.podspec.template index 04b6e7661..19f0337da 100644 --- a/tools/ci_build/github/apple/objectivec/objc.podspec.template +++ b/tools/ci_build/github/apple/objectivec/objc.podspec.template @@ -1,7 +1,7 @@ Pod::Spec.new do |s| - s.name = "@NAME@" - s.version = "@VERSION@" - s.summary = "@SUMMARY@ + s.name = "@NAME@" + s.version = "@VERSION@" + s.summary = "@SUMMARY@" s.description = "@DESCRIPTION@" s.homepage = "https://github.com/microsoft/onnxruntime-genai" @@ -17,10 +17,6 @@ Pod::Spec.new do |s| core.dependency "@C_POD_NAME@", "#{s.version}" core.requires_arc = true core.compiler_flags = "-std=c++17", "-fobjc-arc-exceptions", "-Wall", "-Wextra", "-Werror" - core.pod_target_xcconfig = { - "HEADER_SEARCH_PATHS" => include_dirs.join(" "), - "OTHER_CPLUSPLUSFLAGS" => "-fvisibility=hidden -fvisibility-inlines-hidden", - } include_dirs = [ @INCLUDE_DIR_LIST@ @@ -41,6 +37,11 @@ Pod::Spec.new do |s| @TEST_SOURCE_FILE_LIST@ ] end + + core.pod_target_xcconfig = { + "HEADER_SEARCH_PATHS" => include_dirs.join(" "), + "OTHER_CPLUSPLUSFLAGS" => "-fvisibility=hidden -fvisibility-inlines-hidden", + } end s.dependency 'onnxruntime-objc', '~> @ORT_VERSION@' end From d17fa471987674eacb2bb6b58b8ed9ed92ade72b Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 15 Oct 2024 15:45:46 +0800 Subject: [PATCH 25/59] [skip ci] fix pod file copying --- .../github/apple/c/assemble_c_pod_package.py | 6 ++++-- .../objectivec/assemble_objc_pod_package.py | 8 ++++--- .../github/apple/package_assembly_utils.py | 21 +++++++++++++------ 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/tools/ci_build/github/apple/c/assemble_c_pod_package.py b/tools/ci_build/github/apple/c/assemble_c_pod_package.py index a94fb6195..0186f5729 100644 --- a/tools/ci_build/github/apple/c/assemble_c_pod_package.py +++ b/tools/ci_build/github/apple/c/assemble_c_pod_package.py @@ -11,6 +11,8 @@ _script_dir = pathlib.Path(__file__).parent.resolve(strict=True) sys.path.append(str(_script_dir.parent)) +# the license file +license_file = "LICENSE" from package_assembly_utils import ( # noqa: E402 PackageVariant, @@ -68,7 +70,7 @@ def assemble_c_pod_package( # copy the necessary files to the staging directory shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True, symlinks=True) shutil.copytree(public_headers_dir, staging_dir / public_headers_dir.name, dirs_exist_ok=True, symlinks=True) - copy_repo_relative_to_dir(["LICENSE"], staging_dir) + copy_repo_relative_to_dir(None, [license_file], staging_dir) (ios_deployment_target, macos_deployment_target, weak_framework) = get_podspec_values(framework_info) @@ -78,7 +80,7 @@ def assemble_c_pod_package( # By default, we build both "iphoneos" and "iphonesimulator" architectures, and the deployment target should be the same between these two. "IOS_DEPLOYMENT_TARGET": ios_deployment_target, "MACOSX_DEPLOYMENT_TARGET": macos_deployment_target, - "LICENSE_FILE": "LICENSE", + "LICENSE_FILE": license_file, "NAME": pod_name, "ORTGENAI_C_FRAMEWORK": framework_dir.name, "ORTGENAI_C_HEADERS_DIR": public_headers_dir.name, diff --git a/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py index 895f2f6ee..917f29944 100755 --- a/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py +++ b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py @@ -54,7 +54,7 @@ def get_pod_files(package_variant: PackageVariant): """ filtered_pod_files = {} for key in all_objc_files: - filtered_pod_files[key] = filter_files(all_objc_files[key], []) + filtered_pod_files[key] = filter_files("src", all_objc_files[key], []) return filtered_pod_files @@ -95,11 +95,13 @@ def assemble_objc_pod_package( pod_files = get_pod_files(package_variant) + copy_repo_relative_to_dir(None, [license_file], staging_dir) + # copy the necessary files to the staging directory - need_copy = [license_file, *pod_files["source_files"], *pod_files["test_source_files"]] + need_copy = [*pod_files["source_files"], *pod_files["test_source_files"]] if "test_resource_files" in pod_files: need_copy.append(*pod_files["test_resource_files"]) - copy_repo_relative_to_dir(need_copy, staging_dir,) + copy_repo_relative_to_dir("src", need_copy, staging_dir) # generate the podspec file from the template diff --git a/tools/ci_build/github/apple/package_assembly_utils.py b/tools/ci_build/github/apple/package_assembly_utils.py index 8f1b8d735..23cfc0508 100644 --- a/tools/ci_build/github/apple/package_assembly_utils.py +++ b/tools/ci_build/github/apple/package_assembly_utils.py @@ -64,7 +64,7 @@ def replace_template_variable(match): output.write(content) -def filter_files(all_file_patterns: List[str], excluded_file_patterns: List[str]): +def filter_files(subpath: str, all_file_patterns: List[str], excluded_file_patterns: List[str]): """ Filters file paths based on inclusion and exclusion patterns @@ -74,18 +74,23 @@ def filter_files(all_file_patterns: List[str], excluded_file_patterns: List[str] :return The filtered list of file paths """ # get all files matching the patterns in all_file_patterns - all_files = [str(path.relative_to(repo_root)) for pattern in all_file_patterns for path in repo_root.glob(pattern)] + if subpath: + src_root = repo_root / subpath + else: + src_root = repo_root + + all_files = [str(path.relative_to(src_root)) for pattern in all_file_patterns for path in src_root.glob(pattern)] # get all files matching the patterns in excluded_file_patterns exclude_files = [ - str(path.relative_to(repo_root)) for pattern in excluded_file_patterns for path in repo_root.glob(pattern) + str(path.relative_to(src_root)) for pattern in excluded_file_patterns for path in src_root.glob(pattern) ] # return the difference return list(set(all_files) - set(exclude_files)) -def copy_repo_relative_to_dir(patterns: List[str], dest_dir: pathlib.Path): +def copy_repo_relative_to_dir(subpath: str, patterns: List[str], dest_dir: pathlib.Path): """ Copies file paths relative to the repo root to a directory. The given paths or path patterns are relative to the repo root, and the @@ -94,9 +99,13 @@ def copy_repo_relative_to_dir(patterns: List[str], dest_dir: pathlib.Path): :param patterns The paths or path patterns relative to the repo root. :param dest_dir The destination directory. """ - paths = [path for pattern in patterns for path in repo_root.glob(pattern)] + if subpath: + src_root = repo_root / subpath + else: + src_root = repo_root + paths = [path for pattern in patterns for path in src_root.glob(pattern)] for path in paths: - repo_relative_path = path.relative_to(repo_root) + repo_relative_path = path.relative_to(src_root) dst_path = dest_dir / repo_relative_path os.makedirs(dst_path.parent, exist_ok=True) shutil.copy(path, dst_path) From 5ff6fb97a3a2ef27f13de214eb5d581667aece15 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 15 Oct 2024 15:52:51 +0800 Subject: [PATCH 26/59] [skip ci] yml --- .pipelines/stages/jobs/steps/capi-appleframework-step.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/stages/jobs/steps/capi-appleframework-step.yml b/.pipelines/stages/jobs/steps/capi-appleframework-step.yml index 62a356173..0e947dcb7 100644 --- a/.pipelines/stages/jobs/steps/capi-appleframework-step.yml +++ b/.pipelines/stages/jobs/steps/capi-appleframework-step.yml @@ -12,7 +12,7 @@ steps: submodules: recursive - template: utils/set-genai-version.yml -- template: steps/utils/set-cmake-build-type.yml +- template: utils/set-cmake-build-type.yml parameters: build_config: ${{parameters.build_config}} From 8939e46cc2194937908967422905f3cc4166c801 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:00:24 +0800 Subject: [PATCH 27/59] [skip ci] fix pipeline --- .pipelines/stages/jobs/steps/capi-appleframework-step.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/stages/jobs/steps/capi-appleframework-step.yml b/.pipelines/stages/jobs/steps/capi-appleframework-step.yml index 0e947dcb7..902a3d3c7 100644 --- a/.pipelines/stages/jobs/steps/capi-appleframework-step.yml +++ b/.pipelines/stages/jobs/steps/capi-appleframework-step.yml @@ -23,8 +23,8 @@ steps: python3 -m pip install requests python3 tools/ci_build/github/apple/build_apple_framework.py \ --build_dir "$(Build.BinariesDirectory)/apple_framework" \ + --config $(cmake_build_type) \ ${{ parameters.build_settings_file }} - --config $(cmake_build_type) mkdir $(Build.BinariesDirectory)/artifacts mkdir -p $(Build.BinariesDirectory)/artifacts_staging/onnxruntime-genai.xcframework From 242dd2af37dc0cb06ecec8aaafe844f87b36e77b Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:50:50 +0800 Subject: [PATCH 28/59] nuget authenticate --- .pipelines/stages/jobs/nuget-packaging-job.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/stages/jobs/nuget-packaging-job.yml b/.pipelines/stages/jobs/nuget-packaging-job.yml index 1be6a9cd2..5d2a33b12 100644 --- a/.pipelines/stages/jobs/nuget-packaging-job.yml +++ b/.pipelines/stages/jobs/nuget-packaging-job.yml @@ -179,6 +179,8 @@ jobs: - template: steps/utils/set-genai-version.yml + - task: NuGetAuthenticate@1 + - powershell: | dotnet --info dotnet build Microsoft.ML.OnnxRuntimeGenAI.csproj -p:Configuration="$(buildConfig)" -p:IncludeMobileTargets=true --verbosity normal From 732415c4e345cae810f9ef4de400387ec8bde1cd Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Wed, 16 Oct 2024 14:31:55 +0800 Subject: [PATCH 29/59] [skip ci] pod lint --- src/objectivec/include/ort_genai_objc.h | 18 +++++++++--------- src/objectivec/oga_generator.mm | 2 +- src/objectivec/oga_generator_params.mm | 2 +- src/objectivec/oga_images.mm | 2 +- src/objectivec/oga_internal.h | 6 +++--- src/objectivec/oga_internal.mm | 4 ++-- src/objectivec/oga_model.mm | 2 +- src/objectivec/oga_multi_modal_processor.mm | 2 +- src/objectivec/oga_named_tensors.mm | 2 +- src/objectivec/oga_sequences.mm | 2 +- src/objectivec/oga_tensor.mm | 2 +- src/objectivec/oga_tokenizer.mm | 6 +++--- 12 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 8fa7b95de..134d87f00 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -31,7 +31,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGAModel : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithPath:(NSString*)path +- (nullable instancetype)initWithPath:(NSString*)path error:(NSError**)error NS_DESIGNATED_INITIALIZER; - (nullable OGASequences*)generate:(OGAGeneratorParams*)params @@ -41,7 +41,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGATokenizer : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithModel:(OGAModel*)model +- (nullable instancetype)initWithModel:(OGAModel*)model error:(NSError**)error NS_DESIGNATED_INITIALIZER; - (nullable OGASequences*)encode:(NSString*)str @@ -54,10 +54,10 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGATokenizerStream : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithTokenizer:(OGATokenizer*)tokenizer +- (nullable instancetype)initWithTokenizer:(OGATokenizer*)tokenizer error:(NSError**)error NS_DESIGNATED_INITIALIZER; -- (nullable)initWithMultiModalProcessor:(OGAMultiModalProcessor*)processor +- (nullable instancetype)initWithMultiModalProcessor:(OGAMultiModalProcessor*)processor error:(NSError**)error NS_DESIGNATED_INITIALIZER; - (nullable NSString*)decode:(int32_t)token @@ -91,7 +91,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGAGeneratorParams : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithModel:(OGAModel*)model +- (nullable instancetype)initWithModel:(OGAModel*)model error:(NSError**)error NS_DESIGNATED_INITIALIZER; - (BOOL)setInputs:(OGANamedTensors*)namedTensors @@ -116,7 +116,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGAGenerator : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithModel:(OGAModel*)model +- (nullable instancetype)initWithModel:(OGAModel*)model params:(OGAGeneratorParams*)params error:(NSError**)error NS_DESIGNATED_INITIALIZER; @@ -131,7 +131,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGATensor : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithDataPointer:(void*)data +- (nullable instancetype)initWithDataPointer:(void*)data shape:(OGAInt64Span*)shape type:(OGAElementType)elementType error:(NSError**)error; @@ -148,7 +148,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGAImages : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithPath:(NSArray *)paths +- (nullable instancetype)initWithPath:(NSArray *)paths error:(NSError**)error NS_DESIGNATED_INITIALIZER; @end @@ -156,7 +156,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGAMultiModalProcessor : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable)initWithModel:(OGAModel*)model +- (nullable instancetype)initWithModel:(OGAModel*)model error:(NSError**)error NS_DESIGNATED_INITIALIZER; - (nullable OGANamedTensors*)processImages:(NSString*)prompt diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm index 4db451dfa..02032a604 100644 --- a/src/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -9,7 +9,7 @@ @implementation OGAGenerator { std::unique_ptr _generator; } -- (nullable)initWithModel:(OGAModel*)model +- (nullable instancetype)initWithModel:(OGAModel*)model params:(OGAGeneratorParams*)params error:(NSError**)error { if ((self = [super init]) == nil) { diff --git a/src/objectivec/oga_generator_params.mm b/src/objectivec/oga_generator_params.mm index 95e840944..d67b28796 100644 --- a/src/objectivec/oga_generator_params.mm +++ b/src/objectivec/oga_generator_params.mm @@ -9,7 +9,7 @@ @implementation OGAGeneratorParams { std::unique_ptr _generatorParams; } -- (nullable)initWithModel:(OGAModel*)model error:(NSError**)error { +- (nullable instancetype)initWithModel:(OGAModel*)model error:(NSError**)error { if ((self = [super init]) == nil) { return nil; } diff --git a/src/objectivec/oga_images.mm b/src/objectivec/oga_images.mm index 9f4e89815..57c50bdb6 100644 --- a/src/objectivec/oga_images.mm +++ b/src/objectivec/oga_images.mm @@ -9,7 +9,7 @@ @implementation OGAImages { std::unique_ptr _images; } -- (nullable)initWithPath:(NSArray *)paths error:(NSError**)error { +- (nullable instancetype)initWithPath:(NSArray *)paths error:(NSError**)error { if ((self = [super init]) == nil) { return nil; } diff --git a/src/objectivec/oga_internal.h b/src/objectivec/oga_internal.h index aa9948f61..4ec9b8d8c 100644 --- a/src/objectivec/oga_internal.h +++ b/src/objectivec/oga_internal.h @@ -21,7 +21,7 @@ NS_ASSUME_NONNULL_BEGIN @interface OGAInt32Span () -- (nullable)initWithRawPointer:(const int32_t*)pointer size:(size_t)size; +- (nullable instancetype)initWithRawPointer:(const int32_t*)pointer size:(size_t)size; - (const int32_t*)pointer; - (size_t)size; @@ -30,7 +30,7 @@ NS_ASSUME_NONNULL_BEGIN @interface OGAInt64Span () -- (nullable)initWithRawPointer:(const int64_t*)pointer size:(size_t)size; +- (nullable instancetype)initWithRawPointer:(const int64_t*)pointer size:(size_t)size; - (const int64_t*)pointer; - (size_t)size; @@ -39,7 +39,7 @@ NS_ASSUME_NONNULL_BEGIN @interface OGASequences () -- (nullable)initWithError:(NSError**)error; +- (nullable instancetype)initWithError:(NSError**)error; - (instancetype)initWithNativePointer:(std::unique_ptr)ptr; - (OgaSequences&)CXXAPIOgaSequences; diff --git a/src/objectivec/oga_internal.mm b/src/objectivec/oga_internal.mm index aafb1efd3..04902e4c4 100644 --- a/src/objectivec/oga_internal.mm +++ b/src/objectivec/oga_internal.mm @@ -8,7 +8,7 @@ @implementation OGAInt32Span { size_t _size; } -- (nullable)initWithRawPointer:(const int32_t*)pointer size:(size_t)size { +- (nullable instancetype)initWithRawPointer:(const int32_t*)pointer size:(size_t)size { _ptr = pointer; _size = size; return [self init]; @@ -33,7 +33,7 @@ @implementation OGAInt64Span { size_t _size; } -- (nullable)initWithRawPointer:(const int64_t*)pointer size:(size_t)size { +- (nullable instancetype)initWithRawPointer:(const int64_t*)pointer size:(size_t)size { _ptr = pointer; _size = size; return [self init]; diff --git a/src/objectivec/oga_model.mm b/src/objectivec/oga_model.mm index dbb20d7bb..2d2594fb1 100644 --- a/src/objectivec/oga_model.mm +++ b/src/objectivec/oga_model.mm @@ -9,7 +9,7 @@ @implementation OGAModel { std::unique_ptr _model; } -- (nullable)initWithPath:(NSString*)path error:(NSError**)error { +- (nullable instancetype)initWithPath:(NSString*)path error:(NSError**)error { if ((self = [super init]) == nil) { return nil; } diff --git a/src/objectivec/oga_multi_modal_processor.mm b/src/objectivec/oga_multi_modal_processor.mm index 9790760b9..647bf0ced 100644 --- a/src/objectivec/oga_multi_modal_processor.mm +++ b/src/objectivec/oga_multi_modal_processor.mm @@ -9,7 +9,7 @@ @implementation OGAMultiModalProcessor { std::unique_ptr _processor; } -- (nullable)initWithModel:(OGAModel*)model error:(NSError**)error { +- (nullable instancetype)initWithModel:(OGAModel*)model error:(NSError**)error { if ((self = [super init]) == nil) { return nil; } diff --git a/src/objectivec/oga_named_tensors.mm b/src/objectivec/oga_named_tensors.mm index dab8e55ae..8f22a23ca 100644 --- a/src/objectivec/oga_named_tensors.mm +++ b/src/objectivec/oga_named_tensors.mm @@ -9,7 +9,7 @@ @implementation OGANamedTensors { std::unique_ptr _tensor; } -- (instancetype)initWithNativePointer:(std::unique_ptr)ptr; +- (instancetype)initWithNativePointer:(std::unique_ptr)ptr { _tensor = std::move(ptr); return self; diff --git a/src/objectivec/oga_sequences.mm b/src/objectivec/oga_sequences.mm index 154afb124..1ab96a158 100644 --- a/src/objectivec/oga_sequences.mm +++ b/src/objectivec/oga_sequences.mm @@ -14,7 +14,7 @@ - (instancetype)initWithNativePointer:(std::unique_ptr)ptr { return self; } -- (nullable)initWithError:(NSError**)error { +- (nullable instancetype)initWithError:(NSError**)error { if ((self = [super init]) == nil) { return nil; } diff --git a/src/objectivec/oga_tensor.mm b/src/objectivec/oga_tensor.mm index 617f76425..b9e48aade 100644 --- a/src/objectivec/oga_tensor.mm +++ b/src/objectivec/oga_tensor.mm @@ -9,7 +9,7 @@ @implementation OGATensor { std::unique_ptr _tensor; } -- (nullable)initWithDataPointer:(void*)data +- (nullable instancetype)initWithDataPointer:(void*)data shape:(OGAInt64Span*)shape type:(OGAElementType)elementType error:(NSError**)error { diff --git a/src/objectivec/oga_tokenizer.mm b/src/objectivec/oga_tokenizer.mm index 0cc2b2394..8cd264639 100644 --- a/src/objectivec/oga_tokenizer.mm +++ b/src/objectivec/oga_tokenizer.mm @@ -9,7 +9,7 @@ @implementation OGATokenizer { std::unique_ptr _tokenizer; } -- (nullable)initWithModel:(OGAModel*)model error:(NSError**)error { +- (nullable instancetype)initWithModel:(OGAModel*)model error:(NSError**)error { if ((self = [super init]) == nil) { return nil; } @@ -51,7 +51,7 @@ @implementation OGATokenizerStream { std::unique_ptr _stream; } -- (nullable)initWithTokenizer:(OGATokenizer*)tokenizer error:(NSError**)error { +- (nullable instancetype)initWithTokenizer:(OGATokenizer*)tokenizer error:(NSError**)error { if ((self = [super init]) == nil) { return nil; } @@ -63,7 +63,7 @@ - (nullable)initWithTokenizer:(OGATokenizer*)tokenizer error:(NSError**)error { OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable)initWithMultiModalProcessor:(OGAMultiModalProcessor*)processor error:(NSError**)error { +- (nullable instancetype)initWithMultiModalProcessor:(OGAMultiModalProcessor*)processor error:(NSError**)error { if ((self = [super init]) == nil) { return nil; } From 15f503d0d275d6b27a4b5436f2f66d53a977ee8d Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Wed, 16 Oct 2024 14:35:04 +0800 Subject: [PATCH 30/59] [skip ci] pod testing infra --- .../apple/build_and_assemble_apple_pods.py | 2 + .../github/apple/c/assemble_c_pod_package.py | 3 +- .../github/apple/c/c.podspec.template | 13 +- .../github/apple/get_simulator_device_info.py | 152 ++++++++++ .../apple/objectivec/objc.podspec.template | 23 +- .../github/apple/test_apple_packages.py | 287 ++++++++++++++++++ 6 files changed, 462 insertions(+), 18 deletions(-) create mode 100755 tools/ci_build/github/apple/get_simulator_device_info.py create mode 100644 tools/ci_build/github/apple/test_apple_packages.py diff --git a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py index 795076d0d..7370ee256 100755 --- a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py +++ b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py @@ -137,6 +137,8 @@ def main(): str(build_dir / "framework_out"), "--variant", package_variant.name, + '--ort_version', + args.ort_version ] run(test_apple_packages_args) diff --git a/tools/ci_build/github/apple/c/assemble_c_pod_package.py b/tools/ci_build/github/apple/c/assemble_c_pod_package.py index 0186f5729..72c8fafdf 100644 --- a/tools/ci_build/github/apple/c/assemble_c_pod_package.py +++ b/tools/ci_build/github/apple/c/assemble_c_pod_package.py @@ -133,12 +133,13 @@ def parse_args(): help="Path to the onnxruntime framework directory to include in the pod.", ) parser.add_argument( - "--variant", choices=PackageVariant.all_variant_names(), required=True, help="Pod package variant." + "--variant", choices=PackageVariant.release_variant_names(), required=True, help="Pod package variant." ) parser.add_argument( "--ort_version", required=True, help="The ORT version to depend on." ) + return parser.parse_args() diff --git a/tools/ci_build/github/apple/c/c.podspec.template b/tools/ci_build/github/apple/c/c.podspec.template index c0780425d..ccb4dec31 100644 --- a/tools/ci_build/github/apple/c/c.podspec.template +++ b/tools/ci_build/github/apple/c/c.podspec.template @@ -4,11 +4,12 @@ Pod::Spec.new do |s| s.summary = "@SUMMARY@" s.homepage = "https://github.com/microsoft/onnxruntime-genai" s.license = { :type => "MIT", :file => "@LICENSE_FILE@" } - s.author = {"ONNX Runtime" => "onnxruntime@microsoft.com"} - s.source = { :http => 'file:///Users/chester/Projects/onnxruntime-genai/build/apple_framework/framework_out/archive.zip'} - - s.platform = :ios, "@IOS_DEPLOYMENT_TARGET@", :osx, "@MACOSX_DEPLOYMENT_TARGET@" - + s.author = {"ONNX Runtime" => "onnxruntime@microsoft.com" } + s.source = { :http => 'file:///http_source_placeholder' } + + s.ios.deployment_target = "@IOS_DEPLOYMENT_TARGET@" + s.osx.deployment_target = "@MACOSX_DEPLOYMENT_TARGET@" + s.vendored_frameworks = "@ORTGENAI_C_FRAMEWORK@" s.static_framework = true s.weak_framework = [ @WEAK_FRAMEWORK@ ] @@ -24,4 +25,4 @@ Pod::Spec.new do |s| "OTHER_CPLUSPLUSFLAGS" => "-fvisibility=hidden -fvisibility-inlines-hidden", } end - + diff --git a/tools/ci_build/github/apple/get_simulator_device_info.py b/tools/ci_build/github/apple/get_simulator_device_info.py new file mode 100755 index 000000000..7de9aa139 --- /dev/null +++ b/tools/ci_build/github/apple/get_simulator_device_info.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from __future__ import annotations + +import argparse +import functools +import itertools +import json +import subprocess + + +@functools.total_ordering +class Version: + """ + A simple Version class. + We opt to use this instead of `packaging.version.Version` to avoid depending on the external `packaging` package. + It only supports integer version components. + """ + + def __init__(self, version_string: str): + self._components = tuple(int(component) for component in version_string.split(".")) + + def __eq__(self, other: Version) -> bool: + component_pairs = itertools.zip_longest(self._components, other._components, fillvalue=0) + return all(pair[0] == pair[1] for pair in component_pairs) + + def __lt__(self, other: Version) -> bool: + component_pairs = itertools.zip_longest(self._components, other._components, fillvalue=0) + for self_component, other_component in component_pairs: + if self_component != other_component: + return self_component < other_component + return False + + +def get_simulator_device_info( + requested_runtime_platform: str = "iOS", + requested_device_type_product_family: str = "iPhone", + max_runtime_version_str: str | None = None, +) -> dict[str, str]: + """ + Retrieves simulator device information from Xcode. + This simulator device should be appropriate for running tests on this machine. + + :param requested_runtime_platform: The runtime platform to select. + :param requested_device_type_product_family: The device type product family to select. + :param max_runtime_version_str: The maximum runtime version to allow. + + :return: A dictionary containing information about the selected simulator device. + """ + max_runtime_version = Version(max_runtime_version_str) if max_runtime_version_str is not None else None + + simctl_proc = subprocess.run( + ["xcrun", "simctl", "list", "--json", "--no-escape-slashes"], + text=True, + capture_output=True, + check=True, + ) + + simctl_json = json.loads(simctl_proc.stdout) + + # device type id -> device type structure + device_type_map = {device_type["identifier"]: device_type for device_type in simctl_json["devicetypes"]} + + # runtime id -> runtime structure + runtime_map = {runtime["identifier"]: runtime for runtime in simctl_json["runtimes"]} + + def runtime_filter(runtime) -> bool: + if not runtime["isAvailable"]: + return False + + if runtime["platform"] != requested_runtime_platform: + return False + + if max_runtime_version is not None and Version(runtime["version"]) > max_runtime_version: + return False + + return True + + def runtime_id_filter(runtime_id: str) -> bool: + runtime = runtime_map.get(runtime_id) + if runtime is None: + return False + return runtime_filter(runtime) + + def device_type_filter(device_type) -> bool: + if device_type["productFamily"] != requested_device_type_product_family: + return False + + return True + + def device_filter(device) -> bool: + if not device["isAvailable"]: + return False + + if not device_type_filter(device_type_map[device["deviceTypeIdentifier"]]): + return False + + return True + + # simctl_json["devices"] is a map of runtime id -> list of device structures + # expand this into a list of (runtime id, device structure) and filter out invalid entries + runtime_id_and_device_pairs = [] + for runtime_id, device_list in filter( + lambda runtime_id_and_device_list: runtime_id_filter(runtime_id_and_device_list[0]), + simctl_json["devices"].items(), + ): + runtime_id_and_device_pairs.extend((runtime_id, device) for device in filter(device_filter, device_list)) + + # sort key - tuple of (runtime version, device type min runtime version) + # the secondary device type min runtime version value is to treat more recent device types as greater + def runtime_id_and_device_pair_key(runtime_id_and_device_pair): + runtime_id, device = runtime_id_and_device_pair + + runtime = runtime_map[runtime_id] + device_type = device_type_map[device["deviceTypeIdentifier"]] + + return (Version(runtime["version"]), device_type["minRuntimeVersion"]) + + selected_runtime_id, selected_device = max(runtime_id_and_device_pairs, key=runtime_id_and_device_pair_key) + selected_runtime = runtime_map[selected_runtime_id] + selected_device_type = device_type_map[selected_device["deviceTypeIdentifier"]] + + result = { + "device_name": selected_device["name"], + "device_udid": selected_device["udid"], + "device_type_identifier": selected_device_type["identifier"], + "device_type_name": selected_device_type["name"], + "device_type_product_family": selected_device_type["productFamily"], + "runtime_identifier": selected_runtime["identifier"], + "runtime_platform": selected_runtime["platform"], + "runtime_version": selected_runtime["version"], + } + + return result + + +def main(): + parser = argparse.ArgumentParser(description="Gets simulator info from Xcode and prints it in JSON format.") + parser.add_argument("--max-runtime-version", help="The maximum runtime version to allow.") + args = parser.parse_args() + + info = get_simulator_device_info( + max_runtime_version_str=args.max_runtime_version, + ) + + print(json.dumps(info, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/ci_build/github/apple/objectivec/objc.podspec.template b/tools/ci_build/github/apple/objectivec/objc.podspec.template index 19f0337da..cac187946 100644 --- a/tools/ci_build/github/apple/objectivec/objc.podspec.template +++ b/tools/ci_build/github/apple/objectivec/objc.podspec.template @@ -1,14 +1,15 @@ Pod::Spec.new do |s| - s.name = "@NAME@" - s.version = "@VERSION@" - s.summary = "@SUMMARY@" - s.description = "@DESCRIPTION@" - - s.homepage = "https://github.com/microsoft/onnxruntime-genai" - s.license = { :type => "MIT", :file => "@LICENSE_FILE@" } - s.author = {"ONNX Runtime" => "onnxruntime@microsoft.com"} - s.source = { :http => 'file:///Users/chester/Projects/onnxruntime-genai/src/archive.zip'} - s.platform = :ios, "@IOS_DEPLOYMENT_TARGET@", :osx, "@MACOSX_DEPLOYMENT_TARGET@" + s.name = "@NAME@" + s.version = "@VERSION@" + s.summary = "@SUMMARY@" + s.description = "@DESCRIPTION@" + + s.homepage = "https://github.com/microsoft/onnxruntime-genai" + s.license = { :type => "MIT", :file => "@LICENSE_FILE@" } + s.author = { "ONNX Runtime" => "onnxruntime@microsoft.com" } + s.source = { :http => 'file:///http_source_placeholder' } + s.ios.deployment_target = "@IOS_DEPLOYMENT_TARGET@" + s.osx.deployment_target = "@MACOSX_DEPLOYMENT_TARGET@" s.default_subspec = 'Core' s.static_framework = true @@ -45,4 +46,4 @@ Pod::Spec.new do |s| end s.dependency 'onnxruntime-objc', '~> @ORT_VERSION@' end - + diff --git a/tools/ci_build/github/apple/test_apple_packages.py b/tools/ci_build/github/apple/test_apple_packages.py new file mode 100644 index 000000000..07b6e6525 --- /dev/null +++ b/tools/ci_build/github/apple/test_apple_packages.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import contextlib +import json +import os +import pathlib +import shutil +import subprocess +import sys +import tempfile + +from c.assemble_c_pod_package import assemble_c_pod_package +from package_assembly_utils import PackageVariant, gen_file_from_template, get_ort_version + +SCRIPT_PATH = pathlib.Path(__file__).resolve(strict=True) +REPO_DIR = SCRIPT_PATH.parents[4] + + +def _test_apple_packages(args): + # check if CocoaPods is installed + if shutil.which("pod") is None: + if args.fail_if_cocoapods_missing: + raise ValueError("CocoaPods is required for this test") + else: + print("CocoaPods is not installed, ignore this test") + return + + # Now we need to create a zip file contains the framework and the podspec file, both of these 2 files + # should be under the c_framework_dir + c_framework_dir = args.c_framework_dir.resolve() + if not c_framework_dir.is_dir(): + raise FileNotFoundError(f"c_framework_dir {c_framework_dir} is not a folder.") + + has_framework = (c_framework_dir / "onnxruntime-genai.framework").exists() + has_xcframework = (c_framework_dir / "onnxruntime-genai.xcframework").exists() + + if not has_framework and not has_xcframework: + raise FileNotFoundError(f"{c_framework_dir} does not have onnxruntime-genai.framework/xcframework") + + if has_framework and has_xcframework: + raise ValueError("Cannot proceed when both onnxruntime-genai.framework and onnxruntime-genai.xcframework exist") + + framework_name = "onnxruntime-genai.framework" if has_framework else "onnxruntime-genai.xcframework" + + # create a temp folder + + with contextlib.ExitStack() as context_stack: + if args.test_project_stage_dir is None: + stage_dir = pathlib.Path(context_stack.enter_context(tempfile.TemporaryDirectory())).resolve() + else: + # If we specify the stage dir, then use it to create test project + stage_dir = args.test_project_stage_dir.resolve() + if os.path.exists(stage_dir): + shutil.rmtree(stage_dir) + os.makedirs(stage_dir) + + # assemble the test project here + target_proj_path = stage_dir / "apple_package_test" + + # copy the test project source files to target_proj_path + test_proj_path = pathlib.Path(REPO_DIR, "test/platform/apple/apple_package_test") + shutil.copytree(test_proj_path, target_proj_path) + + # assemble local pod files here + local_pods_dir = stage_dir / "local_pods" + + # We will only publish xcframework, however, assembly of the xcframework is a post process + # and it cannot be done by CMake for now. See, https://gitlab.kitware.com/cmake/cmake/-/issues/21752 + # For a single sysroot and arch built by build.py or cmake, we can only generate framework + # We still need a way to test it. framework_dir and public_headers_dir have different values when testing a + # framework and a xcframework. + framework_dir = args.c_framework_dir / framework_name + public_headers_dir = framework_dir / "Headers" if has_framework else args.c_framework_dir / "Headers" + + pod_name, podspec = assemble_c_pod_package( + staging_dir=local_pods_dir, + pod_version=get_ort_version(), + framework_info_file=args.framework_info_file, + public_headers_dir=public_headers_dir, + framework_dir=framework_dir, + package_variant=PackageVariant[args.variant], + ort_version=args.ort_version + ) + + # move podspec out to target_proj_path first + podspec = shutil.move(podspec, target_proj_path / podspec.name) + + # create a zip file contains the framework + zip_file_path = local_pods_dir / f"{pod_name}.zip" + + # shutil.make_archive doesn't preserve symlinks. we know this is running on macOS so use zip + subprocess.run(["zip", "-r", "-y", str(zip_file_path), "."], cwd=local_pods_dir, check=True) + + # update the podspec to point to the local framework zip file + with open(podspec) as file: + file_data = file.read() + + file_data = file_data.replace("file:///http_source_placeholder", f"file:///{zip_file_path}") + + with open(podspec, "w") as file: + file.write(file_data) + + # generate Podfile to point to pod + gen_file_from_template( + target_proj_path / "Podfile.template", + target_proj_path / "Podfile", + {"C_POD_NAME": pod_name, "C_POD_PODSPEC": f"./{podspec.name}"}, + ) + + # clean the Cocoapods cache first, in case the same pod was cached in previous runs + subprocess.run(["pod", "cache", "clean", "--all"], shell=False, check=True, cwd=target_proj_path) + + # install pods + # set env to skip macos test targets accordingly + env = os.environ.copy() + env["SKIP_MACOS_TEST"] = "true" if args.skip_macos_test else "false" + subprocess.run(["pod", "install"], shell=False, check=True, cwd=target_proj_path, env=env) + + # run the tests + if not args.prepare_test_project_only: + simulator_device_info = subprocess.check_output( + [ + sys.executable, + str(REPO_DIR / "tools" / "ci_build" / "github" / "apple" / "get_simulator_device_info.py"), + ], + text=True, + ).strip() + print(f"Simulator device info:\n{simulator_device_info}") + + simulator_device_info = json.loads(simulator_device_info) + + # Xcode UI tests seem to be flaky: https://github.com/orgs/community/discussions/68807 + # Add a couple of retries if we get this error: + # ios_package_testUITests-Runner Failed to initialize for UI testing: + # Error Domain=com.apple.dt.XCTest.XCTFuture Code=1000 "Timed out while loading Accessibility." + attempts = 0 + cmd = [ + "xcrun", + "xcodebuild", + "test", + "-workspace", + "./apple_package_test.xcworkspace", + "-scheme", + "ios_package_test", + "-destination", + f"platform=iOS Simulator,id={simulator_device_info['device_udid']}", + ] + + while True: + attempts += 1 + completed_process = subprocess.run( + cmd, + shell=False, + capture_output=True, + check=False, + text=True, + cwd=target_proj_path, + ) + + # print so it's in CI output + print(completed_process.stdout) + + if completed_process.returncode != 0: + print(f"Running ios_package_test failed. Return code was {completed_process.returncode}") + print("xcrun xcodebuild test stderr:") + print(completed_process.stderr) + print("---") + + if "Timed out while loading Accessibility" in completed_process.stderr and attempts < 3: + continue + + raise subprocess.CalledProcessError( + completed_process.returncode, " ".join(cmd), completed_process.stdout, completed_process.stderr + ) + + break + + if args.mac_catalyst_enabled: + subprocess.run( + [ + "xcrun", + "xcodebuild", + "test", + "-workspace", + "./apple_package_test.xcworkspace", + "-scheme", + "ios_package_test", + "-destination", + "platform=macOS,variant=Mac Catalyst", + "CODE_SIGNING_ALLOWED=NO", + ], + shell=False, + check=True, + cwd=target_proj_path, + ) + + if not args.skip_macos_test: + subprocess.run( + [ + "xcrun", + "xcodebuild", + "test", + "-workspace", + "./apple_package_test.xcworkspace", + "-scheme", + "macos_package_test", + "-destination", + "platform=macOS", + ], + shell=False, + check=True, + cwd=target_proj_path, + ) + + +def parse_args(): + parser = argparse.ArgumentParser( + os.path.basename(__file__), description="Test iOS framework using CocoaPods package." + ) + + parser.add_argument( + "--fail_if_cocoapods_missing", + action="store_true", + help="This script will fail if CocoaPods is not installed, " + "will not throw error unless fail_if_cocoapod_missing is set.", + ) + + parser.add_argument( + "--framework_info_file", + type=pathlib.Path, + required=True, + help="Path to the framework_info.json or xcframework_info.json file containing additional values for the podspec. " + "This file should be generated by CMake in the build directory.", + ) + + parser.add_argument( + "--c_framework_dir", type=pathlib.Path, required=True, help="Provide the parent directory for C/C++ framework" + ) + + parser.add_argument( + "--variant", + choices=PackageVariant.release_variant_names(), + required=True, + help="Pod package variant.", + ) + + parser.add_argument( + "--test_project_stage_dir", + type=pathlib.Path, + help="The stage dir for the test project, if not specified, will use a temporary path", + ) + + parser.add_argument( + "--prepare_test_project_only", + action="store_true", + help="Prepare the test project only, without running the tests", + ) + + parser.add_argument( + "--skip_macos_test", + action="store_true", + help="Skip macos platform tests. Specify this argument when build targets only contain ios archs. ", + ) + + parser.add_argument( + "--mac_catalyst_enabled", + action="store_true", + help="Run tests for mac catalyst variants. Specify this argument when build targets contains catalyst archs. ", + ) + + parser.add_argument( + "--ort_version", required=True, help="The ORT version to depend on." + ) + + return parser.parse_args() + + +def main(): + args = parse_args() + _test_apple_packages(args) + + +if __name__ == "__main__": + main() From dee8bd23c43b398497bec7c1a7d5761e69d37a28 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:00:25 +0800 Subject: [PATCH 31/59] [skip ci] Init objc testing --- src/objectivec/test/ort_genai_api_test.mm | 41 +++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/objectivec/test/ort_genai_api_test.mm diff --git a/src/objectivec/test/ort_genai_api_test.mm b/src/objectivec/test/ort_genai_api_test.mm new file mode 100644 index 000000000..6c6a46460 --- /dev/null +++ b/src/objectivec/test/ort_genai_api_test.mm @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import + +#import "ort_genai_objc.h" +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface ORTGenAIAPITest : XCTestCase + +@end + +@implementation ORTGenAIAPITest + + +- (void)Tensor_And_AddExtraInput { + // Create a [3 4] shaped tensor + std::array data{0, 1, 2, 3, + 10, 11, 12, 13, + 20, 21, 22, 23}; + std::vector shape{3, 4}; // Use vector so we can easily compare for equality later + + NSBundle* bundle = [NSBundle mainBundle]; + NSString* path = [[bundle resourcePath] stringByAppendingString:@"hf-internal-testing/tiny-random-gpt2-fp32"]; + + + NSError *error = nil; + OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; + OGAGeneratorParams *param = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; + + OGATensor* tensor = [[OGATensor alloc] initWithDataPointer:data.data() shape:shape.data() type:OGAElementTypeFloat32 error:&error]; + + [param setModelInput:@"test_input" tensor:tensor error:&error]; +} + +@end + +NS_ASSUME_NONNULL_END \ No newline at end of file From b3c4868f9518a0ad1d2a67f8312b7a1972fb4f16 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 17 Oct 2024 12:27:44 +0800 Subject: [PATCH 32/59] Fix --- src/objectivec/include/ort_genai_objc.h | 2 ++ src/objectivec/oga_internal.h | 4 ---- src/objectivec/test/ort_genai_api_test.mm | 3 ++- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 134d87f00..1d17e6659 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -67,6 +67,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGAInt32Span : NSObject - (instancetype)init NS_UNAVAILABLE; +- (nullable instancetype)initWithRawPointer:(const int32_t*)pointer size:(size_t)size; - (int32_t)last; @@ -75,6 +76,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGAInt64Span : NSObject - (instancetype)init NS_UNAVAILABLE; +- (nullable instancetype)initWithRawPointer:(const int64_t*)pointer size:(size_t)size; - (int64_t)last; diff --git a/src/objectivec/oga_internal.h b/src/objectivec/oga_internal.h index 4ec9b8d8c..b134333d8 100644 --- a/src/objectivec/oga_internal.h +++ b/src/objectivec/oga_internal.h @@ -21,8 +21,6 @@ NS_ASSUME_NONNULL_BEGIN @interface OGAInt32Span () -- (nullable instancetype)initWithRawPointer:(const int32_t*)pointer size:(size_t)size; - - (const int32_t*)pointer; - (size_t)size; @@ -30,8 +28,6 @@ NS_ASSUME_NONNULL_BEGIN @interface OGAInt64Span () -- (nullable instancetype)initWithRawPointer:(const int64_t*)pointer size:(size_t)size; - - (const int64_t*)pointer; - (size_t)size; diff --git a/src/objectivec/test/ort_genai_api_test.mm b/src/objectivec/test/ort_genai_api_test.mm index 6c6a46460..6b0a24f7b 100644 --- a/src/objectivec/test/ort_genai_api_test.mm +++ b/src/objectivec/test/ort_genai_api_test.mm @@ -31,7 +31,8 @@ - (void)Tensor_And_AddExtraInput { OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; OGAGeneratorParams *param = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; - OGATensor* tensor = [[OGATensor alloc] initWithDataPointer:data.data() shape:shape.data() type:OGAElementTypeFloat32 error:&error]; + OGAInt64Span* shapeData = [[OGAInt64Span alloc] initWithRawPointer:shape.data() size:2]; + OGATensor* tensor = [[OGATensor alloc] initWithDataPointer:data.data() shape:shapeData type:OGAElementTypeFloat32 error:&error]; [param setModelInput:@"test_input" tensor:tensor error:&error]; } From 6e3c975e6dcf54d2c330e064f82f0991c60b8082 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:48:27 +0800 Subject: [PATCH 33/59] Update --- src/objectivec/include/ort_genai_objc.h | 32 ++++++++++++++--------- src/objectivec/oga_generator.mm | 5 ++++ src/objectivec/oga_generator_params.mm | 12 +++++++++ src/objectivec/oga_internal.h | 2 ++ src/objectivec/oga_tensor.mm | 5 ++++ src/objectivec/test/ort_genai_api_test.mm | 29 +++++++++++++++++++- 6 files changed, 72 insertions(+), 13 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 1d17e6659..03e206a76 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -32,7 +32,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (instancetype)init NS_UNAVAILABLE; - (nullable instancetype)initWithPath:(NSString*)path - error:(NSError**)error NS_DESIGNATED_INITIALIZER; + error:(NSError**)error NS_DESIGNATED_INITIALIZER; - (nullable OGASequences*)generate:(OGAGeneratorParams*)params error:(NSError**)error; @@ -42,7 +42,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGATokenizer : NSObject - (instancetype)init NS_UNAVAILABLE; - (nullable instancetype)initWithModel:(OGAModel*)model - error:(NSError**)error NS_DESIGNATED_INITIALIZER; + error:(NSError**)error NS_DESIGNATED_INITIALIZER; - (nullable OGASequences*)encode:(NSString*)str error:(NSError**)error; @@ -55,10 +55,10 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGATokenizerStream : NSObject - (instancetype)init NS_UNAVAILABLE; - (nullable instancetype)initWithTokenizer:(OGATokenizer*)tokenizer - error:(NSError**)error NS_DESIGNATED_INITIALIZER; + error:(NSError**)error NS_DESIGNATED_INITIALIZER; - (nullable instancetype)initWithMultiModalProcessor:(OGAMultiModalProcessor*)processor - error:(NSError**)error NS_DESIGNATED_INITIALIZER; + error:(NSError**)error NS_DESIGNATED_INITIALIZER; - (nullable NSString*)decode:(int32_t)token error:(NSError**)error; @@ -94,11 +94,17 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGAGeneratorParams : NSObject - (instancetype)init NS_UNAVAILABLE; - (nullable instancetype)initWithModel:(OGAModel*)model - error:(NSError**)error NS_DESIGNATED_INITIALIZER; + error:(NSError**)error NS_DESIGNATED_INITIALIZER; - (BOOL)setInputs:(OGANamedTensors*)namedTensors error:(NSError**)error; +- (BOOL)setInputIds:(const int32_t*)rawPointer + inputIdsCount:(size_t)inputIdsCount + sequenceLength:(size_t)sequenceLength + batchSize:(size_t)batchSize + error:(NSError**)error; + - (BOOL)setInputSequences:(OGASequences*)sequences error:(NSError**)error; @@ -119,12 +125,14 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (instancetype)init NS_UNAVAILABLE; - (nullable instancetype)initWithModel:(OGAModel*)model - params:(OGAGeneratorParams*)params - error:(NSError**)error NS_DESIGNATED_INITIALIZER; + params:(OGAGeneratorParams*)params + error:(NSError**)error NS_DESIGNATED_INITIALIZER; - (BOOL)isDone; - (void)computeLogits; - (void)generateNextToken; +- (OGATensor*)getOutput:(NSString*)name; + - (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index; @@ -134,9 +142,9 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (instancetype)init NS_UNAVAILABLE; - (nullable instancetype)initWithDataPointer:(void*)data - shape:(OGAInt64Span*)shape - type:(OGAElementType)elementType - error:(NSError**)error; + shape:(OGAInt64Span*)shape + type:(OGAElementType)elementType + error:(NSError**)error; - (OGAElementType)type; @end @@ -151,7 +159,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (instancetype)init NS_UNAVAILABLE; - (nullable instancetype)initWithPath:(NSArray *)paths - error:(NSError**)error NS_DESIGNATED_INITIALIZER; + error:(NSError**)error NS_DESIGNATED_INITIALIZER; @end @@ -159,7 +167,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (instancetype)init NS_UNAVAILABLE; - (nullable instancetype)initWithModel:(OGAModel*)model - error:(NSError**)error NS_DESIGNATED_INITIALIZER; + error:(NSError**)error NS_DESIGNATED_INITIALIZER; - (nullable OGANamedTensors*)processImages:(NSString*)prompt images:(OGAImages*)images diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm index 02032a604..33b61956c 100644 --- a/src/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -35,6 +35,11 @@ - (void)generateNextToken { _generator->GenerateNextToken(); } +- (OGATensor*)getOutput:(NSString*)name { + std::unique_ptr output = _generator->GetOutput([name UTF8String]); + return [[OGATensor alloc] initWithNativePointer:std::move(output)]; +} + - (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index { try { size_t sequenceLength = _generator->GetSequenceCount(index); diff --git a/src/objectivec/oga_generator_params.mm b/src/objectivec/oga_generator_params.mm index d67b28796..d1ae55bca 100644 --- a/src/objectivec/oga_generator_params.mm +++ b/src/objectivec/oga_generator_params.mm @@ -29,6 +29,18 @@ - (BOOL)setInputs:(OGANamedTensors*)namedTensors error:(NSError**)error { OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } +- (BOOL)setInputIds:(const int32_t*)rawPointer + inputIdsCount:(size_t)inputIdsCount + sequenceLength:(size_t)sequenceLength + batchSize:(size_t)batchSize + error:(NSError**)error { + try { + _generatorParams->SetInputIDs(rawPointer, inputIdsCount, sequenceLength, batchSize); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + - (BOOL)setInputSequences:(OGASequences*)sequences error:(NSError**)error { try { _generatorParams->SetInputSequences([sequences CXXAPIOgaSequences]); diff --git a/src/objectivec/oga_internal.h b/src/objectivec/oga_internal.h index b134333d8..9c107ae41 100644 --- a/src/objectivec/oga_internal.h +++ b/src/objectivec/oga_internal.h @@ -56,6 +56,8 @@ NS_ASSUME_NONNULL_BEGIN @interface OGATensor () +- (instancetype)initWithNativePointer:(std::unique_ptr)ptr; + - (OgaTensor&)CXXAPIOgaTensor; @end diff --git a/src/objectivec/oga_tensor.mm b/src/objectivec/oga_tensor.mm index b9e48aade..ecac5ca1e 100644 --- a/src/objectivec/oga_tensor.mm +++ b/src/objectivec/oga_tensor.mm @@ -9,6 +9,11 @@ @implementation OGATensor { std::unique_ptr _tensor; } +- (instancetype)initWithNativePointer:(std::unique_ptr)ptr { + _tensor = std::move(ptr); + return self; +} + - (nullable instancetype)initWithDataPointer:(void*)data shape:(OGAInt64Span*)shape type:(OGAElementType)elementType diff --git a/src/objectivec/test/ort_genai_api_test.mm b/src/objectivec/test/ort_genai_api_test.mm index 6b0a24f7b..3fea4e576 100644 --- a/src/objectivec/test/ort_genai_api_test.mm +++ b/src/objectivec/test/ort_genai_api_test.mm @@ -26,7 +26,6 @@ - (void)Tensor_And_AddExtraInput { NSBundle* bundle = [NSBundle mainBundle]; NSString* path = [[bundle resourcePath] stringByAppendingString:@"hf-internal-testing/tiny-random-gpt2-fp32"]; - NSError *error = nil; OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; OGAGeneratorParams *param = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; @@ -37,6 +36,34 @@ - (void)Tensor_And_AddExtraInput { [param setModelInput:@"test_input" tensor:tensor error:&error]; } +- (void)GetOutput { + std::vector input_ids_shape{2, 4}; + std::vector input_ids{0, 0, 0, 52, 0, 0, 195, 731}; + auto input_sequence_length = input_ids_shape[1]; + auto batch_size = input_ids_shape[0]; + int max_length = 10; + + NSBundle* bundle = [NSBundle mainBundle]; + NSString* path = [[bundle resourcePath] stringByAppendingString:@"hf-internal-testing/tiny-random-gpt2-fp32"]; + + NSError *error = nil; + OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; + OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; + + [params setInputIds:input_ids.data() + inputIdsCount:input_ids.size() + sequenceLength:input_sequence_length + batchSize:batch_size + error:&error]; + + [params setSearchOption:@"max_length" doubleValue:max_length error:&error]; + + OGAGenerator* generator = [[OGAGenerator alloc] initWithModel:model + params:params + error:&error]; + +} + @end NS_ASSUME_NONNULL_END \ No newline at end of file From 6af7c277aa97a30ef17a16fd5a8db07f8089d23f Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:17:53 +0800 Subject: [PATCH 34/59] getoutput --- src/objectivec/include/ort_genai_objc.h | 6 +++--- src/objectivec/test/ort_genai_api_test.mm | 22 +++++++++++++++++++++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 03e206a76..d280220c8 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -133,7 +133,6 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (void)generateNextToken; - (OGATensor*)getOutput:(NSString*)name; - - (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index; @end @@ -143,9 +142,10 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (instancetype)init NS_UNAVAILABLE; - (nullable instancetype)initWithDataPointer:(void*)data shape:(OGAInt64Span*)shape - type:(OGAElementType)elementType + type:(OGAElementType)elementType error:(NSError**)error; - (OGAElementType)type; +- (void*)data; @end @@ -158,7 +158,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGAImages : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable instancetype)initWithPath:(NSArray *)paths +- (nullable instancetype)initWithPath:(NSArray*)paths error:(NSError**)error NS_DESIGNATED_INITIALIZER; @end diff --git a/src/objectivec/test/ort_genai_api_test.mm b/src/objectivec/test/ort_genai_api_test.mm index 3fea4e576..513d15f82 100644 --- a/src/objectivec/test/ort_genai_api_test.mm +++ b/src/objectivec/test/ort_genai_api_test.mm @@ -61,7 +61,27 @@ - (void)GetOutput { OGAGenerator* generator = [[OGAGenerator alloc] initWithModel:model params:params error:&error]; - + // check prompt + // full logits has shape [2, 4, 1000]. Sample 1 for every 200 tokens and the expected sampled logits has shape [2, 4, 5] + std::vector expected_sampled_logits_prompt{0.29694548f, 0.00955007f, 0.0430819f, 0.10063869f, 0.0437237f, + 0.27329233f, 0.00841076f, -0.1060291f, 0.11328877f, 0.13369876f, + 0.30323744f, 0.0545997f, 0.03894716f, 0.11702324f, 0.0410665f, + -0.12675379f, -0.04443946f, 0.14492269f, 0.03021223f, -0.03212897f, + 0.29694548f, 0.00955007f, 0.0430819f, 0.10063869f, 0.0437237f, + 0.27329233f, 0.00841076f, -0.1060291f, 0.11328877f, 0.13369876f, + -0.04699047f, 0.17915794f, 0.20838135f, 0.10888482f, -0.00277808f, + 0.2938929f, -0.10538938f, -0.00226692f, 0.12050669f, -0.10622668f}; + + [generator computeLogits]; + OGATensor* prompt_logits_ptr = [generator getOutput:@"logits"]; + auto prompt_logits = static_cast([prompt_logits_ptr data]); + int num_prompt_outputs_to_check = 40; + int sample_size = 200; + float tolerance = 0.001f; + // Verify outputs match expected outputs + for (int i = 0; i < num_prompt_outputs_to_check; i++) { + XCTAssertEqualWithAccuracy(expected_sampled_logits_prompt[i], prompt_logits[i * sample_size], tolerance); + } } @end From f759b1a04e05492269d82c65f519aa2783edced1 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:36:37 +0800 Subject: [PATCH 35/59] UPdate --- src/objectivec/test/ort_genai_api_test.mm | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/objectivec/test/ort_genai_api_test.mm b/src/objectivec/test/ort_genai_api_test.mm index 513d15f82..53a3d8023 100644 --- a/src/objectivec/test/ort_genai_api_test.mm +++ b/src/objectivec/test/ort_genai_api_test.mm @@ -82,6 +82,23 @@ - (void)GetOutput { for (int i = 0; i < num_prompt_outputs_to_check; i++) { XCTAssertEqualWithAccuracy(expected_sampled_logits_prompt[i], prompt_logits[i * sample_size], tolerance); } + + [generator generateNextToken]; + + // check for the 1st token generation + // full logits has shape [2, 1, 1000]. Sample 1 for every 200 tokens and the expected sampled logits has shape [2, 1, 5] + std::vector expected_sampled_logits_token_gen{0.03742531f, -0.05752287f, 0.14159015f, 0.04210977f, -0.1484456f, + 0.3041716f, -0.08701379f, -0.03778192f, 0.07471392f, -0.02049096f}; + + [generator computeLogits]; + OGATensor* token_gen_logits_ptr = [generator getOutput:@"logits"]; + auto token_gen_logits = static_cast([token_gen_logits_ptr data]); + int num_token_gen_outputs_to_check = 10; + + for (int i = 0; i < num_token_gen_outputs_to_check; i++) { + XCTAssertEqualWithAccuracy(expected_sampled_logits_token_gen[i], token_gen_logits[i * sample_size], tolerance); + } + [generator generateNextToken]; } @end From 620649825b273a740b5b054f4247205fd8848761 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:59:51 +0800 Subject: [PATCH 36/59] script --- ...lishing.yml => macos-ios-cocoapods-publishing.yml} | 2 +- .../stages/jobs/macos-ios-cocoapods-packaging-job.yml | 11 +++++++---- .../github/apple/build_and_assemble_apple_pods.py | 10 +++++++++- .../ci_build/github/apple/c/assemble_c_pod_package.py | 2 +- .../apple/objectivec/assemble_objc_pod_package.py | 3 ++- 5 files changed, 20 insertions(+), 8 deletions(-) rename .pipelines/{ios-cocoapods-publishing.yml => macos-ios-cocoapods-publishing.yml} (87%) diff --git a/.pipelines/ios-cocoapods-publishing.yml b/.pipelines/macos-ios-cocoapods-publishing.yml similarity index 87% rename from .pipelines/ios-cocoapods-publishing.yml rename to .pipelines/macos-ios-cocoapods-publishing.yml index a79462249..b4dc0be3d 100644 --- a/.pipelines/ios-cocoapods-publishing.yml +++ b/.pipelines/macos-ios-cocoapods-publishing.yml @@ -16,7 +16,7 @@ parameters: trigger: none stages: -- template: stages/ios-cocoapods-packaging-stage.yml +- template: stages/macos-ios-cocoapods-packaging-stage.yml parameters: ort_version: ${{ parameters.ort_version }} build_config: ${{ parameters.build_config }} diff --git a/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml b/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml index 1e32247a6..58b47f72b 100644 --- a/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml +++ b/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml @@ -2,13 +2,15 @@ parameters: - name: build_config type: string default: 'release' +- name: ort_version + type: string jobs: - job: macos_ios_cocoapods_packaging - pool: - vmImage: 'macOS-latest' - variables: - buildSettingsFile: "tools/ci_build/github/apple/default_full_apple_framework_build_settings.json" + pool: + vmImage: 'macOS-latest' + variables: + buildSettingsFile: "tools/ci_build/github/apple/default_full_apple_framework_build_settings.json" workspace: clean: all @@ -35,3 +37,4 @@ jobs: --test \ --variant Full \ --build-settings-file "${{ variables.buildSettingsFile }}" \ + --ort-version ${{parameters.ort_version}} diff --git a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py index 7370ee256..7edb8718e 100755 --- a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py +++ b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py @@ -82,7 +82,11 @@ def parse_args(): ) parser.add_argument( - "--ort_version", required=True, help="The ORT version to depend on." + "--ort-version", required=True, help="The ORT version to depend on." + ) + + parser.add_argument( + "--ort-home", required=False, help="The ORT home for building dependency." ) args = parser.parse_args() @@ -118,6 +122,10 @@ def main(): *args.build_apple_framework_extra_args, ] + if args.ort_home: + build_apple_framework_args.append('--ort_home') + build_apple_framework_args.append(args.ort_home) + if args.include_ops_by_config is not None: build_apple_framework_args += ["--include_ops_by_config", args.include_ops_by_config] diff --git a/tools/ci_build/github/apple/c/assemble_c_pod_package.py b/tools/ci_build/github/apple/c/assemble_c_pod_package.py index 72c8fafdf..878333c38 100644 --- a/tools/ci_build/github/apple/c/assemble_c_pod_package.py +++ b/tools/ci_build/github/apple/c/assemble_c_pod_package.py @@ -137,7 +137,7 @@ def parse_args(): ) parser.add_argument( - "--ort_version", required=True, help="The ORT version to depend on." + "--ort-version", required=True, help="The ORT version to depend on." ) return parser.parse_args() diff --git a/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py index 917f29944..05e144965 100755 --- a/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py +++ b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py @@ -161,7 +161,7 @@ def parse_args(): "--variant", choices=PackageVariant.release_variant_names(), required=True, help="Pod package variant." ) parser.add_argument( - "--ort_version", required=True, help="The ORT version to depend on." + "--ort-version", required=True, help="The ORT version to depend on." ) return parser.parse_args() @@ -174,6 +174,7 @@ def main(): pod_version=args.pod_version, framework_info_file=args.framework_info_file, package_variant=PackageVariant[args.variant], + ort_version=args.ort_versoin ) return 0 From 6b6a8394d15a91d98249efdedc168f0990c7c72e Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:50:37 +0800 Subject: [PATCH 37/59] [skip ci] model test --- src/objectivec/test/model_tests.mm | 64 +++++++++++++++++++++++ src/objectivec/test/ort_genai_api_test.mm | 10 ++-- 2 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 src/objectivec/test/model_tests.mm diff --git a/src/objectivec/test/model_tests.mm b/src/objectivec/test/model_tests.mm new file mode 100644 index 000000000..59da2e43d --- /dev/null +++ b/src/objectivec/test/model_tests.mm @@ -0,0 +1,64 @@ + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import + +#import "ort_genai_objc.h" +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface ORTGenAIModelTest : XCTestCase + +@end + +@implementation ORTGenAIModelTest + + +- (void)GreedySearchGptFp32 { + std::vector input_ids_shape{2, 4}; + std::vector input_ids{0, 0, 0, 52, 0, 0, 195, 731}; + + std::vector expected_output{ + 0, 0, 0, 52, 204, 204, 204, 204, 204, 204, + 0, 0, 195, 731, 731, 114, 114, 114, 114, 114}; + + const auto max_length = 10; + const auto batch_size = input_ids_shape[0]; + const auto input_sequence_length = input_ids_shape[1]; + + NSBundle* bundle = [NSBundle mainBundle]; + NSString* path = [[bundle resourcePath] stringByAppendingString:@"hf-internal-testing/tiny-random-gpt2-fp32"]; + + NSError *error = nil; + OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; + + OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; + [params setSearchOption:@"max_length", max_length]; + [params setSearchOption:@"do_sample", YES]; + [params setSearchOption:@"top_p", 0.25]; + + [params setInputIds:input_ids.data() + inputIdsCount:input_ids.size() + sequenceLength:input_sequence_length + batchSize:batch_size + error:&error]; + OGAGenerator* generator = [[OGAGenerator alloc] initWithModel:model + params:params + error:&error]; + + while (![generator isDone]) { + [generator computeLogits]; + [generator generateNextToken]; + } + + for (int i = 0; i < batch_size; i++) { + OGASequence *sequence = [generator sequenceAtIndex: i]; + auto* expected_output_start = &expected_output[i * max_length]; + XCTAssertTrue(0 == std::memcmp(expected_output_start, [sequence data], max_length * sizeof(int32_t))); + } +} + +@end \ No newline at end of file diff --git a/src/objectivec/test/ort_genai_api_test.mm b/src/objectivec/test/ort_genai_api_test.mm index 53a3d8023..b378a0d44 100644 --- a/src/objectivec/test/ort_genai_api_test.mm +++ b/src/objectivec/test/ort_genai_api_test.mm @@ -39,8 +39,8 @@ - (void)Tensor_And_AddExtraInput { - (void)GetOutput { std::vector input_ids_shape{2, 4}; std::vector input_ids{0, 0, 0, 52, 0, 0, 195, 731}; - auto input_sequence_length = input_ids_shape[1]; - auto batch_size = input_ids_shape[0]; + const auto batch_size = input_ids_shape[0]; + const auto input_sequence_length = input_ids_shape[1]; int max_length = 10; NSBundle* bundle = [NSBundle mainBundle]; @@ -75,9 +75,9 @@ - (void)GetOutput { [generator computeLogits]; OGATensor* prompt_logits_ptr = [generator getOutput:@"logits"]; auto prompt_logits = static_cast([prompt_logits_ptr data]); - int num_prompt_outputs_to_check = 40; - int sample_size = 200; - float tolerance = 0.001f; + const int num_prompt_outputs_to_check = 40; + const int sample_size = 200; + const float tolerance = 0.001f; // Verify outputs match expected outputs for (int i = 0; i < num_prompt_outputs_to_check; i++) { XCTAssertEqualWithAccuracy(expected_sampled_logits_prompt[i], prompt_logits[i * sample_size], tolerance); From cf227f98d0dec022bc9c30e77453bf6278a94f56 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Fri, 18 Oct 2024 15:18:51 +0800 Subject: [PATCH 38/59] [skip ci] beam search --- src/objectivec/test/model_tests.mm | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/objectivec/test/model_tests.mm b/src/objectivec/test/model_tests.mm index 59da2e43d..e399151fd 100644 --- a/src/objectivec/test/model_tests.mm +++ b/src/objectivec/test/model_tests.mm @@ -61,4 +61,44 @@ - (void)GreedySearchGptFp32 { } } +- (void)BeamSearchGptFp32 { + std::vector input_ids_shape{3, 12}; + std::vector input_ids{ + 0, 0, 0, 0, 0, 52, 195, 731, 321, 301, 734, 620, + 41, 554, 74, 622, 206, 222, 75, 223, 221, 198, 224, 572, + 0, 0, 0, 52, 328, 219, 328, 206, 288, 227, 896, 328}; + + std::vector expected_output{ + 0, 0, 0, 0, 0, 52, 195, 731, 321, 301, 734, 620, 131, 131, 131, 181, 638, 638, 638, 638, + 41, 554, 74, 622, 206, 222, 75, 223, 221, 198, 224, 572, 292, 292, 292, 292, 292, 292, 292, 292, + 0, 0, 0, 52, 328, 219, 328, 206, 288, 227, 896, 328, 328, 669, 669, 669, 669, 669, 669, 669}; + + const auto max_length = 20; + const auto batch_size = input_ids_shape[0]; + const auto input_sequence_length = input_ids_shape[1]; + + + NSBundle* bundle = [NSBundle mainBundle]; + NSString* path = [[bundle resourcePath] stringByAppendingString:@"hf-internal-testing/tiny-random-gpt2-fp32"]; + + NSError *error = nil; + OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; + + OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; + [params setSearchOption:@"max_length", max_length]; + [params setSearchOption:@"length_penalty", 1.0f]; + [params setSearchOption:@"num_beams", 4]; + + [params setInputIds:input_ids.data() + inputIdsCount:input_ids.size() + sequenceLength:input_sequence_length + batchSize:batch_size + error:&error]; + + for (int i = 0; i < batch_size; i++) { + OGASequence *sequence = [generator sequenceAtIndex: i]; + auto* expected_output_start = &expected_output[i * max_length]; + XCTAssertTrue(0 == std::memcmp(expected_output_start, [sequence data], max_length * sizeof(int32_t))); + } +} @end \ No newline at end of file From fed0ba13658db21743184725174f5bbaf9193366 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Mon, 21 Oct 2024 11:55:28 +0800 Subject: [PATCH 39/59] [skip ci] fix tests --- src/objectivec/include/ort_genai_objc.h | 3 +++ src/objectivec/oga_internal.h | 14 ----------- src/objectivec/test/model_tests.mm | 32 ++++++++++++++++--------- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index d280220c8..34fbb6b9b 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -69,6 +69,8 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (instancetype)init NS_UNAVAILABLE; - (nullable instancetype)initWithRawPointer:(const int32_t*)pointer size:(size_t)size; +- (const int32_t*)pointer; +- (size_t)size; - (int32_t)last; @end @@ -78,6 +80,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (instancetype)init NS_UNAVAILABLE; - (nullable instancetype)initWithRawPointer:(const int64_t*)pointer size:(size_t)size; +- (const int64_t*)pointer; - (int64_t)last; @end diff --git a/src/objectivec/oga_internal.h b/src/objectivec/oga_internal.h index 9c107ae41..8e671800a 100644 --- a/src/objectivec/oga_internal.h +++ b/src/objectivec/oga_internal.h @@ -19,20 +19,6 @@ NS_ASSUME_NONNULL_BEGIN @end -@interface OGAInt32Span () - -- (const int32_t*)pointer; -- (size_t)size; - -@end - -@interface OGAInt64Span () - -- (const int64_t*)pointer; -- (size_t)size; - -@end - @interface OGASequences () - (nullable instancetype)initWithError:(NSError**)error; diff --git a/src/objectivec/test/model_tests.mm b/src/objectivec/test/model_tests.mm index e399151fd..192ee5832 100644 --- a/src/objectivec/test/model_tests.mm +++ b/src/objectivec/test/model_tests.mm @@ -36,9 +36,9 @@ - (void)GreedySearchGptFp32 { OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; - [params setSearchOption:@"max_length", max_length]; - [params setSearchOption:@"do_sample", YES]; - [params setSearchOption:@"top_p", 0.25]; + [params setSearchOption:@"max_length" doubleValue:max_length error:&error]; + [params setSearchOption:@"do_sample" boolValue:YES error:&error]; + [params setSearchOption:@"top_p" doubleValue:0.25 error:&error]; [params setInputIds:input_ids.data() inputIdsCount:input_ids.size() @@ -55,9 +55,9 @@ - (void)GreedySearchGptFp32 { } for (int i = 0; i < batch_size; i++) { - OGASequence *sequence = [generator sequenceAtIndex: i]; + OGAInt32Span *sequence = [generator sequenceAtIndex: i]; auto* expected_output_start = &expected_output[i * max_length]; - XCTAssertTrue(0 == std::memcmp(expected_output_start, [sequence data], max_length * sizeof(int32_t))); + XCTAssertTrue(0 == std::memcmp(expected_output_start, [sequence pointer], max_length * sizeof(int32_t))); } } @@ -85,9 +85,9 @@ - (void)BeamSearchGptFp32 { OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; - [params setSearchOption:@"max_length", max_length]; - [params setSearchOption:@"length_penalty", 1.0f]; - [params setSearchOption:@"num_beams", 4]; + [params setSearchOption:@"max_length" doubleValue:max_length error:&error]; + [params setSearchOption:@"length_penalty" doubleValue:1.0f error:&error]; + [params setSearchOption:@"num_beams" doubleValue:4 error:&error]; [params setInputIds:input_ids.data() inputIdsCount:input_ids.size() @@ -95,10 +95,20 @@ - (void)BeamSearchGptFp32 { batchSize:batch_size error:&error]; + OGAGenerator* generator = [[OGAGenerator alloc] initWithModel:model + params:params + error:&error]; + while (![generator isDone]) { + [generator computeLogits]; + [generator generateNextToken]; + } + for (int i = 0; i < batch_size; i++) { - OGASequence *sequence = [generator sequenceAtIndex: i]; + OGAInt32Span *sequence = [generator sequenceAtIndex: i]; auto* expected_output_start = &expected_output[i * max_length]; - XCTAssertTrue(0 == std::memcmp(expected_output_start, [sequence data], max_length * sizeof(int32_t))); + XCTAssertTrue(0 == std::memcmp(expected_output_start, [sequence pointer], max_length * sizeof(int32_t))); } } -@end \ No newline at end of file +@end + +NS_ASSUME_NONNULL_END \ No newline at end of file From 0156a168fcbef0fe4e979fb5076263a8b14cfed6 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Mon, 21 Oct 2024 15:29:53 +0800 Subject: [PATCH 40/59] Update --- src/objectivec/include/ort_genai_objc.h | 65 +++++++++++++++++++++++++ src/objectivec/oga_internal.mm | 12 +++++ src/objectivec/oga_tokenizer.mm | 36 -------------- src/objectivec/oga_tokenizer_stream.mm | 43 ++++++++++++++++ 4 files changed, 120 insertions(+), 36 deletions(-) create mode 100644 src/objectivec/oga_tokenizer_stream.mm diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 34fbb6b9b..7b14808e7 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -1,6 +1,26 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +// GenAI Objective-C API +// +// This is a wrapper around the C++ API, and provides for a set of Objective-C classes with automatic resource management + +/* A simple end to end example of how to generate an answer from a prompt: + * + * OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; + * OGATokenizer* tokenizer = [[OGATokenizer alloc] initWithModel:model error:&error]; + * + * OGASequences* sequences = [tokenizer encode:@"A great recipe for Kung Pao chicken is " error:&error]; + * + * OGAGeneratorParams* params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; + * [params setInputSequences:sequences]; + * [params setSearchOption:@"max_length" doubleValue:200 error:&error]; + * + * OGASequences* output_sequences = [model generate:params error:&error]; + * NSString* out_string = [tokenizer decode:[output_sequences sequenceAtIndex:0]]; + * + */ + NS_ASSUME_NONNULL_BEGIN @class OGAInt32Span; @@ -28,25 +48,69 @@ typedef NS_ENUM(NSInteger, OGAElementType) { OGAElementTypeUint64, // maps to c type uint64_t }; + +/** + * An ORT GenAI model. + */ @interface OGAModel : NSObject - (instancetype)init NS_UNAVAILABLE; + +/** + * Creates a model. + * + * @param path The path to the ONNX GenAI model folder. + * @return The instance, or nil if an error occurs. + */ - (nullable instancetype)initWithPath:(NSString*)path error:(NSError**)error NS_DESIGNATED_INITIALIZER; +/** + * Generate sequences with the model. + * The inputs and outputs are pre-allocated. + * + * @param params The generation params to use. + * @param error Optional error information set if an error occurs. + * @return The generated sequences. + */ - (nullable OGASequences*)generate:(OGAGeneratorParams*)params error:(NSError**)error; @end +/** + * An ORT GenAI tokenizer. + */ @interface OGATokenizer : NSObject - (instancetype)init NS_UNAVAILABLE; + +/** + * Creates a tokenizer. + * + * @param model The model to use. + * @param error Optional error information set if an error occurs. + * @return The instance, or nil if an error occurs. + */ - (nullable instancetype)initWithModel:(OGAModel*)model error:(NSError**)error NS_DESIGNATED_INITIALIZER; +/** + * Encode text to sequences + * + * @param str The text to be encoded. + * @param error Optional error information set if an error occurs. + * @return The encoding result, or nil if an error occurs. + */ - (nullable OGASequences*)encode:(NSString*)str error:(NSError**)error; +/** + * Decode sequences to text + * + * @param data The sequences data to be encoded. + * @param error Optional error information set if an error occurs. + * @return The decoding result, or nil if an error occurs. + */ - (nullable NSString*)decode:(OGAInt32Span*)data error:(NSError**)error; @@ -81,6 +145,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (nullable instancetype)initWithRawPointer:(const int64_t*)pointer size:(size_t)size; - (const int64_t*)pointer; +- (size_t)size; - (int64_t)last; @end diff --git a/src/objectivec/oga_internal.mm b/src/objectivec/oga_internal.mm index 04902e4c4..287b7c419 100644 --- a/src/objectivec/oga_internal.mm +++ b/src/objectivec/oga_internal.mm @@ -23,6 +23,12 @@ - (size_t)size { } - (int32_t)last { + if (_size - 1 <= 0) { + NSException* exception = [NSException exceptionWithName:@"onnxruntime-genai" + reason:@"The size of this span is invalid" + userInfo:nil]; + @throw exception; + } return *(_ptr + (_size - 1)); } @@ -48,6 +54,12 @@ - (size_t)size { } - (int64_t)last { + if (_size - 1 <= 0) { + NSException* exception = [NSException exceptionWithName:@"onnxruntime-genai" + reason:@"The size of this span is invalid" + userInfo:nil]; + @throw exception; + } return *(_ptr + (_size - 1)); } diff --git a/src/objectivec/oga_tokenizer.mm b/src/objectivec/oga_tokenizer.mm index 8cd264639..f052efd94 100644 --- a/src/objectivec/oga_tokenizer.mm +++ b/src/objectivec/oga_tokenizer.mm @@ -47,39 +47,3 @@ - (nullable NSString*)decode:(OGAInt32Span*)data error:(NSError**)error { @end -@implementation OGATokenizerStream { - std::unique_ptr _stream; -} - -- (nullable instancetype)initWithTokenizer:(OGATokenizer*)tokenizer error:(NSError**)error { - if ((self = [super init]) == nil) { - return nil; - } - - try { - _stream = OgaTokenizerStream::Create([tokenizer CXXAPIOgaTokenizer]); - return self; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) -} - -- (nullable instancetype)initWithMultiModalProcessor:(OGAMultiModalProcessor*)processor error:(NSError**)error { - if ((self = [super init]) == nil) { - return nil; - } - - try { - _stream = OgaTokenizerStream::Create([processor CXXAPIOgaMultiModalProcessor]); - return self; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) -} - -- (nullable NSString*)decode:(int32_t)token error:(NSError**)error { - try { - return [NSString stringWithUTF8String:_stream->Decode(token)]; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) -} - -@end diff --git a/src/objectivec/oga_tokenizer_stream.mm b/src/objectivec/oga_tokenizer_stream.mm new file mode 100644 index 000000000..b664c37b0 --- /dev/null +++ b/src/objectivec/oga_tokenizer_stream.mm @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import "error_utils.h" +#import "oga_internal.h" +#import "ort_genai_objc.h" + +@implementation OGATokenizerStream { + std::unique_ptr _stream; +} + +- (nullable instancetype)initWithTokenizer:(OGATokenizer*)tokenizer error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _stream = OgaTokenizerStream::Create([tokenizer CXXAPIOgaTokenizer]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (nullable instancetype)initWithMultiModalProcessor:(OGAMultiModalProcessor*)processor error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _stream = OgaTokenizerStream::Create([processor CXXAPIOgaMultiModalProcessor]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (nullable NSString*)decode:(int32_t)token error:(NSError**)error { + try { + return [NSString stringWithUTF8String:_stream->Decode(token)]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +@end \ No newline at end of file From 8133c36ca86364b436866579733e8d149598d1c1 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 22 Oct 2024 15:18:48 +0800 Subject: [PATCH 41/59] reviews --- src/objectivec/error_utils.h | 2 + src/objectivec/error_utils.mm | 2 +- src/objectivec/include/ort_genai_objc.h | 105 ++++++++++++++++++-- src/objectivec/oga_generator.mm | 27 +++-- src/objectivec/oga_images.mm | 12 +-- src/objectivec/oga_internal.h | 6 +- src/objectivec/oga_internal.mm | 26 +++-- src/objectivec/oga_model.mm | 2 +- src/objectivec/oga_multi_modal_processor.mm | 2 +- src/objectivec/oga_named_tensors.mm | 2 +- src/objectivec/oga_sequences.mm | 16 ++- src/objectivec/oga_tensor.mm | 10 +- src/objectivec/test/assertion_utils.h | 46 +++++++++ src/objectivec/test/model_tests.mm | 22 ++-- src/objectivec/test/ort_genai_api_test.mm | 33 ++++-- 15 files changed, 243 insertions(+), 70 deletions(-) create mode 100644 src/objectivec/test/assertion_utils.h diff --git a/src/objectivec/error_utils.h b/src/objectivec/error_utils.h index a9be5b7fc..8c73faa97 100644 --- a/src/objectivec/error_utils.h +++ b/src/objectivec/error_utils.h @@ -9,6 +9,8 @@ NS_ASSUME_NONNULL_BEGIN +extern NSString* const kOgaErrorDomain; + void OGASaveCodeAndDescriptionToError(int code, const char* description, NSError** error); void OGASaveCodeAndDescriptionToError(int code, NSString* description, NSError** error); void OGASaveExceptionToError(const std::exception& e, NSError** error); diff --git a/src/objectivec/error_utils.mm b/src/objectivec/error_utils.mm index cbfdc0a9e..d886ca922 100644 --- a/src/objectivec/error_utils.mm +++ b/src/objectivec/error_utils.mm @@ -5,7 +5,7 @@ NS_ASSUME_NONNULL_BEGIN -static NSString* const kOgaErrorDomain = @"onnxruntime-genai"; +NSString* const kOgaErrorDomain = @"onnxruntime-genai"; void OGASaveCodeAndDescriptionToError(int code, const char* descriptionCstr, NSError** error) { if (!error) return; diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 7b14808e7..d6830b31b 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -17,7 +17,7 @@ * [params setSearchOption:@"max_length" doubleValue:200 error:&error]; * * OGASequences* output_sequences = [model generate:params error:&error]; - * NSString* out_string = [tokenizer decode:[output_sequences sequenceAtIndex:0]]; + * NSString* out_string = [tokenizer decode:[output_sequences sequenceAtIndex:0] error:&error]; * */ @@ -116,46 +116,129 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @end +/** + * A tokenizer stream is to decoded token strings incrementally, one token at a time. + */ @interface OGATokenizerStream : NSObject - (instancetype)init NS_UNAVAILABLE; + +/** + * Creates a tokenizer stream with an underlying tokenizer. + * + * @param tokenizer The underlying tokenizer to use. + * @param error Optional error information set if an error occurs. + * @return The instance, or nil if an error occurs. + */ - (nullable instancetype)initWithTokenizer:(OGATokenizer*)tokenizer error:(NSError**)error NS_DESIGNATED_INITIALIZER; +/** + * Creates a tokenizer stream with a multi modal processor + * + * @param processor The underlying processor to use. + * @param error Optional error information set if an error occurs. + * @return The instance, or nil if an error occurs. + */ - (nullable instancetype)initWithMultiModalProcessor:(OGAMultiModalProcessor*)processor error:(NSError**)error NS_DESIGNATED_INITIALIZER; - +/** + * Decode one token. + * + * @param token The token to be decoded. + * @param error Optional error information set if an error occurs. + * @return The decoding result, or nil if an error occurs. + */ - (nullable NSString*)decode:(int32_t)token error:(NSError**)error; @end +/** + * Wraps a raw int32_t pointer and its size. Represents a data sequence. + */ @interface OGAInt32Span : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable instancetype)initWithRawPointer:(const int32_t*)pointer size:(size_t)size; +/** + * Creates a span with underlying data. + * + * @param pointer The underlying data pointer to use. + * @param size The underlying data size. + * @return The instance, or nil if an error occurs. + */ +- (nullable instancetype)initWithDataPointer:(const int32_t*)pointer size:(size_t)size; + +/** + * The underlying data pointer + */ - (const int32_t*)pointer; +/** + * The underlying data size + */ - (size_t)size; -- (int32_t)last; +/** + * The last element in this data sequence + * @param error Optional error information set if an error occurs. + * @return The last element, or nil if an error occurs. + */ +- (int32_t)lastElementWithError:(NSError**)error NS_SWIFT_NAME(lastElement()); @end +/** + * Wraps a raw int64_t pointer and its size. Represents a data sequence. + */ @interface OGAInt64Span : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable instancetype)initWithRawPointer:(const int64_t*)pointer size:(size_t)size; +/** + * Creates a span with underlying data. + * + * @param pointer The underlying data pointer to use. + * @param size The underlying data size. + * @return The instance, or nil if an error occurs. + */ +- (nullable instancetype)initWithDataPointer:(const int64_t*)pointer size:(size_t)size; +/** + * The underlying data pointer + */ - (const int64_t*)pointer; + +/** + * The underlying data size + */ - (size_t)size; -- (int64_t)last; + +/** + * The last element in this data sequence + * @param error Optional error information set if an error occurs. + * @return The last element, or nil if an error occurs. + */ +- (int64_t)lastElementWithError:(NSError**)error NS_SWIFT_NAME(lastElement());; @end +/** + * A series of generated sequences + */ @interface OGASequences : NSObject - (instancetype)init NS_UNAVAILABLE; +/** + * The count of generated sequences + */ - (size_t)count; -- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index; + +/** + * Retrieve the sequence at the given index + * @param index The index needed. + * @param error Optional error information set if an error occurs. + * @return The last element, or nil if an error occurs. + */ +- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index + error:(NSError**)error; @end @@ -196,9 +279,9 @@ typedef NS_ENUM(NSInteger, OGAElementType) { params:(OGAGeneratorParams*)params error:(NSError**)error NS_DESIGNATED_INITIALIZER; -- (BOOL)isDone; -- (void)computeLogits; -- (void)generateNextToken; +- (nullable NSNumber*)isDoneWithError:(NSError**)error NS_SWIFT_NAME(isDone()); +- (BOOL)computeLogitsWithError:(NSError**)error NS_SWIFT_NAME(computeLogits()); +- (BOOL)generateNextTokenWithError:(NSError**)error NS_SWIFT_NAME(generateNextToken()); - (OGATensor*)getOutput:(NSString*)name; - (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index; @@ -209,7 +292,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (instancetype)init NS_UNAVAILABLE; - (nullable instancetype)initWithDataPointer:(void*)data - shape:(OGAInt64Span*)shape + shape:(NSArray*)shape type:(OGAElementType)elementType error:(NSError**)error; - (OGAElementType)type; diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm index 33b61956c..58479ff7c 100644 --- a/src/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -23,28 +23,39 @@ - (nullable instancetype)initWithModel:(OGAModel*)model OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (BOOL)isDone { - return _generator->IsDone(); +- (NSNumber*)isDoneWithError:(NSError**)error { + try { + return [NSNumber numberWithBool:_generator->IsDone()]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (void)computeLogits { - _generator->ComputeLogits(); +- (BOOL)computeLogitsWithError:(NSError**)error { + try { + _generator->ComputeLogits(); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } -- (void)generateNextToken { - _generator->GenerateNextToken(); +- (BOOL)generateNextTokenWithError:(NSError**)error { + try { + _generator->GenerateNextToken(); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } - (OGATensor*)getOutput:(NSString*)name { std::unique_ptr output = _generator->GetOutput([name UTF8String]); - return [[OGATensor alloc] initWithNativePointer:std::move(output)]; + return [[OGATensor alloc] initWithCXXPointer:std::move(output)]; } - (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index { try { size_t sequenceLength = _generator->GetSequenceCount(index); const int32_t* data = _generator->GetSequenceData(index); - return [[OGAInt32Span alloc] initWithRawPointer:data size:sequenceLength]; + return [[OGAInt32Span alloc] initWithDataPointer:data size:sequenceLength]; } catch (std::exception) { return nil; } diff --git a/src/objectivec/oga_images.mm b/src/objectivec/oga_images.mm index 57c50bdb6..25a056e4e 100644 --- a/src/objectivec/oga_images.mm +++ b/src/objectivec/oga_images.mm @@ -14,14 +14,14 @@ - (nullable instancetype)initWithPath:(NSArray *)paths error:(NSErro return nil; } - std::vector cpp_paths; - cpp_paths.reserve([paths count]); + try { + std::vector cpp_paths; + cpp_paths.reserve([paths count]); - for (NSString* path in paths){ - cpp_paths.push_back([path UTF8String]); - } + for (NSString* path in paths){ + cpp_paths.push_back([path UTF8String]); + } - try { _images = OgaImages::Load(cpp_paths); return self; } diff --git a/src/objectivec/oga_internal.h b/src/objectivec/oga_internal.h index 8e671800a..606cec04f 100644 --- a/src/objectivec/oga_internal.h +++ b/src/objectivec/oga_internal.h @@ -22,7 +22,7 @@ NS_ASSUME_NONNULL_BEGIN @interface OGASequences () - (nullable instancetype)initWithError:(NSError**)error; -- (instancetype)initWithNativePointer:(std::unique_ptr)ptr; +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr; - (OgaSequences&)CXXAPIOgaSequences; @@ -42,7 +42,7 @@ NS_ASSUME_NONNULL_BEGIN @interface OGATensor () -- (instancetype)initWithNativePointer:(std::unique_ptr)ptr; +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr; - (OgaTensor&)CXXAPIOgaTensor; @@ -50,7 +50,7 @@ NS_ASSUME_NONNULL_BEGIN @interface OGANamedTensors () -- (instancetype)initWithNativePointer:(std::unique_ptr)ptr +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr NS_DESIGNATED_INITIALIZER; - (OgaNamedTensors&)CXXAPIOgaNamedTensors; diff --git a/src/objectivec/oga_internal.mm b/src/objectivec/oga_internal.mm index 287b7c419..d7e3bf200 100644 --- a/src/objectivec/oga_internal.mm +++ b/src/objectivec/oga_internal.mm @@ -8,7 +8,7 @@ @implementation OGAInt32Span { size_t _size; } -- (nullable instancetype)initWithRawPointer:(const int32_t*)pointer size:(size_t)size { +- (nullable instancetype)initWithDataPointer:(const int32_t*)pointer size:(size_t)size { _ptr = pointer; _size = size; return [self init]; @@ -22,12 +22,11 @@ - (size_t)size { return _size; } -- (int32_t)last { - if (_size - 1 <= 0) { - NSException* exception = [NSException exceptionWithName:@"onnxruntime-genai" - reason:@"The size of this span is invalid" - userInfo:nil]; - @throw exception; +- (int32_t)lastElementWithError:(NSError**)error { + if (_size == 0) { + NSDictionary *errorDictionary = @{NSLocalizedDescriptionKey : @"The size of this span is invalid"}; + *error = [[NSError alloc] initWithDomain:kOgaErrorDomain code:-1 userInfo:errorDictionary]; + return -1; } return *(_ptr + (_size - 1)); } @@ -39,7 +38,7 @@ @implementation OGAInt64Span { size_t _size; } -- (nullable instancetype)initWithRawPointer:(const int64_t*)pointer size:(size_t)size { +- (nullable instancetype)initWithDataPointer:(const int64_t*)pointer size:(size_t)size { _ptr = pointer; _size = size; return [self init]; @@ -53,12 +52,11 @@ - (size_t)size { return _size; } -- (int64_t)last { - if (_size - 1 <= 0) { - NSException* exception = [NSException exceptionWithName:@"onnxruntime-genai" - reason:@"The size of this span is invalid" - userInfo:nil]; - @throw exception; +- (int64_t)lastElementWithError:(NSError**)error { + if (_size == 0) { + NSDictionary *errorDictionary = @{NSLocalizedDescriptionKey : @"The size of this span is invalid"}; + *error = [[NSError alloc] initWithDomain:kOgaErrorDomain code:-1 userInfo:errorDictionary]; + return -1; } return *(_ptr + (_size - 1)); } diff --git a/src/objectivec/oga_model.mm b/src/objectivec/oga_model.mm index 2d2594fb1..1fdf4c1fc 100644 --- a/src/objectivec/oga_model.mm +++ b/src/objectivec/oga_model.mm @@ -25,7 +25,7 @@ - (nullable OGASequences*)generate:(OGAGeneratorParams*)params error:(NSError**) try { std::unique_ptr output_sequences = _model->Generate([params CXXAPIOgaGeneratorParams]); - return [[OGASequences alloc] initWithNativePointer:std::move(output_sequences)]; + return [[OGASequences alloc] initWithCXXPointer:std::move(output_sequences)]; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } diff --git a/src/objectivec/oga_multi_modal_processor.mm b/src/objectivec/oga_multi_modal_processor.mm index 647bf0ced..1aec2a1d8 100644 --- a/src/objectivec/oga_multi_modal_processor.mm +++ b/src/objectivec/oga_multi_modal_processor.mm @@ -26,7 +26,7 @@ - (nullable OGANamedTensors*)processImages:(NSString*)prompt error:(NSError**)error { try { OGANamedTensors* result = [[OGANamedTensors alloc] - initWithNativePointer:_processor->ProcessImages([prompt UTF8String], + initWithCXXPointer:_processor->ProcessImages([prompt UTF8String], [images CXXAPIOgaImages])]; return result; } diff --git a/src/objectivec/oga_named_tensors.mm b/src/objectivec/oga_named_tensors.mm index 8f22a23ca..fa3556eb0 100644 --- a/src/objectivec/oga_named_tensors.mm +++ b/src/objectivec/oga_named_tensors.mm @@ -9,7 +9,7 @@ @implementation OGANamedTensors { std::unique_ptr _tensor; } -- (instancetype)initWithNativePointer:(std::unique_ptr)ptr +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr { _tensor = std::move(ptr); return self; diff --git a/src/objectivec/oga_sequences.mm b/src/objectivec/oga_sequences.mm index 1ab96a158..353444e06 100644 --- a/src/objectivec/oga_sequences.mm +++ b/src/objectivec/oga_sequences.mm @@ -9,7 +9,7 @@ @implementation OGASequences { std::unique_ptr _sequences; } -- (instancetype)initWithNativePointer:(std::unique_ptr)ptr { +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr { _sequences = std::move(ptr); return self; } @@ -30,13 +30,19 @@ - (size_t)count { return _sequences->Count(); } -- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index { +- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index + error:(NSError**)error { if (index >= [self count]) { + NSDictionary *errorDictionary = @{NSLocalizedDescriptionKey : @"The index is out of bounds"}; + *error = [[NSError alloc] initWithDomain:kOgaErrorDomain code:-1 userInfo:errorDictionary]; return nil; } - size_t sequenceLength = _sequences->SequenceCount(index); - const int32_t* data = _sequences->SequenceData(index); - return [[OGAInt32Span alloc] initWithRawPointer:data size:sequenceLength]; + try { + size_t sequenceLength = _sequences->SequenceCount(index); + const int32_t* data = _sequences->SequenceData(index); + return [[OGAInt32Span alloc] initWithDataPointer:data size:sequenceLength]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } - (OgaSequences&)CXXAPIOgaSequences { diff --git a/src/objectivec/oga_tensor.mm b/src/objectivec/oga_tensor.mm index ecac5ca1e..1b7ffcc26 100644 --- a/src/objectivec/oga_tensor.mm +++ b/src/objectivec/oga_tensor.mm @@ -9,13 +9,13 @@ @implementation OGATensor { std::unique_ptr _tensor; } -- (instancetype)initWithNativePointer:(std::unique_ptr)ptr { +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr { _tensor = std::move(ptr); return self; } - (nullable instancetype)initWithDataPointer:(void*)data - shape:(OGAInt64Span*)shape + shape:(NSArray*)shape type:(OGAElementType)elementType error:(NSError**)error { if ((self = [super init]) == nil) { @@ -23,7 +23,11 @@ - (nullable instancetype)initWithDataPointer:(void*)data } try { - _tensor = OgaTensor::Create(data, shape.pointer, shape.size, + std::vector cxxShape; + for (NSNumber* object in shape) { + cxxShape.push_back([object intValue]); + } + _tensor = OgaTensor::Create(data, cxxShape.data(), cxxShape.size(), static_cast(elementType)); return self; } diff --git a/src/objectivec/test/assertion_utils.h b/src/objectivec/test/assertion_utils.h new file mode 100644 index 000000000..2b72435b9 --- /dev/null +++ b/src/objectivec/test/assertion_utils.h @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +#define ORTAssertNullableResultSuccessful(result, error) \ + do { \ + XCTAssertNotNil(result, @"Expected non-nil result but got nil. Error: %@", error); \ + XCTAssertNil(error); \ + } while (0) + +#define ORTAssertBoolResultSuccessful(result, error) \ + do { \ + XCTAssertTrue(result, @"Expected true result but got false. Error: %@", error); \ + XCTAssertNil(error); \ + } while (0) + +#define ORTAssertNullableResultUnsuccessful(result, error) \ + do { \ + XCTAssertNil(result); \ + XCTAssertNotNil(error); \ + } while (0) + +#define ORTAssertBoolResultUnsuccessful(result, error) \ + do { \ + XCTAssertFalse(result); \ + XCTAssertNotNil(error); \ + } while (0) + +#define ORTAssertEqualFloatAndNoError(expected, result, error) \ + do { \ + XCTAssertEqualWithAccuracy(expected, result, 1e-3f, @"Expected %f but got %f. Error:%@", expected, result, error); \ + XCTAssertNil(error); \ + } while (0) + +#define ORTAssertEqualFloatArrays(expected, result) \ + do { \ + XCTAssertEqual(expected.count, result.count); \ + for (size_t i = 0; i < expected.count; ++i) { \ + XCTAssertEqualWithAccuracy([expected[i] floatValue], [result[i] floatValue], 1e-3f); \ + } \ + } while (0) + +NS_ASSUME_NONNULL_END diff --git a/src/objectivec/test/model_tests.mm b/src/objectivec/test/model_tests.mm index 192ee5832..0fb9dabb4 100644 --- a/src/objectivec/test/model_tests.mm +++ b/src/objectivec/test/model_tests.mm @@ -5,6 +5,7 @@ #import #import "ort_genai_objc.h" +#import "assertion_utils.h" #import #import @@ -34,8 +35,11 @@ - (void)GreedySearchGptFp32 { NSError *error = nil; OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; + ORTAssertNullableResultSuccessful(model, error); OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; + ORTAssertNullableResultSuccessful(params, error); + [params setSearchOption:@"max_length" doubleValue:max_length error:&error]; [params setSearchOption:@"do_sample" boolValue:YES error:&error]; [params setSearchOption:@"top_p" doubleValue:0.25 error:&error]; @@ -48,10 +52,11 @@ - (void)GreedySearchGptFp32 { OGAGenerator* generator = [[OGAGenerator alloc] initWithModel:model params:params error:&error]; + ORTAssertNullableResultSuccessful(generator, error); - while (![generator isDone]) { - [generator computeLogits]; - [generator generateNextToken]; + while (![[generator isDoneWithError:&error] boolValue]) { + [generator computeLogitsWithError:&error]; + [generator generateNextTokenWithError:&error]; } for (int i = 0; i < batch_size; i++) { @@ -83,8 +88,11 @@ - (void)BeamSearchGptFp32 { NSError *error = nil; OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; + ORTAssertNullableResultSuccessful(model, error); OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; + ORTAssertNullableResultSuccessful(params, error); + [params setSearchOption:@"max_length" doubleValue:max_length error:&error]; [params setSearchOption:@"length_penalty" doubleValue:1.0f error:&error]; [params setSearchOption:@"num_beams" doubleValue:4 error:&error]; @@ -98,9 +106,11 @@ - (void)BeamSearchGptFp32 { OGAGenerator* generator = [[OGAGenerator alloc] initWithModel:model params:params error:&error]; - while (![generator isDone]) { - [generator computeLogits]; - [generator generateNextToken]; + + ORTAssertNullableResultSuccessful(generator, error); + while (![[generator isDoneWithError:&error] boolValue]) { + [generator computeLogitsWithError:&error]; + [generator generateNextTokenWithError:&error]; } for (int i = 0; i < batch_size; i++) { diff --git a/src/objectivec/test/ort_genai_api_test.mm b/src/objectivec/test/ort_genai_api_test.mm index b378a0d44..b1c4aa95e 100644 --- a/src/objectivec/test/ort_genai_api_test.mm +++ b/src/objectivec/test/ort_genai_api_test.mm @@ -4,6 +4,7 @@ #import #import "ort_genai_objc.h" +#import "assertion_utils.h" #import #import @@ -21,19 +22,23 @@ - (void)Tensor_And_AddExtraInput { std::array data{0, 1, 2, 3, 10, 11, 12, 13, 20, 21, 22, 23}; - std::vector shape{3, 4}; // Use vector so we can easily compare for equality later + NSArray* shape = @[@3, @4]; NSBundle* bundle = [NSBundle mainBundle]; NSString* path = [[bundle resourcePath] stringByAppendingString:@"hf-internal-testing/tiny-random-gpt2-fp32"]; NSError *error = nil; + BOOL ret = NO; OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; - OGAGeneratorParams *param = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; + ORTAssertNullableResultSuccessful(model, error); - OGAInt64Span* shapeData = [[OGAInt64Span alloc] initWithRawPointer:shape.data() size:2]; - OGATensor* tensor = [[OGATensor alloc] initWithDataPointer:data.data() shape:shapeData type:OGAElementTypeFloat32 error:&error]; + OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; + ORTAssertNullableResultSuccessful(params, error); + + OGATensor* tensor = [[OGATensor alloc] initWithDataPointer:data.data() shape:shape type:OGAElementTypeFloat32 error:&error]; - [param setModelInput:@"test_input" tensor:tensor error:&error]; + ret = [params setModelInput:@"test_input" tensor:tensor error:&error]; + ORTAssertBoolResultSuccessful(ret, error); } - (void)GetOutput { @@ -47,8 +52,12 @@ - (void)GetOutput { NSString* path = [[bundle resourcePath] stringByAppendingString:@"hf-internal-testing/tiny-random-gpt2-fp32"]; NSError *error = nil; + BOOL ret = NO; OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; + ORTAssertNullableResultSuccessful(model, error); + OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; + ORTAssertNullableResultSuccessful(params, error); [params setInputIds:input_ids.data() inputIdsCount:input_ids.size() @@ -72,7 +81,8 @@ - (void)GetOutput { -0.04699047f, 0.17915794f, 0.20838135f, 0.10888482f, -0.00277808f, 0.2938929f, -0.10538938f, -0.00226692f, 0.12050669f, -0.10622668f}; - [generator computeLogits]; + ret = [generator computeLogitsWithError:&error]; + ORTAssertBoolResultSuccessful(ret, error); OGATensor* prompt_logits_ptr = [generator getOutput:@"logits"]; auto prompt_logits = static_cast([prompt_logits_ptr data]); const int num_prompt_outputs_to_check = 40; @@ -83,14 +93,16 @@ - (void)GetOutput { XCTAssertEqualWithAccuracy(expected_sampled_logits_prompt[i], prompt_logits[i * sample_size], tolerance); } - [generator generateNextToken]; - + ret = [generator generateNextTokenWithError:&error]; + ORTAssertBoolResultSuccessful(ret, error); // check for the 1st token generation // full logits has shape [2, 1, 1000]. Sample 1 for every 200 tokens and the expected sampled logits has shape [2, 1, 5] std::vector expected_sampled_logits_token_gen{0.03742531f, -0.05752287f, 0.14159015f, 0.04210977f, -0.1484456f, 0.3041716f, -0.08701379f, -0.03778192f, 0.07471392f, -0.02049096f}; - [generator computeLogits]; + ret = [generator computeLogitsWithError:&error]; + ORTAssertBoolResultSuccessful(ret, error); + OGATensor* token_gen_logits_ptr = [generator getOutput:@"logits"]; auto token_gen_logits = static_cast([token_gen_logits_ptr data]); int num_token_gen_outputs_to_check = 10; @@ -98,7 +110,8 @@ - (void)GetOutput { for (int i = 0; i < num_token_gen_outputs_to_check; i++) { XCTAssertEqualWithAccuracy(expected_sampled_logits_token_gen[i], token_gen_logits[i * sample_size], tolerance); } - [generator generateNextToken]; + [generator generateNextTokenWithError:&error]; + ORTAssertBoolResultSuccessful(ret, error); } @end From 565aad13918080deaec13257087bbb9534e10f7e Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 22 Oct 2024 15:48:42 +0800 Subject: [PATCH 42/59] reviews --- src/objectivec/include/ort_genai_objc.h | 78 ++++++++++++++++++++++--- src/objectivec/oga_generator.mm | 6 +- src/objectivec/test/model_tests.mm | 7 ++- 3 files changed, 78 insertions(+), 13 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index d6830b31b..1c480f36a 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -48,7 +48,6 @@ typedef NS_ENUM(NSInteger, OGAElementType) { OGAElementTypeUint64, // maps to c type uint64_t }; - /** * An ORT GenAI model. */ @@ -167,7 +166,6 @@ typedef NS_ENUM(NSInteger, OGAElementType) { */ - (nullable instancetype)initWithDataPointer:(const int32_t*)pointer size:(size_t)size; - /** * The underlying data pointer */ @@ -215,7 +213,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { * @param error Optional error information set if an error occurs. * @return The last element, or nil if an error occurs. */ -- (int64_t)lastElementWithError:(NSError**)error NS_SWIFT_NAME(lastElement());; +- (int64_t)lastElementWithError:(NSError**)error NS_SWIFT_NAME(lastElement()); @end @@ -242,49 +240,115 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @end +/** + * The parameters for generation. + */ @interface OGAGeneratorParams : NSObject - (instancetype)init NS_UNAVAILABLE; + +/** + * Creates a GeneratorParams from the given model. + * @param model The model to use for generation. + * @param error Optional error information set if an error occurs. + * @return The instance, or nil if an error occurs. + */ - (nullable instancetype)initWithModel:(OGAModel*)model error:(NSError**)error NS_DESIGNATED_INITIALIZER; +/** + * Set input with NamedTensors type. + * @param namedTensors The named tensors. + * @param error Optional error information set if an error occurs. + */ - (BOOL)setInputs:(OGANamedTensors*)namedTensors error:(NSError**)error; +/** + * Set input with raw input ids. + * @param rawPointer The pointer of the input ids data. + * @param inputIdsCount The count of input ids. + * @param sequenceLength The sequence length. + * @param batchSize The batch size. + * @param error Optional error information set if an error occurs. + */ - (BOOL)setInputIds:(const int32_t*)rawPointer inputIdsCount:(size_t)inputIdsCount sequenceLength:(size_t)sequenceLength batchSize:(size_t)batchSize error:(NSError**)error; - +/** + * Set input with sequences type. + * @param sequences The sequences. + * @param error Optional error information set if an error occurs. + */ - (BOOL)setInputSequences:(OGASequences*)sequences error:(NSError**)error; +/** + * Set input with name and corresponding tensor. + * @param name The input name. + * @param tensor The tensor. + * @param error Optional error information set if an error occurs. + */ - (BOOL)setModelInput:(NSString*)name tensor:(OGATensor*)tensor error:(NSError**)error; +/** + * Set double option value. + * @param key The option key. + * @param value The option value. + * @param error Optional error information set if an error occurs. + */ - (BOOL)setSearchOption:(NSString*)key doubleValue:(double)value error:(NSError**)error; - +/** + * Set boolean option value. + * @param key The option key. + * @param value The option value. + * @param error Optional error information set if an error occurs. + */ - (BOOL)setSearchOption:(NSString*)key boolValue:(BOOL)value error:(NSError**)error; @end +/** + * The main generator interface that can be used for generation loop. + */ @interface OGAGenerator : NSObject - (instancetype)init NS_UNAVAILABLE; +/** + * Creates a generator. + * + * @param model The model to use. + * @param params The generation params to use. + * @param error Optional error information set if an error occurs. + * @return The instance, or nil if an error occurs. + */ - (nullable instancetype)initWithModel:(OGAModel*)model params:(OGAGeneratorParams*)params error:(NSError**)error NS_DESIGNATED_INITIALIZER; - +/** + * Whether generation is done + * @param error Optional error information set if an error occurs. + * @return The result, or nil if an error occurs. + */ - (nullable NSNumber*)isDoneWithError:(NSError**)error NS_SWIFT_NAME(isDone()); - (BOOL)computeLogitsWithError:(NSError**)error NS_SWIFT_NAME(computeLogits()); - (BOOL)generateNextTokenWithError:(NSError**)error NS_SWIFT_NAME(generateNextToken()); - (OGATensor*)getOutput:(NSString*)name; -- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index; +/** + * Retrieve the sequence at the given index + * @param index The index needed. + * @param error Optional error information set if an error occurs. + * @return The last element, or nil if an error occurs. + */ +- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index + error:(NSError**)error; @end diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm index 58479ff7c..687dedb75 100644 --- a/src/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -51,14 +51,14 @@ - (OGATensor*)getOutput:(NSString*)name { return [[OGATensor alloc] initWithCXXPointer:std::move(output)]; } -- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index { +- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index + error:(NSError**)error { try { size_t sequenceLength = _generator->GetSequenceCount(index); const int32_t* data = _generator->GetSequenceData(index); return [[OGAInt32Span alloc] initWithDataPointer:data size:sequenceLength]; - } catch (std::exception) { - return nil; } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } @end diff --git a/src/objectivec/test/model_tests.mm b/src/objectivec/test/model_tests.mm index 0fb9dabb4..63868427c 100644 --- a/src/objectivec/test/model_tests.mm +++ b/src/objectivec/test/model_tests.mm @@ -60,7 +60,8 @@ - (void)GreedySearchGptFp32 { } for (int i = 0; i < batch_size; i++) { - OGAInt32Span *sequence = [generator sequenceAtIndex: i]; + OGAInt32Span *sequence = [generator sequenceAtIndex:i error:&error]; + ORTAssertNullableResultSuccessful(sequence, error); auto* expected_output_start = &expected_output[i * max_length]; XCTAssertTrue(0 == std::memcmp(expected_output_start, [sequence pointer], max_length * sizeof(int32_t))); } @@ -82,7 +83,6 @@ - (void)BeamSearchGptFp32 { const auto batch_size = input_ids_shape[0]; const auto input_sequence_length = input_ids_shape[1]; - NSBundle* bundle = [NSBundle mainBundle]; NSString* path = [[bundle resourcePath] stringByAppendingString:@"hf-internal-testing/tiny-random-gpt2-fp32"]; @@ -114,7 +114,8 @@ - (void)BeamSearchGptFp32 { } for (int i = 0; i < batch_size; i++) { - OGAInt32Span *sequence = [generator sequenceAtIndex: i]; + OGAInt32Span *sequence = [generator sequenceAtIndex:i error:&error]; + ORTAssertNullableResultSuccessful(sequence, error); auto* expected_output_start = &expected_output[i * max_length]; XCTAssertTrue(0 == std::memcmp(expected_output_start, [sequence pointer], max_length * sizeof(int32_t))); } From c4f389cfd2e12ff4f05759ae92ea0fa5efe5246b Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:24:10 +0800 Subject: [PATCH 43/59] update --- .../github/apple/get_simulator_device_info.py | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/tools/ci_build/github/apple/get_simulator_device_info.py b/tools/ci_build/github/apple/get_simulator_device_info.py index 7de9aa139..aa693038b 100755 --- a/tools/ci_build/github/apple/get_simulator_device_info.py +++ b/tools/ci_build/github/apple/get_simulator_device_info.py @@ -8,6 +8,7 @@ import functools import itertools import json +import os import subprocess @@ -37,7 +38,7 @@ def __lt__(self, other: Version) -> bool: def get_simulator_device_info( requested_runtime_platform: str = "iOS", requested_device_type_product_family: str = "iPhone", - max_runtime_version_str: str | None = None, + requested_runtime_version_str: str | None = None, ) -> dict[str, str]: """ Retrieves simulator device information from Xcode. @@ -45,11 +46,13 @@ def get_simulator_device_info( :param requested_runtime_platform: The runtime platform to select. :param requested_device_type_product_family: The device type product family to select. - :param max_runtime_version_str: The maximum runtime version to allow. + :param requested_runtime_version_str: The runtime version to select. If unspecified, selects the latest one. :return: A dictionary containing information about the selected simulator device. """ - max_runtime_version = Version(max_runtime_version_str) if max_runtime_version_str is not None else None + requested_runtime_version = ( + Version(requested_runtime_version_str) if requested_runtime_version_str is not None else None + ) simctl_proc = subprocess.run( ["xcrun", "simctl", "list", "--json", "--no-escape-slashes"], @@ -73,7 +76,7 @@ def runtime_filter(runtime) -> bool: if runtime["platform"] != requested_runtime_platform: return False - if max_runtime_version is not None and Version(runtime["version"]) > max_runtime_version: + if requested_runtime_version is not None and Version(runtime["version"]) != requested_runtime_version: return False return True @@ -108,6 +111,9 @@ def device_filter(device) -> bool: ): runtime_id_and_device_pairs.extend((runtime_id, device) for device in filter(device_filter, device_list)) + if len(runtime_id_and_device_pairs) == 0: + raise ValueError("Failed to find requested simulator device info.") + # sort key - tuple of (runtime version, device type min runtime version) # the secondary device type min runtime version value is to treat more recent device types as greater def runtime_id_and_device_pair_key(runtime_id_and_device_pair): @@ -137,13 +143,20 @@ def runtime_id_and_device_pair_key(runtime_id_and_device_pair): def main(): + requested_runtime_version_environment_variable_name = "ORT_GET_SIMULATOR_DEVICE_INFO_REQUESTED_RUNTIME_VERSION" + parser = argparse.ArgumentParser(description="Gets simulator info from Xcode and prints it in JSON format.") - parser.add_argument("--max-runtime-version", help="The maximum runtime version to allow.") + parser.add_argument( + "--requested-runtime-version", + default=os.environ.get(requested_runtime_version_environment_variable_name, None), + help="The requested runtime version. " + f"This may also be specified with the {requested_runtime_version_environment_variable_name} " + "environment variable. The command line option takes precedence. " + "An unspecified value means the latest available runtime version.", + ) args = parser.parse_args() - info = get_simulator_device_info( - max_runtime_version_str=args.max_runtime_version, - ) + info = get_simulator_device_info(requested_runtime_version_str=args.requested_runtime_version) print(json.dumps(info, indent=2)) From 5b1e1ef12cb6399a70c80a290cf85f1c64e8964a Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Wed, 23 Oct 2024 11:43:20 +0800 Subject: [PATCH 44/59] getOutput --- src/objectivec/include/ort_genai_objc.h | 20 +++++++++++++++++--- src/objectivec/oga_generator.mm | 10 +++++++--- src/objectivec/test/ort_genai_api_test.mm | 7 +++++-- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 1c480f36a..d27267024 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -332,17 +332,31 @@ typedef NS_ENUM(NSInteger, OGAElementType) { params:(OGAGeneratorParams*)params error:(NSError**)error NS_DESIGNATED_INITIALIZER; /** - * Whether generation is done + * Whether generation is done. * @param error Optional error information set if an error occurs. * @return The result, or nil if an error occurs. */ - (nullable NSNumber*)isDoneWithError:(NSError**)error NS_SWIFT_NAME(isDone()); +/** + * Compute logits based on the input. + * @param error Optional error information set if an error occurs. + */ - (BOOL)computeLogitsWithError:(NSError**)error NS_SWIFT_NAME(computeLogits()); +/** + * Generate next token + * @param error Optional error information set if an error occurs. + */ - (BOOL)generateNextTokenWithError:(NSError**)error NS_SWIFT_NAME(generateNextToken()); -- (OGATensor*)getOutput:(NSString*)name; +/** + * Get the output tensor. + * @param name The output name. + * @param error Optional error information set if an error occurs. + * @return The result, or nil if an error occurs. + */ +- (nullable OGATensor*)getOutput:(NSString*)name error:(NSError**)error; /** - * Retrieve the sequence at the given index + * Retrieve the sequence at the given index. * @param index The index needed. * @param error Optional error information set if an error occurs. * @return The last element, or nil if an error occurs. diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm index 687dedb75..896fe388f 100644 --- a/src/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -46,9 +46,13 @@ - (BOOL)generateNextTokenWithError:(NSError**)error { OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } -- (OGATensor*)getOutput:(NSString*)name { - std::unique_ptr output = _generator->GetOutput([name UTF8String]); - return [[OGATensor alloc] initWithCXXPointer:std::move(output)]; +- (nullable OGATensor*)getOutput:(NSString*)name + error:(NSError**)error { + try { + std::unique_ptr output = _generator->GetOutput([name UTF8String]); + return [[OGATensor alloc] initWithCXXPointer:std::move(output)]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } - (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index diff --git a/src/objectivec/test/ort_genai_api_test.mm b/src/objectivec/test/ort_genai_api_test.mm index b1c4aa95e..69bb430a6 100644 --- a/src/objectivec/test/ort_genai_api_test.mm +++ b/src/objectivec/test/ort_genai_api_test.mm @@ -83,7 +83,8 @@ - (void)GetOutput { ret = [generator computeLogitsWithError:&error]; ORTAssertBoolResultSuccessful(ret, error); - OGATensor* prompt_logits_ptr = [generator getOutput:@"logits"]; + OGATensor* prompt_logits_ptr = [generator getOutput:@"logits" error:&error]; + ORTAssertNullableResultSuccessful(prompt_logits_ptr, error); auto prompt_logits = static_cast([prompt_logits_ptr data]); const int num_prompt_outputs_to_check = 40; const int sample_size = 200; @@ -103,7 +104,9 @@ - (void)GetOutput { ret = [generator computeLogitsWithError:&error]; ORTAssertBoolResultSuccessful(ret, error); - OGATensor* token_gen_logits_ptr = [generator getOutput:@"logits"]; + OGATensor* token_gen_logits_ptr = [generator getOutput:@"logits" error:&error]; + ORTAssertNullableResultSuccessful(token_gen_logits_ptr, error); + auto token_gen_logits = static_cast([token_gen_logits_ptr data]); int num_token_gen_outputs_to_check = 10; From c9c31e318484e5bb0f2ead1da20c63603cae561b Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 24 Oct 2024 15:50:08 +0800 Subject: [PATCH 45/59] reviews --- src/objectivec/error_utils.mm | 3 +- src/objectivec/include/ort_genai_objc.h | 45 ++----- src/objectivec/oga_generator.mm | 3 + src/objectivec/oga_internal.h | 19 ++- src/objectivec/oga_internal.mm | 24 +++- src/objectivec/oga_sequences.mm | 12 +- src/objectivec/oga_tensor.mm | 12 +- src/objectivec/test/model_tests.mm | 125 ------------------ src/objectivec/test/ort_genai_api_test.mm | 28 ++-- .../objectivec/assemble_objc_pod_package.py | 14 +- .../apple/objectivec/objc.podspec.template | 5 + .../github/apple/package_assembly_utils.py | 5 +- 12 files changed, 96 insertions(+), 199 deletions(-) delete mode 100644 src/objectivec/test/model_tests.mm diff --git a/src/objectivec/error_utils.mm b/src/objectivec/error_utils.mm index d886ca922..f621af328 100644 --- a/src/objectivec/error_utils.mm +++ b/src/objectivec/error_utils.mm @@ -6,6 +6,7 @@ NS_ASSUME_NONNULL_BEGIN NSString* const kOgaErrorDomain = @"onnxruntime-genai"; +const int kOgaErrorCode = 0x0A; void OGASaveCodeAndDescriptionToError(int code, const char* descriptionCstr, NSError** error) { if (!error) return; @@ -27,7 +28,7 @@ void OGASaveCodeAndDescriptionToError(int code, NSString* description, NSError** } void OGASaveExceptionToError(const std::exception& e, NSError** error) { - OGASaveCodeAndDescriptionToError(0x0A, e.what(), error); + OGASaveCodeAndDescriptionToError(kOgaErrorCode, e.what(), error); } NS_ASSUME_NONNULL_END diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index d27267024..c47c50b92 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -177,46 +177,12 @@ typedef NS_ENUM(NSInteger, OGAElementType) { /** * The last element in this data sequence * @param error Optional error information set if an error occurs. - * @return The last element, or nil if an error occurs. + * @return The last element, or -1 if an error occurs. */ - (int32_t)lastElementWithError:(NSError**)error NS_SWIFT_NAME(lastElement()); @end -/** - * Wraps a raw int64_t pointer and its size. Represents a data sequence. - */ -@interface OGAInt64Span : NSObject - -- (instancetype)init NS_UNAVAILABLE; -/** - * Creates a span with underlying data. - * - * @param pointer The underlying data pointer to use. - * @param size The underlying data size. - * @return The instance, or nil if an error occurs. - */ -- (nullable instancetype)initWithDataPointer:(const int64_t*)pointer size:(size_t)size; - -/** - * The underlying data pointer - */ -- (const int64_t*)pointer; - -/** - * The underlying data size - */ -- (size_t)size; - -/** - * The last element in this data sequence - * @param error Optional error information set if an error occurs. - * @return The last element, or nil if an error occurs. - */ -- (int64_t)lastElementWithError:(NSError**)error NS_SWIFT_NAME(lastElement()); - -@end - /** * A series of generated sequences */ @@ -226,8 +192,10 @@ typedef NS_ENUM(NSInteger, OGAElementType) { /** * The count of generated sequences + * @param error Optional error information set if an error occurs. + * @return The count of sequences, or -1 if an error occurs. */ -- (size_t)count; +- (size_t)getCountWithError:(NSError**)error NS_SWIFT_NAME(getCount()); /** * Retrieve the sequence at the given index @@ -364,6 +332,11 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index error:(NSError**)error; +/** + * Clean up the resource before process exits. + */ ++ (void)shutdown; + @end @interface OGATensor : NSObject diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm index 896fe388f..41b1b71b9 100644 --- a/src/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -65,4 +65,7 @@ - (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } ++ (void)shutdown { + OgaShutdown(); +} @end diff --git a/src/objectivec/oga_internal.h b/src/objectivec/oga_internal.h index 606cec04f..921c3b46c 100644 --- a/src/objectivec/oga_internal.h +++ b/src/objectivec/oga_internal.h @@ -40,9 +40,23 @@ NS_ASSUME_NONNULL_BEGIN @end +@interface OGAInt64Span : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +- (nullable instancetype)initWithDataPointer:(const int64_t*)pointer size:(size_t)size NS_DESIGNATED_INITIALIZER; + +- (const int64_t*)pointer; + +- (size_t)size; + +- (int64_t)lastElementWithError:(NSError**)error NS_SWIFT_NAME(lastElement()); + +@end + @interface OGATensor () -- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr; +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr NS_DESIGNATED_INITIALIZER; - (OgaTensor&)CXXAPIOgaTensor; @@ -50,8 +64,7 @@ NS_ASSUME_NONNULL_BEGIN @interface OGANamedTensors () -- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr - NS_DESIGNATED_INITIALIZER; +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr NS_DESIGNATED_INITIALIZER; - (OgaNamedTensors&)CXXAPIOgaNamedTensors; @end diff --git a/src/objectivec/oga_internal.mm b/src/objectivec/oga_internal.mm index d7e3bf200..175a5b185 100644 --- a/src/objectivec/oga_internal.mm +++ b/src/objectivec/oga_internal.mm @@ -9,9 +9,13 @@ @implementation OGAInt32Span { } - (nullable instancetype)initWithDataPointer:(const int32_t*)pointer size:(size_t)size { + if ((self = [super init]) == nil) { + return nil; + } + _ptr = pointer; _size = size; - return [self init]; + return self; } - (const int32_t*)pointer { @@ -24,8 +28,10 @@ - (size_t)size { - (int32_t)lastElementWithError:(NSError**)error { if (_size == 0) { - NSDictionary *errorDictionary = @{NSLocalizedDescriptionKey : @"The size of this span is invalid"}; - *error = [[NSError alloc] initWithDomain:kOgaErrorDomain code:-1 userInfo:errorDictionary]; + if (error != nil) { + NSDictionary *errorDictionary = @{NSLocalizedDescriptionKey : @"The size of this span is invalid"}; + *error = [[NSError alloc] initWithDomain:kOgaErrorDomain code:-1 userInfo:errorDictionary]; + } return -1; } return *(_ptr + (_size - 1)); @@ -39,9 +45,13 @@ @implementation OGAInt64Span { } - (nullable instancetype)initWithDataPointer:(const int64_t*)pointer size:(size_t)size { + if ((self = [super init]) == nil) { + return nil; + } + _ptr = pointer; _size = size; - return [self init]; + return self; } - (const int64_t*)pointer { @@ -54,8 +64,10 @@ - (size_t)size { - (int64_t)lastElementWithError:(NSError**)error { if (_size == 0) { - NSDictionary *errorDictionary = @{NSLocalizedDescriptionKey : @"The size of this span is invalid"}; - *error = [[NSError alloc] initWithDomain:kOgaErrorDomain code:-1 userInfo:errorDictionary]; + if (error != nil) { + NSDictionary *errorDictionary = @{NSLocalizedDescriptionKey : @"The size of this span is invalid"}; + *error = [[NSError alloc] initWithDomain:kOgaErrorDomain code:-1 userInfo:errorDictionary]; + } return -1; } return *(_ptr + (_size - 1)); diff --git a/src/objectivec/oga_sequences.mm b/src/objectivec/oga_sequences.mm index 353444e06..66df209ed 100644 --- a/src/objectivec/oga_sequences.mm +++ b/src/objectivec/oga_sequences.mm @@ -26,17 +26,15 @@ - (nullable instancetype)initWithError:(NSError**)error { OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (size_t)count { - return _sequences->Count(); +- (size_t)getCountWithError:(NSError**)error { + try { + return _sequences->Count(); + } + OGA_OBJC_API_IMPL_CATCH(error, -1) } - (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index error:(NSError**)error { - if (index >= [self count]) { - NSDictionary *errorDictionary = @{NSLocalizedDescriptionKey : @"The index is out of bounds"}; - *error = [[NSError alloc] initWithDomain:kOgaErrorDomain code:-1 userInfo:errorDictionary]; - return nil; - } try { size_t sequenceLength = _sequences->SequenceCount(index); const int32_t* data = _sequences->SequenceData(index); diff --git a/src/objectivec/oga_tensor.mm b/src/objectivec/oga_tensor.mm index 1b7ffcc26..381e29536 100644 --- a/src/objectivec/oga_tensor.mm +++ b/src/objectivec/oga_tensor.mm @@ -10,6 +10,10 @@ @implementation OGATensor { } - (instancetype)initWithCXXPointer:(std::unique_ptr)ptr { + if ((self = [super init]) == nil) { + return nil; + } + _tensor = std::move(ptr); return self; } @@ -18,17 +22,13 @@ - (nullable instancetype)initWithDataPointer:(void*)data shape:(NSArray*)shape type:(OGAElementType)elementType error:(NSError**)error { - if ((self = [super init]) == nil) { - return nil; - } - try { std::vector cxxShape; for (NSNumber* object in shape) { cxxShape.push_back([object intValue]); } - _tensor = OgaTensor::Create(data, cxxShape.data(), cxxShape.size(), - static_cast(elementType)); + self = [self initWithCXXPointer:OgaTensor::Create(data, cxxShape.data(), cxxShape.size(), + static_cast(elementType))]; return self; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) diff --git a/src/objectivec/test/model_tests.mm b/src/objectivec/test/model_tests.mm deleted file mode 100644 index 63868427c..000000000 --- a/src/objectivec/test/model_tests.mm +++ /dev/null @@ -1,125 +0,0 @@ - -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#import - -#import "ort_genai_objc.h" -#import "assertion_utils.h" -#import -#import - -NS_ASSUME_NONNULL_BEGIN - -@interface ORTGenAIModelTest : XCTestCase - -@end - -@implementation ORTGenAIModelTest - - -- (void)GreedySearchGptFp32 { - std::vector input_ids_shape{2, 4}; - std::vector input_ids{0, 0, 0, 52, 0, 0, 195, 731}; - - std::vector expected_output{ - 0, 0, 0, 52, 204, 204, 204, 204, 204, 204, - 0, 0, 195, 731, 731, 114, 114, 114, 114, 114}; - - const auto max_length = 10; - const auto batch_size = input_ids_shape[0]; - const auto input_sequence_length = input_ids_shape[1]; - - NSBundle* bundle = [NSBundle mainBundle]; - NSString* path = [[bundle resourcePath] stringByAppendingString:@"hf-internal-testing/tiny-random-gpt2-fp32"]; - - NSError *error = nil; - OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; - ORTAssertNullableResultSuccessful(model, error); - - OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; - ORTAssertNullableResultSuccessful(params, error); - - [params setSearchOption:@"max_length" doubleValue:max_length error:&error]; - [params setSearchOption:@"do_sample" boolValue:YES error:&error]; - [params setSearchOption:@"top_p" doubleValue:0.25 error:&error]; - - [params setInputIds:input_ids.data() - inputIdsCount:input_ids.size() - sequenceLength:input_sequence_length - batchSize:batch_size - error:&error]; - OGAGenerator* generator = [[OGAGenerator alloc] initWithModel:model - params:params - error:&error]; - ORTAssertNullableResultSuccessful(generator, error); - - while (![[generator isDoneWithError:&error] boolValue]) { - [generator computeLogitsWithError:&error]; - [generator generateNextTokenWithError:&error]; - } - - for (int i = 0; i < batch_size; i++) { - OGAInt32Span *sequence = [generator sequenceAtIndex:i error:&error]; - ORTAssertNullableResultSuccessful(sequence, error); - auto* expected_output_start = &expected_output[i * max_length]; - XCTAssertTrue(0 == std::memcmp(expected_output_start, [sequence pointer], max_length * sizeof(int32_t))); - } -} - -- (void)BeamSearchGptFp32 { - std::vector input_ids_shape{3, 12}; - std::vector input_ids{ - 0, 0, 0, 0, 0, 52, 195, 731, 321, 301, 734, 620, - 41, 554, 74, 622, 206, 222, 75, 223, 221, 198, 224, 572, - 0, 0, 0, 52, 328, 219, 328, 206, 288, 227, 896, 328}; - - std::vector expected_output{ - 0, 0, 0, 0, 0, 52, 195, 731, 321, 301, 734, 620, 131, 131, 131, 181, 638, 638, 638, 638, - 41, 554, 74, 622, 206, 222, 75, 223, 221, 198, 224, 572, 292, 292, 292, 292, 292, 292, 292, 292, - 0, 0, 0, 52, 328, 219, 328, 206, 288, 227, 896, 328, 328, 669, 669, 669, 669, 669, 669, 669}; - - const auto max_length = 20; - const auto batch_size = input_ids_shape[0]; - const auto input_sequence_length = input_ids_shape[1]; - - NSBundle* bundle = [NSBundle mainBundle]; - NSString* path = [[bundle resourcePath] stringByAppendingString:@"hf-internal-testing/tiny-random-gpt2-fp32"]; - - NSError *error = nil; - OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; - ORTAssertNullableResultSuccessful(model, error); - - OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; - ORTAssertNullableResultSuccessful(params, error); - - [params setSearchOption:@"max_length" doubleValue:max_length error:&error]; - [params setSearchOption:@"length_penalty" doubleValue:1.0f error:&error]; - [params setSearchOption:@"num_beams" doubleValue:4 error:&error]; - - [params setInputIds:input_ids.data() - inputIdsCount:input_ids.size() - sequenceLength:input_sequence_length - batchSize:batch_size - error:&error]; - - OGAGenerator* generator = [[OGAGenerator alloc] initWithModel:model - params:params - error:&error]; - - ORTAssertNullableResultSuccessful(generator, error); - while (![[generator isDoneWithError:&error] boolValue]) { - [generator computeLogitsWithError:&error]; - [generator generateNextTokenWithError:&error]; - } - - for (int i = 0; i < batch_size; i++) { - OGAInt32Span *sequence = [generator sequenceAtIndex:i error:&error]; - ORTAssertNullableResultSuccessful(sequence, error); - auto* expected_output_start = &expected_output[i * max_length]; - XCTAssertTrue(0 == std::memcmp(expected_output_start, [sequence pointer], max_length * sizeof(int32_t))); - } -} -@end - -NS_ASSUME_NONNULL_END \ No newline at end of file diff --git a/src/objectivec/test/ort_genai_api_test.mm b/src/objectivec/test/ort_genai_api_test.mm index 69bb430a6..343aa1bdd 100644 --- a/src/objectivec/test/ort_genai_api_test.mm +++ b/src/objectivec/test/ort_genai_api_test.mm @@ -16,20 +16,31 @@ @interface ORTGenAIAPITest : XCTestCase @implementation ORTGenAIAPITest +- (void)setUp { + [super setUp]; + self.continueAfterFailure = NO; +} + ++ (void)tearDown { + [OGAGenerator shutdown]; +} -- (void)Tensor_And_AddExtraInput { ++ (NSString*)getModelPath { + NSBundle* bundle = [NSBundle bundleForClass:[ORTGenAIAPITest class]]; + NSString* path = [[bundle resourcePath] stringByAppendingString:@"/tiny-random-gpt2-fp32"]; + return path; +} + +- (void)testTensor_And_AddExtraInput { // Create a [3 4] shaped tensor std::array data{0, 1, 2, 3, 10, 11, 12, 13, 20, 21, 22, 23}; NSArray* shape = @[@3, @4]; - NSBundle* bundle = [NSBundle mainBundle]; - NSString* path = [[bundle resourcePath] stringByAppendingString:@"hf-internal-testing/tiny-random-gpt2-fp32"]; - NSError *error = nil; BOOL ret = NO; - OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; + OGAModel* model = [[OGAModel alloc] initWithPath:[ORTGenAIAPITest getModelPath] error:&error]; ORTAssertNullableResultSuccessful(model, error); OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; @@ -41,19 +52,16 @@ - (void)Tensor_And_AddExtraInput { ORTAssertBoolResultSuccessful(ret, error); } -- (void)GetOutput { +- (void)testGetOutput { std::vector input_ids_shape{2, 4}; std::vector input_ids{0, 0, 0, 52, 0, 0, 195, 731}; const auto batch_size = input_ids_shape[0]; const auto input_sequence_length = input_ids_shape[1]; int max_length = 10; - NSBundle* bundle = [NSBundle mainBundle]; - NSString* path = [[bundle resourcePath] stringByAppendingString:@"hf-internal-testing/tiny-random-gpt2-fp32"]; - NSError *error = nil; BOOL ret = NO; - OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; + OGAModel* model = [[OGAModel alloc] initWithPath:[ORTGenAIAPITest getModelPath] error:&error]; ORTAssertNullableResultSuccessful(model, error); OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; diff --git a/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py index 05e144965..c2723e398 100755 --- a/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py +++ b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py @@ -45,7 +45,10 @@ "objectivec/test/*.h", "objectivec/test/*.m", "objectivec/test/*.mm", - ] + ], + "test_resource_files": [ + "test/test_models/hf-internal-testing/tiny-random-gpt2-fp32", + ], } def get_pod_files(package_variant: PackageVariant): @@ -55,6 +58,7 @@ def get_pod_files(package_variant: PackageVariant): filtered_pod_files = {} for key in all_objc_files: filtered_pod_files[key] = filter_files("src", all_objc_files[key], []) + filtered_pod_files[key].extend(filter_files(None, all_objc_files[key], [])) return filtered_pod_files @@ -99,10 +103,12 @@ def assemble_objc_pod_package( # copy the necessary files to the staging directory need_copy = [*pod_files["source_files"], *pod_files["test_source_files"]] - if "test_resource_files" in pod_files: - need_copy.append(*pod_files["test_resource_files"]) copy_repo_relative_to_dir("src", need_copy, staging_dir) + if "test_resource_files" in pod_files: + need_copy = [*pod_files["test_resource_files"]] + copy_repo_relative_to_dir(None, need_copy, staging_dir) + # generate the podspec file from the template def path_patterns_as_variable_value(patterns: list[str]): @@ -121,7 +127,7 @@ def path_patterns_as_variable_value(patterns: list[str]): "PUBLIC_HEADER_FILE_LIST": path_patterns_as_variable_value(pod_files["public_header_files"]), "SOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["source_files"]), "SUMMARY": pod_config["summary"], - # "TEST_RESOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_resource_files"]), + "TEST_RESOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_resource_files"]), "TEST_SOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_source_files"]), "VERSION": pod_version, "ORT_VERSION": ort_version diff --git a/tools/ci_build/github/apple/objectivec/objc.podspec.template b/tools/ci_build/github/apple/objectivec/objc.podspec.template index cac187946..33e3b6888 100644 --- a/tools/ci_build/github/apple/objectivec/objc.podspec.template +++ b/tools/ci_build/github/apple/objectivec/objc.podspec.template @@ -37,12 +37,17 @@ Pod::Spec.new do |s| test.source_files = [ @TEST_SOURCE_FILE_LIST@ ] + + test.resources = [ + @TEST_RESOURCE_FILE_LIST@ + ] end core.pod_target_xcconfig = { "HEADER_SEARCH_PATHS" => include_dirs.join(" "), "OTHER_CPLUSPLUSFLAGS" => "-fvisibility=hidden -fvisibility-inlines-hidden", } + end s.dependency 'onnxruntime-objc', '~> @ORT_VERSION@' end diff --git a/tools/ci_build/github/apple/package_assembly_utils.py b/tools/ci_build/github/apple/package_assembly_utils.py index 23cfc0508..8d1e8f2a0 100644 --- a/tools/ci_build/github/apple/package_assembly_utils.py +++ b/tools/ci_build/github/apple/package_assembly_utils.py @@ -108,7 +108,10 @@ def copy_repo_relative_to_dir(subpath: str, patterns: List[str], dest_dir: pathl repo_relative_path = path.relative_to(src_root) dst_path = dest_dir / repo_relative_path os.makedirs(dst_path.parent, exist_ok=True) - shutil.copy(path, dst_path) + if os.path.isdir(path): + shutil.copytree(path, dst_path) + else: + shutil.copy(path, dst_path) def load_json_config(json_config_file: pathlib.Path): From 8d4173946549fee2e7f4afa3148a516de54b764c Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Mon, 28 Oct 2024 11:58:09 +0800 Subject: [PATCH 46/59] review --- .../macos-ios-cocoapods-packaging-job.yml | 1 + src/objectivec/cxx_api.h | 2 +- src/objectivec/include/ort_genai_objc.h | 14 ++++---- src/objectivec/oga_internal.h | 14 -------- src/objectivec/oga_internal.mm | 36 ------------------- src/objectivec/oga_tensor.mm | 16 ++++++--- src/objectivec/oga_tokenizer.mm | 2 +- src/objectivec/test/ort_genai_api_test.mm | 4 +-- .../github/apple/c/c.podspec.template | 1 + .../github/apple/test_apple_packages.py | 12 +++++++ 10 files changed, 36 insertions(+), 66 deletions(-) diff --git a/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml b/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml index 58b47f72b..a2002f0f7 100644 --- a/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml +++ b/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml @@ -37,4 +37,5 @@ jobs: --test \ --variant Full \ --build-settings-file "${{ variables.buildSettingsFile }}" \ + --build-apple-framework-args==--config=$(cmake_build_type) \ --ort-version ${{parameters.ort_version}} diff --git a/src/objectivec/cxx_api.h b/src/objectivec/cxx_api.h index d1f5fa06f..ded5ca4cd 100644 --- a/src/objectivec/cxx_api.h +++ b/src/objectivec/cxx_api.h @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -// wrapper for ORT C/C++ API headers +// wrapper for ORT GenAI C/C++ API headers #if defined(__clang__) #pragma clang diagnostic push diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index c47c50b92..5aa2390bb 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -13,7 +13,7 @@ * OGASequences* sequences = [tokenizer encode:@"A great recipe for Kung Pao chicken is " error:&error]; * * OGAGeneratorParams* params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; - * [params setInputSequences:sequences]; + * [params setInputSequences:sequences error:&error]; * [params setSearchOption:@"max_length" doubleValue:200 error:&error]; * * OGASequences* output_sequences = [model generate:params error:&error]; @@ -116,7 +116,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @end /** - * A tokenizer stream is to decoded token strings incrementally, one token at a time. + * A tokenizer stream is used to decode token strings incrementally, one token at a time. */ @interface OGATokenizerStream : NSObject - (instancetype)init NS_UNAVAILABLE; @@ -195,13 +195,13 @@ typedef NS_ENUM(NSInteger, OGAElementType) { * @param error Optional error information set if an error occurs. * @return The count of sequences, or -1 if an error occurs. */ -- (size_t)getCountWithError:(NSError**)error NS_SWIFT_NAME(getCount()); +- (size_t)getCountWithError:(NSError**)error NS_SWIFT_NAME(count()); /** * Retrieve the sequence at the given index * @param index The index needed. * @param error Optional error information set if an error occurs. - * @return The last element, or nil if an error occurs. + * @return The sequence at the given index, or nil if an error occurs. */ - (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index error:(NSError**)error; @@ -327,7 +327,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { * Retrieve the sequence at the given index. * @param index The index needed. * @param error Optional error information set if an error occurs. - * @return The last element, or nil if an error occurs. + * @return The sequence at the given index, or nil if an error occurs. */ - (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index error:(NSError**)error; @@ -346,8 +346,8 @@ typedef NS_ENUM(NSInteger, OGAElementType) { shape:(NSArray*)shape type:(OGAElementType)elementType error:(NSError**)error; -- (OGAElementType)type; -- (void*)data; +- (OGAElementType)getTypeWithError:(NSError**)error NS_SWIFT_NAME(type()); +- (nullable void*)getDataPointerWithError:(NSError**)error NS_SWIFT_NAME(dataPointer());; @end diff --git a/src/objectivec/oga_internal.h b/src/objectivec/oga_internal.h index 921c3b46c..069268964 100644 --- a/src/objectivec/oga_internal.h +++ b/src/objectivec/oga_internal.h @@ -40,20 +40,6 @@ NS_ASSUME_NONNULL_BEGIN @end -@interface OGAInt64Span : NSObject - -- (instancetype)init NS_UNAVAILABLE; - -- (nullable instancetype)initWithDataPointer:(const int64_t*)pointer size:(size_t)size NS_DESIGNATED_INITIALIZER; - -- (const int64_t*)pointer; - -- (size_t)size; - -- (int64_t)lastElementWithError:(NSError**)error NS_SWIFT_NAME(lastElement()); - -@end - @interface OGATensor () - (instancetype)initWithCXXPointer:(std::unique_ptr)ptr NS_DESIGNATED_INITIALIZER; diff --git a/src/objectivec/oga_internal.mm b/src/objectivec/oga_internal.mm index 175a5b185..765a8b3eb 100644 --- a/src/objectivec/oga_internal.mm +++ b/src/objectivec/oga_internal.mm @@ -38,39 +38,3 @@ - (int32_t)lastElementWithError:(NSError**)error { } @end - -@implementation OGAInt64Span { - const int64_t* _ptr; - size_t _size; -} - -- (nullable instancetype)initWithDataPointer:(const int64_t*)pointer size:(size_t)size { - if ((self = [super init]) == nil) { - return nil; - } - - _ptr = pointer; - _size = size; - return self; -} - -- (const int64_t*)pointer { - return _ptr; -} - -- (size_t)size { - return _size; -} - -- (int64_t)lastElementWithError:(NSError**)error { - if (_size == 0) { - if (error != nil) { - NSDictionary *errorDictionary = @{NSLocalizedDescriptionKey : @"The size of this span is invalid"}; - *error = [[NSError alloc] initWithDomain:kOgaErrorDomain code:-1 userInfo:errorDictionary]; - } - return -1; - } - return *(_ptr + (_size - 1)); -} - -@end diff --git a/src/objectivec/oga_tensor.mm b/src/objectivec/oga_tensor.mm index 381e29536..d8cd28d26 100644 --- a/src/objectivec/oga_tensor.mm +++ b/src/objectivec/oga_tensor.mm @@ -25,7 +25,7 @@ - (nullable instancetype)initWithDataPointer:(void*)data try { std::vector cxxShape; for (NSNumber* object in shape) { - cxxShape.push_back([object intValue]); + cxxShape.push_back([object longLongValue]); } self = [self initWithCXXPointer:OgaTensor::Create(data, cxxShape.data(), cxxShape.size(), static_cast(elementType))]; @@ -34,12 +34,18 @@ - (nullable instancetype)initWithDataPointer:(void*)data OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (OGAElementType)type { - return OGAElementType(_tensor->Type()); +- (OGAElementType)getTypeWithError:(NSError**)error { + try { + return OGAElementType(_tensor->Type()); + } + OGA_OBJC_API_IMPL_CATCH(error, OGAElementTypeUndefined) } -- (void*)data { - return _tensor->Data(); +- (nullable void*)getDataPointerWithError:(NSError**)error { + try { + return _tensor->Data(); + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } - (OgaTensor&)CXXAPIOgaTensor { diff --git a/src/objectivec/oga_tokenizer.mm b/src/objectivec/oga_tokenizer.mm index f052efd94..31f989ef4 100644 --- a/src/objectivec/oga_tokenizer.mm +++ b/src/objectivec/oga_tokenizer.mm @@ -23,7 +23,7 @@ - (nullable instancetype)initWithModel:(OGAModel*)model error:(NSError**)error { - (nullable OGASequences*)encode:(NSString*)str error:(NSError**)error { OGASequences* sequences = [[OGASequences alloc] initWithError:error]; - if (*error) { + if (!sequences) { return nil; } try { diff --git a/src/objectivec/test/ort_genai_api_test.mm b/src/objectivec/test/ort_genai_api_test.mm index 343aa1bdd..30247049c 100644 --- a/src/objectivec/test/ort_genai_api_test.mm +++ b/src/objectivec/test/ort_genai_api_test.mm @@ -93,7 +93,7 @@ - (void)testGetOutput { ORTAssertBoolResultSuccessful(ret, error); OGATensor* prompt_logits_ptr = [generator getOutput:@"logits" error:&error]; ORTAssertNullableResultSuccessful(prompt_logits_ptr, error); - auto prompt_logits = static_cast([prompt_logits_ptr data]); + auto prompt_logits = static_cast([prompt_logits_ptr getDataPointerWithError:&error]); const int num_prompt_outputs_to_check = 40; const int sample_size = 200; const float tolerance = 0.001f; @@ -115,7 +115,7 @@ - (void)testGetOutput { OGATensor* token_gen_logits_ptr = [generator getOutput:@"logits" error:&error]; ORTAssertNullableResultSuccessful(token_gen_logits_ptr, error); - auto token_gen_logits = static_cast([token_gen_logits_ptr data]); + auto token_gen_logits = static_cast([token_gen_logits_ptr getDataPointerWithError:&error]); int num_token_gen_outputs_to_check = 10; for (int i = 0; i < num_token_gen_outputs_to_check; i++) { diff --git a/tools/ci_build/github/apple/c/c.podspec.template b/tools/ci_build/github/apple/c/c.podspec.template index ccb4dec31..7927add89 100644 --- a/tools/ci_build/github/apple/c/c.podspec.template +++ b/tools/ci_build/github/apple/c/c.podspec.template @@ -12,6 +12,7 @@ Pod::Spec.new do |s| s.vendored_frameworks = "@ORTGENAI_C_FRAMEWORK@" s.static_framework = true + s.framework = ['ImageIO', 'CoreGraphics'] s.weak_framework = [ @WEAK_FRAMEWORK@ ] s.source_files = "@ORTGENAI_C_HEADERS_DIR@/*.h" diff --git a/tools/ci_build/github/apple/test_apple_packages.py b/tools/ci_build/github/apple/test_apple_packages.py index 07b6e6525..183c539e8 100644 --- a/tools/ci_build/github/apple/test_apple_packages.py +++ b/tools/ci_build/github/apple/test_apple_packages.py @@ -12,6 +12,8 @@ import sys import tempfile +from huggingface_hub import snapshot_download + from c.assemble_c_pod_package import assemble_c_pod_package from package_assembly_utils import PackageVariant, gen_file_from_template, get_ort_version @@ -119,6 +121,16 @@ def _test_apple_packages(args): env["SKIP_MACOS_TEST"] = "true" if args.skip_macos_test else "false" subprocess.run(["pod", "install"], shell=False, check=True, cwd=target_proj_path, env=env) + # Download phi3 model + model_dir = target_proj_path / "models" / "Phi-3-mini-4k-instruct-onnx" + + print(f"Downloading models:\n{model_dir}") + snapshot_download( + repo_id="microsoft/Phi-3-mini-4k-instruct-onnx", + allow_patterns="cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4/*", + local_dir=model_dir + ) + # run the tests if not args.prepare_test_project_only: simulator_device_info = subprocess.check_output( From 7b068e83502061f203ecab74d37d386625e2ceb2 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Mon, 28 Oct 2024 12:06:43 +0800 Subject: [PATCH 47/59] clang-format --- src/objectivec/include/ort_genai_objc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 5aa2390bb..df11107a3 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -347,7 +347,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { type:(OGAElementType)elementType error:(NSError**)error; - (OGAElementType)getTypeWithError:(NSError**)error NS_SWIFT_NAME(type()); -- (nullable void*)getDataPointerWithError:(NSError**)error NS_SWIFT_NAME(dataPointer());; +- (nullable void*)getDataPointerWithError:(NSError**)error NS_SWIFT_NAME(dataPointer()); @end From 094fb41384c63a6b83c56cd661fffcb6dcfd9a6d Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:01:59 +0800 Subject: [PATCH 48/59] reviews --- src/objectivec/include/ort_genai_objc.h | 6 +++--- src/objectivec/oga_generator.mm | 6 +++--- src/objectivec/oga_sequences.mm | 2 +- .../ci_build/github/apple/build_and_assemble_apple_pods.py | 7 +++---- .../github/apple/objectivec/assemble_objc_pod_package.py | 7 +------ .../ci_build/github/apple/objectivec/objc.podspec.template | 1 - tools/ci_build/github/apple/package_assembly_utils.py | 2 +- tools/ci_build/github/apple/test_apple_packages.py | 4 ++-- 8 files changed, 14 insertions(+), 21 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index df11107a3..dc705357d 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -193,7 +193,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { /** * The count of generated sequences * @param error Optional error information set if an error occurs. - * @return The count of sequences, or -1 if an error occurs. + * @return The count of sequences, or size_t(-1) if an error occurs. */ - (size_t)getCountWithError:(NSError**)error NS_SWIFT_NAME(count()); @@ -302,9 +302,9 @@ typedef NS_ENUM(NSInteger, OGAElementType) { /** * Whether generation is done. * @param error Optional error information set if an error occurs. - * @return The result, or nil if an error occurs. + * @return The result, or false if an error occurs. */ -- (nullable NSNumber*)isDoneWithError:(NSError**)error NS_SWIFT_NAME(isDone()); +- (BOOL)isDoneWithError:(NSError**)error NS_SWIFT_NAME(isDone()) __attribute__((swift_error(nonnull_error))); /** * Compute logits based on the input. * @param error Optional error information set if an error occurs. diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm index 41b1b71b9..6681206af 100644 --- a/src/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -23,11 +23,11 @@ - (nullable instancetype)initWithModel:(OGAModel*)model OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (NSNumber*)isDoneWithError:(NSError**)error { +- (BOOL)isDoneWithError:(NSError**)error { try { - return [NSNumber numberWithBool:_generator->IsDone()]; + return _generator->IsDone(); } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } - (BOOL)computeLogitsWithError:(NSError**)error { diff --git a/src/objectivec/oga_sequences.mm b/src/objectivec/oga_sequences.mm index 66df209ed..6a07abfcb 100644 --- a/src/objectivec/oga_sequences.mm +++ b/src/objectivec/oga_sequences.mm @@ -30,7 +30,7 @@ - (size_t)getCountWithError:(NSError**)error { try { return _sequences->Count(); } - OGA_OBJC_API_IMPL_CATCH(error, -1) + OGA_OBJC_API_IMPL_CATCH(error, size_t(-1)) } - (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index diff --git a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py index 7edb8718e..3f35a66f9 100755 --- a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py +++ b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py @@ -12,7 +12,7 @@ from c.assemble_c_pod_package import assemble_c_pod_package from objectivec.assemble_objc_pod_package import assemble_objc_pod_package -from package_assembly_utils import PackageVariant, get_ort_version +from package_assembly_utils import PackageVariant, get_ort_genai_version SCRIPT_PATH = pathlib.Path(__file__).resolve() SCRIPT_DIR = SCRIPT_PATH.parent @@ -45,7 +45,7 @@ def parse_args(): parser.add_argument( "--pod-version", - default=f"{get_ort_version()}-local", + default=f"{get_ort_genai_version()}-local", help="The version string of the pod. The same version is used for all pods.", ) @@ -181,8 +181,7 @@ def main(): staging_dir=objc_pod_staging_dir, pod_version=args.pod_version, framework_info_file=framework_info_file, - package_variant=package_variant, - ort_version=args.ort_version + package_variant=package_variant ) if args.test: diff --git a/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py index c2723e398..e11774b70 100755 --- a/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py +++ b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py @@ -73,7 +73,7 @@ def get_pod_config_file(package_variant: PackageVariant): def assemble_objc_pod_package( - staging_dir: pathlib.Path, pod_version: str, framework_info_file: pathlib.Path, package_variant: PackageVariant, ort_version: str + staging_dir: pathlib.Path, pod_version: str, framework_info_file: pathlib.Path, package_variant: PackageVariant ): """ Assembles the files for the Objective-C pod package in a staging directory. @@ -130,7 +130,6 @@ def path_patterns_as_variable_value(patterns: list[str]): "TEST_RESOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_resource_files"]), "TEST_SOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_source_files"]), "VERSION": pod_version, - "ORT_VERSION": ort_version } podspec_template = _script_dir / "objc.podspec.template" @@ -166,9 +165,6 @@ def parse_args(): parser.add_argument( "--variant", choices=PackageVariant.release_variant_names(), required=True, help="Pod package variant." ) - parser.add_argument( - "--ort-version", required=True, help="The ORT version to depend on." - ) return parser.parse_args() @@ -180,7 +176,6 @@ def main(): pod_version=args.pod_version, framework_info_file=args.framework_info_file, package_variant=PackageVariant[args.variant], - ort_version=args.ort_versoin ) return 0 diff --git a/tools/ci_build/github/apple/objectivec/objc.podspec.template b/tools/ci_build/github/apple/objectivec/objc.podspec.template index 33e3b6888..a5f543276 100644 --- a/tools/ci_build/github/apple/objectivec/objc.podspec.template +++ b/tools/ci_build/github/apple/objectivec/objc.podspec.template @@ -49,6 +49,5 @@ Pod::Spec.new do |s| } end - s.dependency 'onnxruntime-objc', '~> @ORT_VERSION@' end diff --git a/tools/ci_build/github/apple/package_assembly_utils.py b/tools/ci_build/github/apple/package_assembly_utils.py index 8d1e8f2a0..cca1c6eb1 100644 --- a/tools/ci_build/github/apple/package_assembly_utils.py +++ b/tools/ci_build/github/apple/package_assembly_utils.py @@ -163,7 +163,7 @@ def get_podspec_values(framework_info): return (ios_deployment_target, macos_deployment_target, weak_framework) -def get_ort_version(): +def get_ort_genai_version(): """ Gets the version string from the repo. diff --git a/tools/ci_build/github/apple/test_apple_packages.py b/tools/ci_build/github/apple/test_apple_packages.py index 183c539e8..a16ad89b0 100644 --- a/tools/ci_build/github/apple/test_apple_packages.py +++ b/tools/ci_build/github/apple/test_apple_packages.py @@ -15,7 +15,7 @@ from huggingface_hub import snapshot_download from c.assemble_c_pod_package import assemble_c_pod_package -from package_assembly_utils import PackageVariant, gen_file_from_template, get_ort_version +from package_assembly_utils import PackageVariant, gen_file_from_template, get_ort_genai_version SCRIPT_PATH = pathlib.Path(__file__).resolve(strict=True) REPO_DIR = SCRIPT_PATH.parents[4] @@ -79,7 +79,7 @@ def _test_apple_packages(args): pod_name, podspec = assemble_c_pod_package( staging_dir=local_pods_dir, - pod_version=get_ort_version(), + pod_version=get_ort_genai_version(), framework_info_file=args.framework_info_file, public_headers_dir=public_headers_dir, framework_dir=framework_dir, From 48e4e11f71770aa376d710c67250eb450ed2142e Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 26 Nov 2024 14:45:35 +0800 Subject: [PATCH 49/59] sync main --- src/objectivec/include/ort_genai_objc.h | 57 ++++++++--------------- src/objectivec/oga_generator.mm | 25 +++++++++- src/objectivec/oga_generator_params.mm | 20 -------- src/objectivec/oga_internal.h | 5 ++ src/objectivec/oga_model.mm | 9 ---- src/objectivec/test/ort_genai_api_test.mm | 53 ++++++++++++--------- 6 files changed, 78 insertions(+), 91 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index dc705357d..888ec8f63 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -64,17 +64,6 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (nullable instancetype)initWithPath:(NSString*)path error:(NSError**)error NS_DESIGNATED_INITIALIZER; -/** - * Generate sequences with the model. - * The inputs and outputs are pre-allocated. - * - * @param params The generation params to use. - * @param error Optional error information set if an error occurs. - * @return The generated sequences. - */ -- (nullable OGASequences*)generate:(OGAGeneratorParams*)params - error:(NSError**)error; - @end /** @@ -231,27 +220,6 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (BOOL)setInputs:(OGANamedTensors*)namedTensors error:(NSError**)error; -/** - * Set input with raw input ids. - * @param rawPointer The pointer of the input ids data. - * @param inputIdsCount The count of input ids. - * @param sequenceLength The sequence length. - * @param batchSize The batch size. - * @param error Optional error information set if an error occurs. - */ -- (BOOL)setInputIds:(const int32_t*)rawPointer - inputIdsCount:(size_t)inputIdsCount - sequenceLength:(size_t)sequenceLength - batchSize:(size_t)batchSize - error:(NSError**)error; -/** - * Set input with sequences type. - * @param sequences The sequences. - * @param error Optional error information set if an error occurs. - */ -- (BOOL)setInputSequences:(OGASequences*)sequences - error:(NSError**)error; - /** * Set input with name and corresponding tensor. * @param name The input name. @@ -305,11 +273,28 @@ typedef NS_ENUM(NSInteger, OGAElementType) { * @return The result, or false if an error occurs. */ - (BOOL)isDoneWithError:(NSError**)error NS_SWIFT_NAME(isDone()) __attribute__((swift_error(nonnull_error))); + +/** + * Appends token sequences to the generator. + * @param sequences The sequences to append. + * @param error Optional error information set if an error occurs. + */ +- (BOOL)appendTokenSequences:(OGASequences*)sequences error:(NSError**)error; + /** - * Compute logits based on the input. + * Appends token sequences to the generator. + * @param tokens The tokens to append. * @param error Optional error information set if an error occurs. */ -- (BOOL)computeLogitsWithError:(NSError**)error NS_SWIFT_NAME(computeLogits()); +- (BOOL)appendTokens:(NSArray*)tokens error:(NSError**)error; + +/** + * Rewinds the generator by the specified number of tokens. + * @param length The number of tokens to rewind. + * @param error Optional error information set if an error occurs. + */ +- (BOOL)rewindTo:(NSUInteger)length error:(NSError**)error; + /** * Generate next token * @param error Optional error information set if an error occurs. @@ -342,10 +327,6 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGATensor : NSObject - (instancetype)init NS_UNAVAILABLE; -- (nullable instancetype)initWithDataPointer:(void*)data - shape:(NSArray*)shape - type:(OGAElementType)elementType - error:(NSError**)error; - (OGAElementType)getTypeWithError:(NSError**)error NS_SWIFT_NAME(type()); - (nullable void*)getDataPointerWithError:(NSError**)error NS_SWIFT_NAME(dataPointer()); diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm index 6681206af..1dca0130f 100644 --- a/src/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -30,9 +30,30 @@ - (BOOL)isDoneWithError:(NSError**)error { OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } -- (BOOL)computeLogitsWithError:(NSError**)error { +- (BOOL)appendTokenSequences:(OGASequences*)sequences error:(NSError**)error { try { - _generator->ComputeLogits(); + _generator->AppendTokenSequences([sequences CXXAPIOgaSequences]); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + +- (BOOL)appendTokens:(NSArray*)tokens error:(NSError**)error { + std::vector cxxTokens; + for (NSNumber* object in tokens) { + cxxTokens.push_back([object intValue]); + } + + try { + _generator->AppendTokens(cxxTokens.data(), cxxTokens.size()); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + +- (BOOL)rewindTo:(NSUInteger)length error:(NSError**)error { + try { + _generator->RewindTo(length); return YES; } OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) diff --git a/src/objectivec/oga_generator_params.mm b/src/objectivec/oga_generator_params.mm index d1ae55bca..3e3b6050d 100644 --- a/src/objectivec/oga_generator_params.mm +++ b/src/objectivec/oga_generator_params.mm @@ -29,26 +29,6 @@ - (BOOL)setInputs:(OGANamedTensors*)namedTensors error:(NSError**)error { OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } -- (BOOL)setInputIds:(const int32_t*)rawPointer - inputIdsCount:(size_t)inputIdsCount - sequenceLength:(size_t)sequenceLength - batchSize:(size_t)batchSize - error:(NSError**)error { - try { - _generatorParams->SetInputIDs(rawPointer, inputIdsCount, sequenceLength, batchSize); - return YES; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) -} - -- (BOOL)setInputSequences:(OGASequences*)sequences error:(NSError**)error { - try { - _generatorParams->SetInputSequences([sequences CXXAPIOgaSequences]); - return YES; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) -} - - (BOOL)setModelInput:(NSString*)name tensor:(OGATensor*)tensor error:(NSError**)error { try { _generatorParams->SetModelInput([name UTF8String], [tensor CXXAPIOgaTensor]); diff --git a/src/objectivec/oga_internal.h b/src/objectivec/oga_internal.h index 069268964..6454597be 100644 --- a/src/objectivec/oga_internal.h +++ b/src/objectivec/oga_internal.h @@ -42,6 +42,11 @@ NS_ASSUME_NONNULL_BEGIN @interface OGATensor () +- (nullable instancetype)initWithDataPointer:(void*)data + shape:(NSArray*)shape + type:(OGAElementType)elementType + error:(NSError**)error; + - (instancetype)initWithCXXPointer:(std::unique_ptr)ptr NS_DESIGNATED_INITIALIZER; - (OgaTensor&)CXXAPIOgaTensor; diff --git a/src/objectivec/oga_model.mm b/src/objectivec/oga_model.mm index 1fdf4c1fc..a38414059 100644 --- a/src/objectivec/oga_model.mm +++ b/src/objectivec/oga_model.mm @@ -21,15 +21,6 @@ - (nullable instancetype)initWithPath:(NSString*)path error:(NSError**)error { OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable OGASequences*)generate:(OGAGeneratorParams*)params error:(NSError**)error { - try { - std::unique_ptr output_sequences = - _model->Generate([params CXXAPIOgaGeneratorParams]); - return [[OGASequences alloc] initWithCXXPointer:std::move(output_sequences)]; - } - OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) -} - - (const OgaModel&)CXXAPIOgaModel { return *(_model.get()); } diff --git a/src/objectivec/test/ort_genai_api_test.mm b/src/objectivec/test/ort_genai_api_test.mm index 30247049c..bdf10f426 100644 --- a/src/objectivec/test/ort_genai_api_test.mm +++ b/src/objectivec/test/ort_genai_api_test.mm @@ -10,6 +10,14 @@ NS_ASSUME_NONNULL_BEGIN +@interface OGATensor (Testing) + +- (nullable instancetype)initWithDataPointer:(void*)data + shape:(NSArray*)shape + type:(OGAElementType)elementType + error:(NSError**)error; +@end + @interface ORTGenAIAPITest : XCTestCase @end @@ -47,6 +55,7 @@ - (void)testTensor_And_AddExtraInput { ORTAssertNullableResultSuccessful(params, error); OGATensor* tensor = [[OGATensor alloc] initWithDataPointer:data.data() shape:shape type:OGAElementTypeFloat32 error:&error]; + ORTAssertNullableResultSuccessful(tensor, error); ret = [params setModelInput:@"test_input" tensor:tensor error:&error]; ORTAssertBoolResultSuccessful(ret, error); @@ -54,9 +63,8 @@ - (void)testTensor_And_AddExtraInput { - (void)testGetOutput { std::vector input_ids_shape{2, 4}; - std::vector input_ids{0, 0, 0, 52, 0, 0, 195, 731}; + NSArray* input_ids = @[@0, @0, @0, @52, @0, @0, @195, @731]; const auto batch_size = input_ids_shape[0]; - const auto input_sequence_length = input_ids_shape[1]; int max_length = 10; NSError *error = nil; @@ -67,30 +75,31 @@ - (void)testGetOutput { OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; ORTAssertNullableResultSuccessful(params, error); - [params setInputIds:input_ids.data() - inputIdsCount:input_ids.size() - sequenceLength:input_sequence_length - batchSize:batch_size - error:&error]; - [params setSearchOption:@"max_length" doubleValue:max_length error:&error]; + [params setSearchOption:@"batch_size" doubleValue:batch_size error:&error]; + + XCTAssertNil(error); OGAGenerator* generator = [[OGAGenerator alloc] initWithModel:model params:params error:&error]; + + [generator appendTokens:input_ids error:&error]; + XCTAssertNil(error); + + ORTAssertNullableResultSuccessful(generator, error); + // check prompt // full logits has shape [2, 4, 1000]. Sample 1 for every 200 tokens and the expected sampled logits has shape [2, 4, 5] std::vector expected_sampled_logits_prompt{0.29694548f, 0.00955007f, 0.0430819f, 0.10063869f, 0.0437237f, - 0.27329233f, 0.00841076f, -0.1060291f, 0.11328877f, 0.13369876f, - 0.30323744f, 0.0545997f, 0.03894716f, 0.11702324f, 0.0410665f, - -0.12675379f, -0.04443946f, 0.14492269f, 0.03021223f, -0.03212897f, - 0.29694548f, 0.00955007f, 0.0430819f, 0.10063869f, 0.0437237f, - 0.27329233f, 0.00841076f, -0.1060291f, 0.11328877f, 0.13369876f, - -0.04699047f, 0.17915794f, 0.20838135f, 0.10888482f, -0.00277808f, - 0.2938929f, -0.10538938f, -0.00226692f, 0.12050669f, -0.10622668f}; - - ret = [generator computeLogitsWithError:&error]; - ORTAssertBoolResultSuccessful(ret, error); + 0.27329233f, 0.00841076f, -0.1060291f, 0.11328877f, 0.13369876f, + 0.30323744f, 0.0545997f, 0.03894716f, 0.11702324f, 0.0410665f, + -0.12675379f, -0.04443946f, 0.14492269f, 0.03021223f, -0.03212897f, + 0.29694548f, 0.00955007f, 0.0430819f, 0.10063869f, 0.0437237f, + 0.27329233f, 0.00841076f, -0.1060291f, 0.11328877f, 0.13369876f, + -0.04699047f, 0.17915794f, 0.20838135f, 0.10888482f, -0.00277808f, + 0.2938929f, -0.10538938f, -0.00226692f, 0.12050669f, -0.10622668f}; + OGATensor* prompt_logits_ptr = [generator getOutput:@"logits" error:&error]; ORTAssertNullableResultSuccessful(prompt_logits_ptr, error); auto prompt_logits = static_cast([prompt_logits_ptr getDataPointerWithError:&error]); @@ -104,13 +113,13 @@ - (void)testGetOutput { ret = [generator generateNextTokenWithError:&error]; ORTAssertBoolResultSuccessful(ret, error); + ret = [generator generateNextTokenWithError:&error]; + ORTAssertBoolResultSuccessful(ret, error); + // check for the 1st token generation // full logits has shape [2, 1, 1000]. Sample 1 for every 200 tokens and the expected sampled logits has shape [2, 1, 5] std::vector expected_sampled_logits_token_gen{0.03742531f, -0.05752287f, 0.14159015f, 0.04210977f, -0.1484456f, - 0.3041716f, -0.08701379f, -0.03778192f, 0.07471392f, -0.02049096f}; - - ret = [generator computeLogitsWithError:&error]; - ORTAssertBoolResultSuccessful(ret, error); + 0.3041716f, -0.08701379f, -0.03778192f, 0.07471392f, -0.02049096f}; OGATensor* token_gen_logits_ptr = [generator getOutput:@"logits" error:&error]; ORTAssertNullableResultSuccessful(token_gen_logits_ptr, error); From 0db23bfbe063e14f6a3bf98854eb5f6a03de231a Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 26 Nov 2024 14:58:48 +0800 Subject: [PATCH 50/59] skip macos tests --- src/objectivec/oga_sequences.mm | 2 +- .../apple/build_and_assemble_apple_pods.py | 13 ++++++++++++ ...ulator_apple_framework_build_settings.json | 21 +++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 tools/ci_build/github/apple/default_ios_simulator_apple_framework_build_settings.json diff --git a/src/objectivec/oga_sequences.mm b/src/objectivec/oga_sequences.mm index 6a07abfcb..1d39d278a 100644 --- a/src/objectivec/oga_sequences.mm +++ b/src/objectivec/oga_sequences.mm @@ -28,7 +28,7 @@ - (nullable instancetype)initWithError:(NSError**)error { - (size_t)getCountWithError:(NSError**)error { try { - return _sequences->Count(); + return _sequences->Count(); } OGA_OBJC_API_IMPL_CATCH(error, size_t(-1)) } diff --git a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py index 3f35a66f9..35bbb6ba4 100755 --- a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py +++ b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py @@ -81,6 +81,12 @@ def parse_args(): help="Pass an argument through to build_apple_framework.py. This may be specified multiple times.", ) + parser.add_argument( + "--skip-macos-test", + action="store_true", + help="Skip macos platform tests. Specify this argument when build targets only contain ios archs. ", + ) + parser.add_argument( "--ort-version", required=True, help="The ORT version to depend on." ) @@ -148,6 +154,8 @@ def main(): '--ort_version', args.ort_version ] + if args.skip_macos_test: + test_apple_packages_args.append("--skip_macos_test") run(test_apple_packages_args) @@ -172,6 +180,9 @@ def main(): if args.test: test_c_pod_args = ["pod", "lib", "lint", "--verbose"] + if args.skip_macos_test: + test_c_pod_args.append("--platforms=ios") + run(test_c_pod_args, cwd=c_pod_staging_dir) log.info("Assembling Objective-C pod.") @@ -186,6 +197,8 @@ def main(): if args.test: test_objc_pod_args = ["pod", "lib", "lint", "--verbose", f"--include-podspecs={c_pod_podspec}"] + if args.skip_macos_test: + test_objc_pod_args.append("--platforms=ios") run(test_objc_pod_args, cwd=objc_pod_staging_dir) diff --git a/tools/ci_build/github/apple/default_ios_simulator_apple_framework_build_settings.json b/tools/ci_build/github/apple/default_ios_simulator_apple_framework_build_settings.json new file mode 100644 index 000000000..555f7f6e4 --- /dev/null +++ b/tools/ci_build/github/apple/default_ios_simulator_apple_framework_build_settings.json @@ -0,0 +1,21 @@ +{ + "build_osx_archs": { + "iphonesimulator": [ + "arm64", + "x86_64" + ] + }, + "build_params": { + "base": [ + "--parallel", + "--cmake_generator", + "Xcode", + "--build_apple_framework", + "--skip_tests" + ], + "iphonesimulator": [ + "--ios", + "--apple_deploy_target=13.0" + ] + } +} From 1afacc916ee2cd0dda9cd77aa83eaee51a03ff28 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 28 Nov 2024 15:05:08 +0800 Subject: [PATCH 51/59] comments --- .../macos-ios-cocoapods-packaging-job.yml | 1 - .../jobs/steps/capi-appleframework-step.yml | 2 +- src/objectivec/cxx_api.h | 2 +- src/objectivec/include/ort_genai_objc.h | 49 +++++++++++++------ src/objectivec/oga_generator.mm | 28 ++++++----- src/objectivec/oga_images.mm | 4 +- src/objectivec/oga_internal.h | 4 +- src/objectivec/oga_internal.mm | 2 +- src/objectivec/oga_multi_modal_processor.mm | 2 +- src/objectivec/oga_named_tensors.mm | 4 ++ src/objectivec/oga_sequences.mm | 26 ++++++---- src/objectivec/oga_tokenizer.mm | 4 +- src/objectivec/test/ort_genai_api_test.mm | 7 ++- 13 files changed, 85 insertions(+), 50 deletions(-) diff --git a/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml b/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml index a2002f0f7..e6bc458a8 100644 --- a/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml +++ b/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml @@ -1,7 +1,6 @@ parameters: - name: build_config type: string - default: 'release' - name: ort_version type: string diff --git a/.pipelines/stages/jobs/steps/capi-appleframework-step.yml b/.pipelines/stages/jobs/steps/capi-appleframework-step.yml index 902a3d3c7..4dcf32a8f 100644 --- a/.pipelines/stages/jobs/steps/capi-appleframework-step.yml +++ b/.pipelines/stages/jobs/steps/capi-appleframework-step.yml @@ -24,7 +24,7 @@ steps: python3 tools/ci_build/github/apple/build_apple_framework.py \ --build_dir "$(Build.BinariesDirectory)/apple_framework" \ --config $(cmake_build_type) \ - ${{ parameters.build_settings_file }} + "${{ parameters.build_settings_file }}" mkdir $(Build.BinariesDirectory)/artifacts mkdir -p $(Build.BinariesDirectory)/artifacts_staging/onnxruntime-genai.xcframework diff --git a/src/objectivec/cxx_api.h b/src/objectivec/cxx_api.h index ded5ca4cd..3e423c17e 100644 --- a/src/objectivec/cxx_api.h +++ b/src/objectivec/cxx_api.h @@ -10,7 +10,7 @@ #pragma clang diagnostic ignored "-Wdocumentation" #endif // defined(__clang__) -#import "ort_genai.h" +#include "ort_genai.h" #if defined(__clang__) #pragma clang diagnostic pop diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 888ec8f63..dab04bb06 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -17,7 +17,7 @@ * [params setSearchOption:@"max_length" doubleValue:200 error:&error]; * * OGASequences* output_sequences = [model generate:params error:&error]; - * NSString* out_string = [tokenizer decode:[output_sequences sequenceAtIndex:0] error:&error]; + * NSString* out_string = [tokenizer decode:[output_sequences sequenceDataAtIndex:0] length:[output_sequences sequenceCountAtIndex:0] error:&error]; * */ @@ -96,10 +96,12 @@ typedef NS_ENUM(NSInteger, OGAElementType) { * Decode sequences to text * * @param data The sequences data to be encoded. + * @param tokensLength The length of the sequences data to be encoded. * @param error Optional error information set if an error occurs. * @return The decoding result, or nil if an error occurs. */ -- (nullable NSString*)decode:(OGAInt32Span*)data +- (nullable NSString*)decode:(const int32_t*)data + length:(size_t)tokensLength error:(NSError**)error; @end @@ -168,7 +170,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { * @param error Optional error information set if an error occurs. * @return The last element, or -1 if an error occurs. */ -- (int32_t)lastElementWithError:(NSError**)error NS_SWIFT_NAME(lastElement()); +- (int32_t)getLastElementWithError:(NSError**)error NS_SWIFT_NAME(lastElement()); @end @@ -187,13 +189,22 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (size_t)getCountWithError:(NSError**)error NS_SWIFT_NAME(count()); /** - * Retrieve the sequence at the given index + * Retrieve the sequence data at the given index. * @param index The index needed. * @param error Optional error information set if an error occurs. - * @return The sequence at the given index, or nil if an error occurs. + * @return The sequence data at the given index, or nil if an error occurs. */ -- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index - error:(NSError**)error; +- (nullable const int32_t*)sequenceDataAtIndex:(size_t)index + error:(NSError**)error; + +/** + * Retrieve the sequence count at the given index. + * @param index The index needed. + * @param error Optional error information set if an error occurs. + * @return The sequence count at the given index, or nil if an error occurs. + */ +- (size_t)sequenceCountAtIndex:(size_t)index + error:(NSError**)error; @end @@ -272,7 +283,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { * @param error Optional error information set if an error occurs. * @return The result, or false if an error occurs. */ -- (BOOL)isDoneWithError:(NSError**)error NS_SWIFT_NAME(isDone()) __attribute__((swift_error(nonnull_error))); +- (BOOL)isDoneWithError:(NSError**)error __attribute__((swift_error(nonnull_error))); /** * Appends token sequences to the generator. @@ -293,13 +304,13 @@ typedef NS_ENUM(NSInteger, OGAElementType) { * @param length The number of tokens to rewind. * @param error Optional error information set if an error occurs. */ -- (BOOL)rewindTo:(NSUInteger)length error:(NSError**)error; +- (BOOL)rewindTo:(size_t)length error:(NSError**)error; /** * Generate next token * @param error Optional error information set if an error occurs. */ -- (BOOL)generateNextTokenWithError:(NSError**)error NS_SWIFT_NAME(generateNextToken()); +- (BOOL)generateNextTokenWithError:(NSError**)error; /** * Get the output tensor. * @param name The output name. @@ -309,13 +320,23 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (nullable OGATensor*)getOutput:(NSString*)name error:(NSError**)error; /** - * Retrieve the sequence at the given index. + * Retrieve the sequence data at the given index. * @param index The index needed. * @param error Optional error information set if an error occurs. - * @return The sequence at the given index, or nil if an error occurs. + * @return The sequence data at the given index, or nil if an error occurs. */ -- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index - error:(NSError**)error; +- (nullable const int32_t*)sequenceDataAtIndex:(size_t)index + error:(NSError**)error; + +/** + * Retrieve the sequence count at the given index. + * @param index The index needed. + * @param error Optional error information set if an error occurs. + * @return The sequence length at the given index, or nil if an error occurs. + */ +- (size_t)sequenceCountAtIndex:(size_t)index + error:(NSError**)error; + /** * Clean up the resource before process exits. diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm index 1dca0130f..1eb0ef006 100644 --- a/src/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -39,19 +39,19 @@ - (BOOL)appendTokenSequences:(OGASequences*)sequences error:(NSError**)error { } - (BOOL)appendTokens:(NSArray*)tokens error:(NSError**)error { - std::vector cxxTokens; - for (NSNumber* object in tokens) { - cxxTokens.push_back([object intValue]); - } - try { + std::vector cxxTokens; + for (NSNumber* object in tokens) { + cxxTokens.push_back([object intValue]); + } + _generator->AppendTokens(cxxTokens.data(), cxxTokens.size()); return YES; } OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } -- (BOOL)rewindTo:(NSUInteger)length error:(NSError**)error { +- (BOOL)rewindTo:(size_t)length error:(NSError**)error { try { _generator->RewindTo(length); return YES; @@ -76,16 +76,22 @@ - (nullable OGATensor*)getOutput:(NSString*)name OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index - error:(NSError**)error { +- (nullable const int32_t*)sequenceDataAtIndex:(size_t)index + error:(NSError**)error { try { - size_t sequenceLength = _generator->GetSequenceCount(index); - const int32_t* data = _generator->GetSequenceData(index); - return [[OGAInt32Span alloc] initWithDataPointer:data size:sequenceLength]; + return _generator->GetSequenceData(index); } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } +- (size_t)sequenceCountAtIndex:(size_t)index + error:(NSError**)error { + try { + return _generator->GetSequenceCount(index); + } + OGA_OBJC_API_IMPL_CATCH(error, size_t(-1)) +} + + (void)shutdown { OgaShutdown(); } diff --git a/src/objectivec/oga_images.mm b/src/objectivec/oga_images.mm index 25a056e4e..43a5297a8 100644 --- a/src/objectivec/oga_images.mm +++ b/src/objectivec/oga_images.mm @@ -28,8 +28,8 @@ - (nullable instancetype)initWithPath:(NSArray *)paths error:(NSErro OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (OgaImages*)CXXAPIOgaImages { - return _images.get(); +- (OgaImages&)CXXAPIOgaImages { + return *(_images.get()); } @end \ No newline at end of file diff --git a/src/objectivec/oga_internal.h b/src/objectivec/oga_internal.h index 6454597be..aa03accdb 100644 --- a/src/objectivec/oga_internal.h +++ b/src/objectivec/oga_internal.h @@ -22,7 +22,7 @@ NS_ASSUME_NONNULL_BEGIN @interface OGASequences () - (nullable instancetype)initWithError:(NSError**)error; -- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr; +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr NS_DESIGNATED_INITIALIZER; - (OgaSequences&)CXXAPIOgaSequences; @@ -36,7 +36,7 @@ NS_ASSUME_NONNULL_BEGIN @interface OGAImages () -- (OgaImages*)CXXAPIOgaImages; +- (OgaImages&)CXXAPIOgaImages; @end diff --git a/src/objectivec/oga_internal.mm b/src/objectivec/oga_internal.mm index 765a8b3eb..7d47f68cb 100644 --- a/src/objectivec/oga_internal.mm +++ b/src/objectivec/oga_internal.mm @@ -26,7 +26,7 @@ - (size_t)size { return _size; } -- (int32_t)lastElementWithError:(NSError**)error { +- (int32_t)getLastElementWithError:(NSError**)error { if (_size == 0) { if (error != nil) { NSDictionary *errorDictionary = @{NSLocalizedDescriptionKey : @"The size of this span is invalid"}; diff --git a/src/objectivec/oga_multi_modal_processor.mm b/src/objectivec/oga_multi_modal_processor.mm index 1aec2a1d8..70d499522 100644 --- a/src/objectivec/oga_multi_modal_processor.mm +++ b/src/objectivec/oga_multi_modal_processor.mm @@ -27,7 +27,7 @@ - (nullable OGANamedTensors*)processImages:(NSString*)prompt try { OGANamedTensors* result = [[OGANamedTensors alloc] initWithCXXPointer:_processor->ProcessImages([prompt UTF8String], - [images CXXAPIOgaImages])]; + &[images CXXAPIOgaImages])]; return result; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) diff --git a/src/objectivec/oga_named_tensors.mm b/src/objectivec/oga_named_tensors.mm index fa3556eb0..9a3f1cdd8 100644 --- a/src/objectivec/oga_named_tensors.mm +++ b/src/objectivec/oga_named_tensors.mm @@ -11,6 +11,10 @@ @implementation OGANamedTensors { - (instancetype)initWithCXXPointer:(std::unique_ptr)ptr { + if ((self = [super init]) == nil) { + return nil; + } + _tensor = std::move(ptr); return self; } diff --git a/src/objectivec/oga_sequences.mm b/src/objectivec/oga_sequences.mm index 1d39d278a..c50c8fb9c 100644 --- a/src/objectivec/oga_sequences.mm +++ b/src/objectivec/oga_sequences.mm @@ -10,17 +10,17 @@ @implementation OGASequences { } - (instancetype)initWithCXXPointer:(std::unique_ptr)ptr { + if ((self = [super init]) == nil) { + return nil; + } + _sequences = std::move(ptr); return self; } - (nullable instancetype)initWithError:(NSError**)error { - if ((self = [super init]) == nil) { - return nil; - } - try { - _sequences = OgaSequences::Create(); + self = [self initWithCXXPointer: OgaSequences::Create()]; return self; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) @@ -33,16 +33,22 @@ - (size_t)getCountWithError:(NSError**)error { OGA_OBJC_API_IMPL_CATCH(error, size_t(-1)) } -- (nullable OGAInt32Span*)sequenceAtIndex:(size_t)index - error:(NSError**)error { +- (nullable const int32_t*)sequenceDataAtIndex:(size_t)index + error:(NSError**)error { try { - size_t sequenceLength = _sequences->SequenceCount(index); - const int32_t* data = _sequences->SequenceData(index); - return [[OGAInt32Span alloc] initWithDataPointer:data size:sequenceLength]; + return _sequences->SequenceData(index); } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } +- (size_t)sequenceCountAtIndex:(size_t)index + error:(NSError**)error { + try { + return _sequences->SequenceCount(index); + } + OGA_OBJC_API_IMPL_CATCH(error, size_t(-1)) +} + - (OgaSequences&)CXXAPIOgaSequences { return *(_sequences.get()); } diff --git a/src/objectivec/oga_tokenizer.mm b/src/objectivec/oga_tokenizer.mm index 31f989ef4..4a1c34192 100644 --- a/src/objectivec/oga_tokenizer.mm +++ b/src/objectivec/oga_tokenizer.mm @@ -33,9 +33,9 @@ - (nullable OGASequences*)encode:(NSString*)str error:(NSError**)error { OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable NSString*)decode:(OGAInt32Span*)data error:(NSError**)error { +- (nullable NSString*)decode:(const int32_t*)tokensData length:(size_t)tokensLength error:(NSError**)error { try { - OgaString result = _tokenizer->Decode(data.pointer, data.size); + OgaString result = _tokenizer->Decode(tokensData, tokensLength); return [NSString stringWithUTF8String:result]; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) diff --git a/src/objectivec/test/ort_genai_api_test.mm b/src/objectivec/test/ort_genai_api_test.mm index bdf10f426..622ca5d62 100644 --- a/src/objectivec/test/ort_genai_api_test.mm +++ b/src/objectivec/test/ort_genai_api_test.mm @@ -76,19 +76,18 @@ - (void)testGetOutput { ORTAssertNullableResultSuccessful(params, error); [params setSearchOption:@"max_length" doubleValue:max_length error:&error]; - [params setSearchOption:@"batch_size" doubleValue:batch_size error:&error]; + XCTAssertNil(error); + [params setSearchOption:@"batch_size" doubleValue:batch_size error:&error]; XCTAssertNil(error); OGAGenerator* generator = [[OGAGenerator alloc] initWithModel:model params:params error:&error]; - + ORTAssertNullableResultSuccessful(generator, error); [generator appendTokens:input_ids error:&error]; XCTAssertNil(error); - ORTAssertNullableResultSuccessful(generator, error); - // check prompt // full logits has shape [2, 4, 1000]. Sample 1 for every 200 tokens and the expected sampled logits has shape [2, 4, 5] std::vector expected_sampled_logits_prompt{0.29694548f, 0.00955007f, 0.0430819f, 0.10063869f, 0.0437237f, From d23223005faa92c6534302dba29d4301c37ca691 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 28 Nov 2024 15:07:31 +0800 Subject: [PATCH 52/59] clang-format --- src/objectivec/oga_generator.mm | 13 +++++-------- src/objectivec/oga_images.mm | 6 +++--- src/objectivec/oga_internal.mm | 3 ++- src/objectivec/oga_multi_modal_processor.mm | 2 +- src/objectivec/oga_named_tensors.mm | 3 +-- src/objectivec/oga_sequences.mm | 8 +++----- src/objectivec/oga_tensor.mm | 8 ++++---- src/objectivec/oga_tokenizer.mm | 5 +++-- src/objectivec/oga_tokenizer_stream.mm | 3 ++- 9 files changed, 24 insertions(+), 27 deletions(-) diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm index 1eb0ef006..fd0082769 100644 --- a/src/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -10,8 +10,8 @@ @implementation OGAGenerator { } - (nullable instancetype)initWithModel:(OGAModel*)model - params:(OGAGeneratorParams*)params - error:(NSError**)error { + params:(OGAGeneratorParams*)params + error:(NSError**)error { if ((self = [super init]) == nil) { return nil; } @@ -67,8 +67,7 @@ - (BOOL)generateNextTokenWithError:(NSError**)error { OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } -- (nullable OGATensor*)getOutput:(NSString*)name - error:(NSError**)error { +- (nullable OGATensor*)getOutput:(NSString*)name error:(NSError**)error { try { std::unique_ptr output = _generator->GetOutput([name UTF8String]); return [[OGATensor alloc] initWithCXXPointer:std::move(output)]; @@ -76,16 +75,14 @@ - (nullable OGATensor*)getOutput:(NSString*)name OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable const int32_t*)sequenceDataAtIndex:(size_t)index - error:(NSError**)error { +- (nullable const int32_t*)sequenceDataAtIndex:(size_t)index error:(NSError**)error { try { return _generator->GetSequenceData(index); } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (size_t)sequenceCountAtIndex:(size_t)index - error:(NSError**)error { +- (size_t)sequenceCountAtIndex:(size_t)index error:(NSError**)error { try { return _generator->GetSequenceCount(index); } diff --git a/src/objectivec/oga_images.mm b/src/objectivec/oga_images.mm index 43a5297a8..8b338f958 100644 --- a/src/objectivec/oga_images.mm +++ b/src/objectivec/oga_images.mm @@ -9,16 +9,16 @@ @implementation OGAImages { std::unique_ptr _images; } -- (nullable instancetype)initWithPath:(NSArray *)paths error:(NSError**)error { +- (nullable instancetype)initWithPath:(NSArray*)paths error:(NSError**)error { if ((self = [super init]) == nil) { return nil; } try { - std::vector cpp_paths; + std::vector cpp_paths; cpp_paths.reserve([paths count]); - for (NSString* path in paths){ + for (NSString* path in paths) { cpp_paths.push_back([path UTF8String]); } diff --git a/src/objectivec/oga_internal.mm b/src/objectivec/oga_internal.mm index 7d47f68cb..e8e4324b0 100644 --- a/src/objectivec/oga_internal.mm +++ b/src/objectivec/oga_internal.mm @@ -29,7 +29,8 @@ - (size_t)size { - (int32_t)getLastElementWithError:(NSError**)error { if (_size == 0) { if (error != nil) { - NSDictionary *errorDictionary = @{NSLocalizedDescriptionKey : @"The size of this span is invalid"}; + NSDictionary* errorDictionary = + @{NSLocalizedDescriptionKey : @"The size of this span is invalid"}; *error = [[NSError alloc] initWithDomain:kOgaErrorDomain code:-1 userInfo:errorDictionary]; } return -1; diff --git a/src/objectivec/oga_multi_modal_processor.mm b/src/objectivec/oga_multi_modal_processor.mm index 70d499522..0442e6174 100644 --- a/src/objectivec/oga_multi_modal_processor.mm +++ b/src/objectivec/oga_multi_modal_processor.mm @@ -27,7 +27,7 @@ - (nullable OGANamedTensors*)processImages:(NSString*)prompt try { OGANamedTensors* result = [[OGANamedTensors alloc] initWithCXXPointer:_processor->ProcessImages([prompt UTF8String], - &[images CXXAPIOgaImages])]; + &[images CXXAPIOgaImages])]; return result; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) diff --git a/src/objectivec/oga_named_tensors.mm b/src/objectivec/oga_named_tensors.mm index 9a3f1cdd8..8243f8403 100644 --- a/src/objectivec/oga_named_tensors.mm +++ b/src/objectivec/oga_named_tensors.mm @@ -9,8 +9,7 @@ @implementation OGANamedTensors { std::unique_ptr _tensor; } -- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr -{ +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr { if ((self = [super init]) == nil) { return nil; } diff --git a/src/objectivec/oga_sequences.mm b/src/objectivec/oga_sequences.mm index c50c8fb9c..7dd717b36 100644 --- a/src/objectivec/oga_sequences.mm +++ b/src/objectivec/oga_sequences.mm @@ -20,7 +20,7 @@ - (instancetype)initWithCXXPointer:(std::unique_ptr)ptr { - (nullable instancetype)initWithError:(NSError**)error { try { - self = [self initWithCXXPointer: OgaSequences::Create()]; + self = [self initWithCXXPointer:OgaSequences::Create()]; return self; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) @@ -33,16 +33,14 @@ - (size_t)getCountWithError:(NSError**)error { OGA_OBJC_API_IMPL_CATCH(error, size_t(-1)) } -- (nullable const int32_t*)sequenceDataAtIndex:(size_t)index - error:(NSError**)error { +- (nullable const int32_t*)sequenceDataAtIndex:(size_t)index error:(NSError**)error { try { return _sequences->SequenceData(index); } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (size_t)sequenceCountAtIndex:(size_t)index - error:(NSError**)error { +- (size_t)sequenceCountAtIndex:(size_t)index error:(NSError**)error { try { return _sequences->SequenceCount(index); } diff --git a/src/objectivec/oga_tensor.mm b/src/objectivec/oga_tensor.mm index d8cd28d26..bd936da06 100644 --- a/src/objectivec/oga_tensor.mm +++ b/src/objectivec/oga_tensor.mm @@ -19,16 +19,16 @@ - (instancetype)initWithCXXPointer:(std::unique_ptr)ptr { } - (nullable instancetype)initWithDataPointer:(void*)data - shape:(NSArray*)shape - type:(OGAElementType)elementType - error:(NSError**)error { + shape:(NSArray*)shape + type:(OGAElementType)elementType + error:(NSError**)error { try { std::vector cxxShape; for (NSNumber* object in shape) { cxxShape.push_back([object longLongValue]); } self = [self initWithCXXPointer:OgaTensor::Create(data, cxxShape.data(), cxxShape.size(), - static_cast(elementType))]; + static_cast(elementType))]; return self; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) diff --git a/src/objectivec/oga_tokenizer.mm b/src/objectivec/oga_tokenizer.mm index 4a1c34192..c961faabc 100644 --- a/src/objectivec/oga_tokenizer.mm +++ b/src/objectivec/oga_tokenizer.mm @@ -33,7 +33,9 @@ - (nullable OGASequences*)encode:(NSString*)str error:(NSError**)error { OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable NSString*)decode:(const int32_t*)tokensData length:(size_t)tokensLength error:(NSError**)error { +- (nullable NSString*)decode:(const int32_t*)tokensData + length:(size_t)tokensLength + error:(NSError**)error { try { OgaString result = _tokenizer->Decode(tokensData, tokensLength); return [NSString stringWithUTF8String:result]; @@ -46,4 +48,3 @@ - (nullable NSString*)decode:(const int32_t*)tokensData length:(size_t)tokensLen } @end - diff --git a/src/objectivec/oga_tokenizer_stream.mm b/src/objectivec/oga_tokenizer_stream.mm index b664c37b0..fb10adf99 100644 --- a/src/objectivec/oga_tokenizer_stream.mm +++ b/src/objectivec/oga_tokenizer_stream.mm @@ -21,7 +21,8 @@ - (nullable instancetype)initWithTokenizer:(OGATokenizer*)tokenizer error:(NSErr OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable instancetype)initWithMultiModalProcessor:(OGAMultiModalProcessor*)processor error:(NSError**)error { +- (nullable instancetype)initWithMultiModalProcessor:(OGAMultiModalProcessor*)processor + error:(NSError**)error { if ((self = [super init]) == nil) { return nil; } From bd116dcdebcabbc0a2d3c5ad1fd51a23e98bd5c6 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 28 Nov 2024 15:14:19 +0800 Subject: [PATCH 53/59] clang format --- src/objectivec/include/ort_genai_objc.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index dab04bb06..977f5ba03 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -195,7 +195,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { * @return The sequence data at the given index, or nil if an error occurs. */ - (nullable const int32_t*)sequenceDataAtIndex:(size_t)index - error:(NSError**)error; + error:(NSError**)error; /** * Retrieve the sequence count at the given index. @@ -337,7 +337,6 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (size_t)sequenceCountAtIndex:(size_t)index error:(NSError**)error; - /** * Clean up the resource before process exits. */ From 9ffb63e8d48e7b27c33bd5a2060ebe2d1f0d7854 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:31:52 +0800 Subject: [PATCH 54/59] Update src/objectivec/include/ort_genai_objc.h Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com> --- src/objectivec/include/ort_genai_objc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 977f5ba03..5e9251f00 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -332,7 +332,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { * Retrieve the sequence count at the given index. * @param index The index needed. * @param error Optional error information set if an error occurs. - * @return The sequence length at the given index, or nil if an error occurs. + * @return The sequence count at the given index, or size_t(-1) if an error occurs. */ - (size_t)sequenceCountAtIndex:(size_t)index error:(NSError**)error; From 1586fee721116ad4b634b633fdb7111eec293214 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:32:00 +0800 Subject: [PATCH 55/59] Update src/objectivec/include/ort_genai_objc.h Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com> --- src/objectivec/include/ort_genai_objc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 5e9251f00..7204fa6a3 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -201,7 +201,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { * Retrieve the sequence count at the given index. * @param index The index needed. * @param error Optional error information set if an error occurs. - * @return The sequence count at the given index, or nil if an error occurs. + * @return The sequence count at the given index, or size_t(-1) if an error occurs. */ - (size_t)sequenceCountAtIndex:(size_t)index error:(NSError**)error; From af3cb0239c0994271b351daf34edfff17bed359f Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Mon, 9 Dec 2024 13:22:11 +0800 Subject: [PATCH 56/59] reviews --- src/objectivec/error_utils.mm | 2 +- src/objectivec/include/ort_genai_objc.h | 4 ++-- src/objectivec/oga_generator.mm | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/objectivec/error_utils.mm b/src/objectivec/error_utils.mm index f621af328..9bed15320 100644 --- a/src/objectivec/error_utils.mm +++ b/src/objectivec/error_utils.mm @@ -12,7 +12,7 @@ void OGASaveCodeAndDescriptionToError(int code, const char* descriptionCstr, NSE if (!error) return; NSString* description = [NSString stringWithCString:descriptionCstr - encoding:NSASCIIStringEncoding]; + encoding:NSUTF8StringEncoding]; *error = [NSError errorWithDomain:kOgaErrorDomain code:code diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 7204fa6a3..c658b5681 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -301,10 +301,10 @@ typedef NS_ENUM(NSInteger, OGAElementType) { /** * Rewinds the generator by the specified number of tokens. - * @param length The number of tokens to rewind. + * @param new_length The desired length in tokens after rewinding. * @param error Optional error information set if an error occurs. */ -- (BOOL)rewindTo:(size_t)length error:(NSError**)error; +- (BOOL)rewindTo:(size_t)new_length error:(NSError**)error; /** * Generate next token diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm index fd0082769..cad8bf1ab 100644 --- a/src/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -51,9 +51,9 @@ - (BOOL)appendTokens:(NSArray*)tokens error:(NSError**)error { OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } -- (BOOL)rewindTo:(size_t)length error:(NSError**)error { +- (BOOL)rewindTo:(size_t)new_length error:(NSError**)error { try { - _generator->RewindTo(length); + _generator->RewindTo(new_length); return YES; } OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) From 8a3146be3848d4451d46f8d503819e59b55967ff Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:33:26 +0800 Subject: [PATCH 57/59] Reviews --- src/objectivec/include/ort_genai_objc.h | 4 ++++ src/objectivec/oga_internal.h | 5 ----- src/objectivec/test/ort_genai_api_test.mm | 14 +++++--------- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index c658b5681..146ba494f 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -347,6 +347,10 @@ typedef NS_ENUM(NSInteger, OGAElementType) { @interface OGATensor : NSObject - (instancetype)init NS_UNAVAILABLE; +- (nullable instancetype)initWithDataPointer:(void*)data + shape:(NSArray*)shape + type:(OGAElementType)elementType + error:(NSError**)error; - (OGAElementType)getTypeWithError:(NSError**)error NS_SWIFT_NAME(type()); - (nullable void*)getDataPointerWithError:(NSError**)error NS_SWIFT_NAME(dataPointer()); diff --git a/src/objectivec/oga_internal.h b/src/objectivec/oga_internal.h index aa03accdb..808b02324 100644 --- a/src/objectivec/oga_internal.h +++ b/src/objectivec/oga_internal.h @@ -42,11 +42,6 @@ NS_ASSUME_NONNULL_BEGIN @interface OGATensor () -- (nullable instancetype)initWithDataPointer:(void*)data - shape:(NSArray*)shape - type:(OGAElementType)elementType - error:(NSError**)error; - - (instancetype)initWithCXXPointer:(std::unique_ptr)ptr NS_DESIGNATED_INITIALIZER; - (OgaTensor&)CXXAPIOgaTensor; diff --git a/src/objectivec/test/ort_genai_api_test.mm b/src/objectivec/test/ort_genai_api_test.mm index 622ca5d62..37db97e41 100644 --- a/src/objectivec/test/ort_genai_api_test.mm +++ b/src/objectivec/test/ort_genai_api_test.mm @@ -10,14 +10,6 @@ NS_ASSUME_NONNULL_BEGIN -@interface OGATensor (Testing) - -- (nullable instancetype)initWithDataPointer:(void*)data - shape:(NSArray*)shape - type:(OGAElementType)elementType - error:(NSError**)error; -@end - @interface ORTGenAIAPITest : XCTestCase @end @@ -102,6 +94,8 @@ - (void)testGetOutput { OGATensor* prompt_logits_ptr = [generator getOutput:@"logits" error:&error]; ORTAssertNullableResultSuccessful(prompt_logits_ptr, error); auto prompt_logits = static_cast([prompt_logits_ptr getDataPointerWithError:&error]); + XCTAssertNil(error); + XCTAssertNotEqual(prompt_logits, nullptr); const int num_prompt_outputs_to_check = 40; const int sample_size = 200; const float tolerance = 0.001f; @@ -124,6 +118,8 @@ - (void)testGetOutput { ORTAssertNullableResultSuccessful(token_gen_logits_ptr, error); auto token_gen_logits = static_cast([token_gen_logits_ptr getDataPointerWithError:&error]); + XCTAssertNil(error); + XCTAssertNotEqual(token_gen_logits, nullptr); int num_token_gen_outputs_to_check = 10; for (int i = 0; i < num_token_gen_outputs_to_check; i++) { @@ -135,4 +131,4 @@ - (void)testGetOutput { @end -NS_ASSUME_NONNULL_END \ No newline at end of file +NS_ASSUME_NONNULL_END From 2b15151758270b6fc0c929fc2f1004258ea5f1e0 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Wed, 11 Dec 2024 11:26:35 +0800 Subject: [PATCH 58/59] Update src/objectivec/include/ort_genai_objc.h Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com> --- src/objectivec/include/ort_genai_objc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 146ba494f..86d0d01fd 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -300,7 +300,7 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (BOOL)appendTokens:(NSArray*)tokens error:(NSError**)error; /** - * Rewinds the generator by the specified number of tokens. + * Rewinds the generator to the given length. * @param new_length The desired length in tokens after rewinding. * @param error Optional error information set if an error occurs. */ From 05ed2504abe16e3fb027c55475db7a339baa838a Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:04:37 +0800 Subject: [PATCH 59/59] Reviews --- src/objectivec/include/ort_genai_objc.h | 45 ++++----------------- src/objectivec/oga_generator.mm | 4 +- src/objectivec/oga_internal.mm | 41 ------------------- src/objectivec/oga_multi_modal_processor.mm | 6 ++- 4 files changed, 14 insertions(+), 82 deletions(-) delete mode 100644 src/objectivec/oga_internal.mm diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 86d0d01fd..e17e61c3a 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -23,7 +23,6 @@ NS_ASSUME_NONNULL_BEGIN -@class OGAInt32Span; @class OGATensor; @class OGASequences; @class OGANamedTensors; @@ -95,12 +94,12 @@ typedef NS_ENUM(NSInteger, OGAElementType) { /** * Decode sequences to text * - * @param data The sequences data to be encoded. + * @param tokensData The sequences data to be encoded. * @param tokensLength The length of the sequences data to be encoded. * @param error Optional error information set if an error occurs. * @return The decoding result, or nil if an error occurs. */ -- (nullable NSString*)decode:(const int32_t*)data +- (nullable NSString*)decode:(const int32_t*)tokensData length:(size_t)tokensLength error:(NSError**)error; @@ -142,38 +141,6 @@ typedef NS_ENUM(NSInteger, OGAElementType) { error:(NSError**)error; @end -/** - * Wraps a raw int32_t pointer and its size. Represents a data sequence. - */ -@interface OGAInt32Span : NSObject - -- (instancetype)init NS_UNAVAILABLE; -/** - * Creates a span with underlying data. - * - * @param pointer The underlying data pointer to use. - * @param size The underlying data size. - * @return The instance, or nil if an error occurs. - */ -- (nullable instancetype)initWithDataPointer:(const int32_t*)pointer size:(size_t)size; - -/** - * The underlying data pointer - */ -- (const int32_t*)pointer; -/** - * The underlying data size - */ -- (size_t)size; -/** - * The last element in this data sequence - * @param error Optional error information set if an error occurs. - * @return The last element, or -1 if an error occurs. - */ -- (int32_t)getLastElementWithError:(NSError**)error NS_SWIFT_NAME(lastElement()); - -@end - /** * A series of generated sequences */ @@ -301,10 +268,10 @@ typedef NS_ENUM(NSInteger, OGAElementType) { /** * Rewinds the generator to the given length. - * @param new_length The desired length in tokens after rewinding. + * @param newLength The desired length in tokens after rewinding. * @param error Optional error information set if an error occurs. */ -- (BOOL)rewindTo:(size_t)new_length error:(NSError**)error; +- (BOOL)rewindTo:(size_t)newLength error:(NSError**)error; /** * Generate next token @@ -379,6 +346,10 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (nullable OGANamedTensors*)processImages:(NSString*)prompt images:(OGAImages*)images error:(NSError**)error; + +- (nullable NSString*)decode:(const int32_t*)tokensData + length:(size_t)tokensLength + error:(NSError**)error; @end NS_ASSUME_NONNULL_END diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm index cad8bf1ab..aefcd7be0 100644 --- a/src/objectivec/oga_generator.mm +++ b/src/objectivec/oga_generator.mm @@ -51,9 +51,9 @@ - (BOOL)appendTokens:(NSArray*)tokens error:(NSError**)error { OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) } -- (BOOL)rewindTo:(size_t)new_length error:(NSError**)error { +- (BOOL)rewindTo:(size_t)newLength error:(NSError**)error { try { - _generator->RewindTo(new_length); + _generator->RewindTo(newLength); return YES; } OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) diff --git a/src/objectivec/oga_internal.mm b/src/objectivec/oga_internal.mm deleted file mode 100644 index e8e4324b0..000000000 --- a/src/objectivec/oga_internal.mm +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#import "oga_internal.h" - -@implementation OGAInt32Span { - const int32_t* _ptr; - size_t _size; -} - -- (nullable instancetype)initWithDataPointer:(const int32_t*)pointer size:(size_t)size { - if ((self = [super init]) == nil) { - return nil; - } - - _ptr = pointer; - _size = size; - return self; -} - -- (const int32_t*)pointer { - return _ptr; -} - -- (size_t)size { - return _size; -} - -- (int32_t)getLastElementWithError:(NSError**)error { - if (_size == 0) { - if (error != nil) { - NSDictionary* errorDictionary = - @{NSLocalizedDescriptionKey : @"The size of this span is invalid"}; - *error = [[NSError alloc] initWithDomain:kOgaErrorDomain code:-1 userInfo:errorDictionary]; - } - return -1; - } - return *(_ptr + (_size - 1)); -} - -@end diff --git a/src/objectivec/oga_multi_modal_processor.mm b/src/objectivec/oga_multi_modal_processor.mm index 0442e6174..917e625f2 100644 --- a/src/objectivec/oga_multi_modal_processor.mm +++ b/src/objectivec/oga_multi_modal_processor.mm @@ -33,9 +33,11 @@ - (nullable OGANamedTensors*)processImages:(NSString*)prompt OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) } -- (nullable NSString*)decode:(OGAInt32Span*)data error:(NSError**)error { +- (nullable NSString*)decode:(const int32_t*)tokensData + length:(size_t)tokensLength + error:(NSError**)error { try { - OgaString result = _processor->Decode(data.pointer, data.size); + OgaString result = _processor->Decode(tokensData, tokensLength); return [NSString stringWithUTF8String:result]; } OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error)