Remove tokenizer workaround after upstream Core AI Models fix

CoreAIChat.xcodeproj/project.pbxproj

@@ -7,7 +7,7 @@
 	objects = {
 
 /* Begin PBXBuildFile section */
-		25053A1B2FDB367C00BE6A46 /* CoreAILM in Frameworks */ = {isa = PBXBuildFile; productRef = 25053A1A2FDB367C00BE6A46 /* CoreAILM */; };
+		250EFA772FE19D2B00C6C26F /* CoreAILM in Frameworks */ = {isa = PBXBuildFile; productRef = 250EFA762FE19D2B00C6C26F /* CoreAILM */; };
 		25538FA02FDC63CE00BD3A07 /* Icon.icon in Resources */ = {isa = PBXBuildFile; fileRef = 25538F9F2FDC63CE00BD3A07 /* Icon.icon */; };
 		25F558A92FDC15D80024FCB3 /* gemma_3_4b_it_4bit_dynamic in Resources */ = {isa = PBXBuildFile; fileRef = 25F558A82FDC15D50024FCB3 /* gemma_3_4b_it_4bit_dynamic */; };
 /* End PBXBuildFile section */
@@ -30,7 +30,7 @@
 		2597A74E2FD874DA00808542 /* Frameworks */ = {
 			isa = PBXFrameworksBuildPhase;
 			files = (
-				25053A1B2FDB367C00BE6A46 /* CoreAILM in Frameworks */,
+				250EFA772FE19D2B00C6C26F /* CoreAILM in Frameworks */,
 			);
 		};
 /* End PBXFrameworksBuildPhase section */
@@ -72,7 +72,7 @@
 			);
 			name = CoreAIChat;
 			packageProductDependencies = (
-				25053A1A2FDB367C00BE6A46 /* CoreAILM */,
+				250EFA762FE19D2B00C6C26F /* CoreAILM */,
 			);
 			productName = CoreAIChat;
 			productReference = 2597A7512FD874DA00808542 /* CoreAIChat.app */;
@@ -103,7 +103,7 @@
 			mainGroup = 2597A7482FD874DA00808542;
 			minimizedProjectReferenceProxies = 1;
 			packageReferences = (
-				25053A192FDB367C00BE6A46 /* XCRemoteSwiftPackageReference "coreai-models" */,
+				250EFA752FE19D2B00C6C26F /* XCRemoteSwiftPackageReference "coreai-models" */,
 			);
 			preferredProjectObjectVersion = 77;
 			productRefGroup = 2597A7522FD874DA00808542 /* Products */;
@@ -346,20 +346,20 @@
 /* End XCConfigurationList section */
 
 /* Begin XCRemoteSwiftPackageReference section */
-		25053A192FDB367C00BE6A46 /* XCRemoteSwiftPackageReference "coreai-models" */ = {
+		250EFA752FE19D2B00C6C26F /* XCRemoteSwiftPackageReference "coreai-models" */ = {
 			isa = XCRemoteSwiftPackageReference;
 			repositoryURL = "https://github.com/apple/coreai-models.git";
 			requirement = {
-				kind = upToNextMajorVersion;
-				minimumVersion = 0.1.0;
+				branch = main;
+				kind = branch;
 			};
 		};
 /* End XCRemoteSwiftPackageReference section */
 
 /* Begin XCSwiftPackageProductDependency section */
-		25053A1A2FDB367C00BE6A46 /* CoreAILM */ = {
+		250EFA762FE19D2B00C6C26F /* CoreAILM */ = {
 			isa = XCSwiftPackageProductDependency;
-			package = 25053A192FDB367C00BE6A46 /* XCRemoteSwiftPackageReference "coreai-models" */;
+			package = 250EFA752FE19D2B00C6C26F /* XCRemoteSwiftPackageReference "coreai-models" */;
 			productName = CoreAILM;
 		};
 /* End XCSwiftPackageProductDependency section */

README.md

@@ -12,9 +12,6 @@
 
 **Beta notice:** This project currently relies on beta versions of Apple's development tools and frameworks. Many components are still under active development, so APIs, behavior, and setup instructions may change.
 
-**Known issue:** Due to a [tokenizer issue](https://github.com/apple/coreai-models/issues/25), the `<end_of_turn>` token is not reliably recognized when using the Gemma 3 model as shown in this example.
-As a result, it may continue generating beyond the intended end of a response. Follow [Replace the generated tokenizer](#replace-the-generated-tokenizer) to apply the included workaround.
-
 ## Setup
 
 ### Prepare the export tools
@@ -69,19 +66,6 @@ gemma_3_4b_it_4bit_dynamic/
 └── tokenizer/
 ```
 
-### Replace the generated tokenizer
-
-The tokenizer generated for the Gemma 3 example may not reliably recognize the `<end_of_turn>` token. This repository includes a corrected tokenizer in `tokenizer_overrides/tokenizer`.
-
-After copying the exported model into the repository, replace its generated `tokenizer` directory from the repository root:
-
-```bash
-rm -rf gemma_3_4b_it_4bit_dynamic/tokenizer
-cp -R tokenizer_overrides/tokenizer gemma_3_4b_it_4bit_dynamic/tokenizer
-```
-
-The corrected tokenizer remains tracked in `tokenizer_overrides`, while the copy inside `gemma_3_4b_it_4bit_dynamic` is ignored as part of the local model files.
-
 ### Run the app
 
 Select the `CoreAIChat` scheme in Xcode and run the app. Loading the model may take a short while, especially on the first launch.