Make tokens strings (it's easier)

Author: Lukasa

Sources/StructuredHeaders/ComponentTypes.swift

@@ -32,7 +32,7 @@ public enum BareItem<BaseData: RandomAccessCollection> where BaseData.Element ==
     case decimal(PseudoDecimal)
     case string(String)
     case undecodedByteSequence(BaseData)
-    case token(BaseData)
+    case token(String)
 }
 
 extension BareItem: ExpressibleByBooleanLiteral {
@@ -55,7 +55,11 @@ extension BareItem: ExpressibleByFloatLiteral {
 
 extension BareItem: ExpressibleByStringLiteral {
     public init(stringLiteral value: StringLiteralType) {
-        self = .string(value)
+        if value.isValidToken {
+            self = .token(value)
+        } else {
+            self = .string(value)
+        }
     }
 }
 
@@ -155,3 +159,42 @@ public struct InnerList<BaseData: RandomAccessCollection>: Hashable where BaseDa
         self.parameters = parameters
     }
 }
+
+extension String {
+    var isValidToken: Bool {
+        let view = self.utf8
+
+        switch view.first {
+        case .some(asciiCapitals), .some(asciiLowercases), .some(asciiAsterisk):
+            // Good
+            ()
+        default:
+            return false
+        }
+
+        for byte in view {
+            switch byte {
+            // Valid token characters are RFC 7230 tchar, colon, and slash.
+            // tchar is:
+            //
+            // tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
+            //                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
+            //                / DIGIT / ALPHA
+            //
+            // The following insane case statement covers this. Tokens suck.
+            case asciiExclamationMark, asciiOctothorpe, asciiDollar, asciiPercent,
+                 asciiAmpersand, asciiSquote, asciiAsterisk, asciiPlus, asciiDash,
+                 asciiPeriod, asciiCaret, asciiUnderscore, asciiBacktick, asciiPipe,
+                 asciiTilde, asciiDigits, asciiCapitals, asciiLowercases,
+                 asciiColon, asciiSlash:
+                // Good
+                ()
+            default:
+                // Bad token
+                return false
+            }
+        }
+
+        return true
+    }
+}

Sources/StructuredHeaders/Decoder/BareItemDecoder.swift

@@ -81,7 +81,7 @@ extension BareItemDecoder: SingleValueDecodingContainer {
         case .string(let string):
             return string
         case .token(let token):
-            return String(decoding: token, as: UTF8.self)
+            return token
         default:
             throw StructuredHeaderError.invalidTypeForItem
         }

Sources/StructuredHeaders/FieldParser.swift

@@ -443,7 +443,7 @@ extension StructuredFieldParser {
         // we get to the end of the string.
         let tokenSlice = self.underlyingData[..<index]
         self.underlyingData = self.underlyingData[index...]
-        return .token(tokenSlice)
+        return .token(String(decoding: tokenSlice, as: UTF8.self))
     }
 
     private mutating func _parseParameters() throws -> OrderedMap<BaseData.SubSequence, BareItem> {

Sources/StructuredHeaders/FieldSerializer.swift

@@ -176,38 +176,10 @@ extension StructuredFieldSerializer {
             }
             self.data.append(asciiDquote)
         case .token(let token):
-            switch token.first {
-            case .some(asciiCapitals), .some(asciiLowercases), .some(asciiAsterisk):
-                // Good
-                ()
-            default:
+            guard token.isValidToken else {
                 throw StructuredHeaderError.invalidToken
             }
-
-            for byte in token {
-                switch byte {
-                // Valid token characters are RFC 7230 tchar, colon, and slash.
-                // tchar is:
-                //
-                // tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
-                //                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
-                //                / DIGIT / ALPHA
-                //
-                // The following insane case statement covers this. Tokens suck.
-                case asciiExclamationMark, asciiOctothorpe, asciiDollar, asciiPercent,
-                     asciiAmpersand, asciiSquote, asciiAsterisk, asciiPlus, asciiDash,
-                     asciiPeriod, asciiCaret, asciiUnderscore, asciiBacktick, asciiPipe,
-                     asciiTilde, asciiDigits, asciiCapitals, asciiLowercases,
-                     asciiColon, asciiSlash:
-                    // Good
-                    ()
-                default:
-                    // Bad token
-                    throw StructuredHeaderError.invalidToken
-                }
-            }
-
-            self.data.append(contentsOf: token)
+            self.data.append(contentsOf: token.utf8)
         case .undecodedByteSequence(let bytes):
             // We require the user to have gotten this right.
             self.data.append(asciiColon)

Tests/StructuredHeadersTests/StructuredFieldParserTests.swift

@@ -38,14 +38,14 @@ final class StructuredFieldParserTests: XCTestCase {
             XCTAssertEqual(Double(baseDecimal), Double(jsonInteger), "\(fixtureName): Got \(bareItem), expected \(schema)")
         case (.string(let baseString), .string(let jsonString)):
             XCTAssertEqual(baseString, jsonString, "\(fixtureName): Got \(bareItem), expected \(schema)")
-        case (.token(let baseToken), .dictionary(let typeDictionary)):
+        case (.token(let token), .dictionary(let typeDictionary)):
             guard typeDictionary.count == 2, case .string(let typeName) = typeDictionary["__type"], case .string(let typeValue) = typeDictionary["value"] else {
                 XCTFail("\(fixtureName): Unexpected type dict \(typeDictionary)")
                 return
             }
 
             XCTAssertEqual(typeName, "token", "\(fixtureName): Expected type token, got type \(typeName)")
-            XCTAssertEqual(typeValue, String(decoding: baseToken, as: UTF8.self), "\(fixtureName): Got \(String(decoding: baseToken, as: UTF8.self)), expected \(typeValue)")
+            XCTAssertEqual(typeValue, token, "\(fixtureName): Got \(token), expected \(typeValue)")
         case (.undecodedByteSequence(let binary), .dictionary(let typeDictionary)):
             guard typeDictionary.count == 2, case .string(let typeName) = typeDictionary["__type"], case .string(let typeValue) = typeDictionary["value"] else {
                 XCTFail("\(fixtureName): Unexpected type dict \(typeDictionary)")

Tests/StructuredHeadersTests/StructuredFieldSerializerTests.swift

@@ -193,7 +193,7 @@ extension BareItem where BaseData == ArraySlice<UInt8> {
         case .dictionary(let typeObject):
             switch (typeObject["__type"], typeObject["value"]) {
             case (.some(.string("token")), .some(.string(let value))):
-                self = .token(ArraySlice(value.utf8))
+                self = .token(value)
 
             case (.some(.string("binary")), .some(.string(let value))):
                 let expectedBase64Bytes = Data(base32Encoded: Data(value.utf8)).base64EncodedData()