added: command plugin to download model files.

Package@swift-5.9.swift

@@ -0,0 +1,69 @@
+// swift-tools-version: 5.9
+import PackageDescription
+
+let package = Package(
+    name: "soto-codegenerator",
+    platforms: [.macOS(.v10_15)],
+    products: [
+        .executable(name: "SotoCodeGenerator", targets: ["SotoCodeGenerator"]),
+        .plugin(name: "SotoCodeGeneratorPlugin", targets: ["SotoCodeGeneratorPlugin"]),
+        .plugin(name: "SotoCodeModelDownloaderPlugin",targets: ["SotoCodeModelDownloaderPlugin"])
+    ],
+    dependencies: [
+        .package(url: "https://github.com/soto-project/soto-smithy.git", from: "0.3.1"),
+        .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.0.0"),
+        .package(url: "https://github.com/hummingbird-project/hummingbird-mustache.git", from: "1.0.3"),
+        .package(url: "https://github.com/apple/swift-log.git", from: "1.4.0"),
+        .package(url: "https://github.com/swift-server/async-http-client.git", from: "1.9.0")
+    ],
+    targets: [
+        .executableTarget(
+            name: "SotoCodeGenerator",
+            dependencies: [
+                .byName(name: "SotoCodeGeneratorLib"),
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+                .product(name: "Logging", package: "swift-log")
+            ]
+        ),
+        .executableTarget(
+            name: "SotoModelDownloader",
+            dependencies: [
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+                .product(name: "AsyncHTTPClient", package: "async-http-client")
+            ]
+        ),
+        .target(
+            name: "SotoCodeGeneratorLib",
+            dependencies: [
+                .product(name: "SotoSmithy", package: "soto-smithy"),
+                .product(name: "SotoSmithyAWS", package: "soto-smithy"),
+                .product(name: "HummingbirdMustache", package: "hummingbird-mustache"),
+                .product(name: "Logging", package: "swift-log")
+            ]
+        ),
+        .plugin(
+            name: "SotoCodeGeneratorPlugin",
+            capability: .buildTool(),
+            dependencies: ["SotoCodeGenerator"]
+        ),
+        .plugin(
+            name: "SotoCodeModelDownloaderPlugin",
+            capability: .command(
+             intent: .custom(
+              verb: "get-soto-models",
+              description: "Download the required Model file schema required for soto code genrator to work"),
+             permissions: [
+                .writeToPackageDirectory(reason: "Write the Model files into target project"),
+                .allowNetworkConnections(
+                    scope: .all(ports: []),
+                    reason: "The plugin needs to download resource's from remote server"
+                )
+             ]),
+            dependencies: ["SotoModelDownloader"]
+          ),
+        .testTarget(
+            name: "SotoCodeGeneratorTests",
+            dependencies: ["SotoCodeGeneratorLib"]
+        )
+    ]
+)

Plugins/SotoCodeModelDownloaderPlugin/plugin.swift

@@ -0,0 +1,103 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Soto for AWS open source project
+//
+// Copyright (c) 2017-2024 the Soto project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of Soto project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+import PackagePlugin
+import Foundation
+
+@main
+struct SotoCodeModelDownloader: CommandPlugin {
+
+    struct ModelDownloaderArguments {
+        let inputFolder: String
+        let outputFolder: String
+        let configFile: String?
+
+        func getArguments() -> [String] {
+            // Construct the command line arguments for the model downloader
+            var arguments = ["--input-folder", inputFolder,
+                             "--output-folder", outputFolder]
+            if let configFilePath = configFile {
+                arguments.append(contentsOf: ["--config", configFilePath])
+            }
+            return arguments
+        }
+    }
+
+    func performCommand(context: PackagePlugin.PluginContext, arguments: [String]) async throws {
+        // Get the path to the SotoModelDownloader executable
+        let sotoModelDownloaderTool = try context.tool(named: "SotoModelDownloader")
+        let sotoModelDownloaderURL = URL(filePath: sotoModelDownloaderTool.path.string)
+
+        // Find the config file (soto.config.json) in the package targets
+        let configFile = context.package.targets
+            .compactMap({ $0 as? SourceModuleTarget })
+            .compactMap({ $0.sourceFiles.first(where: { $0.path.lastComponent.contains("soto.config.json") })?.path })
+            .first
+
+        // Ensure the config file is found
+        guard let configFile else {
+            Diagnostics.error("Cannot find the soto.config.json file in the target")
+            return
+        }
+
+        // Determine the main directory and output folder for the downloaded resources
+        let mainDirectory = configFile.removingLastComponent()
+        let outputFolderPath = mainDirectory.appending("aws").string
+
+        // Prepare arguments for downloading model files
+        let modelDownloaderArgs = ModelDownloaderArguments(
+            inputFolder: Repo.modelDirectory,
+            outputFolder: outputFolderPath + "/models",
+            configFile: configFile.string
+        )
+
+        // Prepare arguments for downloading endpoint files
+        let endpointDownloaderArgs = ModelDownloaderArguments(
+            inputFolder: Repo.endpointsDirectory,
+            outputFolder: outputFolderPath,
+            configFile: nil
+        )
+
+        // Iterate over the download tasks (models and endpoints)
+        for resourceArgs in [endpointDownloaderArgs, modelDownloaderArgs] {
+            // Set up the process to run the SotoModelDownloader
+            let process = Process()
+            process.executableURL = sotoModelDownloaderURL
+            process.arguments = resourceArgs.getArguments()
+
+            // Run the process and wait for it to complete
+            try process.run()
+            process.waitUntilExit()
+
+            // Check the process termination status
+            if process.terminationReason == .exit && process.terminationStatus == 0 {
+                print("Downloaded resources to: \(outputFolderPath)")
+            } else {
+                let terminationDescription = "\(process.terminationReason):\(process.terminationStatus)"
+                throw "get-soto-models invocation failed: \(terminationDescription)"
+            }
+        }
+    }
+}
+
+extension String: Error {}
+
+// MARK: - Model and Endpoint URLs -
+
+enum Repo {
+    /// Model files github directory.
+    static let modelDirectory = "https://github.com/soto-project/soto/tree/main/models"
+
+    /// Endpoints github directory.
+    static let endpointsDirectory = "https://github.com/soto-project/soto/tree/main/models/endpoints"
+}

Sources/SotoCodeGeneratorLib/ConfigFile.swift

@@ -12,7 +12,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-struct ConfigFile: Decodable {
+ struct ConfigFile: Decodable {
     struct ServiceConfig: Decodable {
         let operations: [String]?
     }

Sources/SotoModelDownloader/GitHubResource.swift

@@ -0,0 +1,158 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Soto for AWS open source project
+//
+// Copyright (c) 2017-2023 the Soto project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of Soto project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+import AsyncHTTPClient
+import NIOHTTP1
+/// Downloads file from GitHub repositories.
+struct GitHubResource {
+
+    /// Defines errors specific to GitHub resource operations.
+    enum GitHubResourceError: Error, CustomDebugStringConvertible {
+        case invalidURL
+        case missingURLComponents
+        case invalidGitHubResponse
+
+        var debugDescription: String {
+            switch self {
+            case .invalidURL:
+                return "The provided URL is invalid."
+            case .missingURLComponents:
+                return "The URL is missing necessary components."
+            case .invalidGitHubResponse:
+                return "Received an invalid response from GitHub."
+            }
+        }
+    }
+
+    /// The input folder containing the GitHub repository URL.
+    var inputFolder: String
+
+    /// The output folder where the downloaded files will be saved.
+    var outputFolder: String
+
+    /// An array of expected services to be downloaded.
+    var modelFilter: [String] = []
+
+    /// The base URL for the GitHub API.
+    static var gitHubApi: URL { URL(string:  "https://api.github.com/repos")! }
+
+    /// trigger Downloading files from the GitHub repository.
+    func download() async throws {
+
+        // Ensure that the input folder contains a valid GitHub repository URL.
+        guard inputFolder.contains("github.com") else { throw GitHubResourceError.invalidURL }
+        guard let gitHubURL = URL(string: inputFolder) else { throw GitHubResourceError.invalidURL }
+
+        // Extract user, repository, directory, and reference components from the GitHub URL.
+        let (user, repository, directory, reference) = try extractGitHubComponents(from: gitHubURL.absoluteString)
+
+        // Fetch the list of files from the GitHub repository.
+        let files = try await fetchGitHubTree(user: user, repository: repository, directory: directory)
+
+        // Create the output folder if it does not exist.
+        var isDirectory = ObjCBool(false)
+        let exists = FileManager.default.fileExists(atPath: outputFolder, isDirectory: &isDirectory)
+        if !(exists && isDirectory.boolValue) {
+            try FileManager.default.createDirectory(atPath: outputFolder, withIntermediateDirectories: true)
+        }
+
+        // Download files concurrently using a task group.
+        try await withThrowingTaskGroup(of: Void.self) { group in
+            for filePath in files {
+                group.addTask {
+                    let escapedPath = filePath.replacingOccurrences(of: "#", with: "%23")
+                    let downloadPath = [user, repository, reference, escapedPath].joined(separator: "/")
+                    if let fileName = escapedPath.components(separatedBy: "/").last {
+                        print("Downloading: \(fileName)")
+                    }
+                    try await downloadFile(at: downloadPath, to: escapedPath)
+                }
+            }
+            try await group.waitForAll()
+        }
+    }
+
+    /// Fetches the list of files from the GitHub repository using the GitHub Trees API.
+    private func fetchGitHubTree(user: String, repository: String, directory: String) async throws -> [String] {
+
+        let directoryName = directory.components(separatedBy: "/").last ?? ""
+        var requestURLString = GitHubResource.gitHubApi.absoluteString
+        requestURLString.append("/\(user)/\(repository)/git/trees/HEAD?recursive=1")
+
+        var httpRequest = try HTTPClient.Request(url: requestURLString)
+        httpRequest.headers.add(name: "User-Agent", value: "AsyncHttpClient")
+        let response = try await HTTPClient.shared.execute(request: httpRequest).get()
+        guard response.status == .ok, let data = response.body else { throw GitHubResourceError.invalidGitHubResponse }
+
+        let gitHubDirectoryTree = try JSONDecoder().decode(GitHubDirectoryTree.self, from: data)
+
+        var filePaths = [String]()
+        for item in gitHubDirectoryTree.tree {
+            let serviceName = (item.path.components(separatedBy: "/").last?.components(separatedBy: ".").first) ?? ""
+            if !modelFilter.isEmpty, !modelFilter.contains(serviceName) {
+                continue
+            }
+            // Check for subdirectory
+            let currentDirectory = item.path.components(separatedBy: "/").dropLast().last ?? ""
+            if item.type == "blob" && currentDirectory.elementsEqual(directoryName) {
+                filePaths.append(item.path)
+            }
+        }
+
+        return filePaths
+    }
+
+    /// Extracts user, repository, directory, and reference components from the GitHub repository URL.
+     private func extractGitHubComponents(from urlString: String) throws -> (user: String, repository: String, directory: String, reference: String) {
+         guard let url = URL(string: urlString),
+               url.host == "github.com",
+               let pathComponents = URLComponents(url: url, resolvingAgainstBaseURL: false)?.path.components(separatedBy: "/").dropFirst(1).map({ $0 })
+         else {
+             throw GitHubResourceError.invalidURL
+         }
+
+         guard pathComponents.count > 4 else { throw GitHubResourceError.missingURLComponents }
+
+         let user = pathComponents[0]
+         let repository = pathComponents[1]
+         let reference = pathComponents[3]
+         let directory = pathComponents[4...].joined(separator: "/")
+
+         return (user, repository, directory, reference)
+     }
+
+    /// Downloads a raw file from the GitHub repository.
+    private func downloadFile(at path: String, to directory: String) async throws {
+        let client = HTTPClient(eventLoopGroupProvider: .singleton)
+
+        let downloadUrlString = "https://raw.githubusercontent.com/" + path
+        let downloadRequest = try HTTPClient.Request(url: downloadUrlString)
+        let fileName = directory.components(separatedBy: "/").last ?? directory
+        let filePath = outputFolder + "/\(fileName)"
+
+        var downloadStatus: HTTPResponseStatus = .noContent
+        let delegate = try FileDownloadDelegate(path: filePath) { _, response in
+            downloadStatus = response.status
+        }
+
+        _ = try await client.execute(request: downloadRequest, delegate: delegate).get()
+
+        guard downloadStatus == .ok else { throw GitHubResourceError.invalidGitHubResponse }
+
+        try await client.shutdown()
+    }
+}
+
+

Sources/SotoModelDownloader/Model/ConfigFile + JsonDecode.swift

@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Soto for AWS open source project
+//
+// Copyright (c) 2017-2023 the Soto project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of Soto project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+struct ConfigFile: Decodable {
+    struct ServiceConfig: Decodable {
+        let operations: [String]?
+    }
+
+    let services: [String: ServiceConfig]?
+}
+
+extension ConfigFile {
+
+   static func decodeFrom(file configFile: String) throws -> Self {
+        let data = try Data(contentsOf: URL(fileURLWithPath: configFile))
+        let sotoConfig = try JSONDecoder().decode(ConfigFile.self, from: data)
+        return sotoConfig
+    }
+}