first async refactor of fetch/index

Sources/apk/Common/ApkRepositoriesConfig.swift

@@ -0,0 +1,37 @@
+/*
+ * darwin-apk © 2024 Gay Pizza Specifications
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import Foundation
+import ArgumentParser
+
+public struct ApkRepositoriesConfig {
+  public let repositories: [ApkIndexRepository]
+
+  public init() async throws(ExitCode) {
+    do {
+      self.repositories = try await Self.readConfig(name: "repositories").flatMap { repo in
+        Self.readConfig(name: "arch").map { arch in
+          ApkIndexRepository(name: repo, arch: arch)
+        }
+      }.reduce(into: []) { $0.append($1) }
+    } catch {
+      print("Failed to read repository configurations, \(error.localizedDescription)")
+      throw .failure
+    }
+  }
+
+  private static func readConfig(name: String)
+      -> AsyncFilterSequence<AsyncMapSequence<AsyncLineSequence<URL.AsyncBytes>, String>> {
+    return URL(filePath: name, directoryHint: .notDirectory).lines
+      .map { $0.trimmingCharacters(in: .whitespaces) }
+      .filter { !$0.isEmpty && $0.first != "#" }  // Ignore empty & commented lines
+  }
+}
+
+public extension ApkIndex {
+  @inlinable static func resolve(_ config: ApkRepositoriesConfig, fetch: ApkIndexFetchMode) async throws -> Self {
+    try await Self.resolve(config.repositories, fetch: fetch)
+  }
+}

Sources/apk/Index/ApkIndex.swift

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-public struct ApkIndex {
+public struct ApkIndex: Sendable {
   public let packages: [ApkIndexPackage]
 }
 

Sources/apk/Index/ApkIndexDependency.swift

@@ -5,7 +5,7 @@
 
 import Foundation
 
-public struct ApkIndexDependency: Hashable {
+public struct ApkIndexDependency: Hashable, Sendable {
   let requirement: ApkRequirement
 
   init(requirement: ApkRequirement) {

Sources/apk/Index/ApkIndexDigest.swift

@@ -6,7 +6,7 @@
 import Foundation
 import CryptoKit
 
-public struct ApkIndexDigest {
+public struct ApkIndexDigest: Sendable {
   public let type: DigestType
   public let data: Data
 
@@ -89,7 +89,7 @@ extension ApkIndexDigest: Equatable, Hashable {
 }
 
 public extension ApkIndexDigest {
-  enum DigestType {
+  enum DigestType: Sendable {
     case md5, sha1, sha256
   }
 }

Sources/apk/Index/ApkIndexDownloader.swift

@@ -5,8 +5,9 @@
 
 import Foundation
 
-struct ApkIndexDownloader {
-  func downloadFile(remote remoteURL: URL, destination destLocalURL: URL) {
+public struct ApkIndexDownloader {
+  @available(*, deprecated, message: "This is stinky, use ApkIndexDownloader.fetch instead")
+  internal func downloadFile(remote remoteURL: URL, destination destLocalURL: URL) {
     let sem = DispatchSemaphore.init(value: 0)
     let downloadTask = URLSession.shared.downloadTask(with: remoteURL) { url, response, error in
       if let localURL = url {
@@ -26,4 +27,49 @@ struct ApkIndexDownloader {
     downloadTask.resume()
     sem.wait()
   }
+
+  public static func fetch(repository: ApkIndexRepository) async throws(FetchError) -> URL {
+    let localDestinationURL = URL(filePath: repository.localName)
+
+    let tempLocationURL: URL, response: URLResponse
+    do {
+      (tempLocationURL, response) = try await URLSession.shared.download(from: repository.url)
+    } catch {
+      throw .downloadFailed(error)
+    }
+    guard let httpResponse = response as? HTTPURLResponse,
+        httpResponse.statusCode == 200 else {
+      throw .invalidServerResponse((response as? HTTPURLResponse)?.statusCode ?? -1)
+    }
+
+    // Move index repository to destination location
+    do {
+      // Replace existing APKINDEX.tar.gz files
+      if FileManager.default.fileExists(atPath: localDestinationURL.path()) {
+        try FileManager.default.removeItem(at: localDestinationURL)
+      }
+
+      // Move downloaded file to the new location
+      try FileManager.default.moveItem(at: tempLocationURL, to: localDestinationURL)
+      return localDestinationURL
+    } catch let error {
+      throw .moveFailed(error)
+    }
+  }
+}
+
+public extension ApkIndexDownloader {
+  enum FetchError: Error, LocalizedError {
+    case invalidServerResponse(_ code: Int)
+    case downloadFailed(_ err: any Error)
+    case moveFailed(_ err: any Error)
+
+    public var errorDescription: String? {
+      switch self {
+      case .invalidServerResponse(let code): "Server responded with HTTP response code \(code)"
+      case .downloadFailed(let err):         "Failed to create session, \(err.localizedDescription)"
+      case .moveFailed(let err):             "Couldn't move index, \(err.localizedDescription)"
+      }
+    }
+  }
 }

Sources/apk/Index/ApkIndexInstallIf.swift

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-public struct ApkIndexInstallIf: Hashable {
+public struct ApkIndexInstallIf: Hashable, Sendable {
   let requirement: ApkRequirement
 
   init(requirement: ApkRequirement) {

Sources/apk/Index/ApkIndexPackage.swift

@@ -5,7 +5,7 @@
 
 import Foundation
 
-public struct ApkIndexPackage: Hashable {
+public struct ApkIndexPackage: Hashable, Sendable {
   public let indexChecksum: ApkIndexDigest
   public let name: String
   public let version: String

Sources/apk/Index/ApkIndexProvides.swift

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-public struct ApkIndexProvides: Hashable {
+public struct ApkIndexProvides: Hashable, Sendable {
   let name: String
 
   init(requirement: ApkRequirement) {

Sources/apk/Index/ApkIndexReading.swift

@@ -30,6 +30,39 @@ public extension ApkIndex {
   }
 }
 
+public extension ApkIndex {
+  static func resolve<S: Sequence>(_ repositories: S, fetch: ApkIndexFetchMode) async throws -> Self where S.Element == ApkIndexRepository {
+    try await withThrowingTaskGroup(of: Self.self) { group in
+      for repository in repositories {
+        group.addTask(priority: .userInitiated) {
+          let local: URL
+          switch fetch {
+          case .local:
+            local = URL(filePath: repository.localName)
+          case .lazy:
+            if !FileManager.default.fileExists(atPath: repository.localName) {
+              fallthrough
+            }
+            local = URL(filePath: repository.localName)
+          case .update:
+            print("Fetching \"\(repository.name)\"")
+            local = try await ApkIndexDownloader.fetch(repository: repository)
+          }
+          let index = try ApkIndex(readFrom: local)
+          return index
+        }
+      }
+      return try await ApkIndex.merge(group.reduce(into: []) { $0.append($1) })
+    }
+  }
+}
+
+public enum ApkIndexFetchMode: Sendable {
+  case update
+  case lazy
+  case local
+}
+
 public enum ApkIndexReadingError: Error, LocalizedError {
   case missingSignature
   case missingIndex

Sources/apk/Index/ApkIndexRepository.swift

@@ -6,7 +6,7 @@
 import Foundation
 import CryptoKit
 
-public struct ApkIndexRepository {
+public struct ApkIndexRepository: Sendable {
   public let name: String
   public let arch: String
   public let discriminator: String

Sources/apk/Index/ApkIndexUpdate.swift

@@ -29,7 +29,7 @@ public struct ApkIndexUpdater {
 
     let graph: ApkPackageGraph
     do {
-      let tables = try self.repositories.map { try readIndex(URL(filePath: $0.localName)) }
+      let tables = try self.repositories.map { try Self.readIndex(URL(filePath: $0.localName)) }
       graph = ApkPackageGraph(index: ApkIndex.merge(tables))
       graph.buildGraphNode()
 
@@ -46,11 +46,11 @@ public struct ApkIndexUpdater {
     }
   }
 
-  private func readIndex(_ indexURL: URL) throws -> ApkIndex {
+  public static func readIndex(_ indexURL: URL) throws -> ApkIndex {
     let tarSignature: [TarReader.Entry]
     let tarRecords: [TarReader.Entry]
 
-    print("Archive:    \(indexURL.lastPathComponent)")
+    let arcName = indexURL.lastPathComponent
 
     let durFormat = Duration.UnitsFormatStyle(
       allowedUnits: [ .seconds, .milliseconds ],
@@ -69,7 +69,7 @@ public struct ApkIndexUpdater {
       fatalError(error.localizedDescription)
     }
 
-    print("Gzip time:  \((ContinuousClock.now - gzipStart).formatted(durFormat))")
+    print("\(arcName): Gzip time:  \((ContinuousClock.now - gzipStart).formatted(durFormat))")
     let untarStart = ContinuousClock.now
 
     let signatureStream = MemoryInputStream(buffer: tars[0])
@@ -84,10 +84,10 @@ public struct ApkIndexUpdater {
     guard let description = tarRecords.firstFile(name: "DESCRIPTION")
     else { fatalError("DESCRIPTION missing") }
 
-    print("TAR time:   \((ContinuousClock.now - untarStart).formatted(durFormat))")
+    print("\(arcName): TAR time:   \((ContinuousClock.now - untarStart).formatted(durFormat))")
     let indexStart = ContinuousClock.now
     defer {
-      print("Index time: \((ContinuousClock.now - indexStart).formatted(durFormat))")
+      print("\(arcName): Index time: \((ContinuousClock.now - indexStart).formatted(durFormat))")
     }
 
     return try ApkIndex(raw: