From 8c7f0f23d5b6c12d6371addceffb3f8736413130 Mon Sep 17 00:00:00 2001 From: a dinosaur Date: Sun, 17 Nov 2024 02:31:44 +1100 Subject: [PATCH] first async refactor of fetch/index --- .../Common/ApkRepositoriesConfig.swift} | 19 ++++--- Sources/apk/Index/ApkIndex.swift | 2 +- Sources/apk/Index/ApkIndexDependency.swift | 2 +- Sources/apk/Index/ApkIndexDigest.swift | 4 +- Sources/apk/Index/ApkIndexDownloader.swift | 50 ++++++++++++++++++- Sources/apk/Index/ApkIndexInstallIf.swift | 2 +- Sources/apk/Index/ApkIndexPackage.swift | 2 +- Sources/apk/Index/ApkIndexProvides.swift | 2 +- Sources/apk/Index/ApkIndexReading.swift | 33 ++++++++++++ Sources/apk/Index/ApkIndexRepository.swift | 2 +- Sources/apk/Index/ApkIndexUpdate.swift | 12 ++--- .../Subcommands/DpkSearchCommand.swift | 4 +- .../Subcommands/DpkUpdateCommand.swift | 10 ++-- 13 files changed, 112 insertions(+), 32 deletions(-) rename Sources/{dpk-cli/RepositoriesConfig.swift => apk/Common/ApkRepositoriesConfig.swift} (72%) diff --git a/Sources/dpk-cli/RepositoriesConfig.swift b/Sources/apk/Common/ApkRepositoriesConfig.swift similarity index 72% rename from Sources/dpk-cli/RepositoriesConfig.swift rename to Sources/apk/Common/ApkRepositoriesConfig.swift index df4c258..eef075e 100644 --- a/Sources/dpk-cli/RepositoriesConfig.swift +++ b/Sources/apk/Common/ApkRepositoriesConfig.swift @@ -5,12 +5,11 @@ import Foundation import ArgumentParser -import darwin_apk -struct RepositoriesConfig { - let repositories: [ApkIndexRepository] +public struct ApkRepositoriesConfig { + public let repositories: [ApkIndexRepository] - init() async throws(ExitCode) { + public init() async throws(ExitCode) { do { self.repositories = try await Self.readConfig(name: "repositories").flatMap { repo in Self.readConfig(name: "arch").map { arch in @@ -23,12 +22,6 @@ struct RepositoriesConfig { } } - var localRepositories: [URL] { - self.repositories.map { repo in - URL(filePath: repo.localName, directoryHint: .notDirectory) - } - } - private static func readConfig(name: String) -> AsyncFilterSequence, String>> { return URL(filePath: name, directoryHint: .notDirectory).lines @@ -36,3 +29,9 @@ struct RepositoriesConfig { .filter { !$0.isEmpty && $0.first != "#" } // Ignore empty & commented lines } } + +public extension ApkIndex { + @inlinable static func resolve(_ config: ApkRepositoriesConfig, fetch: ApkIndexFetchMode) async throws -> Self { + try await Self.resolve(config.repositories, fetch: fetch) + } +} diff --git a/Sources/apk/Index/ApkIndex.swift b/Sources/apk/Index/ApkIndex.swift index e5c79ef..aa1fbbd 100644 --- a/Sources/apk/Index/ApkIndex.swift +++ b/Sources/apk/Index/ApkIndex.swift @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -public struct ApkIndex { +public struct ApkIndex: Sendable { public let packages: [ApkIndexPackage] } diff --git a/Sources/apk/Index/ApkIndexDependency.swift b/Sources/apk/Index/ApkIndexDependency.swift index 6eabeb0..a675860 100644 --- a/Sources/apk/Index/ApkIndexDependency.swift +++ b/Sources/apk/Index/ApkIndexDependency.swift @@ -5,7 +5,7 @@ import Foundation -public struct ApkIndexDependency: Hashable { +public struct ApkIndexDependency: Hashable, Sendable { let requirement: ApkRequirement init(requirement: ApkRequirement) { diff --git a/Sources/apk/Index/ApkIndexDigest.swift b/Sources/apk/Index/ApkIndexDigest.swift index 032a43c..e560503 100644 --- a/Sources/apk/Index/ApkIndexDigest.swift +++ b/Sources/apk/Index/ApkIndexDigest.swift @@ -6,7 +6,7 @@ import Foundation import CryptoKit -public struct ApkIndexDigest { +public struct ApkIndexDigest: Sendable { public let type: DigestType public let data: Data @@ -89,7 +89,7 @@ extension ApkIndexDigest: Equatable, Hashable { } public extension ApkIndexDigest { - enum DigestType { + enum DigestType: Sendable { case md5, sha1, sha256 } } diff --git a/Sources/apk/Index/ApkIndexDownloader.swift b/Sources/apk/Index/ApkIndexDownloader.swift index 730aebf..bc975d4 100644 --- a/Sources/apk/Index/ApkIndexDownloader.swift +++ b/Sources/apk/Index/ApkIndexDownloader.swift @@ -5,8 +5,9 @@ import Foundation -struct ApkIndexDownloader { - func downloadFile(remote remoteURL: URL, destination destLocalURL: URL) { +public struct ApkIndexDownloader { + @available(*, deprecated, message: "This is stinky, use ApkIndexDownloader.fetch instead") + internal func downloadFile(remote remoteURL: URL, destination destLocalURL: URL) { let sem = DispatchSemaphore.init(value: 0) let downloadTask = URLSession.shared.downloadTask(with: remoteURL) { url, response, error in if let localURL = url { @@ -26,4 +27,49 @@ struct ApkIndexDownloader { downloadTask.resume() sem.wait() } + + public static func fetch(repository: ApkIndexRepository) async throws(FetchError) -> URL { + let localDestinationURL = URL(filePath: repository.localName) + + let tempLocationURL: URL, response: URLResponse + do { + (tempLocationURL, response) = try await URLSession.shared.download(from: repository.url) + } catch { + throw .downloadFailed(error) + } + guard let httpResponse = response as? HTTPURLResponse, + httpResponse.statusCode == 200 else { + throw .invalidServerResponse((response as? HTTPURLResponse)?.statusCode ?? -1) + } + + // Move index repository to destination location + do { + // Replace existing APKINDEX.tar.gz files + if FileManager.default.fileExists(atPath: localDestinationURL.path()) { + try FileManager.default.removeItem(at: localDestinationURL) + } + + // Move downloaded file to the new location + try FileManager.default.moveItem(at: tempLocationURL, to: localDestinationURL) + return localDestinationURL + } catch let error { + throw .moveFailed(error) + } + } +} + +public extension ApkIndexDownloader { + enum FetchError: Error, LocalizedError { + case invalidServerResponse(_ code: Int) + case downloadFailed(_ err: any Error) + case moveFailed(_ err: any Error) + + public var errorDescription: String? { + switch self { + case .invalidServerResponse(let code): "Server responded with HTTP response code \(code)" + case .downloadFailed(let err): "Failed to create session, \(err.localizedDescription)" + case .moveFailed(let err): "Couldn't move index, \(err.localizedDescription)" + } + } + } } diff --git a/Sources/apk/Index/ApkIndexInstallIf.swift b/Sources/apk/Index/ApkIndexInstallIf.swift index 9b0e353..b7280d7 100644 --- a/Sources/apk/Index/ApkIndexInstallIf.swift +++ b/Sources/apk/Index/ApkIndexInstallIf.swift @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -public struct ApkIndexInstallIf: Hashable { +public struct ApkIndexInstallIf: Hashable, Sendable { let requirement: ApkRequirement init(requirement: ApkRequirement) { diff --git a/Sources/apk/Index/ApkIndexPackage.swift b/Sources/apk/Index/ApkIndexPackage.swift index 462aff1..cbdc7f9 100644 --- a/Sources/apk/Index/ApkIndexPackage.swift +++ b/Sources/apk/Index/ApkIndexPackage.swift @@ -5,7 +5,7 @@ import Foundation -public struct ApkIndexPackage: Hashable { +public struct ApkIndexPackage: Hashable, Sendable { public let indexChecksum: ApkIndexDigest public let name: String public let version: String diff --git a/Sources/apk/Index/ApkIndexProvides.swift b/Sources/apk/Index/ApkIndexProvides.swift index 0900887..adcdbc7 100644 --- a/Sources/apk/Index/ApkIndexProvides.swift +++ b/Sources/apk/Index/ApkIndexProvides.swift @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -public struct ApkIndexProvides: Hashable { +public struct ApkIndexProvides: Hashable, Sendable { let name: String init(requirement: ApkRequirement) { diff --git a/Sources/apk/Index/ApkIndexReading.swift b/Sources/apk/Index/ApkIndexReading.swift index f0231c1..08bcb0d 100644 --- a/Sources/apk/Index/ApkIndexReading.swift +++ b/Sources/apk/Index/ApkIndexReading.swift @@ -30,6 +30,39 @@ public extension ApkIndex { } } +public extension ApkIndex { + static func resolve(_ repositories: S, fetch: ApkIndexFetchMode) async throws -> Self where S.Element == ApkIndexRepository { + try await withThrowingTaskGroup(of: Self.self) { group in + for repository in repositories { + group.addTask(priority: .userInitiated) { + let local: URL + switch fetch { + case .local: + local = URL(filePath: repository.localName) + case .lazy: + if !FileManager.default.fileExists(atPath: repository.localName) { + fallthrough + } + local = URL(filePath: repository.localName) + case .update: + print("Fetching \"\(repository.name)\"") + local = try await ApkIndexDownloader.fetch(repository: repository) + } + let index = try ApkIndex(readFrom: local) + return index + } + } + return try await ApkIndex.merge(group.reduce(into: []) { $0.append($1) }) + } + } +} + +public enum ApkIndexFetchMode: Sendable { + case update + case lazy + case local +} + public enum ApkIndexReadingError: Error, LocalizedError { case missingSignature case missingIndex diff --git a/Sources/apk/Index/ApkIndexRepository.swift b/Sources/apk/Index/ApkIndexRepository.swift index 3a7dc4c..c54bbf6 100644 --- a/Sources/apk/Index/ApkIndexRepository.swift +++ b/Sources/apk/Index/ApkIndexRepository.swift @@ -6,7 +6,7 @@ import Foundation import CryptoKit -public struct ApkIndexRepository { +public struct ApkIndexRepository: Sendable { public let name: String public let arch: String public let discriminator: String diff --git a/Sources/apk/Index/ApkIndexUpdate.swift b/Sources/apk/Index/ApkIndexUpdate.swift index 032e244..cf98d01 100644 --- a/Sources/apk/Index/ApkIndexUpdate.swift +++ b/Sources/apk/Index/ApkIndexUpdate.swift @@ -29,7 +29,7 @@ public struct ApkIndexUpdater { let graph: ApkPackageGraph do { - let tables = try self.repositories.map { try readIndex(URL(filePath: $0.localName)) } + let tables = try self.repositories.map { try Self.readIndex(URL(filePath: $0.localName)) } graph = ApkPackageGraph(index: ApkIndex.merge(tables)) graph.buildGraphNode() @@ -46,11 +46,11 @@ public struct ApkIndexUpdater { } } - private func readIndex(_ indexURL: URL) throws -> ApkIndex { + public static func readIndex(_ indexURL: URL) throws -> ApkIndex { let tarSignature: [TarReader.Entry] let tarRecords: [TarReader.Entry] - print("Archive: \(indexURL.lastPathComponent)") + let arcName = indexURL.lastPathComponent let durFormat = Duration.UnitsFormatStyle( allowedUnits: [ .seconds, .milliseconds ], @@ -69,7 +69,7 @@ public struct ApkIndexUpdater { fatalError(error.localizedDescription) } - print("Gzip time: \((ContinuousClock.now - gzipStart).formatted(durFormat))") + print("\(arcName): Gzip time: \((ContinuousClock.now - gzipStart).formatted(durFormat))") let untarStart = ContinuousClock.now let signatureStream = MemoryInputStream(buffer: tars[0]) @@ -84,10 +84,10 @@ public struct ApkIndexUpdater { guard let description = tarRecords.firstFile(name: "DESCRIPTION") else { fatalError("DESCRIPTION missing") } - print("TAR time: \((ContinuousClock.now - untarStart).formatted(durFormat))") + print("\(arcName): TAR time: \((ContinuousClock.now - untarStart).formatted(durFormat))") let indexStart = ContinuousClock.now defer { - print("Index time: \((ContinuousClock.now - indexStart).formatted(durFormat))") + print("\(arcName): Index time: \((ContinuousClock.now - indexStart).formatted(durFormat))") } return try ApkIndex(raw: diff --git a/Sources/dpk-cli/Subcommands/DpkSearchCommand.swift b/Sources/dpk-cli/Subcommands/DpkSearchCommand.swift index 0fa3f5a..b3a0258 100644 --- a/Sources/dpk-cli/Subcommands/DpkSearchCommand.swift +++ b/Sources/dpk-cli/Subcommands/DpkSearchCommand.swift @@ -41,10 +41,10 @@ struct DpkSearchCommand: AsyncParsableCommand { let match: any PatternMatcher match = try matcher.init(patterns: patterns, ignoreCase: !self.caseSensitive) - let localRepositories = try await RepositoriesConfig().localRepositories + let localRepositories = try await ApkRepositoriesConfig() let index: ApkIndex do { - index = ApkIndex.merge(try localRepositories.map(ApkIndex.init)) + index = try await ApkIndex.resolve(localRepositories, fetch: .local) } catch { print("Failed to build package index: \(error.localizedDescription)") throw .failure diff --git a/Sources/dpk-cli/Subcommands/DpkUpdateCommand.swift b/Sources/dpk-cli/Subcommands/DpkUpdateCommand.swift index 07dcaad..7319077 100644 --- a/Sources/dpk-cli/Subcommands/DpkUpdateCommand.swift +++ b/Sources/dpk-cli/Subcommands/DpkUpdateCommand.swift @@ -13,11 +13,13 @@ struct DpkUpdateCommand: AsyncParsableCommand { abstract: "Update the system package repositories.", aliases: [ "u" ]) + @Flag(help: "Index on-disk cache") + var lazyDownload: Bool = false + func run() async throws { + let repositories = try await ApkRepositoriesConfig().repositories print("Updating package repositories") - let repositories = try await RepositoriesConfig().repositories - var updater = ApkIndexUpdater() - updater.repositories.append(contentsOf: repositories) - updater.update() + let index = try await ApkIndex.resolve(repositories, fetch: self.lazyDownload ? .lazy : .update) + print("Indexed \(index.packages.count) package(s)") } }