From be51934334dd74627c7c5fa3339a2997b4deab95 Mon Sep 17 00:00:00 2001 From: a dinosaur Date: Fri, 22 Nov 2024 21:01:01 +1100 Subject: [PATCH] Consolidate update+reader & remove some dead code --- Sources/apk/Index/ApkIndexDownloader.swift | 22 ---- Sources/apk/Index/ApkIndexReader.swift | 102 ++++++++++++++++++ Sources/apk/Index/ApkIndexReading.swift | 79 -------------- Sources/apk/Index/ApkIndexUpdate.swift | 67 +----------- Sources/dpk-cli/ApkRepositoriesConfig.swift | 4 +- .../Subcommands/DpkSearchCommand.swift | 2 +- .../Subcommands/DpkUpdateCommand.swift | 2 +- 7 files changed, 108 insertions(+), 170 deletions(-) create mode 100644 Sources/apk/Index/ApkIndexReader.swift delete mode 100644 Sources/apk/Index/ApkIndexReading.swift diff --git a/Sources/apk/Index/ApkIndexDownloader.swift b/Sources/apk/Index/ApkIndexDownloader.swift index bc975d4..02960aa 100644 --- a/Sources/apk/Index/ApkIndexDownloader.swift +++ b/Sources/apk/Index/ApkIndexDownloader.swift @@ -6,28 +6,6 @@ import Foundation public struct ApkIndexDownloader { - @available(*, deprecated, message: "This is stinky, use ApkIndexDownloader.fetch instead") - internal func downloadFile(remote remoteURL: URL, destination destLocalURL: URL) { - let sem = DispatchSemaphore.init(value: 0) - let downloadTask = URLSession.shared.downloadTask(with: remoteURL) { url, response, error in - if let localURL = url { - do { - // Replace existing APKINDEX.tar.gz files - if FileManager.default.fileExists(atPath: destLocalURL.path()) { - try FileManager.default.removeItem(at: destLocalURL) - } - // Move temporary to the new location - try FileManager.default.moveItem(at: localURL, to: destLocalURL) - } catch { - print("Download error: \(error.localizedDescription)") - } - } - sem.signal() - } - downloadTask.resume() - sem.wait() - } - public static func fetch(repository: ApkIndexRepository) async throws(FetchError) -> URL { let localDestinationURL = URL(filePath: repository.localName) diff --git a/Sources/apk/Index/ApkIndexReader.swift b/Sources/apk/Index/ApkIndexReader.swift new file mode 100644 index 0000000..80b4089 --- /dev/null +++ b/Sources/apk/Index/ApkIndexReader.swift @@ -0,0 +1,102 @@ +/* + * darwin-apk © 2024 Gay Pizza Specifications + * SPDX-License-Identifier: Apache-2.0 + */ + +import Foundation + +public struct ApkIndexReader { + static func read(from indexURL: URL) throws -> ApkIndex { + let timed = false + var timer: ContinuousClock.Instant + let durFormat = Duration.UnitsFormatStyle( + allowedUnits: [ .seconds, .milliseconds ], + width: .condensedAbbreviated, + fractionalPart: .show(length: 3)) + + if timed { + timer = ContinuousClock.now + } + + let file = try FileInputStream(indexURL) + //var file = try MemoryInputStream(buffer: try Data(contentsOf: indexURL)) + var gzip = GZipReader() + var tarRecords = [TarReader.Entry]() + for tarData in try (0..<2).map({ _ in try gzip.read(inStream: file) }) { + let tarStream = MemoryInputStream(buffer: tarData) + tarRecords += try TarReader.read(tarStream) + } + + guard case .file(let signatureName, _) = tarRecords.first else { + throw ReadingError.missingSignature + } + guard let apkIndexFile = tarRecords.firstFile(name: "APKINDEX") else { + throw ReadingError.missingIndex + } + guard let description = tarRecords.firstFile(name: "DESCRIPTION") else { + throw ReadingError.missingDescription + } + + if timed { + print("\(indexURL.lastPathComponent): Extract time: \((ContinuousClock.now - timer).formatted(durFormat))") + timer = ContinuousClock.now + } + + let index = try ApkIndex(raw: + try ApkRawIndex(lines: MemoryInputStream(buffer: apkIndexFile).lines)) + + if timed { + print("\(indexURL.lastPathComponent): Index time: \((ContinuousClock.now - timer).formatted(durFormat))") + } + + return index + } + + public static func resolve>(_ repositories: S, fetch: FetchMode) async throws -> ApkIndex { + try await withThrowingTaskGroup(of: ApkIndex.self) { group in + for repository in repositories { + group.addTask(priority: .userInitiated) { + let local: URL + switch fetch { + case .local: + local = URL(filePath: repository.localName) + case .lazy: + if !FileManager.default.fileExists(atPath: repository.localName) { + fallthrough + } + local = URL(filePath: repository.localName) + case .update: + //FIXME: Don't call print in the lib + print("Fetching \"\(repository.resolved)\"") + local = try await ApkIndexDownloader.fetch(repository: repository) + } + let index = try Self.read(from: local) + return index + } + } + return try await ApkIndex.merge(group.reduce(into: []) { $0.append($1) }) + } + } +} + +public extension ApkIndexReader { + enum FetchMode: Sendable { + case update + case lazy + case local + } + + enum ReadingError: Error, LocalizedError { + case missingSignature + case missingIndex + case missingDescription + + public var errorDescription: String? { + switch self { + case .missingSignature: "Missing signature" + case .missingIndex: "APKINDEX missing" + case .missingDescription: "DESCRIPTION missing" + } + } + } +} diff --git a/Sources/apk/Index/ApkIndexReading.swift b/Sources/apk/Index/ApkIndexReading.swift deleted file mode 100644 index 123b33a..0000000 --- a/Sources/apk/Index/ApkIndexReading.swift +++ /dev/null @@ -1,79 +0,0 @@ -/* - * darwin-apk © 2024 Gay Pizza Specifications - * SPDX-License-Identifier: Apache-2.0 - */ - -import Foundation - -public extension ApkIndex { - init(readFrom indexURL: URL) throws { - let file = try FileInputStream(indexURL) - var gzip = GZipReader() - var tarRecords = [TarReader.Entry]() - for tarData in try (0..<2).map({ _ in try gzip.read(inStream: file) }) { - let tarStream = MemoryInputStream(buffer: tarData) - tarRecords += try TarReader.read(tarStream) - } - - guard case .file(let signatureName, _) = tarRecords.first else { - throw ApkIndexReadingError.missingSignature - } - guard let apkIndexFile = tarRecords.firstFile(name: "APKINDEX") else { - throw ApkIndexReadingError.missingIndex - } - guard let description = tarRecords.firstFile(name: "DESCRIPTION") else { - throw ApkIndexReadingError.missingDescription - } - - try self.init(raw: - try ApkRawIndex(lines: MemoryInputStream(buffer: apkIndexFile).lines)) - } -} - -public extension ApkIndex { - static func resolve(_ repositories: S, fetch: ApkIndexFetchMode) async throws -> Self where S.Element == ApkIndexRepository { - try await withThrowingTaskGroup(of: Self.self) { group in - for repository in repositories { - group.addTask(priority: .userInitiated) { - let local: URL - switch fetch { - case .local: - local = URL(filePath: repository.localName) - case .lazy: - if !FileManager.default.fileExists(atPath: repository.localName) { - fallthrough - } - local = URL(filePath: repository.localName) - case .update: - //FIXME: Don't call print in the lib - print("Fetching \"\(repository.resolved)\"") - local = try await ApkIndexDownloader.fetch(repository: repository) - } - let index = try ApkIndex(readFrom: local) - return index - } - } - return try await ApkIndex.merge(group.reduce(into: []) { $0.append($1) }) - } - } -} - -public enum ApkIndexFetchMode: Sendable { - case update - case lazy - case local -} - -public enum ApkIndexReadingError: Error, LocalizedError { - case missingSignature - case missingIndex - case missingDescription - - public var errorDescription: String? { - switch self { - case .missingSignature: "Missing signature" - case .missingIndex: "APKINDEX missing" - case .missingDescription: "DESCRIPTION missing" - } - } -} diff --git a/Sources/apk/Index/ApkIndexUpdate.swift b/Sources/apk/Index/ApkIndexUpdate.swift index cf98d01..a48bf2a 100644 --- a/Sources/apk/Index/ApkIndexUpdate.swift +++ b/Sources/apk/Index/ApkIndexUpdate.swift @@ -12,25 +12,10 @@ public struct ApkIndexUpdater { self.repositories = [] } - public func update() { - let downloader = ApkIndexDownloader() - for repo in self.repositories { - let localIndex = URL(filePath: repo.localName) -#if false - let shouldDownload = true -#else - let shouldDownload = !FileManager.default.fileExists(atPath: localIndex.path()) -#endif - if shouldDownload { - print("Fetching index for \"\(repo.name)\"") - downloader.downloadFile(remote: repo.url, destination: localIndex) - } - } - + public func buildGraph() async { let graph: ApkPackageGraph do { - let tables = try self.repositories.map { try Self.readIndex(URL(filePath: $0.localName)) } - graph = ApkPackageGraph(index: ApkIndex.merge(tables)) + graph = ApkPackageGraph(index: try await ApkIndexReader.resolve(self.repositories, fetch: .lazy)) graph.buildGraphNode() try graph.pkgIndex.description.write(to: URL(filePath: "packages.txt"), atomically: false, encoding: .utf8) @@ -45,52 +30,4 @@ public struct ApkIndexUpdater { for node in graph.deepIsolates { print(node, to: &out) } } } - - public static func readIndex(_ indexURL: URL) throws -> ApkIndex { - let tarSignature: [TarReader.Entry] - let tarRecords: [TarReader.Entry] - - let arcName = indexURL.lastPathComponent - - let durFormat = Duration.UnitsFormatStyle( - allowedUnits: [ .seconds, .milliseconds ], - width: .condensedAbbreviated, - fractionalPart: .show(length: 3)) - let gzipStart = ContinuousClock.now - - var tars = [Data]() - do { - var file = try FileInputStream(indexURL) - //var file = try MemoryInputStream(buffer: try Data(contentsOf: indexURL)) - var gzip = GZipReader() - tars.append(try gzip.read(inStream: file)) - tars.append(try gzip.read(inStream: file)) - } catch { - fatalError(error.localizedDescription) - } - - print("\(arcName): Gzip time: \((ContinuousClock.now - gzipStart).formatted(durFormat))") - let untarStart = ContinuousClock.now - - let signatureStream = MemoryInputStream(buffer: tars[0]) - tarSignature = try TarReader.read(signatureStream) - let recordsStream = MemoryInputStream(buffer: tars[1]) - tarRecords = try TarReader.read(recordsStream) - - guard case .file(let signatureName, _) = tarSignature.first - else { fatalError("Missing signature") } - guard let apkIndexFile = tarRecords.firstFile(name: "APKINDEX") - else { fatalError("APKINDEX missing") } - guard let description = tarRecords.firstFile(name: "DESCRIPTION") - else { fatalError("DESCRIPTION missing") } - - print("\(arcName): TAR time: \((ContinuousClock.now - untarStart).formatted(durFormat))") - let indexStart = ContinuousClock.now - defer { - print("\(arcName): Index time: \((ContinuousClock.now - indexStart).formatted(durFormat))") - } - - return try ApkIndex(raw: - try ApkRawIndex(lines: MemoryInputStream(buffer: apkIndexFile).lines)) - } } diff --git a/Sources/dpk-cli/ApkRepositoriesConfig.swift b/Sources/dpk-cli/ApkRepositoriesConfig.swift index 0c42479..59b9314 100644 --- a/Sources/dpk-cli/ApkRepositoriesConfig.swift +++ b/Sources/dpk-cli/ApkRepositoriesConfig.swift @@ -31,8 +31,8 @@ public struct ApkRepositoriesConfig { } } -public extension ApkIndex { - @inlinable static func resolve(_ config: ApkRepositoriesConfig, fetch: ApkIndexFetchMode) async throws -> Self { +public extension ApkIndexReader { + @inlinable static func resolve(_ config: ApkRepositoriesConfig, fetch: FetchMode) async throws -> ApkIndex { try await Self.resolve(config.repositories, fetch: fetch) } } diff --git a/Sources/dpk-cli/Subcommands/DpkSearchCommand.swift b/Sources/dpk-cli/Subcommands/DpkSearchCommand.swift index d24e50c..644f3c2 100644 --- a/Sources/dpk-cli/Subcommands/DpkSearchCommand.swift +++ b/Sources/dpk-cli/Subcommands/DpkSearchCommand.swift @@ -44,7 +44,7 @@ struct DpkSearchCommand: AsyncParsableCommand { let localRepositories = try await ApkRepositoriesConfig() let index: ApkIndex do { - index = try await ApkIndex.resolve(localRepositories, fetch: .local) + index = try await ApkIndexReader.resolve(localRepositories, fetch: .local) } catch { eprint("Failed to build package index: \(error.localizedDescription)") throw .failure diff --git a/Sources/dpk-cli/Subcommands/DpkUpdateCommand.swift b/Sources/dpk-cli/Subcommands/DpkUpdateCommand.swift index 16ae139..baf89b3 100644 --- a/Sources/dpk-cli/Subcommands/DpkUpdateCommand.swift +++ b/Sources/dpk-cli/Subcommands/DpkUpdateCommand.swift @@ -19,7 +19,7 @@ struct DpkUpdateCommand: AsyncParsableCommand { func run() async throws { let repositories = try await ApkRepositoriesConfig().repositories eprint("Updating package repositories") - let index = try await ApkIndex.resolve(repositories, fetch: self.lazyDownload ? .lazy : .update) + let index = try await ApkIndexReader.resolve(repositories, fetch: self.lazyDownload ? .lazy : .update) eprint("Indexed \(index.packages.count) package(s)") } }