From c89a6b92e624325cbdcdb3fa0f1e316f9d61b1ce Mon Sep 17 00:00:00 2001 From: a dinosaur Date: Thu, 10 Jul 2025 21:51:30 +1000 Subject: [PATCH] Port index changes to parallel sort --- Sources/apk/Graph/ApkPackageGraph.swift | 51 ++++++--------- Sources/apk/Graph/ApkPackageGraphNode.swift | 62 ++++++++++++++---- .../apk/Index/ApkIndexRequirementRef.swift | 64 ------------------- .../apk/Version/ApkVersionSpecification.swift | 13 +++- .../dpk-cli/Subcommands/DpkGraphCommand.swift | 53 ++++++++++----- 5 files changed, 114 insertions(+), 129 deletions(-) delete mode 100644 Sources/apk/Index/ApkIndexRequirementRef.swift diff --git a/Sources/apk/Graph/ApkPackageGraph.swift b/Sources/apk/Graph/ApkPackageGraph.swift index dd03ba6..2e65127 100644 --- a/Sources/apk/Graph/ApkPackageGraph.swift +++ b/Sources/apk/Graph/ApkPackageGraph.swift @@ -6,61 +6,46 @@ import Foundation public class ApkPackageGraph { - public let pkgIndex: ApkIndex + public var pkgIndex: ApkIndex private var _nodes = [ApkPackageGraphNode]() public var nodes: [ApkPackageGraphNode] { self._nodes } - public var shallowIsolates: [ApkPackageGraphNode] { self._nodes.filter(\.parents.isEmpty) } - public var deepIsolates: [ApkPackageGraphNode] { self._nodes.filter(\.children.isEmpty) } + public var shallowIsolates: [ApkPackageGraphNode] { self._nodes.filter(\.isShallow) } + public var deepIsolates: [ApkPackageGraphNode] { self._nodes.filter(\.isDeep) } public init(index: ApkIndex) { self.pkgIndex = index } public func buildGraphNode() { - var provides = [String: Int]() - - for (idx, package) in self.pkgIndex.packages.enumerated() { - provides[package.name] = idx - for provision in package.provides { - if !provides.keys.contains(provision.name) { - provides[provision.name] = idx - } - } - } - - for (id, package) in pkgIndex.packages.enumerated() { - let children: [ApkIndexRequirementRef] = package.dependencies.compactMap { dependency in - guard !dependency.requirement.versionSpec.conflict, - let id = provides[dependency.requirement.name] else { + for (packageID, package) in pkgIndex.packages.enumerated() { + let children: [ApkPackageGraphNode.ChildRef] = package.dependencies.compactMap { dependency in + guard let providerID = pkgIndex.resolveIndex(requirement: dependency.requirement) else { return nil } - return .init(self, id: id, constraint: .dep(version: dependency.requirement.versionSpec)) + return .init(constraint: .dependency, packageID: providerID, versionSpec: dependency.requirement.versionSpec) } + package.installIf.compactMap { installIf in - guard let id = provides[installIf.requirement.name] else { + guard let prvID = pkgIndex.resolveIndex(requirement: installIf.requirement) else { return nil } - return .init(self, id: id, constraint: .installIf(version: installIf.requirement.versionSpec )) + return .init(constraint: .installIf, packageID: prvID, versionSpec: installIf.requirement.versionSpec) } self._nodes.append(.init(self, - id: id, + id: packageID, children: children )) } - var reverseDependencies = [ApkIndexRequirementRef: [ApkIndexRequirementRef]]() - + var reverseDependencies = [ApkIndex.Index: [ApkIndex.Index]]() for (index, node) in self._nodes.enumerated() { for child in node.children { - reverseDependencies[child, default: []].append( - .init(self, id: index, constraint: child.constraint) - ) + reverseDependencies[child.packageID, default: []].append(index) } } for (ref, parents) in reverseDependencies { - self._nodes[ref.packageID].parents = parents + self._nodes[ref].parentIDs = parents } } } @@ -74,8 +59,8 @@ extension ApkPackageGraph { func findDependencyCycle( node: ApkPackageGraphNode, - _ resolving: inout Set, - _ visited: inout Set + _ resolving: inout Set, + _ visited: inout Set ) -> (ApkPackageGraphNode, ApkPackageGraphNode)? { for dependency in node.children { let depNode = self._nodes[dependency.packageID] @@ -103,8 +88,8 @@ extension ApkPackageGraph { // Map all nodes to all of their children, remove any self dependencies var working = self._nodes.reduce(into: [ApkPackageGraphNode: Set]()) { d, node in d[node] = Set(node.children.filter { child in - if case .dep(let version) = child.constraint { - !version.conflict && child.packageID != node.packageID + if case .dependency = child.constraint { + !child.versionSpec.isConflict && child.packageID != node.packageID } else { false } }.map { self._nodes[$0.packageID] }) } @@ -118,7 +103,7 @@ extension ApkPackageGraph { // Put all extra nodes into the working map, with an empty set extras.forEach { - working[$0] = .init() + working[$0] = [] } while !working.isEmpty { diff --git a/Sources/apk/Graph/ApkPackageGraphNode.swift b/Sources/apk/Graph/ApkPackageGraphNode.swift index a91a521..b82f207 100644 --- a/Sources/apk/Graph/ApkPackageGraphNode.swift +++ b/Sources/apk/Graph/ApkPackageGraphNode.swift @@ -1,24 +1,31 @@ /* - * darwin-apk © 2024 Gay Pizza Specifications + * darwin-apk © 2024, 2025 Gay Pizza Specifications * SPDX-License-Identifier: Apache-2.0 */ import Foundation public class ApkPackageGraphNode { + public let packageID: ApkIndex.Index + public var parentIDs = [ApkIndex.Index]() + public var children: [ChildRef] + private weak var _graph: ApkPackageGraph? - let packageID: Int - //private var _parents = NSHashTable.weakObjects() - //private var _children = NSHashTable.weakObjects() - var parents = [ApkIndexRequirementRef]() - var children: [ApkIndexRequirementRef] - - var package: ApkIndexPackage { + public var package: ApkIndexPackage { self._graph!.pkgIndex.packages[self.packageID] } + public var parents: [ApkIndexPackage] { + self.parentIDs.map { index in self._graph!.pkgIndex.packages[index] } + } + public var childPackages: [ApkIndexPackage] { + self.children.map { child in self._graph!.pkgIndex.packages[child.packageID] } + } - internal init(_ graph: ApkPackageGraph, id: Int, children: [ApkIndexRequirementRef]) { + @inlinable public var isShallow: Bool { self.parentIDs.isEmpty } + @inlinable public var isDeep: Bool { self.children.isEmpty } + + internal init(_ graph: ApkPackageGraph, id: Int, children: [ChildRef]) { self._graph = graph self.packageID = id self.children = children @@ -35,15 +42,42 @@ extension ApkPackageGraphNode: Equatable, Hashable { } } +extension ApkPackageGraphNode { + public struct ChildRef { + let constraint: Constraint + let packageID: Int + let versionSpec: ApkVersionSpecification + } + + public enum Constraint { + case dependency, installIf + } +} + extension ApkPackageGraphNode: CustomStringConvertible { public var description: String { - var result = "node[\(self.package.name)]" - if !self.parents.isEmpty { - result += ", parents[\(self.parents.lazy.map(\.description).joined(separator: ", "))]" + let package = self.package + var result = " \(package.nameDescription):\n" + if !self.parentIDs.isEmpty { + result += " parents:\n" + for parent in self.parents { + result += " \(parent.nameDescription)\n" + } } if !self.children.isEmpty { - result += ", children[\(self.children.lazy.map(\.description).joined(separator: ", "))]" - + result += " children:\n" + for child in self.children { + let childPackage = self._graph!.pkgIndex.packages[child.packageID] + result += " " + switch child.constraint { + case .dependency: result += "dep=" + case .installIf: result += "installIf=" + } + result += childPackage.nameDescription + result += ", " + result += child.versionSpec.description + result += "\n" + } } return result } diff --git a/Sources/apk/Index/ApkIndexRequirementRef.swift b/Sources/apk/Index/ApkIndexRequirementRef.swift deleted file mode 100644 index b8e2144..0000000 --- a/Sources/apk/Index/ApkIndexRequirementRef.swift +++ /dev/null @@ -1,64 +0,0 @@ -/* - * darwin-apk © 2024 Gay Pizza Specifications - * SPDX-License-Identifier: Apache-2.0 - */ - -struct ApkIndexRequirementRef { - private weak var _graph: ApkPackageGraph? - - let packageID: Int - let constraint: Constraint - - init(_ graph: ApkPackageGraph, id: Int, constraint: Constraint) { - self._graph = graph - self.packageID = id - self.constraint = constraint - } - - var package: ApkIndexPackage { - self._graph!.pkgIndex.packages[self.packageID] - } - - func satisfied(by other: ApkVersionRequirement) -> Bool { - true - } - - func normalize() -> ApkIndexRequirementRef { - .init(self._graph!, id: self.packageID, constraint: .dep(version: .any())) - } -} - -extension ApkIndexRequirementRef: Equatable, Hashable { - static func == (lhs: Self, rhs: Self) -> Bool { - lhs.packageID == rhs.packageID && lhs.constraint == rhs.constraint - } - - func hash(into hasher: inout Hasher) { - self.packageID.hash(into: &hasher) - self.constraint.hash(into: &hasher) - } -} - -extension ApkIndexRequirementRef { - enum Constraint: Hashable { - case dep(version: ApkVersionSpecification) - case provision - case installIf(version: ApkVersionSpecification) - } -} - -extension ApkIndexRequirementRef: CustomStringConvertible { - var description: String { - guard let package = self._graph?.pkgIndex.packages[self.packageID] else { - return String() - } - return switch self.constraint { - case .dep(let version): - "dep=\(ApkVersionRequirement(name: package.name, spec: version))" - case .provision: - "provides=\(package.name)" - case .installIf(let version): - "installIf=\(ApkVersionRequirement(name: package.name, spec: version))" - } - } -} diff --git a/Sources/apk/Version/ApkVersionSpecification.swift b/Sources/apk/Version/ApkVersionSpecification.swift index c750380..8c6099e 100644 --- a/Sources/apk/Version/ApkVersionSpecification.swift +++ b/Sources/apk/Version/ApkVersionSpecification.swift @@ -1,5 +1,5 @@ /* - * darwin-apk © 2024 Gay Pizza Specifications + * darwin-apk © 2024, 2025 Gay Pizza Specifications * SPDX-License-Identifier: Apache-2.0 */ @@ -92,3 +92,14 @@ extension ApkVersionSpecification.Operator: CustomStringConvertible { } } } + +extension ApkVersionSpecification: CustomStringConvertible { + var description: String { + switch self { + case .any(invert: false): "depend=any" + case .any(invert: true): "conflict=any" + case .constraint(invert: false, let op, let version): "depend\(op)\(version)" + case .constraint(invert: true, let op, let version): "conflict\(op)\(version)" + } + } +} diff --git a/Sources/dpk-cli/Subcommands/DpkGraphCommand.swift b/Sources/dpk-cli/Subcommands/DpkGraphCommand.swift index 5d34652..49018a5 100644 --- a/Sources/dpk-cli/Subcommands/DpkGraphCommand.swift +++ b/Sources/dpk-cli/Subcommands/DpkGraphCommand.swift @@ -1,5 +1,5 @@ /* - * darwin-apk © 2024 Gay Pizza Specifications + * darwin-apk © 2024, 2025 Gay Pizza Specifications * SPDX-License-Identifier: Apache-2.0 */ @@ -14,28 +14,47 @@ struct DpkGraphCommand: AsyncParsableCommand { let graph: ApkPackageGraph do { let localRepositories = try await ApkRepositoriesConfig() - graph = ApkPackageGraph(index: try await ApkIndexReader.resolve(localRepositories.repositories, fetch: .lazy)) - graph.buildGraphNode() + var timerStart = DispatchTime.now() + graph = ApkPackageGraph(index: + try await ApkIndexReader.resolve(localRepositories.repositories, fetch: .lazy)) + print("Index build took \(timerStart.distance(to: .now()).seconds) seconds") try graph.pkgIndex.description.write(to: URL(filePath: "packages.txt"), atomically: false, encoding: .utf8) - } catch { - fatalError(error.localizedDescription) - } -#if false - if var out = TextFileWriter(URL(filePath: "shallowIsolates.txt")) { - for node in graph.shallowIsolates { print(node, to: &out) } - } - if var out = TextFileWriter(URL(filePath: "deepIsolates.txt")) { - for node in graph.deepIsolates { print(node, to: &out) } - } -#else - do { + timerStart = DispatchTime.now() + graph.buildGraphNode() + print("Graph build took \(timerStart.distance(to: .now()).seconds) seconds") + + try graph.shallowIsolates.map { $0.package.nameDescription }.joined(separator: "\n") + .write(to: URL(filePath: "shallowIsolates.txt"), atomically: false, encoding: .utf8) + try graph.deepIsolates.map { $0.package.nameDescription }.joined(separator: "\n") + .write(to: URL(filePath: "deepIsolates.txt"), atomically: false, encoding: .utf8) + let sorted = try graph.parallelOrderSort() - print(sorted) + if var out = TextFileWriter(URL(filePath: "sorted.txt")) { + for (i, set) in sorted.enumerated() { + print("\(i):\n", to: &out) + for item in set { + print("\(item.description)", to: &out) + } + } + } } catch { fatalError(error.localizedDescription) } -#endif + } +} + +fileprivate extension DispatchTimeInterval { + var seconds: Double { + switch self { + case .seconds(let value): Double(value) + case .milliseconds(let value): Double(value) / 1_000 + case .microseconds(let value): Double(value) / 1_000_000 + case .nanoseconds(let value): Double(value) / 1_000_000_000 + case .never: .infinity + @unknown default: + fatalError("Unsupported") + } } }