Initial implementation of APKINDEX, fetching, reading, parsing, & merging

Sources/apk/Index/ApkIndex.swift

@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: Apache-2.0
+
+struct ApkIndex {
+  let packages: [ApkIndexPackage]
+}
+
+extension ApkIndex {
+  func first(name: String) -> ApkIndexPackage? {
+    self.packages.first {
+      $0.name == name
+    }
+  }
+}
+
+extension ApkIndex {
+  static func merge<S: Sequence>(_ tables: S) -> Self where S.Element == Self {
+    Self.init(packages: tables.flatMap(\.packages))
+  }
+
+  static func merge(_ tables: Self...) -> ApkIndex {
+    Self.init(packages: tables.flatMap(\.packages))
+  }
+}
+
+extension ApkIndex {
+  init(raw: ApkRawIndex) throws {
+    self.packages = try raw.packages.map {
+      try ApkIndexPackage(raw: $0)
+    }
+  }
+}

Sources/apk/Index/ApkIndexDownloader.swift

@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import Foundation
+
+struct ApkIndexDownloader {
+  func downloadFile(remote remoteURL: URL, destination destLocalURL: URL) {
+    let sem = DispatchSemaphore.init(value: 0)
+    let downloadTask = URLSession.shared.downloadTask(with: remoteURL) { url, response, error in
+      if let localURL = url {
+        do {
+          // Replace existing APKINDEX.tar.gz files
+          if FileManager.default.fileExists(atPath: destLocalURL.path()) {
+            try FileManager.default.removeItem(at: destLocalURL)
+          }
+          // Move temporary to the new location
+          try FileManager.default.moveItem(at: localURL, to: destLocalURL)
+        } catch {
+          print("Download error: \(error.localizedDescription)")
+        }
+      }
+      sem.signal()
+    }
+    downloadTask.resume()
+    sem.wait()
+  }
+}

Sources/apk/Index/ApkIndexPackage.swift

@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import Foundation
+
+struct ApkIndexPackage {
+  let indexChecksum: String  //TODO: Decode cus why not
+  let name: String
+  let version: String
+  let architecture: String?
+  let packageSize: UInt64
+  let installedSize: UInt64
+  let packageDescription: String
+  let url: String
+  let license: String
+  let origin: String?
+  let maintainer: String?
+  let buildTime: Date?
+  let commit: String?
+  let providerPriority: UInt16?
+  let dependencies: [String]  //TODO: stuff
+  let provides: [String]  //TODO: stuff
+  let installIf: [String]  //TODO: stuff
+
+  var downloadFilename: String { "\(self.name)-\(version).apk" }
+
+  //TODO: Implementation
+  //lazy var semanticVersion: (Int, Int, Int) = (0, 0, 0)
+}
+
+extension ApkIndexPackage {
+  init(raw rawEntry: ApkRawIndexEntry) throws(Self.ParseError) {
+    // Required fields
+    var indexChecksum: String? = nil
+    var name: String? = nil
+    var version: String? = nil
+    var description: String? = nil
+    var url: String? = nil
+    var license: String? = nil
+    var packageSize: UInt64? = nil
+    var installedSize: UInt64? = nil
+
+    var dependencies = [String]()
+    var provides = [String]()
+    var installIf = [String]()
+
+    // Optional fields
+    var architecture: String? = nil
+    var origin: String? = nil
+    var maintainer: String? = nil
+    var buildTime: Date? = nil
+    var commit: String? = nil
+    var providerPriority: UInt16? = nil
+
+    // Read all the raw records for this entry
+    for record in rawEntry.fields {
+      switch record.key {
+      case "P":
+        name = record.value
+      case "V":
+        version = record.value
+      case "T":
+        description = record.value
+      case "U":
+        url = record.value
+      case "L":
+        license = record.value
+      case "A":
+        architecture = record.value
+      case "D":
+        dependencies = record.value.components(separatedBy: " ")
+      case "C":
+        indexChecksum = record.value  // base64-encoded SHA1 hash prefixed with "Q1"
+      case "S":
+        guard let value = UInt64(record.value, radix: 10) else {
+          throw .badValue(key: record.key)
+        }
+        packageSize = value
+      case "I":
+        guard let value = UInt64(record.value, radix: 10) else {
+          throw .badValue(key: record.key)
+        }
+        installedSize = value
+      case "p":
+        provides = record.value.components(separatedBy: " ")
+      case "i":
+        installIf = record.value.components(separatedBy: " ")
+      case "o":
+        origin = record.value
+      case "m":
+        maintainer = record.value
+      case "t":
+        guard let timet = UInt64(record.value, radix: 10),
+            let timetInterval = TimeInterval(exactly: timet) else {
+          throw .badValue(key: record.key)
+        }
+        buildTime = Date(timeIntervalSince1970: timetInterval)
+      case "c":
+        commit = record.value
+      case "k":
+        guard let value = UInt64(record.value, radix: 10),
+            (0..<UInt64(UInt16.max)).contains(value) else {
+          throw .badValue(key: record.key)
+        }
+        providerPriority = UInt16(truncatingIfNeeded: value)
+      case "F", "M", "R", "Z", "r", "q", "a", "s", "f":
+        break // installed db entries
+      default:
+        // Safe to ignore
+        guard record.key.isLowercase else {
+          throw .badValue(key: record.key)
+        }
+      }
+    }
+
+    self.indexChecksum = try indexChecksum.unwrap(or: Self.ParseError.required(key: "C"))
+    self.name = try name.unwrap(or: Self.ParseError.required(key: "P"))
+    self.version = try version.unwrap(or: Self.ParseError.required(key: "V"))
+    self.packageDescription = try description.unwrap(or: Self.ParseError.required(key: "T"))
+    self.url = try url.unwrap(or: Self.ParseError.required(key: "U"))
+    self.license = try license.unwrap(or: Self.ParseError.required(key: "L"))
+    self.packageSize = try packageSize.unwrap(or: Self.ParseError.required(key: "S"))
+    self.installedSize = try installedSize.unwrap(or: Self.ParseError.required(key: "I"))
+
+    self.architecture = architecture
+    self.origin = origin
+    self.maintainer = maintainer
+    self.buildTime = buildTime
+    self.commit = commit
+    self.providerPriority = providerPriority
+
+    self.dependencies = dependencies
+    self.provides = provides
+    self.installIf = installIf
+  }
+
+  public enum ParseError: Error, LocalizedError {
+    case badValue(key: Character)
+    case unexpectedKey(key: Character)
+    case required(key: Character)
+
+    public var errorDescription: String? {
+      switch self {
+      case .badValue(let key):      "Bad value for key \"\(key)\""
+      case .unexpectedKey(let key): "Unexpected key \"\(key)\""
+      case .required(let key):      "Missing required key \"\(key)\""
+      }
+    }
+  }
+}
+
+extension ApkIndexPackage: CustomStringConvertible {
+  var description: String { "pkg(\(self.name))" }
+}
+
+fileprivate extension Optional {
+  func unwrap<E: Error>(or error: @autoclosure () -> E) throws(E) -> Wrapped {
+    switch self {
+    case .some(let v):
+      return v
+    case .none:
+      throw error()
+    }
+  }
+}

Sources/apk/Index/ApkIndexUpdate.swift

@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import Foundation
+import SWCompression
+import CryptoKit
+
+public struct ApkIndexUpdater {
+  var repositories: [String]
+  var architectures: [String]
+
+  public init() {
+    self.repositories = [
+      "https://dl-cdn.alpinelinux.org/alpine/v3.21/main",
+      "https://dl-cdn.alpinelinux.org/alpine/edge/community"
+    ]
+    // other archs: "armhf", "armv7", "loongarch64", "ppc64le", "riscv64", "s390x", "x86"
+    self.architectures = [ "aarch64", "x86_64" ]
+  }
+
+  public func update() {
+    let repositories = self.repositories.flatMap { repo in
+      self.architectures.map { arch in
+        Repository(name: repo, arch: arch)
+      }
+    }
+
+    let downloader = ApkIndexDownloader()
+    for repo in repositories {
+      let localIndex = URL(filePath: repo.localName)
+#if false
+      let shouldDownload = true
+#else
+      let shouldDownload = !FileManager.default.fileExists(atPath: localIndex.path())
+#endif
+      if shouldDownload {
+        print("Fetching index for \"\(repo.name)\"")
+        downloader.downloadFile(remote: repo.url, destination: localIndex)
+      }
+    }
+
+    let index: ApkIndex
+    do {
+      let tables = try repositories.map { try readIndex(URL(filePath: $0.localName)) }
+      index = ApkIndex.merge(tables)
+    } catch {
+      fatalError(error.localizedDescription)
+    }
+
+    for package in index.packages {
+      print(package)
+    }
+  }
+
+  private func readIndex(_ indexURL: URL) throws -> ApkIndex {
+    let tarSignature: [TarReader.Entry]
+    let tarRecords: [TarReader.Entry]
+
+    let tars = try GzipArchive.multiUnarchive(  // Slow...
+      archive: Data(contentsOf: indexURL))
+    assert(tars.count >= 2)
+
+    var signatureStream = MemoryInputStream(buffer: tars[0].data)
+    tarSignature = try TarReader.read(&signatureStream)
+    var recordsStream = MemoryInputStream(buffer: tars[1].data)
+    tarRecords = try TarReader.read(&recordsStream)
+
+    guard case .file(let signatureName, _) = tarSignature.first
+    else { fatalError("Missing signature") }
+    print(signatureName)
+    guard let apkIndexFile = tarRecords.firstFile(name: "APKINDEX")
+    else { fatalError("APKINDEX missing") }
+    guard let description = tarRecords.firstFile(name: "DESCRIPTION")
+    else { fatalError("DESCRIPTION missing") }
+
+    let reader = TextInputStream(binaryStream: MemoryInputStream(buffer: apkIndexFile))
+    return try ApkIndex(raw:
+      try ApkRawIndex(lines: reader.lines))
+  }
+}
+
+extension ApkIndexUpdater {
+  struct Repository {
+    let name: String
+    let arch: String
+    let discriminator: String
+
+    private static func resolveApkIndex(_ repo: String, _ arch: String)
+      -> String { "\(repo)/\(arch)/APKINDEX.tar.gz" }
+    var url: URL { URL(string: Self.resolveApkIndex(self.name, self.arch))! }
+    var localName: String { "APKINDEX.\(discriminator).tar.gz" }
+
+    init(name repo: String, arch: String) {
+      self.name = repo
+      self.arch = arch
+
+      let urlSHA1Digest = Data(Insecure.SHA1.hash(data: Data(Self.resolveApkIndex(repo, arch).utf8)))
+      self.discriminator = urlSHA1Digest.subdata(in: 0..<3).map { String(format: "%02x", $0) }.joined()
+    }
+  }
+}

Sources/apk/Index/ApkRawIndex.swift

@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import Foundation
+
+struct ApkRawIndex {
+  let packages: [ApkRawIndexEntry]
+
+  init(lines: any Sequence<String>) throws {
+    var packages = [ApkRawIndexEntry]()
+
+    var recordLines = [String]()
+    recordLines.reserveCapacity(15)
+
+    for line in lines {
+      if line.trimmingCharacters(in: .whitespaces).isEmpty {
+        if !recordLines.isEmpty {
+          packages.append(try .init(parsingEntryLines: recordLines))
+          recordLines.removeAll(keepingCapacity: true)
+        }
+      } else {
+        recordLines.append(line)
+      }
+    }
+    if !recordLines.isEmpty {
+      packages.append(try .init(parsingEntryLines: recordLines))
+    }
+
+    self.packages = packages
+  }
+}
+
+struct ApkRawIndexEntry {
+  let fields: [Record]
+
+  struct Record {
+    let key: Character
+    let value: String
+  }
+}
+
+extension ApkRawIndexEntry {
+  init(parsingEntryLines lines: any Sequence<String>) throws {
+    self.fields = try lines.map { line in
+      guard let splitIdx = line.firstIndex(of: ":"),
+          line.distance(from: line.startIndex, to: splitIdx) == 1 else {
+        throw ApkRawIndexError.badPair
+      }
+      return Record(
+        key:   line.first!,
+        value: String(line[line.index(after: splitIdx)...]))
+    }
+  }
+
+  func toMap() -> [Character: String] {
+    Dictionary(uniqueKeysWithValues: self.fields.map { $0.pair })
+  }
+
+  func lookup(_ key: Character) -> String? {
+    fields.first(where: { $0.key == key })?.value
+  }
+}
+
+extension ApkRawIndexEntry.Record {
+  var pair: (Character, String) {
+    (self.key, self.value)
+  }
+}
+
+enum ApkRawIndexError: Error, LocalizedError {
+  case badPair
+
+  var errorDescription: String? {
+    switch self {
+    case .badPair: "Malformed raw key-value pair"
+    }
+  }
+}