diff --git a/Package.swift b/Package.swift index 63f4947..d7fb130 100644 --- a/Package.swift +++ b/Package.swift @@ -8,14 +8,10 @@ let package = Package( ], dependencies: [ .package(url: "https://github.com/apple/swift-argument-parser", from: "1.5.0"), - .package(url: "https://github.com/tsolomko/SWCompression", from: "4.8.6"), ], targets: [ .target( name: "darwin-apk", - dependencies: [ - .product(name: "SWCompression", package: "SWCompression"), - ], path: "Sources/apk"), .executableTarget( name: "dpk", diff --git a/Sources/apk/Index/ApkIndexUpdate.swift b/Sources/apk/Index/ApkIndexUpdate.swift index 19f1557..4f3b7e8 100644 --- a/Sources/apk/Index/ApkIndexUpdate.swift +++ b/Sources/apk/Index/ApkIndexUpdate.swift @@ -4,7 +4,6 @@ */ import Foundation -import SWCompression import CryptoKit public struct ApkIndexUpdater { @@ -64,23 +63,46 @@ public struct ApkIndexUpdater { let tarSignature: [TarReader.Entry] let tarRecords: [TarReader.Entry] - let tars = try GzipArchive.multiUnarchive( // Slow... - archive: Data(contentsOf: indexURL)) - assert(tars.count >= 2) + print("Archive: \(indexURL.lastPathComponent)") - var signatureStream = MemoryInputStream(buffer: tars[0].data) + let durFormat = Duration.UnitsFormatStyle( + allowedUnits: [ .seconds, .milliseconds ], + width: .condensedAbbreviated, + fractionalPart: .show(length: 3)) + let gzipStart = ContinuousClock.now + + var tars = [Data]() + do { + var file: any InputStream = try FileInputStream(indexURL) + //var file: any InputStream = try MemoryInputStream(buffer: try Data(contentsOf: indexURL)) + tars.append(try GZip.read(inStream: &file)) + tars.append(try GZip.read(inStream: &file)) + + } catch { + fatalError(error.localizedDescription) + } + + print("Gzip time: \((ContinuousClock.now - gzipStart).formatted(durFormat))") + let untarStart = ContinuousClock.now + + var signatureStream = MemoryInputStream(buffer: tars[0]) tarSignature = try TarReader.read(&signatureStream) - var recordsStream = MemoryInputStream(buffer: tars[1].data) + var recordsStream = MemoryInputStream(buffer: tars[1]) tarRecords = try TarReader.read(&recordsStream) guard case .file(let signatureName, _) = tarSignature.first else { fatalError("Missing signature") } - print(signatureName) guard let apkIndexFile = tarRecords.firstFile(name: "APKINDEX") else { fatalError("APKINDEX missing") } guard let description = tarRecords.firstFile(name: "DESCRIPTION") else { fatalError("DESCRIPTION missing") } + print("TAR time: \((ContinuousClock.now - untarStart).formatted(durFormat))") + let indexStart = ContinuousClock.now + defer { + print("Index time: \((ContinuousClock.now - indexStart).formatted(durFormat))") + } + let reader = TextInputStream(binaryStream: MemoryInputStream(buffer: apkIndexFile)) return try ApkIndex(raw: try ApkRawIndex(lines: reader.lines)) diff --git a/Sources/apk/Utility/FileInputStream.swift b/Sources/apk/Utility/FileInputStream.swift index 332d597..483b2a1 100644 --- a/Sources/apk/Utility/FileInputStream.swift +++ b/Sources/apk/Utility/FileInputStream.swift @@ -4,6 +4,8 @@ */ import Foundation +import Darwin +import System public struct FileInputStream: InputStream { private var _hnd: FileHandle @@ -72,4 +74,12 @@ public struct FileInputStream: InputStream { throw .fileHandleError(error) } } + + public mutating func read(_ buffer: UnsafeMutablePointer, maxLength len: Int) throws(StreamError) -> Int { + let res = unistd.read(self._hnd.fileDescriptor, buffer, len) + if res < 0 { + throw .fileHandleError(Errno(rawValue: errno)) + } + return res + } } diff --git a/Sources/apk/Utility/GZip.swift b/Sources/apk/Utility/GZip.swift new file mode 100644 index 0000000..e61219c --- /dev/null +++ b/Sources/apk/Utility/GZip.swift @@ -0,0 +1,203 @@ +/* + * darwin-apk © 2024 Gay Pizza Specifications + * SPDX-License-Identifier: Apache-2.0 + */ + +import Foundation +import zlib + +struct GZip { + static let CM_DEFLATE: UInt8 = 8 + + static let FTEXT: UInt8 = 1 << 0 + static let FHCRC: UInt8 = 1 << 1 + static let FEXTRA: UInt8 = 1 << 2 + static let FNAME: UInt8 = 1 << 3 + static let FCOMMENT: UInt8 = 1 << 4 + + static let XFL_BEST: UInt8 = 2 + static let XFL_FASTEST: UInt8 = 4 + + private static func skipString(_ stream: inout any InputStream) throws(GZipError) { + var c: UInt8? + repeat { + c = stream.next() + guard c != nil else { + throw .truncatedStream + } + } while c != 0 + } + + static func read(inStream stream: inout any InputStream) throws(GZipError) -> Data { + // Check Gzip magic signature + guard (try? stream.read(2)) == Data([0x1F, 0x8B]) else { + throw .badMagic + } + + // Check compression field (should only ever be DEFLATE) + guard let compression = stream.next(), + compression == Self.CM_DEFLATE else { + throw .badHeader + } + + guard + let flags = stream.next(), + let modificationTime = stream.readUInt(), + let extraFlags = stream.next(), + let operatingSystemID = stream.next() else { + throw .truncatedStream + } + + + if flags & Self.FEXTRA != 0 { + // Skip "extra" field + guard let extraLength = stream.readUShort() else { + throw.truncatedStream + } + do { + try stream.seek(.current(Int(extraLength))) + } catch { + throw .streamError(error) + } + } + if flags & Self.FNAME != 0 { + // Skip null-terminated name string + try skipString(&stream) + } + if flags & Self.FCOMMENT != 0 { + // Skip null-terminated comment string + try skipString(&stream) + } + if flags & Self.FHCRC != 0 { + guard let crc16 = stream.readUShort() else { + throw .badField("crc16") + } + } + + let deflateBegin: Int + do { + deflateBegin = try stream.tell + } catch { + throw .streamError(error) + } + + var payload = Data() + let (streamLength, computedCRC) = try Self.deflate(payload: &payload, stream: &stream) + + // End-of-stream verification fields + do { + try stream.seek(.set(deflateBegin + streamLength)) + } catch { + throw .streamError(error) + } + guard + let crc = stream.readUInt(), + let inputSizeMod32 = stream.readUInt() else { + throw .truncatedStream + } + + // Perform verification checks + guard UInt32(truncatingIfNeeded: computedCRC) == crc else { + throw .verificationFailed("CRC32 didn't match") + } + guard inputSizeMod32 == UInt32(truncatingIfNeeded: payload.count) else { + throw .verificationFailed("Bad decompressed size") + } + + return payload + } + + private static func deflate(payload: inout Data, stream: inout any InputStream) throws(GZipError) -> (Int, UInt) { + var zstream = z_stream() + var zerr = inflateInit2_(&zstream, -15, ZLIB_VERSION, Int32(MemoryLayout.size)) + guard zerr == Z_OK else { + throw .zlib(zerr) + } + + defer { + inflateEnd(&zstream) + } + + let bufferSize = 0x8000 + var inputBuffer = [UInt8](repeating: 0, count: bufferSize) + var outputBuffer = [UInt8](repeating: 0, count: bufferSize) + + var computeCRC: UInt = crc32(0, nil, 0) + var block = 0 + repeat { + if zstream.avail_in == 0 { + let read: Int + do { + read = try stream.read(&inputBuffer, maxLength: inputBuffer.count) + } catch { + throw .streamError(error) + } + guard read > 0 else { + throw .truncatedStream + } + zstream.avail_in = UInt32(read) + zstream.next_in = inputBuffer.withUnsafeMutableBufferPointer(\.baseAddress!) + } + zstream.avail_out = UInt32(outputBuffer.count) + zstream.next_out = outputBuffer.withUnsafeMutableBufferPointer(\.baseAddress!) + zerr = inflate(&zstream, Z_BLOCK) + + let decodedBytes = outputBuffer.count - Int(zstream.avail_out) + computeCRC = crc32(computeCRC, outputBuffer, UInt32(decodedBytes)) + payload += Data(outputBuffer[.. UInt16? { + guard let buffer = try? self.read(2), buffer.count == 2 else { + return nil + } + return buffer.withUnsafeBytes { $0.load(as: UInt16.self) }.littleEndian + } + + mutating func readUInt() -> UInt32? { + guard let buffer = try? self.read(4), buffer.count == 4 else { + return nil + } + return buffer.withUnsafeBytes { $0.load(as: UInt32.self) }.littleEndian + } +} diff --git a/Sources/apk/Utility/InputStream.swift b/Sources/apk/Utility/InputStream.swift index 5f57c92..0f09fee 100644 --- a/Sources/apk/Utility/InputStream.swift +++ b/Sources/apk/Utility/InputStream.swift @@ -5,10 +5,9 @@ import Foundation -public protocol InputStream: Stream, IteratorProtocol { - associatedtype Element = UInt8 - +public protocol InputStream: Stream, IteratorProtocol where Element == UInt8 { mutating func read(_ count: Int) throws(StreamError) -> Data + mutating func read(_ buffer: UnsafeMutablePointer, maxLength len: Int) throws(StreamError) -> Int } public extension InputStream { diff --git a/Sources/apk/Utility/MemoryInputStream.swift b/Sources/apk/Utility/MemoryInputStream.swift index 3996d06..4f0dd39 100644 --- a/Sources/apk/Utility/MemoryInputStream.swift +++ b/Sources/apk/Utility/MemoryInputStream.swift @@ -52,6 +52,15 @@ public struct MemoryInputStream: InputStream { return bytes } + public mutating func read(_ buffer: UnsafeMutablePointer, maxLength count: Int) throws(StreamError) -> Int { + let beg = min(self._idx, self._len) + let end = min(self._idx + count, self._len) + let len = beg.distance(to: end) + let buf = UnsafeMutableRawBufferPointer(start: buffer, count: len) + self._idx += len + return self._sli.copyBytes(to: buf, from: beg.. UInt8? { if self._idx < self._len { let byte = self._sli[self._idx] diff --git a/Sources/apk/Utility/Stream.swift b/Sources/apk/Utility/Stream.swift index 797a1a9..f14c2d4 100644 --- a/Sources/apk/Utility/Stream.swift +++ b/Sources/apk/Utility/Stream.swift @@ -4,6 +4,7 @@ */ import Foundation +import System public protocol Stream { mutating func seek(_ whence: StreamWhence) throws(StreamError) @@ -21,6 +22,7 @@ public enum StreamError: Error, LocalizedError { case seekRange case overflow case fileHandleError(_ error: any Error) + case fileDescriptorError(_ error: Errno) public var errorDescription: String? { switch self { @@ -28,6 +30,7 @@ public enum StreamError: Error, LocalizedError { case .seekRange: "Seek out of range" case .overflow: "Stream position overflowed" case .fileHandleError(let error): "Error from file handle: \(error.localizedDescription)" + case .fileDescriptorError(let error): "\(error)" } } }