Merge pull request #894 from kiwix/767-range-request-rebased

Support range requests

Model/Entities/Entities.swift

@@ -150,6 +150,8 @@ struct URLContentMetaData {
     let mime: String
     let size: UInt
     let zimTitle: String
+    let lastModified: Date?
+
     var httpContentType: String {
         if mime == "text/plain" {
             return "text/plain;charset=UTf-8"
@@ -172,6 +174,23 @@ struct URLContentMetaData {
         }
         return UTType(mimeType: mime)?.preferredFilenameExtension
     }
+
+    /// Currently using the same eTag value for everything that is the same ZIM file
+    /// since all those resources were created and stored at file creation
+    var eTag: String? {
+        guard let lastModified else { return nil }
+        return "\"\(lastModified.timeIntervalSince1970)\""
+    }
+
+    func contentRange(for requestedRange: ClosedRange<UInt>) -> String {
+        "bytes \(requestedRange.lowerBound)-\(requestedRange.upperBound)/\(size)"
+    }
+}
+
+extension ClosedRange<UInt> {
+    var fullRangeSize: UInt {
+        upperBound - lowerBound
+    }
 }
 
 struct URLContent {

Model/Utilities/ByteRanges.swift

@@ -17,12 +17,13 @@ import Foundation
 
 enum ByteRanges {
 
-    static func rangesFor(contentLength: UInt, rangeSize size: UInt) -> [ClosedRange<UInt>] {
+    static func rangesFor(contentLength: UInt, rangeSize size: UInt, start: UInt = 0) -> [ClosedRange<UInt>] {
         guard size > 0 else {
             return []
         }
-        return stride(from: 0, to: contentLength, by: UInt.Stride(size)).map { point in
-            let endOfRange = min(contentLength - 1, point + size - 1)
+        let end = start + contentLength
+        return stride(from: start, to: end, by: UInt.Stride(size)).map { point in
+            let endOfRange = min(end - 1, point + size - 1)
             return point...endOfRange
         }
     }

Model/Utilities/Date+Extension.swift

@@ -0,0 +1,29 @@
+// This file is part of Kiwix for iOS & macOS.
+//
+// Kiwix is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 3 of the License, or
+// any later version.
+//
+// Kiwix is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Kiwix; If not, see https://www.gnu.org/licenses/.
+
+import Foundation
+
+extension Date {
+
+    /// Format the current date the way as it would come from a server's Last-Modified header
+    /// - Returns: eg: Thu, 16 May 2024 11:38:20 GMT
+    func formatAsGMT() -> String {
+        let formatter = DateFormatter()
+        formatter.dateFormat = "EEE, dd MMM yyyy HH:mm:ss 'GMT'"
+        formatter.locale = Locale(identifier: "en_US")
+        formatter.timeZone = TimeZone(abbreviation: "GMT")
+        return formatter.string(from: self)
+    }
+}

Model/Utilities/HTTPSuccess.swift

@@ -0,0 +1,82 @@
+// This file is part of Kiwix for iOS & macOS.
+//
+// Kiwix is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 3 of the License, or
+// any later version.
+//
+// Kiwix is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Kiwix; If not, see https://www.gnu.org/licenses/.
+
+import Foundation
+
+enum HTTPSuccess {
+    static func response(
+        url: URL,
+        metaData: URLContentMetaData,
+        requestedRange: ClosedRange<UInt>?
+    ) -> HTTPURLResponse? {
+        if let requestedRange {
+            return Self.http206Response(
+                url: url,
+                metaData: metaData,
+                requestedRange: requestedRange
+            )
+        } else {
+            return Self.http200Response(
+                url: url,
+                metaData: metaData
+            )
+        }
+    }
+
+    private static func http200Response(
+        url: URL,
+        metaData: URLContentMetaData
+    ) -> HTTPURLResponse? {
+        var headers = defaultResponseHeaders(for: metaData)
+        headers["Content-Length"] = "\(metaData.size)"
+        return HTTPURLResponse(
+            url: url,
+            statusCode: 200,
+            httpVersion: "HTTP/1.1",
+            headerFields: headers
+        )
+    }
+
+    private static func http206Response(
+        url: URL,
+        metaData: URLContentMetaData,
+        requestedRange: ClosedRange<UInt>
+    ) -> HTTPURLResponse? {
+        var headers = defaultResponseHeaders(for: metaData)
+        headers["Content-Length"] = "\(requestedRange.fullRangeSize)"
+        headers["Content-Range"] = metaData.contentRange(for: requestedRange)
+        return HTTPURLResponse(
+            url: url,
+            statusCode: 206,
+            httpVersion: "HTTP/1.1",
+            headerFields: headers
+        )
+    }
+
+    private static func defaultResponseHeaders(for metaData: URLContentMetaData) -> [String: String] {
+        var headers = [
+            "Accept-Ranges": "bytes",
+            "Content-Type": metaData.httpContentType,
+            "Date": Date().formatAsGMT()
+        ]
+        if let modifiedDate = metaData.lastModified {
+            headers["Last-Modified"] = modifiedDate.formatAsGMT()
+        }
+        if let eTag = metaData.eTag {
+            headers["ETag"] = eTag
+        }
+        return headers
+    }
+}

Model/Utilities/WebKitHandler.swift

@@ -16,6 +16,10 @@
 import os
 import WebKit
 
+enum RangeRequestError: Error {
+    case invalidRange
+}
+
 /// Skipping handling for HTTP 206 Partial Content
 /// For video playback, WebKit makes a large amount of requests with small byte range (e.g. 8 bytes)
 /// to retrieve content of the video.
@@ -77,18 +81,29 @@ final class KiwixURLSchemeHandler: NSObject, WKURLSchemeHandler {
             return
         }
         guard let metaData = await contentMetaData(for: url) else {
-            sendHTTP404Response(urlSchemeTask, url: url)
+            sendHTTPErrorResponse(urlSchemeTask, url: url, status: .code404)
+            return
+        }
+        let requestedRange: ClosedRange<UInt>?
+        do {
+            requestedRange = try rangeFrom(request)
+        } catch {
+            sendHTTPErrorResponse(urlSchemeTask, url: url, status: .code400)
             return
         }
 
-        guard let dataStream = await dataStream(for: url, metaData: metaData) else {
-            sendHTTP404Response(urlSchemeTask, url: url)
+        guard let dataStream = await dataStream(for: url, metaData: metaData, requestedRange: requestedRange) else {
+            sendHTTPErrorResponse(urlSchemeTask, url: url, status: .code404)
             return
         }
 
         // send the headers
         guard isStartedFor(urlSchemeTask.hash) else { return }
-        guard let responseHeaders = http200Response(urlSchemeTask, url: url, metaData: metaData) else {
+        guard let responseHeaders = HTTPSuccess.response(
+            url: url,
+            metaData: metaData,
+            requestedRange: requestedRange
+        ) else {
             urlSchemeTask.didFailWithError(URLError(.badServerResponse, userInfo: ["url": url]))
             stopFor(urlSchemeTask.hash)
             return
@@ -107,33 +122,57 @@ final class KiwixURLSchemeHandler: NSObject, WKURLSchemeHandler {
         stopFor(urlSchemeTask.hash)
     }
 
+    // MARK: Range request detection
+
+    private func rangeFrom(_ request: URLRequest) throws -> ClosedRange<UInt>? {
+        guard let range = request.allHTTPHeaderFields?["Range"] as? String else {
+            return nil
+        }
+        let parts = range.components(separatedBy: ["=", "-"])
+        guard parts.count > 1, let rangeStart = UInt(parts[1]) else {
+            throw RangeRequestError.invalidRange
+        }
+        let rangeEnd = parts.count == 3 ? UInt(parts[2]) ?? 0 : 0
+        return rangeStart...rangeEnd+1
+    }
+
     // MARK: Reading content
 
-    private func dataStream(for url: URL, metaData: URLContentMetaData) async -> DataStream<URLContent>? {
+    private func dataStream(
+        for url: URL,
+        metaData: URLContentMetaData,
+        requestedRange: ClosedRange<UInt>?
+    ) async -> DataStream<URLContent>? {
         let dataProvider: any DataProvider<URLContent>
-        let ranges: [ClosedRange<UInt>] // the list of ranges we should use to stream data
-        let size2MB: UInt = 2097152 // 2MB
+        let ranges: [ClosedRange<UInt>] = rangesForDataStreaming(metaData, requestedRange: requestedRange)
         if metaData.isMediaType, let directAccess = await directAccessInfo(for: url) {
             dataProvider = ZimDirectContentProvider(directAccess: directAccess,
                                                     contentSize: metaData.size)
-            ranges = ByteRanges.rangesFor(
-                contentLength: metaData.size,
-                rangeSize: size2MB
-            )
         } else {
             dataProvider = ZimContentProvider(for: url)
-            // if the data is larger than 2MB, read it "in chunks"
-            if metaData.size > size2MB {
-                ranges = ByteRanges.rangesFor(
-                    contentLength: metaData.size,
-                    rangeSize: size2MB
-                )
-            } else { // use the full range and read it in one go
-                ranges = [0...metaData.size]
-            }
         }
         return DataStream(dataProvider: dataProvider, ranges: ranges)
     }
+
+    /// The list of ranges we should use to stream data
+    /// - Parameter metaData: the URLContentMetaData from the ZIM file content
+    /// - Returns: If the data is larger than 2MB, it returns the "chunks" that should be read,
+    /// otherwise returns the full range 0...metaData.size
+    private func rangesForDataStreaming(
+        _ metaData: URLContentMetaData,
+        requestedRange: ClosedRange<UInt>?
+    ) -> [ClosedRange<UInt>] {
+        let size2MB: UInt = 2_097_152 // 2MB
+        if let requested = requestedRange {
+            return ByteRanges.rangesFor(
+                contentLength: requested.upperBound - requested.lowerBound,
+                rangeSize: size2MB,
+                start: requested.lowerBound
+            )
+        } else {
+            return ByteRanges.rangesFor(contentLength: metaData.size, rangeSize: size2MB)
+        }
+    }
 
     private func contentMetaData(for url: URL) async -> URLContentMetaData? {
         return await withCheckedContinuation { continuation in
@@ -154,6 +193,7 @@ final class KiwixURLSchemeHandler: NSObject, WKURLSchemeHandler {
     }
 
     // MARK: Writing content
+
     private func writeContent(
         to urlSchemeTask: WKURLSchemeTask,
         from dataStream: DataStream<URLContent>
@@ -166,33 +206,27 @@ final class KiwixURLSchemeHandler: NSObject, WKURLSchemeHandler {
         }
     }
 
-    // MARK: Success responses
-    private func http200Response(
-        _ urlSchemeTask: WKURLSchemeTask,
-        url: URL,
-        metaData: URLContentMetaData
-    ) -> HTTPURLResponse? {
-        let headers = ["Content-Type": metaData.httpContentType,
-                       "Content-Length": "\(metaData.size)"]
-        return HTTPURLResponse(
-            url: url,
-            statusCode: 200,
-            httpVersion: "HTTP/1.1",
-            headerFields: headers
-        )
-    }
-
-    // MARK: Error responses
+    // MARK: Error response
 
     @MainActor
-    private func sendHTTP404Response(_ urlSchemeTask: WKURLSchemeTask, url: URL) {
+    private func sendHTTPErrorResponse(_ urlSchemeTask: WKURLSchemeTask, url: URL, status: StatusCode) {
         guard isStartedFor(urlSchemeTask.hash) else { return }
-        if let response = HTTPURLResponse(url: url, statusCode: 404, httpVersion: "HTTP/1.1", headerFields: nil) {
+        if let response = HTTPURLResponse(
+            url: url,
+            statusCode: status.rawValue,
+            httpVersion: "HTTP/1.1",
+            headerFields: nil
+        ) {
             urlSchemeTask.didReceive(response)
             urlSchemeTask.didFinish()
         } else {
             urlSchemeTask.didFailWithError(URLError(.badServerResponse, userInfo: ["url": url]))
         }
         stopFor(urlSchemeTask.hash)
     }
+
+    private enum StatusCode: Int {
+        case code400 = 400
+        case code404 = 404
+    }
 }

Model/ZimFileService/ZimFileService.mm

@@ -179,7 +179,8 @@ - (NSDictionary *)getMetaData:(NSUUID *)zimFileID contentPath:(NSString *)conten
         return @{
             @"mime": [NSString stringWithUTF8String:item.getMimetype().c_str()],
             @"size": [NSNumber numberWithUnsignedLongLong:item.getSize()],
-            @"title": [NSString stringWithUTF8String:item.getTitle().c_str()]
+            @"title": [NSString stringWithUTF8String:item.getTitle().c_str()],
+            @"zimFileDate": [self getModificationDateOf: zimFileID]
         };
     } catch (std::exception) {
         return nil;
@@ -247,4 +248,21 @@ - (NSDictionary *_Nullable) getDirectAccess: (NSUUID *)zimFileID contentPath:(NS
     return entry.getItem(entry.isRedirect());
 }
 
+/// get the modification date of the ZIM file itself
+- (NSDate *_Nullable) getModificationDateOf: (NSUUID *_Nonnull) zimFileID {
+    NSURL *fileURL = [self getFileURL: zimFileID];
+    if (fileURL == nil) {
+        return nil;
+    }
+    NSFileManager *fileManager = [NSFileManager defaultManager];
+    NSError *error = nil;
+    NSDictionary *fileAttributes = [fileManager attributesOfItemAtPath:[fileURL path] error:&error];
+    if (fileAttributes) {
+        return [fileAttributes objectForKey:NSFileModificationDate];
+    } else {
+        NSLog(@"Error retrieving file modification date: %@", [error localizedDescription]);
+        return nil;
+    }
+}
+
 @end

Model/ZimFileService/ZimFileService.swift

@@ -140,7 +140,13 @@ extension ZimFileService {
               let mime = content["mime"] as? String,
               let size = content["size"] as? UInt,
               let title = content["title"] as? String else { return nil }
-        return URLContentMetaData(mime: mime, size: size, zimTitle: title)
+        let zimFileModificationDate = content["zimFileDate"] as? Date
+        return URLContentMetaData(
+            mime: mime,
+            size: size,
+            zimTitle: title,
+            lastModified: zimFileModificationDate
+        )
     }
 
     func getURLContent(zimFileID: String, contentPath: String, start: UInt = 0, end: UInt = 0) -> URLContent? {