Scraping Comments

ios/HackerNews.xcodeproj/project.pbxproj

@@ -46,6 +46,7 @@
 		EC0B1C752D1A34110000C3AC /* CommentsHeader.swift in Sources */ = {isa = PBXBuildFile; fileRef = EC0B1C742D1A34110000C3AC /* CommentsHeader.swift */; };
 		EC29FDA72CFFD074007B1AE9 /* BookmarksScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = EC29FDA62CFFD074007B1AE9 /* BookmarksScreen.swift */; };
 		EC29FDA92CFFD0B5007B1AE9 /* SettingsScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = EC29FDA82CFFD0B5007B1AE9 /* SettingsScreen.swift */; };
+		EC70E1612D1DD82B00582023 /* HNWebClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = EC70E1602D1DD82B00582023 /* HNWebClient.swift */; };
 		ECCE8F262D03815300349733 /* Pager.swift in Sources */ = {isa = PBXBuildFile; fileRef = ECCE8F252D03815300349733 /* Pager.swift */; };
 /* End PBXBuildFile section */
 
@@ -145,6 +146,7 @@
 		EC0B1C742D1A34110000C3AC /* CommentsHeader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CommentsHeader.swift; sourceTree = "<group>"; };
 		EC29FDA62CFFD074007B1AE9 /* BookmarksScreen.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarksScreen.swift; sourceTree = "<group>"; };
 		EC29FDA82CFFD0B5007B1AE9 /* SettingsScreen.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsScreen.swift; sourceTree = "<group>"; };
+		EC70E1602D1DD82B00582023 /* HNWebClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HNWebClient.swift; sourceTree = "<group>"; };
 		ECCE8F252D03815300349733 /* Pager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Pager.swift; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
@@ -213,6 +215,7 @@
 			children = (
 				A42705AC2A429D2E0057E439 /* HNApi.swift */,
 				A423B0672BAE05FB00267DDB /* NetworkDebugger.swift */,
+				EC70E1602D1DD82B00582023 /* HNWebClient.swift */,
 			);
 			path = Network;
 			sourceTree = "<group>";
@@ -585,6 +588,7 @@
 				EC0B1C752D1A34110000C3AC /* CommentsHeader.swift in Sources */,
 				A413E8592A8D868500C0F867 /* ThemedButtonStyle.swift in Sources */,
 				A427057D2A4293B10057E439 /* HNApp.swift in Sources */,
+				EC70E1612D1DD82B00582023 /* HNWebClient.swift in Sources */,
 				A434C2E02A8E75960002F488 /* WebView.swift in Sources */,
 				A49933942AA28B5900DED8B1 /* StoryViewModel.swift in Sources */,
 				A47309B62AA7D1F600201376 /* CommentRow.swift in Sources */,

ios/HackerNews/Components/CommentRow.swift

@@ -9,23 +9,21 @@ import Foundation
 import SwiftUI
 
 struct CommentRow: View {
-  let comment: Comment
-  let level: Int
+  let comment: CommentInfo
   let maxIndentationLevel: Int = 5
 
   var body: some View {
     VStack(alignment: .leading) {
       // first row
       HStack {
         // author
-        let author = comment.by != nil ? comment.by! : ""
-        Text("@\(author)")
+        Text("@\(comment.user)")
           .font(.caption)
           .fontWeight(.bold)
         // time
         HStack(alignment: .center, spacing: 4.0) {
           Image(systemName: "clock")
-          Text(comment.displayableDate)
+          Text(comment.age)
         }
         .font(.caption)
         // collapse/expand
@@ -52,8 +50,7 @@ struct CommentRow: View {
       }
 
       // Comment Body
-      let commentText = comment.text != nil ? comment.text! : ""
-      Text(commentText.strippingHTML())
+      Text(comment.text.strippingHTML())
         .font(.caption)
     }
     .padding(8.0)
@@ -62,7 +59,7 @@ struct CommentRow: View {
     .padding(
       EdgeInsets(
         top: 0,
-        leading: min(CGFloat(level * 20), CGFloat(maxIndentationLevel * 20)),
+        leading: min(CGFloat(comment.level * 20), CGFloat(maxIndentationLevel * 20)),
         bottom: 0,
         trailing: 0)
     )
@@ -72,7 +69,7 @@ struct CommentRow: View {
 struct CommentView_Preview: PreviewProvider {
   static var previews: some View {
     PreviewVariants {
-        CommentRow(comment: PreviewHelpers.makeFakeComment(), level: 0)
+      CommentRow(comment: PreviewHelpers.makeFakeComment())
     }
   }
 }
@@ -81,7 +78,7 @@ struct CommentViewIndentation_Preview: PreviewProvider {
   static var previews: some View {
     Group {
       ForEach(0..<6) { index in
-        CommentRow(comment: PreviewHelpers.makeFakeComment(), level: index)
+        CommentRow(comment: PreviewHelpers.makeFakeComment(level: index))
           .previewLayout(.sizeThatFits)
           .previewDisplayName("Indentation \(index)")
       }

ios/HackerNews/Models/StoryViewModel.swift

@@ -15,13 +15,12 @@ struct StoryUiState {
 enum CommentsState {
   case notStarted
   case loading
-  case loaded(comments: [FlattenedComment])
+  case loaded(comments: [CommentInfo])
 }
 
 struct CommentsHeaderState {
   let story: Story
   var expanded: Bool = false
-
 }
 
 @MainActor
@@ -31,6 +30,8 @@ class StoryViewModel: ObservableObject {
 
   private let story: Story
 
+  private let webClient = HNWebClient()
+
   init(story: Story) {
     self.story = story
     self.state = StoryUiState(
@@ -46,48 +47,12 @@ class StoryViewModel: ObservableObject {
 
   func fetchComments() async {
     state.comments = .loading
-
-    var commentsToRequest = story.comments
-    var commentsById = [Int64 : Comment]()
-    while !commentsToRequest.isEmpty {
-      let items = await HNApi().fetchItems(ids: commentsToRequest)
-      commentsToRequest.removeAll()
-      for item in items {
-        if let comment = item as? Comment {
-          commentsById[comment.id] = comment
-          commentsToRequest.append(contentsOf: comment.replies ?? [])
-        } else {
-          print("Found not comment \(item)")
-        }
-      }
+    let page = await webClient.getStoryPage(id: story.id)
+    switch page {
+    case .success(let data):
+      state.comments = .loaded(comments: data.comments)
+    case .error:
+      state.comments = .loaded(comments: [])
     }
-
-    var flattenedComments = [FlattenedComment]()
-    flattenComments(
-      ids: story.comments,
-      flattened: &flattenedComments,
-      commentsById: &commentsById
-    )
-
-    state.comments = .loaded(comments: flattenedComments)
-  }
-
-  private func flattenComments(ids: [Int64], depth: Int = 0, flattened: inout [FlattenedComment], commentsById: inout [Int64 : Comment]) {
-    for id in ids {
-      if let foundComment = commentsById[id] {
-        flattened.append(FlattenedComment(comment: foundComment, depth: depth))
-        flattenComments(ids: foundComment.replies ?? [], depth: depth + 1, flattened: &flattened, commentsById: &commentsById)
-      } else {
-        print("Could not find comment \(id)")
-      }
-    }
-}
-}
-
-struct FlattenedComment: Identifiable {
-  let comment: Comment
-  let depth: Int
-  var id: Int64 {
-    comment.id
   }
 }

ios/HackerNews/Network/HNWebClient.swift

@@ -0,0 +1,72 @@
+//
+//  HNWebClient.swift
+//  HackerNews
+//
+//  Created by Rikin Marfatia on 12/26/24.
+//
+
+import Foundation
+import SwiftSoup
+
+let BASE_WEB_URL = "https://news.ycombinator.com/"
+private let LOGIN_URL = BASE_WEB_URL + "login"
+private let ITEM_URL = BASE_WEB_URL + "item"
+private let COMMENT_URL = BASE_WEB_URL + "comment"
+
+enum PostPage {
+  case success(data: PostPageResponse)
+  case error
+}
+
+struct PostPageResponse {
+  let comments: [CommentInfo]
+}
+
+struct CommentInfo {
+  let id: Int64
+  let upvoted: Bool
+  let upvoteUrl: String?
+  let text: String
+  let user: String
+  let age: String
+  let level: Int
+}
+
+class HNWebClient {
+  func getStoryPage(id: Int64) async -> PostPage {
+    // make request for page
+    let url = URL(string:"\(ITEM_URL)?id=\(id)")
+    do {
+      let (data, _) = try await URLSession.shared.data(from: url!)
+      guard let html = String(data: data, encoding: .utf8) else { return .error }
+      let document: Document = try SwiftSoup.parse(html)
+      let commentTree = try document.select("table.comment-tree tr.athing.comtr")
+      let comments: [CommentInfo] = try commentTree.map { comment in
+        let commentId = try Int64(comment.id(), format: .number)
+        let commentLevel = try comment.select("td.ind").attr("indent")
+        let commentText = try comment.select("div.commtext").text()
+        let commentAuthor = try comment.select("a.hnuser").text()
+        let commentDate = try comment.select("span.age").attr("title").split(separator: " ").first!
+        let upvoteLinkElement = try comment.select("a[id^=up_").first()
+        let upvoteUrl = try upvoteLinkElement?.attr("href")
+        let upvoted = upvoteLinkElement?.hasClass("nosee") ?? false
+
+        let date = String(commentDate).asDate()
+
+        return CommentInfo(
+          id: commentId,
+          upvoted: upvoted,
+          upvoteUrl: upvoteUrl,
+          text: commentText,
+          user: commentAuthor,
+          age: date?.timeAgoDisplay() ?? "",
+          level: Int(commentLevel)!
+        )
+      }
+      return .success(data: PostPageResponse(comments: comments))
+    } catch {
+      print("Error fetching post IDs: \(error)")
+      return .error
+    }
+  }
+}

ios/HackerNews/Screens/StoryScreen.swift

@@ -34,11 +34,8 @@ struct StoryScreen: View {
             .scaleEffect(2)
         case .loaded(let comments):
           VStack {
-            List(comments, id: \.id) { flattenedComment in
-              CommentRow(
-                comment: flattenedComment.comment,
-                level: flattenedComment.depth
-              )
+            List(comments, id: \.id) { commentInfo in
+              CommentRow(comment: commentInfo)
               .listRowBackground(Color.clear)
               .listRowSeparator(.hidden)
               .listRowInsets(.init(top: 0, leading: 0, bottom: 0, trailing: 0))
@@ -71,12 +68,12 @@ struct StoryScreen: View {
 struct StoryScreen_Preview: PreviewProvider {
   static var previews: some View {
     let comments = [
-      PreviewHelpers.makeFakeFlattenedComment(),
-      PreviewHelpers.makeFakeFlattenedComment(),
-      PreviewHelpers.makeFakeFlattenedComment(),
-      PreviewHelpers.makeFakeFlattenedComment()
+      PreviewHelpers.makeFakeComment(),
+      PreviewHelpers.makeFakeComment(),
+      PreviewHelpers.makeFakeComment(),
+      PreviewHelpers.makeFakeComment(),
     ]
-    let viewModel = StoryViewModel(story: PreviewHelpers.makeFakeStory(kids: comments.map { $0.comment.id }))
+    let viewModel = StoryViewModel(story: PreviewHelpers.makeFakeStory(kids: comments.map { $0.id }))
     viewModel.state.comments = .loaded(comments: comments)
     return PreviewVariants {
       PreviewHelpers.withNavigationView {

ios/HackerNews/Utils/DateUtils.swift

@@ -8,9 +8,29 @@
 import Foundation
 
 extension Date {
-    func timeAgoDisplay() -> String {
-        let formatter = RelativeDateTimeFormatter()
-        formatter.unitsStyle = .full
-        return formatter.localizedString(for: self, relativeTo: Date())
-    }
+  func timeAgoDisplay() -> String {
+    let formatter = RelativeDateTimeFormatter()
+    formatter.unitsStyle = .full
+    return formatter.localizedString(for: self, relativeTo: Date())
+  }
+}
+
+extension String {
+  /**
+   Handle various time strings that we can get from the HN API / Web
+   2024-12-27T13:59:29
+   2024-09-05T17:48:25.000000Z
+   */
+  func asDate() -> Date? {
+    let regularFormatter = DateFormatter()
+    regularFormatter.dateFormat = "yyyy-MM-dd'T'HH:mm:ss"
+    regularFormatter.timeZone = TimeZone(identifier: "UTC")
+
+    let secondaryFormatter = ISO8601DateFormatter()
+    secondaryFormatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
+    secondaryFormatter.timeZone = TimeZone(identifier: "UTC")
+
+    let date = regularFormatter.date(from: self) ?? secondaryFormatter.date(from: self)
+    return date
+  }
 }

ios/HackerNews/Utils/Previews.swift

@@ -117,12 +117,11 @@ struct PreviewHelpers {
     )
   }
 
-  static func makeFakeComment() -> Comment {
-    Comment(
+  static func makeFakeComment(level: Int = 0) -> CommentInfo {
+    CommentInfo(
       id: 1,
-      by: "dang",
-      time: referenceTimestamp,
-      type: .comment,
+      upvoted: false,
+      upvoteUrl: "",
       text: """
         Totally useless commentary:
         It makes me deeply happy to hear success stories like this for a project that's moving in the correctly opposite direction to that of the rest of the world.
@@ -131,14 +130,9 @@ struct PreviewHelpers {
 
         My soul was also satisfied by the Sleeping At Night post which, along with the recent "Lie Still in Bed" article, makes for very simple options to attempt to fix sleep (discipline) issues
         """,
-      parent: nil,
-      kids: nil
+      user: "dang",
+      age: "10 minutes ago",
+      level: level
     )
   }
-
-  static func makeFakeFlattenedComment(
-    comment: Comment = makeFakeComment(), depth: Int = 0
-  ) -> FlattenedComment {
-    FlattenedComment(comment: comment, depth: depth)
-  }
 }