Skip to content

Commit

Permalink
Don't query topics when there is no topic ids
Browse files Browse the repository at this point in the history
  • Loading branch information
lenguyenthanh committed Jun 18, 2024
1 parent 427c818 commit 1a5f0bf
Showing 1 changed file with 13 additions and 11 deletions.
24 changes: 13 additions & 11 deletions modules/ingestor/src/main/scala/ingestor.forum.scala
Original file line number Diff line number Diff line change
Expand Up @@ -110,17 +110,19 @@ object ForumIngestor:
extension (events: List[ChangeStreamDocument[Document]])
private def toSources: IO[List[(String, ForumSource)]] =
val topicIds = events.flatMap(_.topicId).distinct
topicByIds(topicIds)
.flatMap: topicMap =>
events
.traverse: event =>
(event.id, event.topicId, event.fullDocument)
.flatMapN: (id, topicId, doc) =>
doc.toSource(topicName = topicMap.get(topicId)).map(id -> _)
.match
case Some(value) => value.some.pure[IO]
case _ => info"failed to convert document to source: $event".as(none)
.map(_.flatten)
if topicIds.isEmpty then info"no topics found for posts: $events".as(Nil)
else
topicByIds(topicIds)
.flatMap: topicMap =>
events
.traverse: event =>
(event.id, event.topicId, event.fullDocument)
.flatMapN: (id, topicId, doc) =>
doc.toSource(topicName = topicMap.get(topicId)).map(id -> _)
.match
case Some(value) => value.some.pure[IO]
case _ => info"failed to convert document to source: $event".as(none)
.map(_.flatten)

extension (doc: Document)
private def toSource(topicName: Option[String]): Option[ForumSource] =
Expand Down

0 comments on commit 1a5f0bf

Please sign in to comment.