Refines the EOSE strategy to make sure relays that have not replied have a chance to

app/src/main/java/com/vitorpamplona/amethyst/service/NostrSingleEventDataSource.kt

@@ -2,6 +2,7 @@ package com.vitorpamplona.amethyst.service
 
 import com.vitorpamplona.amethyst.model.AddressableNote
 import com.vitorpamplona.amethyst.model.Note
+import com.vitorpamplona.amethyst.model.User
 import com.vitorpamplona.amethyst.service.relays.COMMON_FEED_TYPES
 import com.vitorpamplona.amethyst.service.relays.EOSETime
 import com.vitorpamplona.amethyst.service.relays.JsonFilter
@@ -21,34 +22,32 @@ object NostrSingleEventDataSource : NostrDataSource("SingleEventFeed") {
     private var addressesToWatch = setOf<Note>()
 
     private fun createReactionsToWatchInAddressFilter(): List<TypedFilter>? {
-        val addressesToWatch = eventsToWatch.filter { it.address() != null } + addressesToWatch
+        val addressesToWatch = eventsToWatch.filter { it.address() != null } + addressesToWatch.filter { it.address() != null }
 
         if (addressesToWatch.isEmpty()) {
             return null
         }
 
-        return addressesToWatch.mapNotNull {
+        return groupByEOSEPresence(eventsToWatch).mapNotNull {
-            it.address()?.let { aTag ->
+            TypedFilter(
-                TypedFilter(
+                types = COMMON_FEED_TYPES,
-                    types = COMMON_FEED_TYPES,
+                filter = JsonFilter(
-                    filter = JsonFilter(
+                    kinds = listOf(
-                        kinds = listOf(
+                        TextNoteEvent.kind,
-                            TextNoteEvent.kind,
+                        ReactionEvent.kind,
-                            ReactionEvent.kind,
+                        RepostEvent.kind,
-                            RepostEvent.kind,
+                        GenericRepostEvent.kind,
-                            GenericRepostEvent.kind,
+                        ReportEvent.kind,
-                            ReportEvent.kind,
+                        LnZapEvent.kind,
-                            LnZapEvent.kind,
+                        PollNoteEvent.kind,
-                            PollNoteEvent.kind,
+                        CommunityPostApprovalEvent.kind,
-                            CommunityPostApprovalEvent.kind,
+                        LiveActivitiesChatMessageEvent.kind
-                            LiveActivitiesChatMessageEvent.kind
+                    ),
-                        ),
+                    tags = mapOf("a" to it.mapNotNull { it.address()?.toTag() }),
-                        tags = mapOf("a" to listOf(aTag.toTag())),
+                    since = findMinimumEOSEs(it),
-                        since = it.lastReactionsDownloadTime,
+                    limit = 1000 // Max amount of "replies" to download on a specific event.
-                        limit = 1000 // Max amount of "replies" to download on a specific event.
-                    )
                 )
-            }
+            )
         }
     }
 
@@ -67,7 +66,7 @@ object NostrSingleEventDataSource : NostrDataSource("SingleEventFeed") {
                         filter = JsonFilter(
                             kinds = listOf(aTag.kind),
                             authors = listOf(aTag.pubKeyHex),
-                            limit = 1000 // Max amount of "replies" to download on a specific event.
+                            limit = 5
                         )
                     )
                 } else {
@@ -77,7 +76,7 @@ object NostrSingleEventDataSource : NostrDataSource("SingleEventFeed") {
                             kinds = listOf(aTag.kind),
                             tags = mapOf("d" to listOf(aTag.dTag)),
                             authors = listOf(aTag.pubKeyHex),
-                            limit = 1000 // Max amount of "replies" to download on a specific event.
+                            limit = 5
                         )
                     )
                 }
@@ -86,13 +85,11 @@ object NostrSingleEventDataSource : NostrDataSource("SingleEventFeed") {
     }
 
     private fun createRepliesAndReactionsFilter(): List<TypedFilter>? {
-        val reactionsToWatch = eventsToWatch
+        if (eventsToWatch.isEmpty()) {
-
-        if (reactionsToWatch.isEmpty()) {
             return null
         }
 
-        return reactionsToWatch.map {
+        return groupByEOSEPresence(eventsToWatch).map {
             TypedFilter(
                 types = COMMON_FEED_TYPES,
                 filter = JsonFilter(
@@ -105,8 +102,8 @@ object NostrSingleEventDataSource : NostrDataSource("SingleEventFeed") {
                         LnZapEvent.kind,
                         PollNoteEvent.kind
                     ),
-                    tags = mapOf("e" to listOf(it.idHex)),
+                    tags = mapOf("e" to it.map { it.idHex }),
-                    since = it.lastReactionsDownloadTime,
+                    since = findMinimumEOSEs(it),
                     limit = 1000 // Max amount of "replies" to download on a specific event.
                 )
             )
@@ -207,3 +204,45 @@ object NostrSingleEventDataSource : NostrDataSource("SingleEventFeed") {
         }
     }
 }
+
+fun groupByEOSEPresence(notes: Set<Note>): Collection<List<Note>> {
+    return notes.groupBy { it.lastReactionsDownloadTime.keys.sorted().joinToString(",") }.values
+}
+
+fun groupByEOSEPresence(users: Iterable<User>): Collection<List<User>> {
+    return users.groupBy { it.latestEOSEs.keys.sorted().joinToString(",") }.values
+}
+
+fun findMinimumEOSEs(notes: List<Note>): Map<String, EOSETime> {
+    val minLatestEOSEs = mutableMapOf<String, EOSETime>()
+
+    notes.forEach {
+        it.lastReactionsDownloadTime.forEach {
+            val minEose = minLatestEOSEs[it.key]
+            if (minEose == null) {
+                minLatestEOSEs.put(it.key, EOSETime(it.value.time))
+            } else if (it.value.time < minEose.time) {
+                minEose.time = it.value.time
+            }
+        }
+    }
+
+    return minLatestEOSEs
+}
+
+fun findMinimumEOSEsForUsers(users: List<User>): Map<String, EOSETime> {
+    val minLatestEOSEs = mutableMapOf<String, EOSETime>()
+
+    users.forEach {
+        it.latestEOSEs.forEach {
+            val minEose = minLatestEOSEs[it.key]
+            if (minEose == null) {
+                minLatestEOSEs.put(it.key, EOSETime(it.value.time))
+            } else if (it.value.time < minEose.time) {
+                minEose.time = it.value.time
+            }
+        }
+    }
+
+    return minLatestEOSEs
+}

app/src/main/java/com/vitorpamplona/amethyst/service/NostrSingleUserDataSource.kt

@@ -17,99 +17,71 @@ object NostrSingleUserDataSource : NostrDataSource("SingleUserFeed") {
 
         val firstTimers = usersToWatch.filter { it.info?.latestMetadata == null }.map { it.pubkeyHex }
 
+        if (firstTimers.isEmpty()) return null
+
         return listOf(
             TypedFilter(
                 types = COMMON_FEED_TYPES,
                 filter = JsonFilter(
-                    kinds = listOf(MetadataEvent.kind, StatusEvent.kind),
+                    kinds = listOf(MetadataEvent.kind),
-                    authors = firstTimers,
+                    authors = firstTimers
-                    limit = 10 * firstTimers.size
                 )
             )
         )
     }
 
-    fun createUserMetadataFilter(minLatestEOSEs: Map<String, EOSETime>): TypedFilter? {
+    fun createUserMetadataStatusReportFilter(): List<TypedFilter>? {
         if (usersToWatch.isEmpty()) return null
 
-        return TypedFilter(
+        val secondTimers = usersToWatch.filter { it.info?.latestMetadata != null }
-            types = COMMON_FEED_TYPES,
+
-            filter = JsonFilter(
+        if (secondTimers.isEmpty()) return null
-                kinds = listOf(StatusEvent.kind),
+
-                authors = usersToWatch.map { it.pubkeyHex },
+        return groupByEOSEPresence(secondTimers).map { group ->
-                since = minLatestEOSEs
+            val groupIds = group.map { it.pubkeyHex }
+            val minEOSEs = findMinimumEOSEsForUsers(group)
+            listOf(
+                TypedFilter(
+                    types = COMMON_FEED_TYPES,
+                    filter = JsonFilter(
+                        kinds = listOf(MetadataEvent.kind, StatusEvent.kind),
+                        authors = groupIds,
+                        since = minEOSEs
+                    )
+                ),
+                TypedFilter(
+                    types = COMMON_FEED_TYPES,
+                    filter = JsonFilter(
+                        kinds = listOf(ReportEvent.kind),
+                        tags = mapOf("p" to groupIds),
+                        since = minEOSEs
+                    )
+                )
             )
-        )
+        }.flatten()
-    }
-
-    fun createUserStatusFilter(minLatestEOSEs: Map<String, EOSETime>): TypedFilter? {
-        if (usersToWatch.isEmpty()) return null
-
-        return TypedFilter(
-            types = COMMON_FEED_TYPES,
-            filter = JsonFilter(
-                kinds = listOf(StatusEvent.kind),
-                authors = usersToWatch.map { it.pubkeyHex },
-                since = minLatestEOSEs
-            )
-        )
-    }
-
-    fun createUserReportFilter(minLatestEOSEs: Map<String, EOSETime>): TypedFilter? {
-        if (usersToWatch.isEmpty()) return null
-
-        return TypedFilter(
-            types = COMMON_FEED_TYPES,
-            filter = JsonFilter(
-                kinds = listOf(ReportEvent.kind),
-                tags = mapOf("p" to usersToWatch.map { it.pubkeyHex }),
-                since = minLatestEOSEs
-            )
-        )
     }
 
     val userChannel = requestNewChannel() { time, relayUrl ->
-        usersToWatch.forEach {
+        checkNotInMainThread()
-            val eose = it.latestEOSEs[relayUrl]
-            if (eose == null) {
-                it.latestEOSEs = it.latestEOSEs + Pair(relayUrl, EOSETime(time))
-            } else {
-                eose.time = time
-            }
-        }
-    }
 
-    val userChannelFirstTimers = requestNewChannel() { time, relayUrl ->
-        // Many relays operate with limits in the amount of filters.
-        // As information comes, the filters will be rotated to get more data.
-        invalidateFilters()
-    }
-
-    override fun updateChannelFilters() {
-        val minLatestEOSEs = mutableMapOf<String, EOSETime>()
-        val neverGottenAnEOSE = mutableSetOf<String>()
         usersToWatch.forEach {
-            if (it.latestEOSEs.isEmpty()) { // first time
+            if (it.info?.latestMetadata != null) {
-                neverGottenAnEOSE.add(it.pubkeyHex)
+                val eose = it.latestEOSEs[relayUrl]
-            } else {
+                if (eose == null) {
-                it.latestEOSEs.forEach {
+                    it.latestEOSEs = it.latestEOSEs + Pair(relayUrl, EOSETime(time))
-                    val minEose = minLatestEOSEs[it.key]
+                } else {
-                    if (minEose == null) {
+                    eose.time = time
-                        minLatestEOSEs.put(it.key, EOSETime(it.value.time))
-                    } else if (it.value.time < minEose.time) {
-                        minEose.time = it.value.time
-                    }
                 }
             }
         }
+    }
+
+    override fun updateChannelFilters() {
+        checkNotInMainThread()
 
         userChannel.typedFilters = listOfNotNull(
-            createUserMetadataFilter(minLatestEOSEs),
+            createUserMetadataFilter(),
-            createUserStatusFilter(minLatestEOSEs),
+            createUserMetadataStatusReportFilter()
-            createUserReportFilter(minLatestEOSEs)
-        ).ifEmpty { null }
-        userChannelFirstTimers.typedFilters = listOfNotNull(
-            createUserMetadataFilter()
         ).flatten().ifEmpty { null }
     }