Restructures Data Access filters and LocalCache to use a ConcurrentSkipList instead of ConcurrentHashMap

app/src/main/java/com/vitorpamplona/amethyst/model/LargeCache.kt

@@ -0,0 +1,323 @@
+/**
+ * Copyright (c) 2024 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.amethyst.model
+
+import java.util.concurrent.ConcurrentSkipListMap
+import java.util.function.BiConsumer
+
+class LargeCache<K, V> {
+    val cache = ConcurrentSkipListMap<K, V>()
+
+    fun get(key: K) = cache.get(key)
+
+    fun remove(key: K) = cache.remove(key)
+
+    fun size() = cache.size
+
+    fun getOrCreate(
+        key: K,
+        builder: (key: K) -> V,
+    ): V {
+        val value = cache.get(key)
+
+        return if (value != null) {
+            value
+        } else {
+            val newObject = builder(key)
+            cache.putIfAbsent(key, newObject) ?: newObject
+        }
+    }
+
+    fun forEach(consumer: BiConsumer<K, V>) {
+        cache.forEach(consumer)
+    }
+
+    fun filter(consumer: BiFilter<K, V>): List<V> {
+        val runner = BiFilterCollector(consumer)
+        cache.forEach(runner)
+        return runner.results
+    }
+
+    fun filterIntoSet(consumer: BiFilter<K, V>): Set<V> {
+        val runner = BiFilterUniqueCollector(consumer)
+        cache.forEach(runner)
+        return runner.results
+    }
+
+    fun <R> mapNotNull(consumer: BiMapper<K, V, R?>): List<R> {
+        val runner = BiMapCollector(consumer)
+        cache.forEach(runner)
+        return runner.results
+    }
+
+    fun <R> mapNotNullIntoSet(consumer: BiMapper<K, V, R?>): Set<R> {
+        val runner = BiMapUniqueCollector(consumer)
+        cache.forEach(runner)
+        return runner.results
+    }
+
+    fun <R> mapFlatten(consumer: BiMapper<K, V, Collection<R>?>): List<R> {
+        val runner = BiMapFlattenCollector(consumer)
+        cache.forEach(runner)
+        return runner.results
+    }
+
+    fun <R> mapFlattenIntoSet(consumer: BiMapper<K, V, Collection<R>?>): Set<R> {
+        val runner = BiMapFlattenUniqueCollector(consumer)
+        cache.forEach(runner)
+        return runner.results
+    }
+
+    fun <R> map(consumer: BiNotNullMapper<K, V, R>): List<R> {
+        val runner = BiNotNullMapCollector(consumer)
+        cache.forEach(runner)
+        return runner.results
+    }
+
+    fun sumOf(consumer: BiSumOf<K, V>): Int {
+        val runner = BiSumOfCollector(consumer)
+        cache.forEach(runner)
+        return runner.sum
+    }
+
+    fun sumOfLong(consumer: BiSumOfLong<K, V>): Long {
+        val runner = BiSumOfLongCollector(consumer)
+        cache.forEach(runner)
+        return runner.sum
+    }
+
+    fun <R> groupBy(consumer: BiNotNullMapper<K, V, R>): Map<R, List<V>> {
+        val runner = BiGroupByCollector(consumer)
+        cache.forEach(runner)
+        return runner.results
+    }
+
+    fun <R> countByGroup(consumer: BiNotNullMapper<K, V, R>): Map<R, Int> {
+        val runner = BiCountByGroupCollector(consumer)
+        cache.forEach(runner)
+        return runner.results
+    }
+
+    fun count(consumer: BiFilter<K, V>): Int {
+        val runner = BiCountIfCollector(consumer)
+        cache.forEach(runner)
+        return runner.count
+    }
+}
+
+fun interface BiFilter<K, V> {
+    fun filter(
+        k: K,
+        v: V,
+    ): Boolean
+}
+
+class BiFilterCollector<K, V>(val filter: BiFilter<K, V>) : BiConsumer<K, V> {
+    var results: ArrayList<V> = ArrayList()
+
+    override fun accept(
+        k: K,
+        v: V,
+    ) {
+        if (filter.filter(k, v)) {
+            results.add(v)
+        }
+    }
+}
+
+class BiFilterUniqueCollector<K, V>(val filter: BiFilter<K, V>) : BiConsumer<K, V> {
+    var results: HashSet<V> = HashSet()
+
+    override fun accept(
+        k: K,
+        v: V,
+    ) {
+        if (filter.filter(k, v)) {
+            results.add(v)
+        }
+    }
+}
+
+fun interface BiMapper<K, V, R> {
+    fun map(
+        k: K,
+        v: V,
+    ): R?
+}
+
+class BiMapCollector<K, V, R>(val mapper: BiMapper<K, V, R?>) : BiConsumer<K, V> {
+    var results: ArrayList<R> = ArrayList()
+
+    override fun accept(
+        k: K,
+        v: V,
+    ) {
+        val result = mapper.map(k, v)
+        if (result != null) {
+            results.add(result)
+        }
+    }
+}
+
+class BiMapUniqueCollector<K, V, R>(val mapper: BiMapper<K, V, R?>) : BiConsumer<K, V> {
+    var results: HashSet<R> = HashSet()
+
+    override fun accept(
+        k: K,
+        v: V,
+    ) {
+        val result = mapper.map(k, v)
+        if (result != null) {
+            results.add(result)
+        }
+    }
+}
+
+class BiMapFlattenCollector<K, V, R>(val mapper: BiMapper<K, V, Collection<R>?>) : BiConsumer<K, V> {
+    var results: ArrayList<R> = ArrayList()
+
+    override fun accept(
+        k: K,
+        v: V,
+    ) {
+        val result = mapper.map(k, v)
+        if (result != null) {
+            results.addAll(result)
+        }
+    }
+}
+
+class BiMapFlattenUniqueCollector<K, V, R>(val mapper: BiMapper<K, V, Collection<R>?>) : BiConsumer<K, V> {
+    var results: HashSet<R> = HashSet()
+
+    override fun accept(
+        k: K,
+        v: V,
+    ) {
+        val result = mapper.map(k, v)
+        if (result != null) {
+            results.addAll(result)
+        }
+    }
+}
+
+fun interface BiNotNullMapper<K, V, R> {
+    fun map(
+        k: K,
+        v: V,
+    ): R
+}
+
+class BiNotNullMapCollector<K, V, R>(val mapper: BiNotNullMapper<K, V, R>) : BiConsumer<K, V> {
+    var results: ArrayList<R> = ArrayList()
+
+    override fun accept(
+        k: K,
+        v: V,
+    ) {
+        results.add(mapper.map(k, v))
+    }
+}
+
+fun interface BiSumOf<K, V> {
+    fun map(
+        k: K,
+        v: V,
+    ): Int
+}
+
+class BiSumOfCollector<K, V>(val mapper: BiSumOf<K, V>) : BiConsumer<K, V> {
+    var sum = 0
+
+    override fun accept(
+        k: K,
+        v: V,
+    ) {
+        sum += mapper.map(k, v)
+    }
+}
+
+fun interface BiSumOfLong<K, V> {
+    fun map(
+        k: K,
+        v: V,
+    ): Long
+}
+
+class BiSumOfLongCollector<K, V>(val mapper: BiSumOfLong<K, V>) : BiConsumer<K, V> {
+    var sum = 0L
+
+    override fun accept(
+        k: K,
+        v: V,
+    ) {
+        sum += mapper.map(k, v)
+    }
+}
+
+class BiGroupByCollector<K, V, R>(val mapper: BiNotNullMapper<K, V, R>) : BiConsumer<K, V> {
+    var results = HashMap<R, ArrayList<V>>()
+
+    override fun accept(
+        k: K,
+        v: V,
+    ) {
+        val group = mapper.map(k, v)
+
+        val list = results[group]
+        if (list == null) {
+            val answer = ArrayList<V>()
+            answer.add(v)
+            results[group] = answer
+        } else {
+            list.add(v)
+        }
+    }
+}
+
+class BiCountByGroupCollector<K, V, R>(val mapper: BiNotNullMapper<K, V, R>) : BiConsumer<K, V> {
+    var results = HashMap<R, Int>()
+
+    override fun accept(
+        k: K,
+        v: V,
+    ) {
+        val group = mapper.map(k, v)
+
+        val count = results[group]
+        if (count == null) {
+            results[group] = 1
+        } else {
+            results[group] = count + 1
+        }
+    }
+}
+
+class BiCountIfCollector<K, V>(val filter: BiFilter<K, V>) : BiConsumer<K, V> {
+    var count = 0
+
+    override fun accept(
+        k: K,
+        v: V,
+    ) {
+        if (filter.filter(k, v)) count++
+    }
+}

app/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt

@@ -120,30 +120,16 @@ import java.time.Instant
 import java.time.ZoneId
 import java.time.format.DateTimeFormatter
 import java.util.concurrent.ConcurrentHashMap
-import kotlin.time.measureTimedValue
 
 object LocalCache {
     val antiSpam = AntiSpamFilter()
 
-    private val users = ConcurrentHashMap<HexKey, User>(5000)
+    val users = LargeCache<HexKey, User>()
-    private val notes = ConcurrentHashMap<HexKey, Note>(5000)
+    val notes = LargeCache<HexKey, Note>()
+    val addressables = LargeCache<String, AddressableNote>()
+
     val channels = ConcurrentHashMap<HexKey, Channel>()
-    val addressables = ConcurrentHashMap<String, AddressableNote>(100)
+    val awaitingPaymentRequests = ConcurrentHashMap<HexKey, Pair<Note?, (LnZapPaymentResponseEvent) -> Unit>>(10)
-
-    val awaitingPaymentRequests =
-        ConcurrentHashMap<HexKey, Pair<Note?, (LnZapPaymentResponseEvent) -> Unit>>(10)
-
-    var noteListCache: List<Note> = emptyList()
-    var userListCache: List<User> = emptyList()
-
-    fun updateListCache() {
-        val (value, elapsed) =
-            measureTimedValue {
-                noteListCache = ArrayList(notes.values)
-                userListCache = ArrayList(users.values)
-            }
-        Log.d("LocalCache", "UpdateListCache $elapsed")
-    }
 
     fun checkGetOrCreateUser(key: String): User? {
         // checkNotInMainThread()
@@ -156,27 +142,24 @@ object LocalCache {
 
     fun getOrCreateUser(key: HexKey): User {
         // checkNotInMainThread()
+        require(isValidHex(key = key)) { "$key is not a valid hex" }
 
-        return users[key]
+        return users.getOrCreate(key) {
-            ?: run {
+            User(it)
-                require(isValidHex(key = key)) { "$key is not a valid hex" }
+        }
-
-                val newObject = User(key)
-                users.putIfAbsent(key, newObject) ?: newObject
-            }
     }
 
     fun getUserIfExists(key: String): User? {
         if (key.isEmpty()) return null
-        return users[key]
+        return users.get(key)
     }
 
     fun getAddressableNoteIfExists(key: String): AddressableNote? {
-        return addressables[key]
+        return addressables.get(key)
     }
 
     fun getNoteIfExists(key: String): Note? {
-        return addressables[key] ?: notes[key]
+        return addressables.get(key) ?: notes.get(key)
     }
 
     fun getChannelIfExists(key: String): Channel? {
@@ -216,24 +199,21 @@ object LocalCache {
     ): Note {
         checkNotInMainThread()
 
-        return notes.get(idHex)
+        require(isValidHex(idHex)) { "$idHex is not a valid hex" }
-            ?: run {
-                require(isValidHex(idHex)) { "$idHex is not a valid hex" }
 
-                notes.putIfAbsent(idHex, note) ?: note
+        return notes.getOrCreate(idHex) {
-            }
+            note
+        }
     }
 
     fun getOrCreateNote(idHex: String): Note {
         checkNotInMainThread()
 
-        return notes.get(idHex)
+        require(isValidHex(idHex)) { "$idHex is not a valid hex" }
-            ?: run {
-                require(isValidHex(idHex)) { "$idHex is not a valid hex" }
 
-                val newObject = Note(idHex)
+        return notes.getOrCreate(idHex) {
-                notes.putIfAbsent(idHex, newObject) ?: newObject
+            Note(idHex)
-            }
+        }
     }
 
     fun checkGetOrCreateChannel(key: String): Channel? {
@@ -288,11 +268,9 @@ object LocalCache {
 
         // we can't use naddr here because naddr might include relay info and
         // the preferred relay should not be part of the index.
-        return addressables[key.toTag()]
+        return addressables.getOrCreate(key.toTag()) {
-            ?: run {
+            AddressableNote(key)
-                val newObject = AddressableNote(key)
+        }
-                addressables.putIfAbsent(key.toTag(), newObject) ?: newObject
-            }
     }
 
     fun getOrCreateAddressableNote(key: ATag): AddressableNote {
@@ -761,9 +739,6 @@ object LocalCache {
 
         if (version.event == null) {
             version.loadEvent(event, author, emptyList())
-            if (version.liveSet != null) {
-                updateListCache()
-            }
             version.liveSet?.innerOts?.invalidateData()
         }
 
@@ -1423,9 +1398,6 @@ object LocalCache {
             checkGetOrCreateNote(it)?.let { editedNote ->
                 modificationCache.remove(editedNote.idHex)
                 // must update list of Notes to quickly update the user.
-                if (editedNote.liveSet != null) {
-                    updateListCache()
-                }
                 editedNote.liveSet?.innerModifications?.invalidateData()
             }
         }
@@ -1636,10 +1608,10 @@ object LocalCache {
             }
         }
 
-        return userListCache.filter {
+        return users.filter { _, user: User ->
-            (it.anyNameStartsWith(username)) ||
+            (user.anyNameStartsWith(username)) ||
-                it.pubkeyHex.startsWith(username, true) ||
+                user.pubkeyHex.startsWith(username, true) ||
-                it.pubkeyNpub().startsWith(username, true)
+                user.pubkeyNpub().startsWith(username, true)
         }
     }
 
@@ -1655,39 +1627,39 @@ object LocalCache {
             }
         }
 
-        return noteListCache.filter {
+        return notes.filter { _, note ->
             (
-                it.event !is GenericRepostEvent &&
+                note.event !is GenericRepostEvent &&
-                    it.event !is RepostEvent &&
+                    note.event !is RepostEvent &&
-                    it.event !is CommunityPostApprovalEvent &&
+                    note.event !is CommunityPostApprovalEvent &&
-                    it.event !is ReactionEvent &&
+                    note.event !is ReactionEvent &&
-                    it.event !is GiftWrapEvent &&
+                    note.event !is GiftWrapEvent &&
-                    it.event !is SealedGossipEvent &&
+                    note.event !is SealedGossipEvent &&
-                    it.event !is OtsEvent &&
+                    note.event !is OtsEvent &&
-                    it.event !is LnZapEvent &&
+                    note.event !is LnZapEvent &&
-                    it.event !is LnZapRequestEvent
+                    note.event !is LnZapRequestEvent
             ) &&
                 (
-                    it.event?.content()?.contains(text, true)
+                    note.event?.content()?.contains(text, true)
                         ?: false ||
-                        it.event?.matchTag1With(text) ?: false ||
+                        note.event?.matchTag1With(text) ?: false ||
-                        it.idHex.startsWith(text, true) ||
+                        note.idHex.startsWith(text, true) ||
-                        it.idNote().startsWith(text, true)
+                        note.idNote().startsWith(text, true)
                 )
         } +
-            addressables.values.filter {
+            addressables.filter { _, addressable ->
                 (
-                    it.event !is GenericRepostEvent &&
+                    addressable.event !is GenericRepostEvent &&
-                        it.event !is RepostEvent &&
+                        addressable.event !is RepostEvent &&
-                        it.event !is CommunityPostApprovalEvent &&
+                        addressable.event !is CommunityPostApprovalEvent &&
-                        it.event !is ReactionEvent &&
+                        addressable.event !is ReactionEvent &&
-                        it.event !is GiftWrapEvent &&
+                        addressable.event !is GiftWrapEvent &&
-                        it.event !is LnZapEvent &&
+                        addressable.event !is LnZapEvent &&
-                        it.event !is LnZapRequestEvent
+                        addressable.event !is LnZapRequestEvent
                 ) &&
                     (
-                        it.event?.content()?.contains(text, true)
+                        addressable.event?.content()?.contains(text, true)
-                            ?: false || it.event?.matchTag1With(text) ?: false || it.idHex.startsWith(text, true)
+                            ?: false || addressable.event?.matchTag1With(text) ?: false || addressable.idHex.startsWith(text, true)
                     )
             }
     }
@@ -1710,17 +1682,15 @@ object LocalCache {
     suspend fun findStatusesForUser(user: User): ImmutableList<AddressableNote> {
         checkNotInMainThread()
 
-        return addressables
+        return addressables.filter { _, it ->
-            .filter {
+            val noteEvent = it.event
-                val noteEvent = it.value.event
+            (
-                (
+                noteEvent is StatusEvent &&
-                    noteEvent is StatusEvent &&
+                    noteEvent.pubKey == user.pubkeyHex &&
-                        noteEvent.pubKey == user.pubkeyHex &&
+                    !noteEvent.isExpired() &&
-                        !noteEvent.isExpired() &&
+                    noteEvent.content.isNotBlank()
-                        noteEvent.content.isNotBlank()
+            )
-                )
+        }
-            }
-            .values
             .sortedWith(compareBy({ it.event?.expiration() ?: it.event?.createdAt() }, { it.idHex }))
             .reversed()
             .toImmutableList()
@@ -1732,7 +1702,7 @@ object LocalCache {
         var minTime: Long? = null
         val time = TimeUtils.now()
 
-        noteListCache.forEach { item ->
+        notes.forEach { _, item ->
             val noteEvent = item.event
             if ((noteEvent is OtsEvent && noteEvent.isTaggedEvent(note.idHex) && !noteEvent.isExpirationBefore(time))) {
                 noteEvent.verifiedTime?.let { stampedTime ->
@@ -1764,7 +1734,7 @@ object LocalCache {
         val time = TimeUtils.now()
 
         val newNotes =
-            noteListCache.filter { item ->
+            notes.filter { _, item ->
                 val noteEvent = item.event
 
                 noteEvent is TextNoteModificationEvent && noteEvent.pubKey == originalAuthor && noteEvent.isTaggedEvent(note.idHex) && !noteEvent.isExpirationBefore(time)
@@ -1776,11 +1746,9 @@ object LocalCache {
     }
 
     fun cleanObservers() {
-        noteListCache.forEach { it.clearLive() }
+        notes.forEach { _, it -> it.clearLive() }
-
+        addressables.forEach { _, it -> it.clearLive() }
-        addressables.forEach { it.value.clearLive() }
+        users.forEach { _, it -> it.clearLive() }
-
-        userListCache.forEach { it.clearLive() }
     }
 
     fun pruneOldAndHiddenMessages(account: Account) {
@@ -1806,8 +1774,8 @@ object LocalCache {
             }
         }
 
-        userListCache.forEach { userPair ->
+        users.forEach { _, user ->
-            userPair.privateChatrooms.values.map {
+            user.privateChatrooms.values.map {
                 val toBeRemoved = it.pruneMessagesToTheLatestOnly()
 
                 val childrenToBeRemoved = mutableListOf<Note>()
@@ -1822,7 +1790,7 @@ object LocalCache {
 
                 if (toBeRemoved.size > 1) {
                     println(
-                        "PRUNE: ${toBeRemoved.size} private messages with ${userPair.toBestDisplayName()} removed. ${it.roomMessages.size} kept",
+                        "PRUNE: ${toBeRemoved.size} private messages with ${user.toBestDisplayName()} removed. ${it.roomMessages.size} kept",
                     )
                 }
             }
@@ -1831,21 +1799,20 @@ object LocalCache {
 
     fun prunePastVersionsOfReplaceables() {
         val toBeRemoved =
-            noteListCache
+            notes.filter { _, note ->
-                .filter {
+                val noteEvent = note.event
-                    val noteEvent = it.event
+                if (noteEvent is AddressableEvent) {
-                    if (noteEvent is AddressableEvent) {
+                    noteEvent.createdAt() <
-                        noteEvent.createdAt() <
+                        (addressables.get(noteEvent.address().toTag())?.event?.createdAt() ?: 0)
-                            (addressables[noteEvent.address().toTag()]?.event?.createdAt() ?: 0)
+                } else {
-                    } else {
+                    false
-                        false
-                    }
                 }
+            }
 
         val childrenToBeRemoved = mutableListOf<Note>()
 
         toBeRemoved.forEach {
-            val newerVersion = addressables[(it.event as? AddressableEvent)?.address()?.toTag()]
+            val newerVersion = (it.event as? AddressableEvent)?.address()?.toTag()?.let { tag -> addressables.get(tag) }
             if (newerVersion != null) {
                 it.moveAllReferencesTo(newerVersion)
             }
@@ -1865,23 +1832,22 @@ object LocalCache {
         checkNotInMainThread()
 
         val toBeRemoved =
-            noteListCache
+            notes.filter { _, note ->
-                .filter {
+                (
-                    (
+                    (note.event is TextNoteEvent && !note.isNewThread()) ||
-                        (it.event is TextNoteEvent && !it.isNewThread()) ||
+                        note.event is ReactionEvent ||
-                            it.event is ReactionEvent ||
+                        note.event is LnZapEvent ||
-                            it.event is LnZapEvent ||
+                        note.event is LnZapRequestEvent ||
-                            it.event is LnZapRequestEvent ||
+                        note.event is ReportEvent ||
-                            it.event is ReportEvent ||
+                        note.event is GenericRepostEvent
-                            it.event is GenericRepostEvent
+                ) &&
-                    ) &&
+                    note.replyTo?.any { it.liveSet?.isInUse() == true } != true &&
-                        it.replyTo?.any { it.liveSet?.isInUse() == true } != true &&
+                    note.liveSet?.isInUse() != true && // don't delete if observing.
-                        it.liveSet?.isInUse() != true && // don't delete if observing.
+                    note.author?.pubkeyHex !in
-                        it.author?.pubkeyHex !in
+                    accounts && // don't delete if it is the logged in account
-                        accounts && // don't delete if it is the logged in account
+                    note.event?.isTaggedUsers(accounts) !=
-                        it.event?.isTaggedUsers(accounts) !=
+                    true // don't delete if it's a notification to the logged in user
-                        true // don't delete if it's a notification to the logged in user
+            }
-                }
 
         val childrenToBeRemoved = mutableListOf<Note>()
 
@@ -1948,7 +1914,7 @@ object LocalCache {
         checkNotInMainThread()
 
         val now = TimeUtils.now()
-        val toBeRemoved = noteListCache.filter { it.event?.isExpirationBefore(now) == true }
+        val toBeRemoved = notes.filter { _, it -> it.event?.isExpirationBefore(now) == true }
 
         val childrenToBeRemoved = mutableListOf<Note>()
 
@@ -1973,11 +1939,7 @@ object LocalCache {
             account.liveHiddenUsers.value
                 ?.hiddenUsers
                 ?.map { userHex ->
-                    (
+                    (notes.filter { _, it -> it.event?.pubKey() == userHex } + addressables.filter { _, it -> it.event?.pubKey() == userHex }).toSet()
-                        noteListCache.filter { it.event?.pubKey() == userHex } +
-                            addressables.values.filter { it.event?.pubKey() == userHex }
-                    )
-                        .toSet()
                 }
                 ?.flatten()
                 ?: emptyList()
@@ -1996,13 +1958,13 @@ object LocalCache {
         checkNotInMainThread()
 
         var removingContactList = 0
-        userListCache.forEach {
+        users.forEach { _, user ->
             if (
-                it.pubkeyHex !in loggedIn &&
+                user.pubkeyHex !in loggedIn &&
-                (it.liveSet == null || it.liveSet?.isInUse() == false) &&
+                (user.liveSet == null || user.liveSet?.isInUse() == false) &&
-                it.latestContactList != null
+                user.latestContactList != null
             ) {
-                it.latestContactList = null
+                user.latestContactList = null
                 removingContactList++
             }
         }
@@ -2154,7 +2116,6 @@ class LocalCacheLiveData {
     fun invalidateData(newNote: Note) {
         bundler.invalidateList(newNote) {
                 bundledNewNotes ->
-            LocalCache.updateListCache()
             _newEventBundles.emit(bundledNewNotes)
         }
     }

app/src/main/java/com/vitorpamplona/amethyst/model/User.kt

@@ -354,7 +354,7 @@ class User(val pubkeyHex: String) {
     }
 
     suspend fun transientFollowerCount(): Int {
-        return LocalCache.userListCache.count { it.latestContactList?.isTaggedUser(pubkeyHex) ?: false }
+        return LocalCache.users.count { _, it -> it.latestContactList?.isTaggedUser(pubkeyHex) ?: false }
     }
 
     fun cachedFollowingKeySet(): Set<HexKey> {
@@ -378,7 +378,7 @@ class User(val pubkeyHex: String) {
     }
 
     suspend fun cachedFollowerCount(): Int {
-        return LocalCache.userListCache.count { it.latestContactList?.isTaggedUser(pubkeyHex) ?: false }
+        return LocalCache.users.count { _, it -> it.latestContactList?.isTaggedUser(pubkeyHex) ?: false }
     }
 
     fun hasSentMessagesTo(key: ChatroomKey?): Boolean {

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/BookmarkPrivateFeedFilter.kt

@@ -45,7 +45,6 @@ class BookmarkPrivateFeedFilter(val account: Account) : FeedFilter<Note>() {
         return notes
             .plus(addresses)
             .toSet()
-            .sortedWith(compareBy({ it.createdAt() }, { it.idHex }))
+            .sortedWith(DefaultFeedOrder)
-            .reversed()
     }
 }

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/BookmarkPublicFeedFilter.kt

@@ -40,7 +40,6 @@ class BookmarkPublicFeedFilter(val account: Account) : FeedFilter<Note>() {
         return notes
             .plus(addresses)
             .toSet()
-            .sortedWith(compareBy({ it.createdAt() }, { it.idHex }))
+            .sortedWith(DefaultFeedOrder)
-            .reversed()
     }
 }

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/ChannelFeedFilter.kt

@@ -44,6 +44,6 @@ class ChannelFeedFilter(val channel: Channel, val account: Account) : AdditiveFe
     }
 
     override fun sort(collection: Set<Note>): List<Note> {
-        return collection.sortedWith(compareBy({ it.createdAt() }, { it.idHex })).reversed()
+        return collection.sortedWith(DefaultFeedOrder)
     }
 }

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/ChatroomFeedFilter.kt

@@ -47,6 +47,6 @@ class ChatroomFeedFilter(val withUser: ChatroomKey, val account: Account) :
     }
 
     override fun sort(collection: Set<Note>): List<Note> {
-        return collection.sortedWith(compareBy({ it.createdAt() }, { it.idHex })).reversed()
+        return collection.sortedWith(DefaultFeedOrder)
     }
 }

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/ChatroomListKnownFeedFilter.kt

@@ -197,6 +197,6 @@ class ChatroomListKnownFeedFilter(val account: Account) : AdditiveFeedFilter<Not
     }
 
     override fun sort(collection: Set<Note>): List<Note> {
-        return collection.sortedWith(compareBy({ it.createdAt() }, { it.idHex })).reversed()
+        return collection.sortedWith(DefaultFeedOrder)
     }
 }

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/ChatroomListNewFeedFilter.kt

@@ -138,6 +138,6 @@ class ChatroomListNewFeedFilter(val account: Account) : AdditiveFeedFilter<Note>
     }
 
     override fun sort(collection: Set<Note>): List<Note> {
-        return collection.sortedWith(compareBy({ it.createdAt() }, { it.idHex })).reversed()
+        return collection.sortedWith(DefaultFeedOrder)
     }
 }

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/CommunityFeedFilter.kt

@@ -24,16 +24,22 @@ import com.vitorpamplona.amethyst.model.Account
 import com.vitorpamplona.amethyst.model.AddressableNote
 import com.vitorpamplona.amethyst.model.LocalCache
 import com.vitorpamplona.amethyst.model.Note
+import com.vitorpamplona.quartz.encoders.HexKey
 import com.vitorpamplona.quartz.events.CommunityPostApprovalEvent
 
-class CommunityFeedFilter(val note: AddressableNote, val account: Account) :
+class CommunityFeedFilter(val note: AddressableNote, val account: Account) : AdditiveFeedFilter<Note>() {
-    AdditiveFeedFilter<Note>() {
     override fun feedKey(): String {
         return account.userProfile().pubkeyHex + "-" + note.idHex
     }
 
     override fun feed(): List<Note> {
-        return sort(innerApplyFilter(LocalCache.noteListCache))
+        val myPubKey = account.userProfile().pubkeyHex
+        val result =
+            LocalCache.notes.mapFlattenIntoSet { _, it ->
+                filterMap(it, myPubKey)
+            }
+
+        return sort(result)
     }
 
     override fun applyFilter(collection: Set<Note>): Set<Note> {
@@ -41,31 +47,36 @@ class CommunityFeedFilter(val note: AddressableNote, val account: Account) :
     }
 
     private fun innerApplyFilter(collection: Collection<Note>): Set<Note> {
-        val myUnapprovedPosts =
+        val myPubKey = account.userProfile().pubkeyHex
-            collection
-                .asSequence()
-                .filter { it.event is CommunityPostApprovalEvent } // Only Approvals
-                .filter {
-                    it.author?.pubkeyHex == account.userProfile().pubkeyHex
-                } // made by the logged in user
-                .filter { it.event?.isTaggedAddressableNote(note.idHex) == true } // for this community
-                .filter { it.isNewThread() } // check if it is a new thread
-                .toSet()
 
-        val approvedPosts =
+        return collection.mapNotNull {
-            collection
+            filterMap(it, myPubKey)
-                .asSequence()
+        }.flatten().toSet()
-                .filter { it.event is CommunityPostApprovalEvent } // Only Approvals
+    }
-                .filter { it.event?.isTaggedAddressableNote(note.idHex) == true } // Of the given community
-                .mapNotNull { it.replyTo }
-                .flatten() // get approved posts
-                .filter { it.isNewThread() } // check if it is a new thread
-                .toSet()
 
-        return myUnapprovedPosts + approvedPosts
+    private fun filterMap(
+        note: Note,
+        myPubKey: HexKey,
+    ): List<Note>? {
+        return if (
+            // Only Approvals
+            note.event is CommunityPostApprovalEvent &&
+            // Of the given community
+            note.event?.isTaggedAddressableNote(this.note.idHex) == true
+        ) {
+            // if it is my post, bring on
+            if (note.author?.pubkeyHex == myPubKey && note.isNewThread()) {
+                listOf(note)
+            } else {
+                // brings the actual posts, not the approvals
+                note.replyTo?.filter { it.isNewThread() }
+            }
+        } else {
+            null
+        }
     }
 
     override fun sort(collection: Set<Note>): List<Note> {
-        return collection.sortedWith(compareBy({ it.createdAt() }, { it.idHex })).reversed()
+        return collection.sortedWith(DefaultFeedOrder)
     }
 }

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/DefaultFeedOrder.kt

@@ -20,36 +20,6 @@
  */
 package com.vitorpamplona.amethyst.ui.dal
 
-import com.vitorpamplona.amethyst.model.Account
-import com.vitorpamplona.amethyst.model.GLOBAL_FOLLOWS
-import com.vitorpamplona.amethyst.model.KIND3_FOLLOWS
 import com.vitorpamplona.amethyst.model.Note
-import com.vitorpamplona.amethyst.service.OnlineChecker
-import com.vitorpamplona.quartz.events.LiveActivitiesEvent
-import com.vitorpamplona.quartz.events.LiveActivitiesEvent.Companion.STATUS_LIVE
 
-class DiscoverLiveNowFeedFilter(
+val DefaultFeedOrder = compareBy<Note>({ it.createdAt() }, { it.idHex }).reversed()
-    account: Account,
-) : DiscoverLiveFeedFilter(account) {
-    override fun followList(): String {
-        // uses follows by default, but other lists if they were selected in the top bar
-        val currentList = super.followList()
-        return if (currentList == GLOBAL_FOLLOWS) {
-            KIND3_FOLLOWS
-        } else {
-            currentList
-        }
-    }
-
-    override fun innerApplyFilter(collection: Collection<Note>): Set<Note> {
-        val allItems = super.innerApplyFilter(collection)
-
-        val onlineOnly =
-            allItems.filter {
-                val noteEvent = it.event as? LiveActivitiesEvent
-                noteEvent?.status() == STATUS_LIVE && OnlineChecker.isOnline(noteEvent.streaming())
-            }
-
-        return onlineOnly.toSet()
-    }
-}

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/DiscoverChatFeedFilter.kt

@@ -21,7 +21,6 @@
 package com.vitorpamplona.amethyst.ui.dal
 
 import com.vitorpamplona.amethyst.model.Account
-import com.vitorpamplona.amethyst.model.GLOBAL_FOLLOWS
 import com.vitorpamplona.amethyst.model.LocalCache
 import com.vitorpamplona.amethyst.model.Note
 import com.vitorpamplona.amethyst.model.ParticipantListBuilder
@@ -29,7 +28,6 @@ import com.vitorpamplona.quartz.events.ChannelCreateEvent
 import com.vitorpamplona.quartz.events.IsInPublicChatChannel
 import com.vitorpamplona.quartz.events.MuteListEvent
 import com.vitorpamplona.quartz.events.PeopleListEvent
-import com.vitorpamplona.quartz.utils.TimeUtils
 
 open class DiscoverChatFeedFilter(val account: Account) : AdditiveFeedFilter<Note>() {
     override fun feedKey(): String {
@@ -56,39 +54,34 @@ open class DiscoverChatFeedFilter(val account: Account) : AdditiveFeedFilter<Not
         return innerApplyFilter(collection)
     }
 
+    fun buildFilterParams(account: Account): FilterByListParams {
+        return FilterByListParams.create(
+            userHex = account.userProfile().pubkeyHex,
+            selectedListName = account.defaultDiscoveryFollowList.value,
+            followLists = account.liveDiscoveryFollowLists.value,
+            hiddenUsers = account.flowHiddenUsers.value,
+        )
+    }
+
     protected open fun innerApplyFilter(collection: Collection<Note>): Set<Note> {
-        val now = TimeUtils.now()
+        val params = buildFilterParams(account)
-        val isGlobal = account.defaultDiscoveryFollowList.value == GLOBAL_FOLLOWS
-        val isHiddenList = showHiddenKey()
 
-        val followingKeySet = account.liveDiscoveryFollowLists.value?.users ?: emptySet()
+        return collection.mapNotNullTo(HashSet()) { note ->
-        val followingTagSet = account.liveDiscoveryFollowLists.value?.hashtags ?: emptySet()
+            // note event here will never be null
-        val followingGeohashSet = account.liveDiscoveryFollowLists.value?.geotags ?: emptySet()
+            val noteEvent = note.event
-
+            if (noteEvent is ChannelCreateEvent && params.match(noteEvent)) {
-        val createEvents = collection.filter { it.event is ChannelCreateEvent }
+                note
-        val anyOtherChannelEvent =
+            } else if (noteEvent is IsInPublicChatChannel) {
-            collection
+                val channel = noteEvent.channel()?.let { LocalCache.checkGetOrCreateNote(it) }
-                .asSequence()
+                if (channel != null && (channel.event == null || params.match(channel.event))) {
-                .filter { it.event is IsInPublicChatChannel }
+                    channel
-                .mapNotNull { (it.event as? IsInPublicChatChannel)?.channel() }
+                } else {
-                .mapNotNull { LocalCache.checkGetOrCreateNote(it) }
+                    null
-                .toSet()
-
-        val activities =
-            (createEvents + anyOtherChannelEvent)
-                .asSequence()
-                // .filter { it.event is ChannelCreateEvent } // Event heads might not be loaded yet.
-                .filter {
-                    isGlobal ||
-                        it.author?.pubkeyHex in followingKeySet ||
-                        it.event?.isTaggedHashes(followingTagSet) == true ||
-                        it.event?.isTaggedGeoHashes(followingGeohashSet) == true
                 }
-                .filter { isHiddenList || it.author?.let { !account.isHidden(it.pubkeyHex) } ?: true }
+            } else {
-                .filter { (it.createdAt() ?: 0) <= now }
+                null
-                .toSet()
+            }
-
+        }
-        return activities
     }
 
     override fun sort(collection: Set<Note>): List<Note> {

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/DiscoverCommunityFeedFilter.kt

@@ -21,15 +21,14 @@
 package com.vitorpamplona.amethyst.ui.dal
 
 import com.vitorpamplona.amethyst.model.Account
-import com.vitorpamplona.amethyst.model.GLOBAL_FOLLOWS
 import com.vitorpamplona.amethyst.model.LocalCache
 import com.vitorpamplona.amethyst.model.Note
 import com.vitorpamplona.amethyst.model.ParticipantListBuilder
+import com.vitorpamplona.quartz.encoders.ATag
 import com.vitorpamplona.quartz.events.CommunityDefinitionEvent
 import com.vitorpamplona.quartz.events.CommunityPostApprovalEvent
 import com.vitorpamplona.quartz.events.MuteListEvent
 import com.vitorpamplona.quartz.events.PeopleListEvent
-import com.vitorpamplona.quartz.utils.TimeUtils
 
 open class DiscoverCommunityFeedFilter(val account: Account) : AdditiveFeedFilter<Note>() {
     override fun feedKey(): String {
@@ -44,9 +43,27 @@ open class DiscoverCommunityFeedFilter(val account: Account) : AdditiveFeedFilte
     }
 
     override fun feed(): List<Note> {
-        val allNotes = LocalCache.addressables.values
+        val filterParams =
+            FilterByListParams.create(
+                userHex = account.userProfile().pubkeyHex,
+                selectedListName = account.defaultDiscoveryFollowList.value,
+                followLists = account.liveDiscoveryFollowLists.value,
+                hiddenUsers = account.flowHiddenUsers.value,
+            )
 
-        val notes = innerApplyFilter(allNotes)
+        // Here we only need to look for CommunityDefinition Events
+        val notes =
+            LocalCache.addressables.mapNotNullIntoSet { key, note ->
+                val noteEvent = note.event
+                if (noteEvent == null && shouldInclude(ATag.parseAtagUnckecked(key), filterParams)) {
+                    // send unloaded communities to the screen
+                    note
+                } else if (noteEvent is CommunityDefinitionEvent && filterParams.match(noteEvent)) {
+                    note
+                } else {
+                    null
+                }
+            }
 
         return sort(notes)
     }
@@ -56,41 +73,44 @@ open class DiscoverCommunityFeedFilter(val account: Account) : AdditiveFeedFilte
     }
 
     protected open fun innerApplyFilter(collection: Collection<Note>): Set<Note> {
-        val now = TimeUtils.now()
+        // here, we need to look for CommunityDefinition in new collection AND new CommunityDefinition from Post Approvals
-        val isGlobal = account.defaultDiscoveryFollowList.value == GLOBAL_FOLLOWS
+        val filterParams =
-        val isHiddenList = showHiddenKey()
+            FilterByListParams.create(
+                userHex = account.userProfile().pubkeyHex,
+                selectedListName = account.defaultDiscoveryFollowList.value,
+                followLists = account.liveDiscoveryFollowLists.value,
+                hiddenUsers = account.flowHiddenUsers.value,
+            )
 
-        val followingKeySet = account.liveDiscoveryFollowLists.value?.users ?: emptySet()
+        return collection.mapNotNull { note ->
-        val followingTagSet = account.liveDiscoveryFollowLists.value?.hashtags ?: emptySet()
+            // note event here will never be null
-        val followingGeohashSet = account.liveDiscoveryFollowLists.value?.geotags ?: emptySet()
+            val noteEvent = note.event
+            if (noteEvent is CommunityDefinitionEvent && filterParams.match(noteEvent)) {
+                listOf(note)
+            } else if (noteEvent is CommunityPostApprovalEvent) {
+                noteEvent.communities().mapNotNull {
+                    val definitionNote = LocalCache.getOrCreateAddressableNote(it)
+                    val definitionEvent = definitionNote.event
 
-        val createEvents = collection.filter { it.event is CommunityDefinitionEvent }
+                    if (definitionEvent == null && shouldInclude(it, filterParams)) {
-        val anyOtherCommunityEvent =
+                        definitionNote
-            collection
+                    } else if (definitionEvent is CommunityDefinitionEvent && filterParams.match(definitionEvent)) {
-                .asSequence()
+                        definitionNote
-                .filter { it.event is CommunityPostApprovalEvent }
+                    } else {
-                .mapNotNull { (it.event as? CommunityPostApprovalEvent)?.communities() }
+                        null
-                .flatten()
+                    }
-                .map { LocalCache.getOrCreateAddressableNote(it) }
-                .toSet()
-
-        val activities =
-            (createEvents + anyOtherCommunityEvent)
-                .asSequence()
-                .filter { it.event is CommunityDefinitionEvent }
-                .filter {
-                    isGlobal ||
-                        it.author?.pubkeyHex in followingKeySet ||
-                        it.event?.isTaggedHashes(followingTagSet) == true ||
-                        it.event?.isTaggedGeoHashes(followingGeohashSet) == true
                 }
-                .filter { isHiddenList || it.author?.let { !account.isHidden(it.pubkeyHex) } ?: true }
+            } else {
-                .filter { (it.createdAt() ?: 0) <= now }
+                null
-                .toSet()
+            }
-
+        }.flatten().toSet()
-        return activities
     }
 
+    private fun shouldInclude(
+        aTag: ATag?,
+        params: FilterByListParams,
+    ) = aTag != null && aTag.kind == CommunityDefinitionEvent.KIND && params.match(aTag)
+
     override fun sort(collection: Set<Note>): List<Note> {
         val followingKeySet =
             account.liveDiscoveryFollowLists.value?.users ?: account.liveKind3Follows.value.users

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/DiscoverLiveFeedFilter.kt

@@ -21,7 +21,6 @@
 package com.vitorpamplona.amethyst.ui.dal
 
 import com.vitorpamplona.amethyst.model.Account
-import com.vitorpamplona.amethyst.model.GLOBAL_FOLLOWS
 import com.vitorpamplona.amethyst.model.LocalCache
 import com.vitorpamplona.amethyst.model.Note
 import com.vitorpamplona.amethyst.model.ParticipantListBuilder
@@ -31,7 +30,6 @@ import com.vitorpamplona.quartz.events.LiveActivitiesEvent.Companion.STATUS_LIVE
 import com.vitorpamplona.quartz.events.LiveActivitiesEvent.Companion.STATUS_PLANNED
 import com.vitorpamplona.quartz.events.MuteListEvent
 import com.vitorpamplona.quartz.events.PeopleListEvent
-import com.vitorpamplona.quartz.utils.TimeUtils
 
 open class DiscoverLiveFeedFilter(
     val account: Account,
@@ -64,33 +62,15 @@ open class DiscoverLiveFeedFilter(
     }
 
     protected open fun innerApplyFilter(collection: Collection<Note>): Set<Note> {
-        val now = TimeUtils.now()
+        val filterParams =
-        val isGlobal = account.defaultDiscoveryFollowList.value == GLOBAL_FOLLOWS
+            FilterByListParams.create(
-        val isHiddenList = showHiddenKey()
+                userHex = account.userProfile().pubkeyHex,
+                selectedListName = account.defaultDiscoveryFollowList.value,
+                followLists = account.liveDiscoveryFollowLists.value,
+                hiddenUsers = account.flowHiddenUsers.value,
+            )
 
-        val followingKeySet = account.liveDiscoveryFollowLists.value?.users ?: emptySet()
+        return collection.filterTo(HashSet()) { it.event is LiveActivitiesEvent && filterParams.match(it.event) }
-        val followingTagSet = account.liveDiscoveryFollowLists.value?.hashtags ?: emptySet()
-        val followingGeohashSet = account.liveDiscoveryFollowLists.value?.geotags ?: emptySet()
-
-        val activities =
-            collection
-                .asSequence()
-                .filter { it.event is LiveActivitiesEvent }
-                .filter {
-                    isGlobal ||
-                        (it.event as LiveActivitiesEvent).participantsIntersect(followingKeySet) ||
-                        it.event?.isTaggedHashes(
-                            followingTagSet,
-                        ) == true ||
-                        it.event?.isTaggedGeoHashes(
-                            followingGeohashSet,
-                        ) == true
-                }
-                .filter { isHiddenList || it.author?.let { !account.isHidden(it.pubkeyHex) } ?: true }
-                .filter { (it.createdAt() ?: 0) <= now }
-                .toSet()
-
-        return activities
     }
 
     override fun sort(collection: Set<Note>): List<Note> {

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/DiscoverMarketplaceFeedFilter.kt

@@ -21,13 +21,11 @@
 package com.vitorpamplona.amethyst.ui.dal
 
 import com.vitorpamplona.amethyst.model.Account
-import com.vitorpamplona.amethyst.model.GLOBAL_FOLLOWS
 import com.vitorpamplona.amethyst.model.LocalCache
 import com.vitorpamplona.amethyst.model.Note
 import com.vitorpamplona.quartz.events.ClassifiedsEvent
 import com.vitorpamplona.quartz.events.MuteListEvent
 import com.vitorpamplona.quartz.events.PeopleListEvent
-import com.vitorpamplona.quartz.utils.TimeUtils
 
 open class DiscoverMarketplaceFeedFilter(
     val account: Account,
@@ -46,10 +44,13 @@ open class DiscoverMarketplaceFeedFilter(
     }
 
     override fun feed(): List<Note> {
-        val classifieds =
+        val params = buildFilterParams(account)
-            LocalCache.addressables.filter { it.value.event is ClassifiedsEvent }.map { it.value }
 
-        val notes = innerApplyFilter(classifieds)
+        val notes =
+            LocalCache.addressables.filterIntoSet { _, it ->
+                val noteEvent = it.event
+                noteEvent is ClassifiedsEvent && noteEvent.isWellFormed() && params.match(noteEvent)
+            }
 
         return sort(notes)
     }
@@ -58,35 +59,22 @@ open class DiscoverMarketplaceFeedFilter(
         return innerApplyFilter(collection)
     }
 
+    fun buildFilterParams(account: Account): FilterByListParams {
+        return FilterByListParams.create(
+            account.userProfile().pubkeyHex,
+            account.defaultDiscoveryFollowList.value,
+            account.liveDiscoveryFollowLists.value,
+            account.flowHiddenUsers.value,
+        )
+    }
+
     protected open fun innerApplyFilter(collection: Collection<Note>): Set<Note> {
-        val now = TimeUtils.now()
+        val params = buildFilterParams(account)
-        val isGlobal = account.defaultDiscoveryFollowList.value == GLOBAL_FOLLOWS
-        val isHiddenList = showHiddenKey()
 
-        val followingKeySet = account.liveDiscoveryFollowLists.value?.users ?: emptySet()
+        return collection.filterTo(HashSet()) {
-        val followingTagSet = account.liveDiscoveryFollowLists.value?.hashtags ?: emptySet()
+            val noteEvent = it.event
-        val followingGeohashSet = account.liveDiscoveryFollowLists.value?.geotags ?: emptySet()
+            noteEvent is ClassifiedsEvent && noteEvent.isWellFormed() && params.match(noteEvent)
-
+        }
-        val activities =
-            collection
-                .asSequence()
-                .filter {
-                    it.event is ClassifiedsEvent &&
-                        it.event?.hasTagWithContent("image") == true &&
-                        it.event?.hasTagWithContent("price") == true &&
-                        it.event?.hasTagWithContent("title") == true
-                }
-                .filter {
-                    isGlobal ||
-                        it.author?.pubkeyHex in followingKeySet ||
-                        it.event?.isTaggedHashes(followingTagSet) == true ||
-                        it.event?.isTaggedGeoHashes(followingGeohashSet) == true
-                }
-                .filter { isHiddenList || it.author?.let { !account.isHidden(it.pubkeyHex) } ?: true }
-                .filter { (it.createdAt() ?: 0) <= now }
-                .toSet()
-
-        return activities
     }
 
     override fun sort(collection: Set<Note>): List<Note> {

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/FilterByListParams.kt

@@ -0,0 +1,103 @@
+/**
+ * Copyright (c) 2024 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.amethyst.ui.dal
+
+import com.vitorpamplona.amethyst.model.Account
+import com.vitorpamplona.amethyst.model.GLOBAL_FOLLOWS
+import com.vitorpamplona.quartz.encoders.ATag
+import com.vitorpamplona.quartz.events.Event
+import com.vitorpamplona.quartz.events.EventInterface
+import com.vitorpamplona.quartz.events.LiveActivitiesEvent
+import com.vitorpamplona.quartz.events.MuteListEvent
+import com.vitorpamplona.quartz.events.PeopleListEvent
+import com.vitorpamplona.quartz.utils.TimeUtils
+
+class FilterByListParams(
+    val isGlobal: Boolean,
+    val isHiddenList: Boolean,
+    val followLists: Account.LiveFollowLists?,
+    val hiddenLists: Account.LiveHiddenUsers,
+    val now: Long = TimeUtils.now(),
+) {
+    fun isNotHidden(userHex: String) = !(hiddenLists.hiddenUsers.contains(userHex) || hiddenLists.spammers.contains(userHex))
+
+    fun isNotInTheFuture(noteEvent: Event) = noteEvent.createdAt <= now
+
+    fun isEventInList(noteEvent: Event): Boolean {
+        if (followLists == null) return false
+
+        return if (noteEvent is LiveActivitiesEvent) {
+            noteEvent.participantsIntersect(followLists.users) ||
+                noteEvent.isTaggedHashes(followLists.hashtags) ||
+                noteEvent.isTaggedGeoHashes(followLists.users) ||
+                noteEvent.isTaggedAddressableNotes(followLists.communities)
+        } else {
+            noteEvent.pubKey in followLists.users ||
+                noteEvent.isTaggedHashes(followLists.hashtags) ||
+                noteEvent.isTaggedGeoHashes(followLists.users) ||
+                noteEvent.isTaggedAddressableNotes(followLists.communities)
+        }
+    }
+
+    fun isATagInList(aTag: ATag): Boolean {
+        if (followLists == null) return false
+
+        return aTag.pubKeyHex in followLists.users
+    }
+
+    fun match(
+        noteEvent: EventInterface?,
+        isGlobalRelay: Boolean = true,
+    ) = if (noteEvent is Event) match(noteEvent, isGlobalRelay) else false
+
+    fun match(
+        noteEvent: Event,
+        isGlobalRelay: Boolean = true,
+    ) = ((isGlobal && isGlobalRelay) || isEventInList(noteEvent)) &&
+        (isHiddenList || isNotHidden(noteEvent.pubKey)) &&
+        isNotInTheFuture(noteEvent)
+
+    fun match(aTag: ATag?) =
+        aTag != null &&
+            (isGlobal || isATagInList(aTag)) &&
+            (isHiddenList || isNotHidden(aTag.pubKeyHex))
+
+    companion object {
+        fun showHiddenKey(
+            selectedListName: String,
+            userHex: String,
+        ) = selectedListName == PeopleListEvent.blockListFor(userHex) || selectedListName == MuteListEvent.blockListFor(userHex)
+
+        fun create(
+            userHex: String,
+            selectedListName: String,
+            followLists: Account.LiveFollowLists?,
+            hiddenUsers: Account.LiveHiddenUsers,
+        ): FilterByListParams {
+            return FilterByListParams(
+                isGlobal = selectedListName == GLOBAL_FOLLOWS,
+                isHiddenList = showHiddenKey(selectedListName, userHex),
+                followLists = followLists,
+                hiddenLists = hiddenUsers,
+            )
+        }
+    }
+}

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/GeoHashFeedFilter.kt

@@ -36,7 +36,12 @@ class GeoHashFeedFilter(val tag: String, val account: Account) : AdditiveFeedFil
     }
 
     override fun feed(): List<Note> {
-        return sort(innerApplyFilter(LocalCache.noteListCache))
+        val notes =
+            LocalCache.notes.filterIntoSet { _, it ->
+                acceptableEvent(it, tag)
+            }
+
+        return sort(notes)
     }
 
     override fun applyFilter(collection: Set<Note>): Set<Note> {
@@ -44,25 +49,24 @@ class GeoHashFeedFilter(val tag: String, val account: Account) : AdditiveFeedFil
     }
 
     private fun innerApplyFilter(collection: Collection<Note>): Set<Note> {
-        val myTag = tag ?: return emptySet()
+        return collection.filterTo(HashSet<Note>()) { acceptableEvent(it, tag) }
+    }
 
-        return collection
+    fun acceptableEvent(
-            .asSequence()
+        it: Note,
-            .filter {
+        geoTag: String,
-                (
+    ): Boolean {
-                    it.event is TextNoteEvent ||
+        return (
-                        it.event is LongTextNoteEvent ||
+            it.event is TextNoteEvent ||
-                        it.event is ChannelMessageEvent ||
+                it.event is LongTextNoteEvent ||
-                        it.event is PrivateDmEvent ||
+                it.event is ChannelMessageEvent ||
-                        it.event is PollNoteEvent ||
+                it.event is PrivateDmEvent ||
-                        it.event is AudioHeaderEvent
+                it.event is PollNoteEvent ||
-                ) && it.event?.isTaggedGeoHash(myTag) == true
+                it.event is AudioHeaderEvent
-            }
+        ) && it.event?.isTaggedGeoHash(geoTag) == true && account.isAcceptable(it)
-            .filter { account.isAcceptable(it) }
-            .toSet()
     }
 
     override fun sort(collection: Set<Note>): List<Note> {
-        return collection.sortedWith(compareBy({ it.createdAt() }, { it.idHex })).reversed()
+        return collection.sortedWith(DefaultFeedOrder)
     }
 }

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/HashtagFeedFilter.kt

@@ -36,7 +36,12 @@ class HashtagFeedFilter(val tag: String, val account: Account) : AdditiveFeedFil
     }
 
     override fun feed(): List<Note> {
-        return sort(innerApplyFilter(LocalCache.noteListCache))
+        val notes =
+            LocalCache.notes.filterIntoSet { _, it ->
+                acceptableEvent(it, tag)
+            }
+
+        return sort(notes)
     }
 
     override fun applyFilter(collection: Set<Note>): Set<Note> {
@@ -44,25 +49,24 @@ class HashtagFeedFilter(val tag: String, val account: Account) : AdditiveFeedFil
     }
 
     private fun innerApplyFilter(collection: Collection<Note>): Set<Note> {
-        val myTag = tag ?: return emptySet()
+        return collection.filterTo(HashSet<Note>()) { acceptableEvent(it, tag) }
+    }
 
-        return collection
+    fun acceptableEvent(
-            .asSequence()
+        it: Note,
-            .filter {
+        hashTag: String,
-                (
+    ): Boolean {
-                    it.event is TextNoteEvent ||
+        return (
-                        it.event is LongTextNoteEvent ||
+            it.event is TextNoteEvent ||
-                        it.event is ChannelMessageEvent ||
+                it.event is LongTextNoteEvent ||
-                        it.event is PrivateDmEvent ||
+                it.event is ChannelMessageEvent ||
-                        it.event is PollNoteEvent ||
+                it.event is PrivateDmEvent ||
-                        it.event is AudioHeaderEvent
+                it.event is PollNoteEvent ||
-                ) && it.event?.isTaggedHash(myTag) == true
+                it.event is AudioHeaderEvent
-            }
+        ) && it.event?.isTaggedHash(hashTag) == true && account.isAcceptable(it)
-            .filter { account.isAcceptable(it) }
-            .toSet()
     }
 
     override fun sort(collection: Set<Note>): List<Note> {
-        return collection.sortedWith(compareBy({ it.createdAt() }, { it.idHex })).reversed()
+        return collection.sortedWith(DefaultFeedOrder)
     }
 }

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/HomeConversationsFeedFilter.kt

@@ -21,7 +21,6 @@
 package com.vitorpamplona.amethyst.ui.dal
 
 import com.vitorpamplona.amethyst.model.Account
-import com.vitorpamplona.amethyst.model.GLOBAL_FOLLOWS
 import com.vitorpamplona.amethyst.model.LocalCache
 import com.vitorpamplona.amethyst.model.Note
 import com.vitorpamplona.quartz.events.ChannelMessageEvent
@@ -30,7 +29,6 @@ import com.vitorpamplona.quartz.events.MuteListEvent
 import com.vitorpamplona.quartz.events.PeopleListEvent
 import com.vitorpamplona.quartz.events.PollNoteEvent
 import com.vitorpamplona.quartz.events.TextNoteEvent
-import com.vitorpamplona.quartz.utils.TimeUtils
 
 class HomeConversationsFeedFilter(val account: Account) : AdditiveFeedFilter<Note>() {
     override fun feedKey(): String {
@@ -38,55 +36,54 @@ class HomeConversationsFeedFilter(val account: Account) : AdditiveFeedFilter<Not
     }
 
     override fun showHiddenKey(): Boolean {
-        return account.defaultHomeFollowList.value ==
+        return account.defaultHomeFollowList.value == PeopleListEvent.blockListFor(account.userProfile().pubkeyHex) ||
-            PeopleListEvent.blockListFor(account.userProfile().pubkeyHex) ||
+            account.defaultHomeFollowList.value == MuteListEvent.blockListFor(account.userProfile().pubkeyHex)
-            account.defaultHomeFollowList.value ==
-            MuteListEvent.blockListFor(account.userProfile().pubkeyHex)
     }
 
     override fun feed(): List<Note> {
-        return sort(innerApplyFilter(LocalCache.noteListCache))
+        val filterParams = buildFilterParams(account)
+
+        return sort(
+            LocalCache.notes.filterIntoSet { _, it ->
+                acceptableEvent(it, filterParams)
+            },
+        )
     }
 
     override fun applyFilter(collection: Set<Note>): Set<Note> {
         return innerApplyFilter(collection)
     }
 
+    fun buildFilterParams(account: Account): FilterByListParams {
+        return FilterByListParams.create(
+            userHex = account.userProfile().pubkeyHex,
+            selectedListName = account.defaultHomeFollowList.value,
+            followLists = account.liveHomeFollowLists.value,
+            hiddenUsers = account.flowHiddenUsers.value,
+        )
+    }
+
     private fun innerApplyFilter(collection: Collection<Note>): Set<Note> {
-        val isGlobal = account.defaultHomeFollowList.value == GLOBAL_FOLLOWS
+        val filterParams = buildFilterParams(account)
-        val isHiddenList = showHiddenKey()
 
-        val followingKeySet = account.liveHomeFollowLists.value?.users ?: emptySet()
+        return collection.filterTo(HashSet()) {
-        val followingTagSet = account.liveHomeFollowLists.value?.hashtags ?: emptySet()
+            acceptableEvent(it, filterParams)
-        val followingGeohashSet = account.liveHomeFollowLists.value?.geotags ?: emptySet()
+        }
+    }
 
-        val now = TimeUtils.now()
+    fun acceptableEvent(
-
+        it: Note,
-        return collection
+        filterParams: FilterByListParams,
-            .asSequence()
+    ): Boolean {
-            .filter {
+        return (
-                (
+            it.event is TextNoteEvent ||
-                    it.event is TextNoteEvent ||
+                it.event is PollNoteEvent ||
-                        it.event is PollNoteEvent ||
+                it.event is ChannelMessageEvent ||
-                        it.event is ChannelMessageEvent ||
+                it.event is LiveActivitiesChatMessageEvent
-                        it.event is LiveActivitiesChatMessageEvent
+        ) && filterParams.match(it.event) && !it.isNewThread()
-                ) &&
-                    (
-                        isGlobal ||
-                            it.author?.pubkeyHex in followingKeySet ||
-                            it.event?.isTaggedHashes(followingTagSet) ?: false ||
-                            it.event?.isTaggedGeoHashes(followingGeohashSet) ?: false
-                    ) &&
-                    // && account.isAcceptable(it)  // This filter follows only. No need to check if
-                    // acceptable
-                    (isHiddenList || it.author?.let { !account.isHidden(it) } ?: true) &&
-                    ((it.event?.createdAt() ?: 0) < now) &&
-                    !it.isNewThread()
-            }
-            .toSet()
     }
 
     override fun sort(collection: Set<Note>): List<Note> {
-        return collection.sortedWith(compareBy({ it.createdAt() }, { it.idHex })).reversed()
+        return collection.sortedWith(DefaultFeedOrder)
     }
 }

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/HomeNewThreadFeedFilter.kt

@@ -21,7 +21,6 @@
 package com.vitorpamplona.amethyst.ui.dal
 
 import com.vitorpamplona.amethyst.model.Account
-import com.vitorpamplona.amethyst.model.GLOBAL_FOLLOWS
 import com.vitorpamplona.amethyst.model.LocalCache
 import com.vitorpamplona.amethyst.model.Note
 import com.vitorpamplona.quartz.events.AudioHeaderEvent
@@ -35,7 +34,6 @@ import com.vitorpamplona.quartz.events.PeopleListEvent
 import com.vitorpamplona.quartz.events.PollNoteEvent
 import com.vitorpamplona.quartz.events.RepostEvent
 import com.vitorpamplona.quartz.events.TextNoteEvent
-import com.vitorpamplona.quartz.utils.TimeUtils
 
 class HomeNewThreadFeedFilter(val account: Account) : AdditiveFeedFilter<Note>() {
     override fun feedKey(): String {
@@ -43,69 +41,70 @@ class HomeNewThreadFeedFilter(val account: Account) : AdditiveFeedFilter<Note>()
     }
 
     override fun showHiddenKey(): Boolean {
-        return account.defaultHomeFollowList.value ==
+        return account.defaultHomeFollowList.value == PeopleListEvent.blockListFor(account.userProfile().pubkeyHex) ||
-            PeopleListEvent.blockListFor(account.userProfile().pubkeyHex) ||
+            account.defaultHomeFollowList.value == MuteListEvent.blockListFor(account.userProfile().pubkeyHex)
-            account.defaultHomeFollowList.value ==
+    }
-            MuteListEvent.blockListFor(account.userProfile().pubkeyHex)
+
+    fun buildFilterParams(account: Account): FilterByListParams {
+        return FilterByListParams.create(
+            userHex = account.userProfile().pubkeyHex,
+            selectedListName = account.defaultHomeFollowList.value,
+            followLists = account.liveHomeFollowLists.value,
+            hiddenUsers = account.flowHiddenUsers.value,
+        )
     }
 
     override fun feed(): List<Note> {
-        val notes = innerApplyFilter(LocalCache.noteListCache, true)
+        val gRelays = account.activeGlobalRelays().toSet()
-        val longFormNotes = innerApplyFilter(LocalCache.addressables.values, false)
+        val filterParams = buildFilterParams(account)
+
+        val notes =
+            LocalCache.notes.filterIntoSet { _, note ->
+                // Avoids processing addressables twice.
+                (note.event?.kind() ?: 99999) < 10000 && acceptableEvent(note, gRelays, filterParams)
+            }
+
+        val longFormNotes =
+            LocalCache.addressables.filterIntoSet { _, note ->
+                acceptableEvent(note, gRelays, filterParams)
+            }
 
         return sort(notes + longFormNotes)
     }
 
     override fun applyFilter(collection: Set<Note>): Set<Note> {
-        return innerApplyFilter(collection, false)
+        return innerApplyFilter(collection)
     }
 
-    private fun innerApplyFilter(
+    private fun innerApplyFilter(collection: Collection<Note>): Set<Note> {
-        collection: Collection<Note>,
+        val gRelays = account.activeGlobalRelays().toSet()
-        ignoreAddressables: Boolean,
+        val filterParams = buildFilterParams(account)
-    ): Set<Note> {
-        val isGlobal = account.defaultHomeFollowList.value == GLOBAL_FOLLOWS
-        val gRelays = account.activeGlobalRelays()
-        val isHiddenList = showHiddenKey()
 
-        val followingKeySet = account.liveHomeFollowLists.value?.users ?: emptySet()
+        return collection.filterTo(HashSet()) {
-        val followingTagSet = account.liveHomeFollowLists.value?.hashtags ?: emptySet()
+            acceptableEvent(it, gRelays, filterParams)
-        val followingGeohashSet = account.liveHomeFollowLists.value?.geotags ?: emptySet()
+        }
-        val followingCommunities = account.liveHomeFollowLists.value?.communities ?: emptySet()
+    }
 
-        val oneMinuteInTheFuture = TimeUtils.now() + (1 * 60) // one minute in the future.
+    fun acceptableEvent(
-
+        it: Note,
-        return collection
+        globalRelays: Set<String>,
-            .asSequence()
+        filterParams: FilterByListParams,
-            .filter { it ->
+    ): Boolean {
-                val noteEvent = it.event
+        val noteEvent = it.event
-                val isGlobalRelay = it.relays.any { gRelays.contains(it.url) }
+        val isGlobalRelay = it.relays.any { globalRelays.contains(it.url) }
-                (
+        return (
-                    noteEvent is TextNoteEvent ||
+            noteEvent is TextNoteEvent ||
-                        noteEvent is ClassifiedsEvent ||
+                noteEvent is ClassifiedsEvent ||
-                        noteEvent is RepostEvent ||
+                noteEvent is RepostEvent ||
-                        noteEvent is GenericRepostEvent ||
+                noteEvent is GenericRepostEvent ||
-                        noteEvent is LongTextNoteEvent ||
+                noteEvent is LongTextNoteEvent ||
-                        noteEvent is PollNoteEvent ||
+                noteEvent is PollNoteEvent ||
-                        noteEvent is HighlightEvent ||
+                noteEvent is HighlightEvent ||
-                        noteEvent is AudioTrackEvent ||
+                noteEvent is AudioTrackEvent ||
-                        noteEvent is AudioHeaderEvent
+                noteEvent is AudioHeaderEvent
-                ) &&
+        ) &&
-                    (!ignoreAddressables || noteEvent.kind() < 10000) &&
+            filterParams.match(noteEvent, isGlobalRelay) &&
-                    (
+            it.isNewThread()
-                        (isGlobal && isGlobalRelay) ||
-                            it.author?.pubkeyHex in followingKeySet ||
-                            noteEvent.isTaggedHashes(followingTagSet) ||
-                            noteEvent.isTaggedGeoHashes(followingGeohashSet) ||
-                            noteEvent.isTaggedAddressableNotes(followingCommunities)
-                    ) &&
-                    // && account.isAcceptable(it)  // This filter follows only. No need to check if
-                    // acceptable
-                    (isHiddenList || it.author?.let { !account.isHidden(it.pubkeyHex) } ?: true) &&
-                    ((it.event?.createdAt() ?: 0) < oneMinuteInTheFuture) &&
-                    it.isNewThread()
-            }
-            .toSet()
     }
 
     override fun sort(collection: Set<Note>): List<Note> {
@@ -115,6 +114,6 @@ class HomeNewThreadFeedFilter(val account: Account) : AdditiveFeedFilter<Note>()
             } else {
                 it.idHex
             }
-        }.sortedWith(compareBy({ it.createdAt() }, { it.idHex })).reversed()
+        }.sortedWith(DefaultFeedOrder)
     }
 }

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/NotificationFeedFilter.kt

@@ -21,7 +21,6 @@
 package com.vitorpamplona.amethyst.ui.dal
 
 import com.vitorpamplona.amethyst.model.Account
-import com.vitorpamplona.amethyst.model.GLOBAL_FOLLOWS
 import com.vitorpamplona.amethyst.model.LocalCache
 import com.vitorpamplona.amethyst.model.Note
 import com.vitorpamplona.quartz.encoders.HexKey
@@ -35,7 +34,6 @@ import com.vitorpamplona.quartz.events.GiftWrapEvent
 import com.vitorpamplona.quartz.events.GitIssueEvent
 import com.vitorpamplona.quartz.events.GitPatchEvent
 import com.vitorpamplona.quartz.events.HighlightEvent
-import com.vitorpamplona.quartz.events.LnZapEvent
 import com.vitorpamplona.quartz.events.LnZapRequestEvent
 import com.vitorpamplona.quartz.events.MuteListEvent
 import com.vitorpamplona.quartz.events.PeopleListEvent
@@ -54,8 +52,24 @@ class NotificationFeedFilter(val account: Account) : AdditiveFeedFilter<Note>()
             MuteListEvent.blockListFor(account.userProfile().pubkeyHex)
     }
 
+    fun buildFilterParams(account: Account): FilterByListParams {
+        return FilterByListParams.create(
+            userHex = account.userProfile().pubkeyHex,
+            selectedListName = account.defaultNotificationFollowList.value,
+            followLists = account.liveNotificationFollowLists.value,
+            hiddenUsers = account.flowHiddenUsers.value,
+        )
+    }
+
     override fun feed(): List<Note> {
-        return sort(innerApplyFilter(LocalCache.noteListCache))
+        val filterParams = buildFilterParams(account)
+
+        val notifications =
+            LocalCache.notes.filterIntoSet { _, note ->
+                acceptableEvent(note, filterParams)
+            }
+
+        return sort(notifications)
     }
 
     override fun applyFilter(collection: Set<Note>): Set<Note> {
@@ -63,32 +77,31 @@ class NotificationFeedFilter(val account: Account) : AdditiveFeedFilter<Note>()
     }
 
     private fun innerApplyFilter(collection: Collection<Note>): Set<Note> {
-        val isGlobal = account.defaultNotificationFollowList.value == GLOBAL_FOLLOWS
+        val filterParams = buildFilterParams(account)
-        val isHiddenList = showHiddenKey()
 
-        val followingKeySet = account.liveNotificationFollowLists.value?.users ?: emptySet()
+        return collection.filterTo(HashSet()) { acceptableEvent(it, filterParams) }
+    }
 
-        val loggedInUser = account.userProfile()
+    fun acceptableEvent(
-        val loggedInUserHex = loggedInUser.pubkeyHex
+        it: Note,
+        filterParams: FilterByListParams,
+    ): Boolean {
+        val loggedInUserHex = account.userProfile().pubkeyHex
 
-        return collection
+        return it.event !is ChannelCreateEvent &&
-            .filterTo(HashSet()) {
+            it.event !is ChannelMetadataEvent &&
-                it.event !is ChannelCreateEvent &&
+            it.event !is LnZapRequestEvent &&
-                    it.event !is ChannelMetadataEvent &&
+            it.event !is BadgeDefinitionEvent &&
-                    it.event !is LnZapRequestEvent &&
+            it.event !is BadgeProfilesEvent &&
-                    it.event !is BadgeDefinitionEvent &&
+            it.event !is GiftWrapEvent &&
-                    it.event !is BadgeProfilesEvent &&
+            (filterParams.isGlobal || filterParams.followLists?.users?.contains(it.author?.pubkeyHex) == true) &&
-                    it.event !is GiftWrapEvent &&
+            it.event?.isTaggedUser(loggedInUserHex) ?: false &&
-                    (it.event is LnZapEvent || it.author !== loggedInUser) &&
+            (filterParams.isHiddenList || it.author == null || !account.isHidden(it.author!!.pubkeyHex)) &&
-                    (isGlobal || it.author?.pubkeyHex in followingKeySet) &&
+            tagsAnEventByUser(it, loggedInUserHex)
-                    it.event?.isTaggedUser(loggedInUserHex) ?: false &&
-                    (isHiddenList || it.author == null || !account.isHidden(it.author!!.pubkeyHex)) &&
-                    tagsAnEventByUser(it, loggedInUserHex)
-            }
     }
 
     override fun sort(collection: Set<Note>): List<Note> {
-        return collection.sortedWith(compareBy({ it.createdAt() }, { it.idHex })).reversed()
+        return collection.sortedWith(DefaultFeedOrder)
     }
 
     fun tagsAnEventByUser(

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/UserProfileAppRecommendationsFeedFilter.kt

@@ -31,7 +31,12 @@ class UserProfileAppRecommendationsFeedFilter(val user: User) : AdditiveFeedFilt
     }
 
     override fun feed(): List<Note> {
-        return sort(innerApplyFilter(LocalCache.addressables.values))
+        val recommendations =
+            LocalCache.addressables.mapFlattenIntoSet { _, note ->
+                filterMap(note)
+            }
+
+        return sort(recommendations)
     }
 
     override fun applyFilter(collection: Set<Note>): Set<Note> {
@@ -39,26 +44,21 @@ class UserProfileAppRecommendationsFeedFilter(val user: User) : AdditiveFeedFilt
     }
 
     private fun innerApplyFilter(collection: Collection<Note>): Set<Note> {
-        val recommendations =
+        return collection.mapNotNull { filterMap(it) }.flatten().toSet()
-            collection
+    }
-                .asSequence()
-                .filter { it.event is AppRecommendationEvent }
-                .mapNotNull {
-                    val noteEvent = it.event as? AppRecommendationEvent
-                    if (noteEvent != null && noteEvent.pubKey == user.pubkeyHex) {
-                        noteEvent.recommendations()
-                    } else {
-                        null
-                    }
-                }
-                .flatten()
-                .map { LocalCache.getOrCreateAddressableNote(it) }
-                .toSet()
 
-        return recommendations
+    fun filterMap(it: Note): List<Note>? {
+        val noteEvent = it.event
+        if (noteEvent is AppRecommendationEvent) {
+            if (noteEvent.pubKey == user.pubkeyHex) {
+                return noteEvent.recommendations().map { LocalCache.getOrCreateAddressableNote(it) }
+            }
+        }
+
+        return null
     }
 
     override fun sort(collection: Set<Note>): List<Note> {
-        return collection.sortedWith(compareBy({ it.createdAt() }, { it.idHex }))
+        return collection.sortedWith(DefaultFeedOrder)
     }
 }

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/UserProfileBookmarksFeedFilter.kt

@@ -47,7 +47,6 @@ class UserProfileBookmarksFeedFilter(val user: User, val account: Account) : Fee
 
         return (notes + addresses)
             .filter { account.isAcceptable(it) }
-            .sortedWith(compareBy({ it.createdAt() }, { it.idHex }))
+            .sortedWith(DefaultFeedOrder)
-            .reversed()
     }
 }

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/UserProfileConversationsFeedFilter.kt

@@ -36,7 +36,17 @@ class UserProfileConversationsFeedFilter(val user: User, val account: Account) :
     }
 
     override fun feed(): List<Note> {
-        return sort(innerApplyFilter(LocalCache.noteListCache))
+        val notes =
+            LocalCache.notes.filterIntoSet { _, it ->
+                acceptableEvent(it)
+            }
+
+        val longFormNotes =
+            LocalCache.addressables.filterIntoSet { _, it ->
+                acceptableEvent(it)
+            }
+
+        return sort(notes + longFormNotes)
     }
 
     override fun applyFilter(collection: Set<Note>): Set<Note> {
@@ -44,23 +54,21 @@ class UserProfileConversationsFeedFilter(val user: User, val account: Account) :
     }
 
     private fun innerApplyFilter(collection: Collection<Note>): Set<Note> {
-        return collection
+        return collection.filterTo(HashSet()) { acceptableEvent(it) }
-            .filter {
+    }
-                it.author == user &&
+
-                    (
+    fun acceptableEvent(it: Note): Boolean {
-                        it.event is TextNoteEvent ||
+        return it.author == user &&
-                            it.event is PollNoteEvent ||
+            (
-                            it.event is ChannelMessageEvent ||
+                it.event is TextNoteEvent ||
-                            it.event is LiveActivitiesChatMessageEvent
+                    it.event is PollNoteEvent ||
-                    ) &&
+                    it.event is ChannelMessageEvent ||
-                    !it.isNewThread() &&
+                    it.event is LiveActivitiesChatMessageEvent
-                    account.isAcceptable(it) == true
+            ) && !it.isNewThread() && account.isAcceptable(it)
-            }
-            .toSet()
     }
 
     override fun sort(collection: Set<Note>): List<Note> {
-        return collection.sortedWith(compareBy({ it.createdAt() }, { it.idHex })).reversed()
+        return collection.sortedWith(DefaultFeedOrder)
     }
 
     override fun limit() = 200

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/UserProfileFollowersFeedFilter.kt

@@ -30,7 +30,9 @@ class UserProfileFollowersFeedFilter(val user: User, val account: Account) : Fee
     }
 
     override fun feed(): List<User> {
-        return LocalCache.userListCache.filter { it.isFollowing(user) && !account.isHidden(it) }
+        return LocalCache.users.filter { _, it ->
+            it.isFollowing(user) && !account.isHidden(it)
+        }
     }
 
     override fun limit() = 400

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/UserProfileNewThreadFeedFilter.kt

@@ -41,8 +41,15 @@ class UserProfileNewThreadFeedFilter(val user: User, val account: Account) :
     }
 
     override fun feed(): List<Note> {
-        val notes = innerApplyFilter(LocalCache.noteListCache)
+        val notes =
-        val longFormNotes = innerApplyFilter(LocalCache.addressables.values)
+            LocalCache.notes.filterIntoSet { _, it ->
+                acceptableEvent(it)
+            }
+
+        val longFormNotes =
+            LocalCache.addressables.filterIntoSet { _, it ->
+                acceptableEvent(it)
+            }
 
         return sort(notes + longFormNotes)
     }
@@ -52,28 +59,26 @@ class UserProfileNewThreadFeedFilter(val user: User, val account: Account) :
     }
 
     private fun innerApplyFilter(collection: Collection<Note>): Set<Note> {
-        return collection
+        return collection.filterTo(HashSet()) { acceptableEvent(it) }
-            .filter {
+    }
-                it.author == user &&
+
-                    (
+    fun acceptableEvent(it: Note): Boolean {
-                        it.event is TextNoteEvent ||
+        return it.author == user &&
-                            it.event is ClassifiedsEvent ||
+            (
-                            it.event is RepostEvent ||
+                it.event is TextNoteEvent ||
-                            it.event is GenericRepostEvent ||
+                    it.event is ClassifiedsEvent ||
-                            it.event is LongTextNoteEvent ||
+                    it.event is RepostEvent ||
-                            it.event is PollNoteEvent ||
+                    it.event is GenericRepostEvent ||
-                            it.event is HighlightEvent ||
+                    it.event is LongTextNoteEvent ||
-                            it.event is AudioTrackEvent ||
+                    it.event is PollNoteEvent ||
-                            it.event is AudioHeaderEvent
+                    it.event is HighlightEvent ||
-                    ) &&
+                    it.event is AudioTrackEvent ||
-                    it.isNewThread() &&
+                    it.event is AudioHeaderEvent
-                    account.isAcceptable(it) == true
+            ) && it.isNewThread() && account.isAcceptable(it)
-            }
-            .toSet()
     }
 
     override fun sort(collection: Set<Note>): List<Note> {
-        return collection.sortedWith(compareBy({ it.createdAt() }, { it.idHex })).reversed()
+        return collection.sortedWith(DefaultFeedOrder)
     }
 
     override fun limit() = 200

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/UserProfileReportsFeedFilter.kt

@@ -44,7 +44,7 @@ class UserProfileReportsFeedFilter(val user: User) : AdditiveFeedFilter<Note>()
     }
 
     override fun sort(collection: Set<Note>): List<Note> {
-        return collection.sortedWith(compareBy({ it.createdAt() }, { it.idHex })).reversed()
+        return collection.sortedWith(DefaultFeedOrder)
     }
 
     override fun limit() = 400

app/src/main/java/com/vitorpamplona/amethyst/ui/dal/VideoFeedFilter.kt

@@ -21,14 +21,12 @@
 package com.vitorpamplona.amethyst.ui.dal
 
 import com.vitorpamplona.amethyst.model.Account
-import com.vitorpamplona.amethyst.model.GLOBAL_FOLLOWS
 import com.vitorpamplona.amethyst.model.LocalCache
 import com.vitorpamplona.amethyst.model.Note
 import com.vitorpamplona.quartz.events.FileHeaderEvent
 import com.vitorpamplona.quartz.events.FileStorageHeaderEvent
 import com.vitorpamplona.quartz.events.MuteListEvent
 import com.vitorpamplona.quartz.events.PeopleListEvent
-import com.vitorpamplona.quartz.utils.TimeUtils
 
 class VideoFeedFilter(val account: Account) : AdditiveFeedFilter<Note>() {
     override fun feedKey(): String {
@@ -36,14 +34,17 @@ class VideoFeedFilter(val account: Account) : AdditiveFeedFilter<Note>() {
     }
 
     override fun showHiddenKey(): Boolean {
-        return account.defaultStoriesFollowList.value ==
+        return account.defaultStoriesFollowList.value == PeopleListEvent.blockListFor(account.userProfile().pubkeyHex) ||
-            PeopleListEvent.blockListFor(account.userProfile().pubkeyHex) ||
+            account.defaultStoriesFollowList.value == MuteListEvent.blockListFor(account.userProfile().pubkeyHex)
-            account.defaultStoriesFollowList.value ==
-            MuteListEvent.blockListFor(account.userProfile().pubkeyHex)
     }
 
     override fun feed(): List<Note> {
-        val notes = innerApplyFilter(LocalCache.noteListCache)
+        val params = buildFilterParams(account)
+
+        val notes =
+            LocalCache.notes.filterIntoSet { _, it ->
+                acceptableEvent(it, params)
+            }
 
         return sort(notes)
     }
@@ -53,36 +54,32 @@ class VideoFeedFilter(val account: Account) : AdditiveFeedFilter<Note>() {
     }
 
     private fun innerApplyFilter(collection: Collection<Note>): Set<Note> {
-        val now = TimeUtils.now()
+        val params = buildFilterParams(account)
-        val isGlobal = account.defaultStoriesFollowList.value == GLOBAL_FOLLOWS
-        val isHiddenList =
-            account.defaultStoriesFollowList.value ==
-                PeopleListEvent.blockListFor(account.userProfile().pubkeyHex) ||
-                account.defaultStoriesFollowList.value ==
-                MuteListEvent.blockListFor(account.userProfile().pubkeyHex)
 
-        val followingKeySet = account.liveStoriesFollowLists.value?.users ?: emptySet()
+        return collection.filterTo(HashSet()) { acceptableEvent(it, params) }
-        val followingTagSet = account.liveStoriesFollowLists.value?.hashtags ?: emptySet()
+    }
-        val followingGeohashSet = account.liveStoriesFollowLists.value?.geotags ?: emptySet()
 
-        return collection
+    fun acceptableEvent(
-            .asSequence()
+        it: Note,
-            .filter {
+        params: FilterByListParams,
-                (it.event is FileHeaderEvent && (it.event as FileHeaderEvent).hasUrl()) ||
+    ): Boolean {
-                    it.event is FileStorageHeaderEvent
+        val noteEvent = it.event
-            }
+
-            .filter {
+        return ((noteEvent is FileHeaderEvent && noteEvent.hasUrl()) || noteEvent is FileStorageHeaderEvent) &&
-                isGlobal ||
+            params.match(noteEvent) &&
-                    it.author?.pubkeyHex in followingKeySet ||
+            account.isAcceptable(it)
-                    (it.event?.isTaggedHashes(followingTagSet) ?: false) ||
+    }
-                    (it.event?.isTaggedGeoHashes(followingGeohashSet) ?: false)
+
-            }
+    fun buildFilterParams(account: Account): FilterByListParams {
-            .filter { isHiddenList || account.isAcceptable(it) }
+        return FilterByListParams.create(
-            .filter { it.createdAt()!! <= now }
+            userHex = account.userProfile().pubkeyHex,
-            .toSet()
+            selectedListName = account.defaultStoriesFollowList.value,
+            followLists = account.liveStoriesFollowLists.value,
+            hiddenUsers = account.flowHiddenUsers.value,
+        )
     }
 
     override fun sort(collection: Set<Note>): List<Note> {
-        return collection.sortedWith(compareBy({ it.createdAt() }, { it.idHex })).reversed()
+        return collection.sortedWith(DefaultFeedOrder)
     }
 }

app/src/main/java/com/vitorpamplona/amethyst/ui/navigation/AppTopBar.kt

@@ -655,15 +655,15 @@ class FollowListViewModel(val account: Account) : ViewModel() {
 
         val newFollowLists =
             LocalCache.addressables
-                .mapNotNull {
+                .mapNotNull { _, addressableNote ->
-                    val event = (it.value.event as? PeopleListEvent)
+                    val event = (addressableNote.event as? PeopleListEvent)
                     // Has to have an list
                     if (
                         event != null &&
                         event.pubKey == account.userProfile().pubkeyHex &&
                         (event.tags.size > 1 || event.content.length > 50)
                     ) {
-                        CodeName(event.address().toTag(), PeopleListName(it.value), CodeNameType.PEOPLE_LIST)
+                        CodeName(event.address().toTag(), PeopleListName(addressableNote), CodeNameType.PEOPLE_LIST)
                     } else {
                         null
                     }
@@ -974,44 +974,44 @@ fun debugState(context: Context) {
     Log.d(
         "STATE DUMP",
         "Notes: " +
-            LocalCache.noteListCache.filter { it.liveSet != null }.size +
+            LocalCache.notes.filter { _, it -> it.liveSet != null }.size +
             " / " +
-            LocalCache.noteListCache.filter { it.event != null }.size +
+            LocalCache.notes.filter { _, it -> it.event != null }.size +
             " / " +
-            LocalCache.noteListCache.size,
+            LocalCache.notes.size(),
     )
     Log.d(
         "STATE DUMP",
         "Addressables: " +
-            LocalCache.addressables.filter { it.value.liveSet != null }.size +
+            LocalCache.addressables.filter { _, it -> it.liveSet != null }.size +
             " / " +
-            LocalCache.addressables.filter { it.value.event != null }.size +
+            LocalCache.addressables.filter { _, it -> it.event != null }.size +
             " / " +
-            LocalCache.addressables.size,
+            LocalCache.addressables.size(),
     )
     Log.d(
         "STATE DUMP",
         "Users: " +
-            LocalCache.userListCache.filter { it.liveSet != null }.size +
+            LocalCache.users.filter { _, it -> it.liveSet != null }.size +
             " / " +
-            LocalCache.userListCache.filter { it.latestMetadata != null }.size +
+            LocalCache.users.filter { _, it -> it.latestMetadata != null }.size +
             " / " +
-            LocalCache.userListCache.size,
+            LocalCache.users.size(),
     )
 
     Log.d(
         "STATE DUMP",
         "Memory used by Events: " +
-            LocalCache.noteListCache.sumOf { it.event?.countMemory() ?: 0 } / (1024 * 1024) +
+            LocalCache.notes.sumOfLong { _, note -> note.event?.countMemory() ?: 0L } / (1024 * 1024) +
             " MB",
     )
 
-    LocalCache.noteListCache
+    LocalCache.notes
-        .groupBy { it.event?.kind() }
+        .countByGroup { _, it -> it.event?.kind() }
-        .forEach { Log.d("STATE DUMP", "Kind ${it.key}: \t${it.value.size} elements ") }
+        .forEach { Log.d("STATE DUMP", "Kind ${it.key}: \t${it.value} elements ") }
-    LocalCache.addressables.values
+    LocalCache.addressables
-        .groupBy { it.event?.kind() }
+        .countByGroup { _, it -> it.event?.kind() }
-        .forEach { Log.d("STATE DUMP", "Kind ${it.key}: \t${it.value.size} elements ") }
+        .forEach { Log.d("STATE DUMP", "Kind ${it.key}: \t${it.value} elements ") }
 }
 
 @OptIn(ExperimentalMaterial3Api::class)

app/src/main/java/com/vitorpamplona/amethyst/ui/navigation/Routes.kt

@@ -38,7 +38,6 @@ import com.vitorpamplona.amethyst.model.Note
 import com.vitorpamplona.amethyst.service.checkNotInMainThread
 import com.vitorpamplona.amethyst.ui.dal.AdditiveFeedFilter
 import com.vitorpamplona.amethyst.ui.dal.ChatroomListKnownFeedFilter
-import com.vitorpamplona.amethyst.ui.dal.DiscoverLiveNowFeedFilter
 import com.vitorpamplona.amethyst.ui.dal.HomeNewThreadFeedFilter
 import com.vitorpamplona.amethyst.ui.dal.NotificationFeedFilter
 import com.vitorpamplona.amethyst.ui.theme.Size20dp
@@ -46,7 +45,6 @@ import com.vitorpamplona.amethyst.ui.theme.Size23dp
 import com.vitorpamplona.amethyst.ui.theme.Size24dp
 import com.vitorpamplona.amethyst.ui.theme.Size25dp
 import com.vitorpamplona.quartz.events.ChatroomKeyable
-import com.vitorpamplona.quartz.events.LiveActivitiesEvent
 import kotlinx.collections.immutable.ImmutableList
 import kotlinx.collections.immutable.persistentListOf
 import kotlinx.collections.immutable.toImmutableList
@@ -309,30 +307,6 @@ object HomeLatestItem : LatestItem() {
     }
 }
 
-object DiscoverLatestItem : LatestItem() {
-    fun hasNewItems(
-        account: Account,
-        newNotes: Set<Note>,
-    ): Boolean {
-        checkNotInMainThread()
-
-        val lastTime = account.loadLastRead(Route.Discover.base + "Live")
-
-        val newestItem = updateNewestItem(newNotes, account, DiscoverLiveNowFeedFilter(account))
-
-        val noteEvent = newestItem?.event
-
-        val dateToUse =
-            if (noteEvent is LiveActivitiesEvent) {
-                noteEvent.starts() ?: newestItem.createdAt()
-            } else {
-                newestItem?.createdAt()
-            }
-
-        return (dateToUse ?: 0) > lastTime
-    }
-}
-
 object NotificationLatestItem : LatestItem() {
     fun hasNewItems(
         account: Account,

app/src/main/java/com/vitorpamplona/amethyst/ui/note/UserReactionsRow.kt

@@ -232,7 +232,7 @@ class UserReactionsViewModel(val account: Account) : ViewModel() {
         val replies = mutableMapOf<String, Int>()
         val takenIntoAccount = mutableSetOf<HexKey>()
 
-        LocalCache.noteListCache.forEach {
+        LocalCache.notes.forEach { _, it ->
             val noteEvent = it.event
             if (noteEvent != null && !takenIntoAccount.contains(noteEvent.id())) {
                 if (noteEvent is ReactionEvent) {

app/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountViewModel.kt

@@ -911,7 +911,7 @@ class AccountViewModel(val account: Account, val settings: SettingsState) : View
     }
 
     fun getAddressableNoteIfExists(key: String): AddressableNote? {
-        return LocalCache.addressables[key]
+        return LocalCache.getAddressableNoteIfExists(key)
     }
 
     suspend fun findStatusesForUser(