diff --git a/paimon-core/src/main/java/org/apache/paimon/lookup/memory/InMemorySetState.java b/paimon-core/src/main/java/org/apache/paimon/lookup/memory/InMemorySetState.java
index f694a576431a..adc59f9eff22 100644
--- a/paimon-core/src/main/java/org/apache/paimon/lookup/memory/InMemorySetState.java
+++ b/paimon-core/src/main/java/org/apache/paimon/lookup/memory/InMemorySetState.java
@@ -56,7 +56,12 @@ public List<V> get(K key) throws IOException {
 
     @Override
     public void retract(K key, V value) throws IOException {
-        values.get(wrapBytes(serializeKey(key))).remove(wrapBytes(serializeValue(value)));
+        Set<ByteArray> set = values.get(wrapBytes(serializeKey(key)));
+        // the key may never have been added, for example when only predicate-passing
+        // rows are indexed and the retracted row was filtered out
+        if (set != null) {
+            set.remove(wrapBytes(serializeValue(value)));
+        }
     }
 
     @Override
diff --git a/paimon-core/src/test/java/org/apache/paimon/lookup/memory/InMemorySetStateTest.java b/paimon-core/src/test/java/org/apache/paimon/lookup/memory/InMemorySetStateTest.java
new file mode 100644
index 000000000000..1b1dab11586a
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/lookup/memory/InMemorySetStateTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.lookup.memory;
+
+import org.apache.paimon.data.serializer.IntSerializer;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
+
+/** Tests for {@link InMemorySetState}. */
+class InMemorySetStateTest {
+
+    @Test
+    void testRetractOnAbsentKeyIsNoOp() throws Exception {
+        // predicate-filtered inserts never add the secondary key, but the refresh
+        // path retracts DELETE/UPDATE_BEFORE rows unconditionally
+        InMemorySetState<Integer, Integer> state =
+                new InMemorySetState<>(IntSerializer.INSTANCE, IntSerializer.INSTANCE);
+
+        state.add(1, 10);
+        state.add(1, 20);
+
+        assertThatCode(() -> state.retract(2, 10)).doesNotThrowAnyException();
+        assertThatCode(() -> state.retract(1, 99)).doesNotThrowAnyException();
+        assertThat(state.get(1)).containsExactlyInAnyOrder(10, 20);
+        assertThat(state.get(2)).isEmpty();
+
+        state.retract(1, 10);
+        assertThat(state.get(1)).containsExactly(20);
+    }
+}