Skip to content
This repository was archived by the owner on Jul 31, 2023. It is now read-only.

Commit 3808014

Browse files
Baohe Zhanggengliangwang
Baohe Zhang
authored andcommitted
[SPARK-31584][WEBUI] Fix NullPointerException when parsing event log with InMemoryStore
### What changes were proposed in this pull request? apache#27716 introduced parent index for InMemoryStore. When the method "deleteParentIndex(Object key)" in InMemoryStore.java is called and the key is not contained in "NaturalKeys v",  A java.lang.NullPointerException will be thrown. This patch fixed the issue by updating the if condition. ### Why are the changes needed? Fixed a minor bug. ### Does this PR introduce any user-facing change? No. ### How was this patch tested? Added a unit test for deleteParentIndex. Closes apache#28378 from baohe-zhang/SPARK-31584. Authored-by: Baohe Zhang <[email protected]> Signed-off-by: Gengliang Wang <[email protected]>
1 parent d34cb59 commit 3808014

File tree

3 files changed

+93
-1
lines changed

3 files changed

+93
-1
lines changed

common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,7 @@ public boolean delete(Object key, T value) {
293293
private void deleteParentIndex(Object key) {
294294
if (hasNaturalParentIndex) {
295295
for (NaturalKeys v : parentToChildrenMap.values()) {
296-
if (v.remove(asKey(key))) {
296+
if (v.remove(asKey(key)) != null) {
297297
// `v` can be empty after removing the natural key and we can remove it from
298298
// `parentToChildrenMap`. However, `parentToChildrenMap` is a ConcurrentMap and such
299299
// checking and deleting can be slow.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.util.kvstore;
19+
20+
public class CustomType2 {
21+
22+
@KVIndex(parent = "parentId")
23+
public String key;
24+
25+
@KVIndex("id")
26+
public String id;
27+
28+
@KVIndex("parentId")
29+
public String parentId;
30+
31+
@Override
32+
public boolean equals(Object o) {
33+
if (o instanceof CustomType2) {
34+
CustomType2 other = (CustomType2) o;
35+
return id.equals(other.id) && parentId.equals(other.parentId);
36+
}
37+
return false;
38+
}
39+
40+
@Override
41+
public int hashCode() {
42+
return id.hashCode() ^ parentId.hashCode();
43+
}
44+
45+
@Override
46+
public String toString() {
47+
return "CustomType2[key=" + key + ",id=" + id + ",parentId=" + parentId;
48+
}
49+
50+
}

common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java

+42
Original file line numberDiff line numberDiff line change
@@ -210,4 +210,46 @@ public void testBasicIteration() throws Exception {
210210
assertFalse(store.view(t1.getClass()).first(t2.id).skip(1).iterator().hasNext());
211211
}
212212

213+
@Test
214+
public void testDeleteParentIndex() throws Exception {
215+
KVStore store = new InMemoryStore();
216+
217+
CustomType2 t1 = new CustomType2();
218+
t1.key = "key1";
219+
t1.id = "id1";
220+
t1.parentId = "parentId1";
221+
store.write(t1);
222+
223+
CustomType2 t2 = new CustomType2();
224+
t2.key = "key2";
225+
t2.id = "id2";
226+
t2.parentId = "parentId1";
227+
store.write(t2);
228+
229+
CustomType2 t3 = new CustomType2();
230+
t3.key = "key3";
231+
t3.id = "id1";
232+
t3.parentId = "parentId2";
233+
store.write(t3);
234+
235+
CustomType2 t4 = new CustomType2();
236+
t4.key = "key4";
237+
t4.id = "id2";
238+
t4.parentId = "parentId2";
239+
store.write(t4);
240+
241+
assertEquals(4, store.count(CustomType2.class));
242+
243+
store.delete(t1.getClass(), t1.key);
244+
assertEquals(3, store.count(CustomType2.class));
245+
246+
store.delete(t2.getClass(), t2.key);
247+
assertEquals(2, store.count(CustomType2.class));
248+
249+
store.delete(t3.getClass(), t3.key);
250+
assertEquals(1, store.count(CustomType2.class));
251+
252+
store.delete(t4.getClass(), t4.key);
253+
assertEquals(0, store.count(CustomType2.class));
254+
}
213255
}

0 commit comments

Comments
 (0)