Skip to content

Commit b5bd017

Browse files
fix: prevent deadlock on concurrent push disconnection (#16544) (#16580)
If the AtmospherePushConnection.disconnect method is invoked concurrently by multiple threads, a deadlock may happen if the servlet container somehow locks HTTP session, as closing AtmosphereResource may cause accesses to the HTTP session. For example, is a thread (A) is invalidating the HTTP session, thus closing Vaadin UIs and disconnecting push, and another thread (B) is also requesting a push disconnect, then B will be blocked when closing the atmosphere resource by the lock on the session held by A, but A is actually waiting to acquire the lock on AtmospherePushConnection held by B, causing the deadlock. This change allows a single thread to perform the disconnect operation, to avoid potential deadlocks. Fixes #16293 Co-authored-by: Marco Collovati <marco@vaadin.com>
1 parent 15f747d commit b5bd017

2 files changed

Lines changed: 145 additions & 31 deletions

File tree

‎flow-server/src/main/java/com/vaadin/flow/server/communication/AtmospherePushConnection.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,8 @@ public class AtmospherePushConnection implements PushConnection {
5454
private transient AtmosphereResource resource;
5555
private transient FragmentedMessage incomingMessage;
5656
private transient Future<Object> outgoingMessage;
57-
5857
private transient Object lock = new Object();
58+
private volatile boolean disconnecting;
5959

6060
/**
6161
* Represents a message that can arrive as multiple fragments.
@@ -314,10 +314,22 @@ protected AtmosphereResource getResource() {
314314

315315
@Override
316316
public void disconnect() {
317+
// If a disconnection is already happening on another thread it is safe
318+
// to skip the operation. This also prevents potential deadlocks if the
319+
// container acquires locks during operations on HTTP session, as
320+
// closing the AtmosphereResource may cause HTTP session access
321+
if (disconnecting) {
322+
getLogger().debug(
323+
"Disconnection already in progress, ignoring request");
324+
return;
325+
}
326+
317327
synchronized (lock) {
328+
disconnecting = true;
318329
assert isConnected();
319330
if (resource == null) {
320-
// Already disconnected. Should not happen but if it does, we
331+
// Already disconnected. Should not happen but if it does,
332+
// we
321333
// don't
322334
// want to cause NPEs
323335
getLogger().debug(
@@ -353,6 +365,7 @@ public void disconnect() {
353365
getLogger().info("Error when closing push connection", e);
354366
}
355367
connectionLost();
368+
disconnecting = false;
356369
}
357370
}
358371

@@ -396,6 +409,7 @@ private void readObject(ObjectInputStream stream)
396409
throws IOException, ClassNotFoundException {
397410
stream.defaultReadObject();
398411
state = State.DISCONNECTED;
412+
disconnecting = false;
399413
lock = new Object();
400414
}
401415

‎flow-server/src/test/java/com/vaadin/flow/server/communication/AtmospherePushConnectionTest.java‎

Lines changed: 129 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,16 @@
2020
import java.io.ObjectInputStream;
2121
import java.io.ObjectOutputStream;
2222
import java.util.concurrent.CompletableFuture;
23+
import java.util.concurrent.CompletionException;
2324
import java.util.concurrent.CountDownLatch;
2425
import java.util.concurrent.TimeUnit;
2526

27+
import java.util.concurrent.atomic.AtomicReference;
28+
import java.util.concurrent.locks.ReentrantLock;
2629
import org.atmosphere.cpr.AtmosphereResource;
2730
import org.atmosphere.cpr.Broadcaster;
2831
import org.junit.Assert;
32+
import org.junit.Before;
2933
import org.junit.Test;
3034
import org.mockito.ArgumentMatchers;
3135
import org.mockito.Mockito;
@@ -39,6 +43,36 @@
3943
* @since 1.0
4044
*/
4145
public class AtmospherePushConnectionTest {
46+
47+
private MockVaadinSession vaadinSession;
48+
private Broadcaster broadcaster;
49+
private AtmosphereResource resource;
50+
private AtmospherePushConnection connection;
51+
52+
@Before
53+
public void setup() throws Exception {
54+
UI ui = Mockito.spy(new UI());
55+
vaadinSession = new MockVaadinSession();
56+
Mockito.when(ui.getSession()).thenReturn(vaadinSession);
57+
broadcaster = Mockito.mock(Broadcaster.class);
58+
resource = Mockito.mock(AtmosphereResource.class);
59+
Mockito.when(resource.getBroadcaster()).thenReturn(broadcaster);
60+
Mockito.doAnswer(i -> {
61+
// Introduce a small delay to hold the lock during disconnect
62+
Thread.sleep(5);
63+
return null;
64+
}).when(resource).close();
65+
Mockito.doAnswer(i -> {
66+
// Introduce a small delay to hold the lock during message push
67+
Thread.sleep(5);
68+
return CompletableFuture.completedFuture(null);
69+
}).when(broadcaster).broadcast(ArgumentMatchers.any(),
70+
ArgumentMatchers.any(AtmosphereResource.class));
71+
72+
connection = new AtmospherePushConnection(ui);
73+
connection.connect(resource);
74+
}
75+
4276
@Test
4377
public void testSerialization() throws Exception {
4478

@@ -63,29 +97,20 @@ public void testSerialization() throws Exception {
6397
@Test
6498
public void pushWhileDisconnect_disconnectedWithoutSendingMessage()
6599
throws Exception {
66-
67-
UI ui = Mockito.spy(new UI());
68-
MockVaadinSession vaadinSession = new MockVaadinSession();
69-
Mockito.when(ui.getSession()).thenReturn(vaadinSession);
70-
Broadcaster broadcaster = Mockito.mock(Broadcaster.class);
71-
AtmosphereResource resource = Mockito.mock(AtmosphereResource.class);
72-
Mockito.when(resource.getBroadcaster()).thenReturn(broadcaster);
73-
74-
AtmospherePushConnection connection = new AtmospherePushConnection(ui);
75-
connection.connect(resource);
76-
77100
CountDownLatch latch = new CountDownLatch(1);
78101
CompletableFuture.runAsync(() -> {
79102
try {
80103
vaadinSession.runWithLock(() -> {
81104
connection.push();
82105
return null;
83106
});
107+
latch.countDown();
84108
} catch (Exception ex) {
85109
throw new RuntimeException(ex);
86-
} finally {
87-
latch.countDown();
88110
}
111+
}).exceptionally(error -> {
112+
error.printStackTrace();
113+
return null;
89114
});
90115
connection.disconnect();
91116
Assert.assertTrue("AtmospherePushConnection not disconnected",
@@ -97,35 +122,110 @@ public void pushWhileDisconnect_disconnectedWithoutSendingMessage()
97122
@Test
98123
public void disconnectWhilePush_messageSentAndThenDisconnected()
99124
throws Exception {
100-
101-
UI ui = Mockito.spy(new UI());
102-
MockVaadinSession vaadinSession = new MockVaadinSession();
103-
Mockito.when(ui.getSession()).thenReturn(vaadinSession);
104-
Broadcaster broadcaster = Mockito.mock(Broadcaster.class);
105-
AtmosphereResource resource = Mockito.mock(AtmosphereResource.class);
106-
Mockito.when(resource.getBroadcaster()).thenReturn(broadcaster);
107-
108-
AtmospherePushConnection connection = new AtmospherePushConnection(ui);
109-
connection.connect(resource);
110-
111-
CountDownLatch latch = new CountDownLatch(1);
125+
CountDownLatch latch = new CountDownLatch(2);
112126
CompletableFuture.runAsync(() -> {
113127
try {
114128
vaadinSession.runWithLock(() -> {
115-
CompletableFuture.runAsync(connection::disconnect);
129+
CompletableFuture.runAsync(() -> {
130+
connection.disconnect();
131+
latch.countDown();
132+
}, CompletableFuture.delayedExecutor(1,
133+
TimeUnit.MILLISECONDS)).exceptionally(error -> {
134+
error.printStackTrace();
135+
return null;
136+
});
116137
connection.push();
117138
return null;
118139
});
140+
latch.countDown();
119141
} catch (Throwable ex) {
120142
throw new RuntimeException(ex);
121-
} finally {
122-
latch.countDown();
123143
}
144+
}).exceptionally(error -> {
145+
error.printStackTrace();
146+
return null;
124147
});
148+
125149
Assert.assertTrue("Push not completed",
126-
latch.await(2, TimeUnit.SECONDS));
150+
latch.await(3, TimeUnit.SECONDS));
127151
Mockito.verify(broadcaster).broadcast(ArgumentMatchers.any(),
128152
ArgumentMatchers.eq(resource));
129153
}
130154

155+
@Test
156+
public void disconnect_concurrentRequests_preventDeadlocks()
157+
throws Exception {
158+
// A deadlock may happen when an HTTP session is invalidated in a
159+
// thread, causing VaadinSession and UIs to be closed and push
160+
// connections to be disconnected, but a push disconnection is
161+
// concurrently requested by another thread.
162+
// This happens for example with MPR and Vaadin 8: when HTTP session is
163+
// invalidated Flow closes UI and disconnects PUSH. At the same time V8
164+
// VaadinSession is nullified on the V8 UI, that in turn spawns a thread
165+
// to disconnect PUSH connection.
166+
// So, there are two concurrent calls to disconnect; one of those
167+
// acquires the AtmospherePushConnection lock and the other one waits
168+
// for it to be released.
169+
// Unfortunately, it may happen that HTTP session operation also require
170+
// locks.
171+
// In the above case, a lock is held by the thread that is performing
172+
// session invalidation, but when AtmospherePushConnection.disconnect is
173+
// invoked on the other thread, it calls AtmosphereResource.close that
174+
// tries to access the HTTP session attributes, but it gets stuck
175+
// waiting for the HTTP session lock to be released.
176+
// At the end, there's a thread holding the AtmosphereConnection lock
177+
// unable to release it because it cannot complete disconnection due to
178+
// the HTTP session lock, that is however hold by the other thread that
179+
// is waiting for AtmosphereConnection lock to be released.
180+
ReentrantLock sessionLock = new ReentrantLock();
181+
Mockito.doAnswer(i -> {
182+
// simulate HTTP session lock attempt because of atmosphere resource
183+
// accesses session attributes
184+
// It does not wait indefinitely, but triggers an error if the lock
185+
// is held by the main thread
186+
if (sessionLock.tryLock(2, TimeUnit.SECONDS)) {
187+
sessionLock.unlock();
188+
} else {
189+
throw new AssertionError(
190+
"Deadlock on AtmosphereResource.close");
191+
}
192+
return null;
193+
}).when(resource).close();
194+
195+
CountDownLatch latch = new CountDownLatch(2);
196+
sessionLock.lock();
197+
CompletableFuture<Throwable> threadErrorFuture;
198+
try {
199+
// Simulate PUSH disconnection from a separate thread
200+
threadErrorFuture = CompletableFuture
201+
.<Throwable> supplyAsync(() -> {
202+
connection.disconnect();
203+
latch.countDown();
204+
return null;
205+
}).exceptionally(t -> {
206+
if (t instanceof CompletionException) {
207+
return t.getCause();
208+
}
209+
return t;
210+
});
211+
// Simulate main thread PUSH disconnection because of session
212+
// invalidation, delayed a bit to allow the other thread to start
213+
// disconnection
214+
Thread.sleep(1);
215+
connection.disconnect();
216+
latch.countDown();
217+
} finally {
218+
sessionLock.unlock();
219+
}
220+
221+
Throwable threadError = threadErrorFuture.get(2, TimeUnit.SECONDS);
222+
if (threadError != null) {
223+
Assert.fail("Disconnection on spawned thread failed: "
224+
+ threadError.getMessage());
225+
}
226+
Assert.assertTrue("Disconnect calls not completed, missing "
227+
+ latch.getCount() + " call", latch.await(3, TimeUnit.SECONDS));
228+
Mockito.verify(resource, Mockito.times(1)).close();
229+
}
230+
131231
}

0 commit comments

Comments
 (0)