View Javadoc
1   /*
2    * Copyright (C) 2012 The Guava Authors
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    * http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.google.common.util.concurrent;
17  
18  import static com.google.common.truth.Truth.assertThat;
19  import static java.util.Arrays.asList;
20  
21  import com.google.common.collect.ImmutableMap;
22  import com.google.common.collect.ImmutableSet;
23  import com.google.common.collect.Lists;
24  import com.google.common.collect.Sets;
25  import com.google.common.testing.NullPointerTester;
26  import com.google.common.testing.TestLogHandler;
27  import com.google.common.util.concurrent.ServiceManager.Listener;
28  
29  import junit.framework.TestCase;
30  
31  import java.util.Arrays;
32  import java.util.Collection;
33  import java.util.List;
34  import java.util.Set;
35  import java.util.concurrent.CountDownLatch;
36  import java.util.concurrent.Executor;
37  import java.util.concurrent.TimeUnit;
38  import java.util.concurrent.TimeoutException;
39  import java.util.logging.Formatter;
40  import java.util.logging.Level;
41  import java.util.logging.LogRecord;
42  import java.util.logging.Logger;
43  
44  /**
45   * Tests for {@link ServiceManager}.
46   *
47   * @author Luke Sandberg
48   * @author Chris Nokleberg
49   */
50  public class ServiceManagerTest extends TestCase {
51  
52    private static class NoOpService extends AbstractService {
53      @Override protected void doStart() {
54        notifyStarted();
55      }
56  
57      @Override protected void doStop() {
58        notifyStopped();
59      }
60    }
61  
62    /*
63     * A NoOp service that will delay the startup and shutdown notification for a configurable amount
64     * of time.
65     */
66    private static class NoOpDelayedService extends NoOpService {
67      private long delay;
68  
69      public NoOpDelayedService(long delay) {
70        this.delay = delay;
71      }
72  
73      @Override protected void doStart() {
74        new Thread() {
75          @Override public void run() {
76            Uninterruptibles.sleepUninterruptibly(delay, TimeUnit.MILLISECONDS);
77            notifyStarted();
78          }
79        }.start();
80      }
81  
82      @Override protected void doStop() {
83        new Thread() {
84          @Override public void run() {
85            Uninterruptibles.sleepUninterruptibly(delay, TimeUnit.MILLISECONDS);
86            notifyStopped();
87          }
88        }.start();
89      }
90    }
91  
92    private static class FailStartService extends NoOpService {
93      @Override protected void doStart() {
94        notifyFailed(new IllegalStateException("failed"));
95      }
96    }
97  
98    private static class FailRunService extends NoOpService {
99      @Override protected void doStart() {
100       super.doStart();
101       notifyFailed(new IllegalStateException("failed"));
102     }
103   }
104 
105   private static class FailStopService extends NoOpService {
106     @Override protected void doStop() {
107       notifyFailed(new IllegalStateException("failed"));
108     }
109   }
110 
111   public void testServiceStartupTimes() {
112     Service a = new NoOpDelayedService(150);
113     Service b = new NoOpDelayedService(353);
114     ServiceManager serviceManager = new ServiceManager(asList(a, b));
115     serviceManager.startAsync().awaitHealthy();
116     ImmutableMap<Service, Long> startupTimes = serviceManager.startupTimes();
117     assertEquals(2, startupTimes.size());
118     assertThat(startupTimes.get(a)).isInclusivelyInRange(150, Long.MAX_VALUE);
119     assertThat(startupTimes.get(b)).isInclusivelyInRange(353, Long.MAX_VALUE);
120   }
121 
122   public void testServiceStartupTimes_selfStartingServices() {
123     // This tests to ensure that:
124     // 1. service times are accurate when the service is started by the manager
125     // 2. service times are recorded when the service is not started by the manager (but they may
126     // not be accurate).
127     final Service b = new NoOpDelayedService(353) {
128       @Override protected void doStart() {
129         super.doStart();
130         // This will delay service listener execution at least 150 milliseconds
131         Uninterruptibles.sleepUninterruptibly(150, TimeUnit.MILLISECONDS);
132       }
133     };
134     Service a = new NoOpDelayedService(150) {
135       @Override protected void doStart() {
136         b.startAsync();
137         super.doStart();
138       }
139     };
140     ServiceManager serviceManager = new ServiceManager(asList(a, b));
141     serviceManager.startAsync().awaitHealthy();
142     ImmutableMap<Service, Long> startupTimes = serviceManager.startupTimes();
143     assertEquals(2, startupTimes.size());
144     assertThat(startupTimes.get(a)).isInclusivelyInRange(150, Long.MAX_VALUE);
145     // Service b startup takes at least 353 millis, but starting the timer is delayed by at least
146     // 150 milliseconds. so in a perfect world the timing would be 353-150=203ms, but since either
147     // of our sleep calls can be arbitrarily delayed we should just assert that there is a time
148     // recorded.
149     assertThat(startupTimes.get(b)).isNotNull();
150   }
151 
152   public void testServiceStartStop() {
153     Service a = new NoOpService();
154     Service b = new NoOpService();
155     ServiceManager manager = new ServiceManager(asList(a, b));
156     RecordingListener listener = new RecordingListener();
157     manager.addListener(listener);
158     assertState(manager, Service.State.NEW, a, b);
159     assertFalse(manager.isHealthy());
160     manager.startAsync().awaitHealthy();
161     assertState(manager, Service.State.RUNNING, a, b);
162     assertTrue(manager.isHealthy());
163     assertTrue(listener.healthyCalled);
164     assertFalse(listener.stoppedCalled);
165     assertTrue(listener.failedServices.isEmpty());
166     manager.stopAsync().awaitStopped();
167     assertState(manager, Service.State.TERMINATED, a, b);
168     assertFalse(manager.isHealthy());
169     assertTrue(listener.stoppedCalled);
170     assertTrue(listener.failedServices.isEmpty());
171   }
172 
173   public void testFailStart() throws Exception {
174     Service a = new NoOpService();
175     Service b = new FailStartService();
176     Service c = new NoOpService();
177     Service d = new FailStartService();
178     Service e = new NoOpService();
179     ServiceManager manager = new ServiceManager(asList(a, b, c, d, e));
180     RecordingListener listener = new RecordingListener();
181     manager.addListener(listener);
182     assertState(manager, Service.State.NEW, a, b, c, d, e);
183     try {
184       manager.startAsync().awaitHealthy();
185       fail();
186     } catch (IllegalStateException expected) {
187     }
188     assertFalse(listener.healthyCalled);
189     assertState(manager, Service.State.RUNNING, a, c, e);
190     assertEquals(ImmutableSet.of(b, d), listener.failedServices);
191     assertState(manager, Service.State.FAILED, b, d);
192     assertFalse(manager.isHealthy());
193 
194     manager.stopAsync().awaitStopped();
195     assertFalse(manager.isHealthy());
196     assertFalse(listener.healthyCalled);
197     assertTrue(listener.stoppedCalled);
198   }
199 
200   public void testFailRun() throws Exception {
201     Service a = new NoOpService();
202     Service b = new FailRunService();
203     ServiceManager manager = new ServiceManager(asList(a, b));
204     RecordingListener listener = new RecordingListener();
205     manager.addListener(listener);
206     assertState(manager, Service.State.NEW, a, b);
207     try {
208       manager.startAsync().awaitHealthy();
209       fail();
210     } catch (IllegalStateException expected) {
211     }
212     assertTrue(listener.healthyCalled);
213     assertEquals(ImmutableSet.of(b), listener.failedServices);
214 
215     manager.stopAsync().awaitStopped();
216     assertState(manager, Service.State.FAILED, b);
217     assertState(manager, Service.State.TERMINATED, a);
218 
219     assertTrue(listener.stoppedCalled);
220   }
221 
222   public void testFailStop() throws Exception {
223     Service a = new NoOpService();
224     Service b = new FailStopService();
225     Service c = new NoOpService();
226     ServiceManager manager = new ServiceManager(asList(a, b, c));
227     RecordingListener listener = new RecordingListener();
228     manager.addListener(listener);
229 
230     manager.startAsync().awaitHealthy();
231     assertTrue(listener.healthyCalled);
232     assertFalse(listener.stoppedCalled);
233     manager.stopAsync().awaitStopped();
234 
235     assertTrue(listener.stoppedCalled);
236     assertEquals(ImmutableSet.of(b), listener.failedServices);
237     assertState(manager, Service.State.FAILED, b);
238     assertState(manager, Service.State.TERMINATED, a, c);
239   }
240 
241   public void testToString() throws Exception {
242     Service a = new NoOpService();
243     Service b = new FailStartService();
244     ServiceManager manager = new ServiceManager(asList(a, b));
245     String toString = manager.toString();
246     assertTrue(toString.contains("NoOpService"));
247     assertTrue(toString.contains("FailStartService"));
248   }
249 
250   public void testTimeouts() throws Exception {
251     Service a = new NoOpDelayedService(50);
252     ServiceManager manager = new ServiceManager(asList(a));
253     manager.startAsync();
254     try {
255       manager.awaitHealthy(1, TimeUnit.MILLISECONDS);
256       fail();
257     } catch (TimeoutException expected) {
258     }
259     manager.awaitHealthy(100, TimeUnit.MILLISECONDS); // no exception thrown
260 
261     manager.stopAsync();
262     try {
263       manager.awaitStopped(1, TimeUnit.MILLISECONDS);
264       fail();
265     } catch (TimeoutException expected) {
266     }
267     manager.awaitStopped(100, TimeUnit.MILLISECONDS);  // no exception thrown
268   }
269 
270   /**
271    * This covers a case where if the last service to stop failed then the stopped callback would
272    * never be called.
273    */
274   public void testSingleFailedServiceCallsStopped() {
275     Service a = new FailStartService();
276     ServiceManager manager = new ServiceManager(asList(a));
277     RecordingListener listener = new RecordingListener();
278     manager.addListener(listener);
279     try {
280       manager.startAsync().awaitHealthy();
281       fail();
282     } catch (IllegalStateException expected) {
283     }
284     assertTrue(listener.stoppedCalled);
285   }
286 
287   /**
288    * This covers a bug where listener.healthy would get called when a single service failed during
289    * startup (it occurred in more complicated cases also).
290    */
291   public void testFailStart_singleServiceCallsHealthy() {
292     Service a = new FailStartService();
293     ServiceManager manager = new ServiceManager(asList(a));
294     RecordingListener listener = new RecordingListener();
295     manager.addListener(listener);
296     try {
297       manager.startAsync().awaitHealthy();
298       fail();
299     } catch (IllegalStateException expected) {
300     }
301     assertFalse(listener.healthyCalled);
302   }
303 
304   /**
305    * This covers a bug where if a listener was installed that would stop the manager if any service
306    * fails and something failed during startup before service.start was called on all the services,
307    * then awaitStopped would deadlock due to an IllegalStateException that was thrown when trying to
308    * stop the timer(!).
309    */
310   public void testFailStart_stopOthers() throws TimeoutException {
311     Service a = new FailStartService();
312     Service b = new NoOpService();
313     final ServiceManager manager = new ServiceManager(asList(a, b));
314     manager.addListener(new Listener() {
315       @Override public void failure(Service service) {
316         manager.stopAsync();
317       }});
318     manager.startAsync();
319     manager.awaitStopped(10, TimeUnit.MILLISECONDS);
320   }
321 
322   private static void assertState(
323       ServiceManager manager, Service.State state, Service... services) {
324     Collection<Service> managerServices = manager.servicesByState().get(state);
325     for (Service service : services) {
326       assertEquals(service.toString(), state, service.state());
327       assertEquals(service.toString(), service.isRunning(), state == Service.State.RUNNING);
328       assertTrue(managerServices + " should contain " + service, managerServices.contains(service));
329     }
330   }
331 
332   /**
333    * This is for covering a case where the ServiceManager would behave strangely if constructed
334    * with no service under management.  Listeners would never fire because the ServiceManager was
335    * healthy and stopped at the same time.  This test ensures that listeners fire and isHealthy
336    * makes sense.
337    */
338   public void testEmptyServiceManager() {
339     Logger logger = Logger.getLogger(ServiceManager.class.getName());
340     logger.setLevel(Level.FINEST);
341     TestLogHandler logHandler = new TestLogHandler();
342     logger.addHandler(logHandler);
343     ServiceManager manager = new ServiceManager(Arrays.<Service>asList());
344     RecordingListener listener = new RecordingListener();
345     manager.addListener(listener);
346     manager.startAsync().awaitHealthy();
347     assertTrue(manager.isHealthy());
348     assertTrue(listener.healthyCalled);
349     assertFalse(listener.stoppedCalled);
350     assertTrue(listener.failedServices.isEmpty());
351     manager.stopAsync().awaitStopped();
352     assertFalse(manager.isHealthy());
353     assertTrue(listener.stoppedCalled);
354     assertTrue(listener.failedServices.isEmpty());
355     // check that our NoOpService is not directly observable via any of the inspection methods or
356     // via logging.
357     assertEquals("ServiceManager{services=[]}", manager.toString());
358     assertTrue(manager.servicesByState().isEmpty());
359     assertTrue(manager.startupTimes().isEmpty());
360     Formatter logFormatter = new Formatter() {
361       @Override public String format(LogRecord record) {
362         return formatMessage(record);
363       }
364     };
365     for (LogRecord record : logHandler.getStoredLogRecords()) {
366       assertFalse(logFormatter.format(record).contains("NoOpService"));
367     }
368   }
369 
370   /**
371    * Tests that a ServiceManager can be fully shut down if one of its failure listeners is slow or
372    * even permanently blocked.
373    */
374 
375   public void testListenerDeadlock() throws InterruptedException {
376     final CountDownLatch failEnter = new CountDownLatch(1);
377     final CountDownLatch failLeave = new CountDownLatch(1);
378     final CountDownLatch afterStarted = new CountDownLatch(1);
379     Service failRunService = new AbstractService() {
380       @Override protected void doStart() {
381         new Thread() {
382           @Override public void run() {
383             notifyStarted();
384             // We need to wait for the main thread to leave the ServiceManager.startAsync call to
385             // ensure that the thread running the failure callbacks is not the main thread.
386             Uninterruptibles.awaitUninterruptibly(afterStarted);
387             notifyFailed(new Exception("boom"));
388           }
389         }.start();
390       }
391       @Override protected void doStop() {
392         notifyStopped();
393       }
394     };
395     final ServiceManager manager = new ServiceManager(
396         Arrays.asList(failRunService, new NoOpService()));
397     manager.addListener(new ServiceManager.Listener() {
398       @Override public void failure(Service service) {
399         failEnter.countDown();
400         // block until after the service manager is shutdown
401         Uninterruptibles.awaitUninterruptibly(failLeave);
402       }
403     });
404     manager.startAsync();
405     afterStarted.countDown();
406     // We do not call awaitHealthy because, due to races, that method may throw an exception.  But
407     // we really just want to wait for the thread to be in the failure callback so we wait for that
408     // explicitly instead.
409     failEnter.await();
410     assertFalse("State should be updated before calling listeners", manager.isHealthy());
411     // now we want to stop the services.
412     Thread stoppingThread = new Thread() {
413       @Override public void run() {
414         manager.stopAsync().awaitStopped();
415       }
416     };
417     stoppingThread.start();
418     // this should be super fast since the only non stopped service is a NoOpService
419     stoppingThread.join(1000);
420     assertFalse("stopAsync has deadlocked!.", stoppingThread.isAlive());
421     failLeave.countDown();  // release the background thread
422   }
423 
424   /**
425    * Catches a bug where when constructing a service manager failed, later interactions with the
426    * service could cause IllegalStateExceptions inside the partially constructed ServiceManager.
427    * This ISE wouldn't actually bubble up but would get logged by ExecutionQueue.  This obfuscated
428    * the original error (which was not constructing ServiceManager correctly).
429    */
430   public void testPartiallyConstructedManager() {
431     Logger logger = Logger.getLogger("global");
432     logger.setLevel(Level.FINEST);
433     TestLogHandler logHandler = new TestLogHandler();
434     logger.addHandler(logHandler);
435     NoOpService service = new NoOpService();
436     service.startAsync();
437     try {
438       new ServiceManager(Arrays.asList(service));
439       fail();
440     } catch (IllegalArgumentException expected) {}
441     service.stopAsync();
442     // Nothing was logged!
443     assertEquals(0, logHandler.getStoredLogRecords().size());
444   }
445 
446   public void testPartiallyConstructedManager_transitionAfterAddListenerBeforeStateIsReady() {
447     // The implementation of this test is pretty sensitive to the implementation :( but we want to
448     // ensure that if weird things happen during construction then we get exceptions.
449     final NoOpService service1 = new NoOpService();
450     // This service will start service1 when addListener is called.  This simulates service1 being
451     // started asynchronously.
452     Service service2 = new Service() {
453       final NoOpService delegate = new NoOpService();
454       @Override public final void addListener(Listener listener, Executor executor) {
455         service1.startAsync();
456         delegate.addListener(listener, executor);
457       }
458       // Delegates from here on down
459       @Override public final Service startAsync() {
460         return delegate.startAsync();
461       }
462 
463       @Override public final Service stopAsync() {
464         return delegate.stopAsync();
465       }
466 
467       @Override public final void awaitRunning() {
468         delegate.awaitRunning();
469       }
470 
471       @Override public final void awaitRunning(long timeout, TimeUnit unit)
472           throws TimeoutException {
473         delegate.awaitRunning(timeout, unit);
474       }
475 
476       @Override public final void awaitTerminated() {
477         delegate.awaitTerminated();
478       }
479 
480       @Override public final void awaitTerminated(long timeout, TimeUnit unit)
481           throws TimeoutException {
482         delegate.awaitTerminated(timeout, unit);
483       }
484 
485       @Override public final boolean isRunning() {
486         return delegate.isRunning();
487       }
488 
489       @Override public final State state() {
490         return delegate.state();
491       }
492 
493       @Override public final Throwable failureCause() {
494         return delegate.failureCause();
495       }
496     };
497     try {
498       new ServiceManager(Arrays.asList(service1, service2));
499       fail();
500     } catch (IllegalArgumentException expected) {
501       assertTrue(expected.getMessage().contains("started transitioning asynchronously"));
502     }
503   }
504 
505   /**
506    * This test is for a case where two Service.Listener callbacks for the same service would call
507    * transitionService in the wrong order due to a race.  Due to the fact that it is a race this
508    * test isn't guaranteed to expose the issue, but it is at least likely to become flaky if the
509    * race sneaks back in, and in this case flaky means something is definitely wrong.
510    *
511    * <p>Before the bug was fixed this test would fail at least 30% of the time.
512    */
513 
514   public void testTransitionRace() throws TimeoutException {
515     for (int k = 0; k < 1000; k++) {
516       List<Service> services = Lists.newArrayList();
517       for (int i = 0; i < 5; i++) {
518         services.add(new SnappyShutdownService(i));
519       }
520       ServiceManager manager = new ServiceManager(services);
521       manager.startAsync().awaitHealthy();
522       manager.stopAsync().awaitStopped(1, TimeUnit.SECONDS);
523     }
524   }
525 
526   /**
527    * This service will shutdown very quickly after stopAsync is called and uses a background thread
528    * so that we know that the stopping() listeners will execute on a different thread than the
529    * terminated() listeners.
530    */
531   private static class SnappyShutdownService extends AbstractExecutionThreadService {
532     final int index;
533     final CountDownLatch latch = new CountDownLatch(1);
534 
535     SnappyShutdownService(int index) {
536       this.index = index;
537     }
538 
539     @Override protected void run() throws Exception {
540       latch.await();
541     }
542 
543     @Override protected void triggerShutdown() {
544       latch.countDown();
545     }
546 
547     @Override protected String serviceName() {
548       return this.getClass().getSimpleName() + "[" + index + "]";
549     }
550   }
551 
552   public void testNulls() {
553     ServiceManager manager = new ServiceManager(Arrays.<Service>asList());
554     new NullPointerTester()
555         .setDefault(ServiceManager.Listener.class, new RecordingListener())
556         .testAllPublicInstanceMethods(manager);
557   }
558 
559   private static final class RecordingListener extends ServiceManager.Listener {
560     volatile boolean healthyCalled;
561     volatile boolean stoppedCalled;
562     final Set<Service> failedServices = Sets.newConcurrentHashSet();
563 
564     @Override public void healthy() {
565       healthyCalled = true;
566     }
567 
568     @Override public void stopped() {
569       stoppedCalled = true;
570     }
571 
572     @Override public void failure(Service service) {
573       failedServices.add(service);
574     }
575   }
576 }