public class BatchProcess extends java.lang.Object implements java.util.concurrent.Callable<ParallelFileProcessingResult>
FileResourceCrawler and FileResourceConsumers, and it can also
support a StatusReporter and an Interrupter.
This is designed to shutdown if a parser has timed out or if there is
an OutOfMemoryError. Consider using BatchProcessDriverCLI
as a daemon/watchdog that monitors and can restart this batch process;
Note that this classs redirects stderr to stdout so that it can communicate without interference with the parent process on stderr.
| Modifier and Type | Class and Description |
|---|---|
static class |
BatchProcess.BATCH_CONSTANTS |
private static class |
BatchProcess.CAUSE_FOR_TERMINATION |
private class |
BatchProcess.State |
private class |
BatchProcess.TimeoutChecker |
private class |
BatchProcess.TimeoutFutureResult |
| Modifier and Type | Field and Description |
|---|---|
private boolean |
alreadyExecuted |
private ConsumersManager |
consumersManager |
private long |
consumersManagerMaxMillis |
private FileResourceCrawler |
fileResourceCrawler |
private Interrupter |
interrupter |
private static org.slf4j.Logger |
LOG |
private int |
maxAliveTimeSeconds |
private java.io.PrintStream |
outputStreamWriter |
private long |
pauseOnEarlyTerminationMillis |
private StatusReporter |
reporter |
private java.util.concurrent.ArrayBlockingQueue<FileStarted> |
timedOuts |
private long |
timeoutCheckPulseMillis |
private long |
timeoutThresholdMillis |
| Constructor and Description |
|---|
BatchProcess(FileResourceCrawler fileResourceCrawler,
ConsumersManager consumersManager,
StatusReporter reporter,
Interrupter interrupter) |
| Modifier and Type | Method and Description |
|---|---|
private boolean |
aliveTooLong(long started) |
private boolean |
areResourcesPotentiallyRemaining() |
ParallelFileProcessingResult |
call()
Runs main execution loop.
|
private int |
countActiveConsumers() |
private int |
getExitStatus(BatchProcess.CAUSE_FOR_TERMINATION causeForTermination,
java.lang.String restartMsg) |
private boolean |
isNonRestart(java.lang.Throwable e) |
private BatchProcess.State |
mainLoop(java.util.concurrent.CompletionService<IFileProcessorFutureResult> completionService,
BatchProcess.TimeoutChecker timeoutChecker) |
private void |
politelyAwaitTermination(BatchProcess.CAUSE_FOR_TERMINATION causeForTermination)
This is used instead of awaitTermination(), because that interrupts
the thread and then waits for its termination.
|
void |
setMaxAliveTimeSeconds(int maxAliveTimeSeconds)
The maximum amount of time that this process can be alive.
|
void |
setPauseOnEarlyTerminationMillis(long pauseOnEarlyTerminationMillis)
If there is an early termination via an interrupt or too many timed out consumers
or because a consumer or other Runnable threw a Throwable, pause this long
before killing the consumers and other threads.
|
void |
setTimeoutCheckPulseMillis(long timeoutCheckPulseMillis) |
void |
setTimeoutThresholdMillis(long timeoutThresholdMillis)
The amount of time allowed before a consumer should be timed out.
|
private ParallelFileProcessingResult |
shutdown(java.util.concurrent.ExecutorService ex,
java.util.concurrent.CompletionService<IFileProcessorFutureResult> completionService,
BatchProcess.TimeoutChecker timeoutChecker,
BatchProcess.State state) |
private void |
shutdownConsumersManager() |
private void |
startConsumersManager() |
private static final org.slf4j.Logger LOG
private java.io.PrintStream outputStreamWriter
private long timeoutThresholdMillis
private long timeoutCheckPulseMillis
private long pauseOnEarlyTerminationMillis
private final long consumersManagerMaxMillis
private int maxAliveTimeSeconds
private final FileResourceCrawler fileResourceCrawler
private final ConsumersManager consumersManager
private final StatusReporter reporter
private final Interrupter interrupter
private final java.util.concurrent.ArrayBlockingQueue<FileStarted> timedOuts
private boolean alreadyExecuted
public BatchProcess(FileResourceCrawler fileResourceCrawler, ConsumersManager consumersManager, StatusReporter reporter, Interrupter interrupter)
public ParallelFileProcessingResult call() throws java.lang.InterruptedException
Redirects stdout to stderr to keep clean communications over stdout with parent process
call in interface java.util.concurrent.Callable<ParallelFileProcessingResult>java.lang.InterruptedExceptionprivate BatchProcess.State mainLoop(java.util.concurrent.CompletionService<IFileProcessorFutureResult> completionService, BatchProcess.TimeoutChecker timeoutChecker)
private ParallelFileProcessingResult shutdown(java.util.concurrent.ExecutorService ex, java.util.concurrent.CompletionService<IFileProcessorFutureResult> completionService, BatchProcess.TimeoutChecker timeoutChecker, BatchProcess.State state)
private void startConsumersManager()
private void shutdownConsumersManager()
private void politelyAwaitTermination(BatchProcess.CAUSE_FOR_TERMINATION causeForTermination)
causeForTermination - reason for termination.private boolean isNonRestart(java.lang.Throwable e)
private int getExitStatus(BatchProcess.CAUSE_FOR_TERMINATION causeForTermination, java.lang.String restartMsg)
private boolean areResourcesPotentiallyRemaining()
private boolean aliveTooLong(long started)
private int countActiveConsumers()
public void setPauseOnEarlyTerminationMillis(long pauseOnEarlyTerminationMillis)
pauseOnEarlyTerminationMillis - how long to pause if there is an early terminationpublic void setTimeoutThresholdMillis(long timeoutThresholdMillis)
timeoutThresholdMillis - threshold in milliseconds before declaring a consumer timed outpublic void setTimeoutCheckPulseMillis(long timeoutCheckPulseMillis)
public void setMaxAliveTimeSeconds(int maxAliveTimeSeconds)
maxAliveTimeSeconds - maximum amount of time in seconds to remain alive