diff --git a/.divinerconfig b/.divinerconfig index e746f43..6fa0104 100644 --- a/.divinerconfig +++ b/.divinerconfig @@ -1,21 +1,22 @@ { "name" : "libphutil", "src_base" : "https://github.com/facebook/libphutil/blob/master", "groups" : { "overview" : "Overview", "contrib" : "Contributing to libphutil", "working" : "Working with libphutil", "util" : "Core Utilities", "library" : "Phutil Module System", "filesystem" : "Filesystem", "exec" : "Command Execution", "futures" : "Futures", "markup" : "Markup", "console" : "Console Utilities", "xhpast" : "XHPAST (PHP/XHP Parser)", "conduit" : "Conduit (Service API)", + "daemon" : "Daemons", "parser" : "Other Parsers", "testcase" : "Test Cases" } } diff --git a/scripts/__init_script__.php b/scripts/__init_script__.php new file mode 100644 index 0000000..011831a --- /dev/null +++ b/scripts/__init_script__.php @@ -0,0 +1,20 @@ +execute(); diff --git a/scripts/daemon/launch_daemon.php b/scripts/daemon/launch_daemon.php new file mode 100755 index 0000000..672b00a --- /dev/null +++ b/scripts/daemon/launch_daemon.php @@ -0,0 +1,37 @@ +#!/usr/bin/env php +\n"; + exit(1); +} + +$daemon = $argv[1]; +unset($argv[1]); +$argv = array_values($argv); + +echo "Running daemon ".$daemon."...\n"; + +$overseer = new PhutilDaemonOverseer($daemon, $argv); +$overseer->run(); diff --git a/scripts/daemon/torture/resist-death.php b/scripts/daemon/torture/resist-death.php new file mode 100755 index 0000000..f1eea22 --- /dev/null +++ b/scripts/daemon/torture/resist-death.php @@ -0,0 +1,35 @@ +#!/usr/bin/env php + array( 'CommandException' => 'future/exec', 'ConduitClient' => 'conduit/client', 'ConduitClientException' => 'conduit/client', 'ConduitFuture' => 'conduit/client', 'ExecFuture' => 'future/exec', 'FileFinder' => 'filesystem/filefinder', 'FileList' => 'filesystem/filelist', 'Filesystem' => 'filesystem', 'FilesystemException' => 'filesystem', 'Future' => 'future', 'FutureIterator' => 'future', 'FutureProxy' => 'future/proxy', 'HTTPFuture' => 'future/http', 'HTTPSFuture' => 'future/https', 'PhutilConsoleFormatter' => 'console', + 'PhutilDaemon' => 'daemon/base', + 'PhutilDaemonOverseer' => 'daemon/overseer', 'PhutilDefaultSyntaxHighlighterEngine' => 'markup/syntax/engine/default', 'PhutilDocblockParser' => 'parser/docblock', 'PhutilDocblockParserTestCase' => 'parser/docblock/__tests__', + 'PhutilFatalDaemon' => 'daemon/torture/fatal', + 'PhutilHangForeverDaemon' => 'daemon/torture/hangforever', 'PhutilInteractiveEditor' => 'console/editor', 'PhutilMarkupEngine' => 'markup/engine', 'PhutilMissingSymbolException' => 'symbols/exception/missing', + 'PhutilNiceDaemon' => 'daemon/torture/nice', + 'PhutilProcessGroupDaemon' => 'daemon/torture/processgroup', 'PhutilRemarkupBlockStorage' => 'markup/engine/remarkup/blockstorage', 'PhutilRemarkupEngine' => 'markup/engine/remarkup', 'PhutilRemarkupEngineBlockRule' => 'markup/engine/remarkup/blockrule/base', 'PhutilRemarkupEngineRemarkupCodeBlockRule' => 'markup/engine/remarkup/blockrule/remarkupcode', 'PhutilRemarkupEngineRemarkupDefaultBlockRule' => 'markup/engine/remarkup/blockrule/remarkupdefault', 'PhutilRemarkupEngineRemarkupHeaderBlockRule' => 'markup/engine/remarkup/blockrule/remarkupheader', 'PhutilRemarkupEngineRemarkupInlineBlockRule' => 'markup/engine/remarkup/blockrule/remarkupinline', 'PhutilRemarkupEngineRemarkupListBlockRule' => 'markup/engine/remarkup/blockrule/remarkuplist', 'PhutilRemarkupRule' => 'markup/engine/remarkup/markuprule/base', 'PhutilRemarkupRuleBold' => 'markup/engine/remarkup/markuprule/bold', 'PhutilRemarkupRuleEscapeHTML' => 'markup/engine/remarkup/markuprule/escapehtml', 'PhutilRemarkupRuleEscapeRemarkup' => 'markup/engine/remarkup/markuprule/escaperemarkup', 'PhutilRemarkupRuleHyperlink' => 'markup/engine/remarkup/markuprule/hyperlink', 'PhutilRemarkupRuleItalic' => 'markup/engine/remarkup/markuprule/italics', 'PhutilRemarkupRuleLinebreaks' => 'markup/engine/remarkup/markuprule/linebreaks', 'PhutilRemarkupRuleMonospace' => 'markup/engine/remarkup/markuprule/monospace', + 'PhutilSaturateStdoutDaemon' => 'daemon/torture/saturatestdout', 'PhutilSymbolLoader' => 'symbols', 'PhutilSyntaxHighlighter' => 'markup/syntax/highlighter/base', 'PhutilSyntaxHighlighterEngine' => 'markup/syntax/engine/base', + 'PhutilTortureTestDaemon' => 'daemon/torture/base', 'PhutilURI' => 'parser/uri', 'PhutilXHPASTSyntaxHighlighter' => 'markup/syntax/highlighter/xhpast', 'TempFile' => 'filesystem/tempfile', 'XHPASTNode' => 'parser/xhpast/api/node', 'XHPASTNodeList' => 'parser/xhpast/api/list', 'XHPASTSyntaxErrorException' => 'parser/xhpast/api/exception', 'XHPASTToken' => 'parser/xhpast/api/token', 'XHPASTTree' => 'parser/xhpast/api/tree', 'XHPASTTreeTestCase' => 'parser/xhpast/api/tree/__tests__', ), 'function' => array( 'Futures' => 'future', 'array_select_keys' => 'utils', 'coalesce' => 'utils', 'csprintf' => 'xsprintf/csprintf', 'exec_manual' => 'future/exec', 'execx' => 'future/exec', 'id' => 'utils', 'idx' => 'utils', 'igroup' => 'utils', 'ipull' => 'utils', 'isort' => 'utils', 'jsprintf' => 'xsprintf/jsprintf', 'mgroup' => 'utils', 'mpull' => 'utils', 'msort' => 'utils', 'newv' => 'utils', 'nonempty' => 'utils', 'phutil_autoload_class' => 'autoload', 'phutil_console_confirm' => 'console', 'phutil_console_format' => 'console', 'phutil_console_prompt' => 'console', 'phutil_console_wrap' => 'console', 'phutil_escape_html' => 'markup', 'phutil_escape_uri' => 'markup', 'phutil_get_library_name_for_root' => 'moduleutils', 'phutil_get_library_root' => 'moduleutils', 'phutil_get_library_root_for_path' => 'moduleutils', 'phutil_render_tag' => 'markup', 'vcsprintf' => 'xsprintf/csprintf', 'vjsprintf' => 'xsprintf/jsprintf', 'xhp_parser_node_constants' => 'parser/xhpast/constants', 'xhpast_get_binary_path' => 'parser/xhpast/bin', 'xhpast_get_build_instructions' => 'parser/xhpast/bin', 'xhpast_get_parser_future' => 'parser/xhpast/bin', 'xhpast_is_available' => 'parser/xhpast/bin', 'xhpast_parser_token_constants' => 'parser/xhpast/constants', 'xsprintf' => 'xsprintf', 'xsprintf_callback_example' => 'xsprintf', 'xsprintf_command' => 'xsprintf/csprintf', 'xsprintf_javascript' => 'xsprintf/jsprintf', ), 'requires_class' => array( 'ConduitFuture' => 'FutureProxy', 'ExecFuture' => 'Future', 'FutureProxy' => 'Future', 'HTTPFuture' => 'Future', 'HTTPSFuture' => 'Future', 'PhutilDefaultSyntaxHighlighterEngine' => 'PhutilSyntaxHighlighterEngine', 'PhutilDocblockParserTestCase' => 'ArcanistPhutilTestCase', + 'PhutilFatalDaemon' => 'PhutilTortureTestDaemon', + 'PhutilHangForeverDaemon' => 'PhutilTortureTestDaemon', + 'PhutilNiceDaemon' => 'PhutilTortureTestDaemon', + 'PhutilProcessGroupDaemon' => 'PhutilTortureTestDaemon', 'PhutilRemarkupEngine' => 'PhutilMarkupEngine', 'PhutilRemarkupEngineRemarkupCodeBlockRule' => 'PhutilRemarkupEngineBlockRule', 'PhutilRemarkupEngineRemarkupDefaultBlockRule' => 'PhutilRemarkupEngineBlockRule', 'PhutilRemarkupEngineRemarkupHeaderBlockRule' => 'PhutilRemarkupEngineBlockRule', 'PhutilRemarkupEngineRemarkupInlineBlockRule' => 'PhutilRemarkupEngineBlockRule', 'PhutilRemarkupEngineRemarkupListBlockRule' => 'PhutilRemarkupEngineBlockRule', 'PhutilRemarkupRuleBold' => 'PhutilRemarkupRule', 'PhutilRemarkupRuleEscapeHTML' => 'PhutilRemarkupRule', 'PhutilRemarkupRuleEscapeRemarkup' => 'PhutilRemarkupRule', 'PhutilRemarkupRuleHyperlink' => 'PhutilRemarkupRule', 'PhutilRemarkupRuleItalic' => 'PhutilRemarkupRule', 'PhutilRemarkupRuleLinebreaks' => 'PhutilRemarkupRule', 'PhutilRemarkupRuleMonospace' => 'PhutilRemarkupRule', + 'PhutilSaturateStdoutDaemon' => 'PhutilTortureTestDaemon', + 'PhutilTortureTestDaemon' => 'PhutilDaemon', 'XHPASTTreeTestCase' => 'ArcanistPhutilTestCase', ), 'requires_interface' => array( ), )); diff --git a/src/daemon/base/PhutilDaemon.php b/src/daemon/base/PhutilDaemon.php new file mode 100644 index 0000000..12469b9 --- /dev/null +++ b/src/daemon/base/PhutilDaemon.php @@ -0,0 +1,68 @@ +argv = $argv; + + if (!self::$sighandlerInstalled) { + self::$sighandlerInstalled = true; + pcntl_signal(SIGINT, __CLASS__.'::__exitOnSignal'); + pcntl_signal(SIGTERM, __CLASS__.'::__exitOnSignal'); + } + } + + final public function stillWorking() { + posix_kill(posix_getppid(), SIGUSR1); + } + + public static function __exitOnSignal($signo) { + // Normally, PHP doesn't invoke destructors when existing in response to + // a signal. This forces it to do so, so we have a fighting chance of + // releasing any locks on our way out. + exit(128 + $signo); + } + + final protected function getArgv() { + return $this->argv; + } + + final public function execute() { + $this->willRun(); + $this->run(); + } + + protected function willRun() { + + } + + abstract protected function run(); + +} diff --git a/src/daemon/base/__init__.php b/src/daemon/base/__init__.php new file mode 100644 index 0000000..a75f48a --- /dev/null +++ b/src/daemon/base/__init__.php @@ -0,0 +1,10 @@ +daemon = $daemon; + $this->argv = array_slice($argv, 1); + + if (self::$instance) { + throw new Exception( + "You may not instantiate more than one Overseer per process."); + } + + self::$instance = $this; + + declare(ticks = 1); + pcntl_signal(SIGUSR1, array($this, 'didReceiveKeepaliveSignal')); + + pcntl_signal(SIGINT, array($this, 'didReceiveTerminalSignal')); + pcntl_signal(SIGTERM, array($this, 'didReceiveTerminalSignal')); + } + + public function run() { + + $root = phutil_get_library_root('phutil'); + $root = dirname($root); + $exec_daemon = $root.'/scripts/daemon/exec/exec_daemon.php'; + + $argv = $this->argv; + array_unshift($argv, $exec_daemon, $this->daemon); + foreach ($argv as $k => $arg) { + $argv[$k] = escapeshellarg($arg); + } + + $command = 'exec '.implode(' ', $argv); + + while (true) { + $this->logMessage('[INIT] Starting process.'); + + $future = new ExecFuture($command); + $future->setStdoutSizeLimit($this->captureBufferSize); + $future->setStderrSizeLimit($this->captureBufferSize); + + $this->deadline = time() + $this->deadlineTimeout; + + $future->isReady(); + $this->childPID = $future->getPID(); + + do { + do { + // $memuse = number_format(memory_get_usage() / 1024, 1); + // $this->logMessage('[STAT] Memory Usage: '.$memuse.' KB'); + + // We need a shortish timeout here so we can run the tick handler + // frequently in order to process signals. + $result = $future->resolve(1); + + if ($result !== null) { + list($err) = $result; + if ($err) { + $this->logMessage('[FAIL] Process edited with error '.$err.'.'); + } else { + $this->logMessage('[DONE] Process exited successfully.'); + } + break 2; + } + } while (time() < $this->deadline); + + $this->logMessage('[HANG] Hang detected. Restarting process.'); + $this->annihilateProcessGroup(); + } while (false); + + $this->logMessage('[WAIT] Waiting to restart process.'); + sleep($this->restartDelay); + } + } + + public function didReceiveKeepaliveSignal($signo) { + $this->deadline = time() + $this->deadlineTimeout; + } + + public function didReceiveTerminalSignal($signo) { + if ($this->signaled) { + exit(128 + $signo); + } + echo "\n>>> Shutting down...\n"; + $this->signaled = true; + $this->annihilateProcessGroup(); + exit(128 + $signo); + } + + private function logMessage($message) { + echo date('Y-m-d g:i:s A').' '.$message."\n"; + } + + private function annihilateProcessGroup() { + $pid = $this->childPID; + $pgid = posix_getpgid($pid); + if ($pid && $pgid) { + exec("kill -TERM -- -{$pgid}"); + sleep($this->killDelay); + exec("kill -KILL -- -{$pgid}"); + $this->childPID = null; + } + } + +} diff --git a/src/daemon/overseer/__init__.php b/src/daemon/overseer/__init__.php new file mode 100644 index 0000000..947d7a9 --- /dev/null +++ b/src/daemon/overseer/__init__.php @@ -0,0 +1,13 @@ +stillWorking(); + sleep(1); + } + } + +} diff --git a/src/daemon/torture/nice/__init__.php b/src/daemon/torture/nice/__init__.php new file mode 100644 index 0000000..0c8546e --- /dev/null +++ b/src/daemon/torture/nice/__init__.php @@ -0,0 +1,12 @@ +resolve(); * * // ...or, throw on nonzero exit with 'resolvex()'. * list($stdout, $stderr) = $future->resolvex(); * * @group exec */ class ExecFuture extends Future { const TIMED_OUT_EXIT_CODE = 142; protected $pipes = array(); protected $proc = null; protected $start = null; protected $timeout = null; + protected $pid; protected $stdout = null; protected $stderr = null; protected $stdin = null; protected $closePipe = false; protected $stdoutPos = 0; protected $stderrPos = 0; protected $command = null; protected $stdoutSizeLimit = PHP_INT_MAX; protected $stderrSizeLimit = PHP_INT_MAX; protected static $echoMode = array(); protected static $descriptorSpec = array( 0 => array('pipe', 'r'), // stdin 1 => array('pipe', 'w'), // stdout 2 => array('pipe', 'w'), // stderr ); - public static function pushEchoMode($mode) { self::$echoMode[] = $mode; } public static function popEchoMode() { array_pop(self::$echoMode); } public static function peekEchoMode() { return end(self::$echoMode); } public function setStdoutSizeLimit($limit) { $this->stdoutSizeLimit = $limit; } public function getStdoutSizeLimit() { return $this->stdoutSizeLimit; } public function setStderrSizeLimit($limit) { $this->stderrSizeLimit = $limit; } public function getStderrSizeLimit() { return $this->stderrSizeLimit; } + public function getPID() { + return $this->pid; + } + public function __construct($command) { $argv = func_get_args(); $this->command = call_user_func_array('csprintf', $argv); } public function __destruct() { foreach ($this->pipes as $pipe) { if (isset($pipe)) { @fclose($pipe); } } $this->pipes = array(null, null, null); if ($this->proc) { @proc_close($this->proc); $this->proc = null; } $this->stdin = null; } public function getCommand() { return $this->command; } public function read() { if ($this->start) { $this->isReady(); // Sync } $result = array( (string)substr($this->stdout, $this->stdoutPos), (string)substr($this->stderr, $this->stderrPos), ); $this->stdoutPos = strlen($this->stdout); $this->stderrPos = strlen($this->stderr); return $result; } public function write($data, $keep_pipe = false) { $this->stdin .= $data; if (!$keep_pipe) { $this->closePipe = true; } if ($this->start) { $this->isReady(); // Sync } return $this; } public function getReadSockets() { list($stdin, $stdout, $stderr) = $this->pipes; $sockets = array(); if (isset($stdout) && !feof($stdout)) { $sockets[] = $stdout; } if (isset($stderr) && !feof($stderr)) { $sockets[] = $stderr; } return $sockets; } /** * Reads some bytes from a stream, discarding output once a certain amount * has been accumulated. * * @param resource $stream * Stream to read from. * @param int $limit * Maximum number of bytes to return from $stream. If additional bytes * are available, they will be read and discarded. * @param string $description * Human-readable description of stream, for exception message. * @return string * The data read from the stream. * @throws Exception * The stream was not readable. */ protected function readAndDiscard($stream, $limit, $description) { $output = ''; do { $data = fread($stream, 4096); if (false === $data) { throw new Exception('Failed to read from '.$description); } $read_bytes = strlen($data); if ($read_bytes > 0 && $limit > 0) { if ($read_bytes > $limit) { $data = substr($data, 0, $limit); } $output .= $data; $limit -= strlen($data); } } while ($read_bytes > 0); return $output; } public function getWriteSockets() { list($stdin, $stdout, $stderr) = $this->pipes; $sockets = array(); if (isset($stdin) && strlen($this->stdin) && !feof($stdin)) { $sockets[] = $stdin; } return $sockets; } public function isReady() { if (!$this->pipes) { if (self::peekEchoMode()) { echo " >>> \$ {$this->command}\n"; } $pipes = array(); $proc = proc_open($this->command, self::$descriptorSpec, $pipes); if (!is_resource($proc)) { throw new Exception('Failed to open process.'); } + $status = proc_get_status($proc); + if (!$status) { + throw new Exception('Failed to get process status.'); + } + + $this->pid = $status['pid']; + $this->start = time(); $this->pipes = $pipes; $this->proc = $proc; list($stdin, $stdout, $stderr) = $pipes; if ((!stream_set_blocking($stdout, false)) || (!stream_set_blocking($stderr, false)) || (!stream_set_blocking($stdin, false))) { $this->__destruct(); throw new Exception('Failed to set streams nonblocking.'); } return false; } if (!$this->proc) { return true; } list($stdin, $stdout, $stderr) = $this->pipes; if (isset($this->stdin) && strlen($this->stdin)) { $bytes = fwrite($stdin, $this->stdin); if ($bytes === false) { throw new Exception('Unable to write to stdin!'); } else if ($bytes) { $this->stdin = substr($this->stdin, $bytes); } if (!strlen($this->stdin) && $this->closePipe) { @fclose($stdin); $this->pipes[0] = null; } } else { // make sure to remove any references to the pipe in the case when stdin // length was zero. avoid the overhead of calling fclose() as it is not // necessary. $this->pipes[0] = null; } // Read status before reading pipes so that we can never miss data that // arrives between our last read and the process exiting. $status = proc_get_status($this->proc); $this->stdout .= $this->readAndDiscard( $stdout, $this->getStdoutSizeLimit() - strlen($this->stdout), 'stdout'); $this->stderr .= $this->readAndDiscard( $stderr, $this->getStderrSizeLimit() - strlen($this->stderr), 'stderr'); if (!$status['running']) { $this->result = array( $status['exitcode'], $this->stdout, $this->stderr, ); $this->__destruct(); return true; } if ($this->timeout && ((time() - $this->start) >= $this->timeout)) { if (defined('SIGKILL')) { $signal = SIGKILL; } else { $signal = 9; } proc_terminate($this->proc, $signal); $this->result = array( self::TIMED_OUT_EXIT_CODE, $this->stdout, $this->stderr."\n". "(This process was prematurely terminated by timeout.)"); $this->__destruct(); return true; } } public function setTimeout($seconds) { $this->timeout = $seconds; return $this; } public function resolve($timeout = null) { if (null === $timeout) { $timeout = $this->timeout; } return parent::resolve($timeout); } public function resolvex($timeout = null) { list($err, $stdout, $stderr) = $this->resolve($timeout); if ($err) { $cmd = $this->command; throw new CommandException( "Command '{$cmd}' failed with error #{$err}:\n". "stdout:\n{$stdout}\n". "stderr:\n{$stderr}\n", $cmd, $err, $stdout, $stderr); } return array($stdout, $stderr); } public function resolveJSON($timeout = null) { list($stdout, $stderr) = $this->resolvex($timeout); if (strlen($stderr)) { $cmd = $this->command; throw new CommandException( "JSON command '{$cmd}' emitted text to stderr when none was expected: ". $stderr, $cmd, 0, $stdout, $stderr); } $object = json_decode($stdout, true); if (!is_array($object)) { $cmd = $this->command; throw new CommandException( "JSON command '{$cmd}' did not produce a valid JSON object on stdout: ". $stdout, $cmd, 0, $stdout, $stderr); } return $object; } }