diff --git a/src/filesystem/__tests__/PhutilDeferredLogTestCase.php b/src/filesystem/__tests__/PhutilDeferredLogTestCase.php
index 481a2a4..83797be 100644
--- a/src/filesystem/__tests__/PhutilDeferredLogTestCase.php
+++ b/src/filesystem/__tests__/PhutilDeferredLogTestCase.php
@@ -1,166 +1,167 @@
<?php
final class PhutilDeferredLogTestCase extends PhutilTestCase {
public function testLogging() {
$this->checkLog(
"derp\n",
'derp',
array());
$this->checkLog(
"[20 Aug 1984] alincoln\n",
'[%T] %u',
array(
'T' => '20 Aug 1984',
'u' => 'alincoln',
));
$this->checkLog(
"%%%%%\n",
'%%%%%%%%%%',
array(
'%' => '%',
));
$this->checkLog(
"\\000\\001\\002\n",
'%a%b%c',
array(
'a' => chr(0),
'b' => chr(1),
'c' => chr(2),
));
$this->checkLog(
"Download: 100%\n",
'Download: %C',
array(
'C' => '100%',
));
$this->checkLog(
"- bee -\n",
'%a %b %c',
array(
'b' => 'bee',
));
$this->checkLog(
"\\\\\n",
'%b',
array(
'b' => '\\',
));
$this->checkLog(
"a\t\\t\n",
"%a\t%b",
array(
'a' => 'a',
'b' => "\t",
));
$this->checkLog(
"\1ab\n",
"\1a%a",
array(
'a' => 'b',
));
$this->checkLog(
"a % xb\n",
'%a %% x%b',
array(
'a' => 'a',
'b' => 'b',
));
}
public function testLogWriteFailure() {
$caught = null;
try {
if (phutil_is_hiphop_runtime()) {
// In HipHop exceptions thrown in destructors are not normally
// catchable, so call __destruct() explicitly.
$log = new PhutilDeferredLog('/derp/derp/derp/derp/derp', 'derp');
$log->__destruct();
} else {
new PhutilDeferredLog('/derp/derp/derp/derp/derp', 'derp');
}
} catch (Exception $ex) {
$caught = $ex;
}
$this->assertTrue($caught instanceof Exception);
}
public function testManyWriters() {
$root = phutil_get_library_root('phutil').'/../';
$bin = $root.'scripts/test/deferred_log.php';
$n_writers = 3;
$n_lines = 8;
$tmp = new TempFile();
$futures = array();
for ($ii = 0; $ii < $n_writers; $ii++) {
$futures[] = new ExecFuture('%s %d %s', $bin, $n_lines, (string)$tmp);
}
- Futures($futures)->resolveAll();
+ id(new FutureIterator($futures))
+ ->resolveAll();
$this->assertEqual(
str_repeat("abcdefghijklmnopqrstuvwxyz\n", $n_writers * $n_lines),
Filesystem::readFile($tmp));
}
public function testNoWrite() {
$tmp = new TempFile();
$log = new PhutilDeferredLog($tmp, 'xyz');
$log->setFile(null);
unset($log);
$this->assertEqual('', Filesystem::readFile($tmp), 'No Write');
}
public function testDoubleWrite() {
$tmp = new TempFile();
$log = new PhutilDeferredLog($tmp, 'xyz');
$log->write();
$log->write();
unset($log);
$this->assertEqual("xyz\n", Filesystem::readFile($tmp), 'Double Write');
}
public function testSetAfterWrite() {
$tmp1 = new TempFile();
$tmp2 = new TempFile();
$log = new PhutilDeferredLog($tmp1, 'xyz');
$log->write();
$caught = null;
try {
$log->setFile($tmp2);
} catch (Exception $ex) {
$caught = $ex;
}
$this->assertTrue($caught instanceof Exception, 'Set After Write');
}
private function checkLog($expect, $format, $data) {
$tmp = new TempFile();
$log = new PhutilDeferredLog($tmp, $format);
$log->setData($data);
unset($log);
$this->assertEqual($expect, Filesystem::readFile($tmp), $format);
}
}
diff --git a/src/future/FutureIterator.php b/src/future/FutureIterator.php
index 10165d8..147757f 100644
--- a/src/future/FutureIterator.php
+++ b/src/future/FutureIterator.php
@@ -1,325 +1,329 @@
<?php
/**
* FutureIterator aggregates @{class:Future}s and allows you to respond to them
* in the order they resolve. This is useful because it minimizes the amount of
* time your program spends waiting on parallel processes.
*
* $futures = array(
* 'a.txt' => new ExecFuture('wc -c a.txt'),
* 'b.txt' => new ExecFuture('wc -c b.txt'),
* 'c.txt' => new ExecFuture('wc -c c.txt'),
* );
*
- * foreach (Futures($futures) as $key => $future) {
+ * foreach (new FutureIterator($futures) as $key => $future) {
* // IMPORTANT: keys are preserved but the order of elements is not. This
* // construct iterates over the futures in the order they resolve, so the
* // fastest future is the one you'll get first. This allows you to start
* // doing followup processing as soon as possible.
*
* list($err, $stdout) = $future->resolve();
* do_some_processing($stdout);
* }
*
* For a general overview of futures, see @{article:Using Futures}.
*
* @task basics Basics
* @task config Configuring Iteration
* @task iterator Iterator Interface
* @task internal Internals
*/
final class FutureIterator implements Iterator {
protected $wait = array();
protected $work = array();
protected $futures = array();
protected $key;
protected $limit;
protected $timeout;
protected $isTimeout = false;
/* -( Basics )------------------------------------------------------------- */
/**
* Create a new iterator over a list of futures. By convention, use the
* convenience function @{function:Futures} instead of instantiating this
* class directly.
*
* @param list List of @{class:Future}s to resolve.
* @task basics
*/
public function __construct(array $futures) {
assert_instances_of($futures, 'Future');
$this->futures = $futures;
}
/**
* Block until all futures resolve.
*
* @return void
* @task basics
*/
public function resolveAll() {
foreach ($this as $future) {
$future->resolve();
}
}
/**
* Add another future to the set of futures. This is useful if you have a
* set of futures to run mostly in parallel, but some futures depend on
* others.
*
* @param Future @{class:Future} to add to iterator
* @task basics
*/
public function addFuture(Future $future, $key = null) {
if ($key === null) {
$this->futures[] = $future;
$this->wait[] = last_key($this->futures);
} else if (!isset($this->futures[$key])) {
$this->futures[$key] = $future;
$this->wait[] = $key;
} else {
throw new Exception("Invalid key {$key}");
}
// Start running the future if we don't have $this->limit futures running
// already. updateWorkingSet() won't start running the future if there's no
// limit, so we'll manually poke it here in that case.
$this->updateWorkingSet();
if (!$this->limit) {
$future->isReady();
}
return $this;
}
/* -( Configuring Iteration )---------------------------------------------- */
/**
* Set a maximum amount of time you want to wait before the iterator will
* yield a result. If no future has resolved yet, the iterator will yield
* null for key and value. Among other potential uses, you can use this to
* show some busy indicator:
*
- * foreach (Futures($futures)->setUpdateInterval(1) as $future) {
+ * $futures = id(new FutureIterator($futures))
+ * ->setUpdateInterval(1);
+ * foreach ($futures as $future) {
* if ($future === null) {
* echo "Still working...\n";
* } else {
* // ...
* }
* }
*
* This will echo "Still working..." once per second as long as futures are
* resolving. By default, FutureIterator never yields null.
*
* @param float Maximum number of seconds to block waiting on futures before
* yielding null.
* @return this
*
* @task config
*/
public function setUpdateInterval($interval) {
$this->timeout = $interval;
return $this;
}
/**
* Limit the number of simultaneously executing futures.
*
- * foreach (Futures($futures)->limit(4) as $future) {
+ * $futures = id(new FutureIterator($futures))
+ * ->limit(4);
+ * foreach ($futures as $future) {
* // Run no more than 4 futures simultaneously.
* }
*
* @param int Maximum number of simultaneous jobs allowed.
* @return this
*
* @task config
*/
public function limit($max) {
$this->limit = $max;
return $this;
}
/* -( Iterator Interface )------------------------------------------------- */
/**
* @task iterator
*/
public function rewind() {
$this->wait = array_keys($this->futures);
$this->work = null;
$this->updateWorkingSet();
$this->next();
}
/**
* @task iterator
*/
public function next() {
$this->key = null;
if (!count($this->wait)) {
return;
}
$read_sockets = array();
$write_sockets = array();
$start = microtime(true);
$timeout = $this->timeout;
$this->isTimeout = false;
$check = $this->getWorkingSet();
$resolve = null;
do {
$read_sockets = array();
$write_sockets = array();
$can_use_sockets = true;
$wait_time = 1;
foreach ($check as $wait => $key) {
$future = $this->futures[$key];
try {
if ($future->getException()) {
$resolve = $wait;
continue;
}
if ($future->isReady()) {
if ($resolve === null) {
$resolve = $wait;
}
continue;
}
$got_sockets = false;
$socks = $future->getReadSockets();
if ($socks) {
$got_sockets = true;
foreach ($socks as $socket) {
$read_sockets[] = $socket;
}
}
$socks = $future->getWriteSockets();
if ($socks) {
$got_sockets = true;
foreach ($socks as $socket) {
$write_sockets[] = $socket;
}
}
// If any currently active future had neither read nor write sockets,
// we can't wait for the current batch of items using sockets.
if (!$got_sockets) {
$can_use_sockets = false;
} else {
$wait_time = min($wait_time, $future->getDefaultWait());
}
} catch (Exception $ex) {
$this->futures[$key]->setException($ex);
$resolve = $wait;
break;
}
}
if ($resolve === null) {
// Check for a setUpdateInterval() timeout.
if ($timeout !== null) {
$elapsed = microtime(true) - $start;
if ($elapsed > $timeout) {
$this->isTimeout = true;
return;
} else {
$wait_time = $timeout - $elapsed;
}
}
if ($can_use_sockets) {
Future::waitForSockets($read_sockets, $write_sockets, $wait_time);
} else {
usleep(1000);
}
}
} while ($resolve === null);
$this->key = $this->wait[$resolve];
unset($this->wait[$resolve]);
$this->updateWorkingSet();
}
/**
* @task iterator
*/
public function current() {
if ($this->isTimeout) {
return null;
}
return $this->futures[$this->key];
}
/**
* @task iterator
*/
public function key() {
if ($this->isTimeout) {
return null;
}
return $this->key;
}
/**
* @task iterator
*/
public function valid() {
if ($this->isTimeout) {
return true;
}
return ($this->key !== null);
}
/* -( Internals )---------------------------------------------------------- */
/**
* @task internal
*/
protected function getWorkingSet() {
if ($this->work === null) {
return $this->wait;
}
return $this->work;
}
/**
* @task internal
*/
protected function updateWorkingSet() {
if (!$this->limit) {
return;
}
$old = $this->work;
$this->work = array_slice($this->wait, 0, $this->limit, true);
// If we're using a limit, our futures are sleeping and need to be polled
// to begin execution, so poll any futures which weren't in our working set
// before.
foreach ($this->work as $work => $key) {
if (!isset($old[$work])) {
$this->futures[$key]->isReady();
}
}
}
}
diff --git a/src/future/exec/__tests__/ExecFutureTestCase.php b/src/future/exec/__tests__/ExecFutureTestCase.php
index 70513c1..2d668a0 100644
--- a/src/future/exec/__tests__/ExecFutureTestCase.php
+++ b/src/future/exec/__tests__/ExecFutureTestCase.php
@@ -1,156 +1,156 @@
<?php
final class ExecFutureTestCase extends PhutilTestCase {
public function testEmptyWrite() {
// NOTE: This is mostly testing that we don't hang while doing an empty
// write.
list($stdout) = id(new ExecFuture('cat'))->write('')->resolvex();
$this->assertEqual('', $stdout);
}
public function testKeepPipe() {
// NOTE: This is mostly testing the semantics of $keep_pipe in write().
list($stdout) = id(new ExecFuture('cat'))
->write('', true)
->start()
->write('x', true)
->write('y', true)
->write('z', false)
->resolvex();
$this->assertEqual('xyz', $stdout);
}
public function testLargeBuffer() {
// NOTE: This is mostly a coverage test to hit branches where we're still
// flushing a buffer.
$data = str_repeat('x', 1024 * 1024 * 4);
list($stdout) = id(new ExecFuture('cat'))->write($data)->resolvex();
$this->assertEqual($data, $stdout);
}
public function testBufferLimit() {
$data = str_repeat('x', 1024 * 1024);
list($stdout) = id(new ExecFuture('cat'))
->setStdoutSizeLimit(1024)
->write($data)
->resolvex();
$this->assertEqual(substr($data, 0, 1024), $stdout);
}
public function testResolveTimeoutTestShouldRunLessThan1Sec() {
// NOTE: This tests interactions between the resolve() timeout and the
// ExecFuture timeout, which are similar but not identical.
$future = id(new ExecFuture('sleep 32000'))->start();
$future->setTimeout(32000);
// We expect this to return in 0.01s.
$result = $future->resolve(0.01);
$this->assertEqual($result, null);
// We expect this to now force the time out / kill immediately. If we don't
// do this, we'll hang when exiting until our subprocess exits (32000
// seconds!)
$future->setTimeout(0.01);
$future->resolve();
}
public function testTimeoutTestShouldRunLessThan1Sec() {
// NOTE: This is partly testing that we choose appropriate select wait
// times; this test should run for significantly less than 1 second.
$future = new ExecFuture('sleep 32000');
list($err) = $future->setTimeout(0.01)->resolve();
$this->assertTrue($err > 0);
$this->assertTrue($future->getWasKilledByTimeout());
}
public function testMultipleTimeoutsTestShouldRunLessThan1Sec() {
$futures = array();
for ($ii = 0; $ii < 4; $ii++) {
$futures[] = id(new ExecFuture('sleep 32000'))->setTimeout(0.01);
}
- foreach (Futures($futures) as $future) {
+ foreach (new FutureIterator($futures) as $future) {
list ($err) = $future->resolve();
$this->assertTrue($err > 0);
$this->assertTrue($future->getWasKilledByTimeout());
}
}
public function testNoHangOnExecFutureDestructionWithRunningChild() {
$start = microtime(true);
$future = new ExecFuture('sleep 30');
$future->start();
unset($future);
$end = microtime(true);
// If ExecFuture::__destruct() hangs until the child closes, we won't make
// it here in time.
$this->assertTrue(($end - $start) < 5);
}
public function testMultipleResolves() {
// It should be safe to call resolve(), resolvex(), resolveKill(), etc.,
// as many times as you want on the same process.
$future = new ExecFuture('echo quack');
$future->resolve();
$future->resolvex();
list($err) = $future->resolveKill();
$this->assertEqual(0, $err);
}
public function testReadBuffering() {
$str_len_8 = 'abcdefgh';
$str_len_4 = 'abcd';
// This is a write/read with no read buffer.
$future = new ExecFuture('cat');
$future->write($str_len_8);
do {
$future->isReady();
list($read) = $future->read();
if (strlen($read)) {
break;
}
} while (true);
// We expect to get the entire string back in the read.
$this->assertEqual($str_len_8, $read);
$future->resolve();
// This is a write/read with a read buffer.
$future = new ExecFuture('cat');
$future->write($str_len_8);
// Set the read buffer size.
$future->setReadBufferSize(4);
do {
$future->isReady();
list($read) = $future->read();
if (strlen($read)) {
break;
}
} while (true);
// We expect to get the entire string back in the read.
$this->assertEqual($str_len_4, $read);
$future->resolve();
}
}
diff --git a/src/future/query/QueryFuture.php b/src/future/query/QueryFuture.php
index 8ccc23f..9878581 100644
--- a/src/future/query/QueryFuture.php
+++ b/src/future/query/QueryFuture.php
@@ -1,129 +1,129 @@
<?php
/**
* This class provides several approaches for querying data from the database:
*
* # Async queries: Used under MySQLi with MySQLnd.
* # Parallel queries: Used under HPHP.
* # Multi queries: Used under MySQLi or HPHP.
* # Single queries: Used under MySQL.
*
* The class automatically decides which approach to use. Usage is like with
* other futures:
*
* $futures = array();
* $futures[] = new QueryFuture($conn1, 'SELECT 1');
* $futures[] = new QueryFuture($conn1, 'DELETE FROM table');
* $futures[] = new QueryFuture($conn2, 'SELECT 2');
*
- * foreach (Futures($futures) as $future) {
+ * foreach (new FutureIterator($futures) as $future) {
* try {
* $result = $future->resolve();
* } catch (AphrontQueryException $ex) {
* }
* }
*
* `$result` contains a list of dicts for select queries or number of modified
* rows for modification queries.
*/
final class QueryFuture extends Future {
private static $futures = array();
private $conn;
private $query;
private $id;
private $async;
private $profilerCallID;
public function __construct(
AphrontDatabaseConnection $conn,
$pattern/* , ... */) {
$this->conn = $conn;
$args = func_get_args();
$args = array_slice($args, 2);
$this->query = vqsprintf($conn, $pattern, $args);
self::$futures[] = $this;
$this->id = last_key(self::$futures);
}
public function isReady() {
if ($this->result !== null || $this->exception) {
return true;
}
if (!$this->conn->supportsAsyncQueries()) {
if ($this->conn->supportsParallelQueries()) {
$queries = array();
$conns = array();
foreach (self::$futures as $id => $future) {
$queries[$id] = $future->query;
$conns[$id] = $future->conn;
}
$results = $this->conn->executeParallelQueries($queries, $conns);
$this->processResults($results);
return true;
}
$conns = array();
$conn_queries = array();
foreach (self::$futures as $id => $future) {
$hash = spl_object_hash($future->conn);
$conns[$hash] = $future->conn;
$conn_queries[$hash][$id] = $future->query;
}
foreach ($conn_queries as $hash => $queries) {
$this->processResults($conns[$hash]->executeRawQueries($queries));
}
return true;
}
if (!$this->async) {
$profiler = PhutilServiceProfiler::getInstance();
$this->profilerCallID = $profiler->beginServiceCall(
array(
'type' => 'query',
'query' => $this->query,
'async' => true,
));
$this->async = $this->conn->asyncQuery($this->query);
return false;
}
$conns = array();
$asyncs = array();
foreach (self::$futures as $id => $future) {
if ($future->async) {
$conns[$id] = $future->conn;
$asyncs[$id] = $future->async;
}
}
$this->processResults($this->conn->resolveAsyncQueries($conns, $asyncs));
if ($this->result !== null || $this->exception) {
return true;
}
return false;
}
private function processResults(array $results) {
foreach ($results as $id => $result) {
$future = self::$futures[$id];
if ($result instanceof Exception) {
$future->exception = $result;
} else {
$future->result = $result;
}
unset(self::$futures[$id]);
if ($future->profilerCallID) {
$profiler = PhutilServiceProfiler::getInstance();
$profiler->endServiceCall($future->profilerCallID, array());
}
}
}
}
diff --git a/src/moduleutils/PhutilLibraryMapBuilder.php b/src/moduleutils/PhutilLibraryMapBuilder.php
index b2b25c0..5c2681a 100644
--- a/src/moduleutils/PhutilLibraryMapBuilder.php
+++ b/src/moduleutils/PhutilLibraryMapBuilder.php
@@ -1,486 +1,488 @@
<?php
/**
* Build maps of libphutil libraries. libphutil uses the library map to locate
* and load classes and functions in the library.
*
* @task map Mapping libphutil Libraries
* @task path Path Management
* @task symbol Symbol Analysis and Caching
* @task source Source Management
*/
final class PhutilLibraryMapBuilder {
private $root;
private $quiet = true;
private $subprocessLimit = 8;
private $fileSymbolMap;
private $librarySymbolMap;
const LIBRARY_MAP_VERSION_KEY = '__library_version__';
const LIBRARY_MAP_VERSION = 2;
const SYMBOL_CACHE_VERSION_KEY = '__symbol_cache_version__';
const SYMBOL_CACHE_VERSION = 11;
/* -( Mapping libphutil Libraries )---------------------------------------- */
/**
* Create a new map builder for a library.
*
* @param string Path to the library root.
*
* @task map
*/
public function __construct($root) {
$this->root = $root;
}
/**
* Control status output. Use `--quiet` to set this.
*
* @param bool If true, don't show status output.
* @return this
*
* @task map
*/
public function setQuiet($quiet) {
$this->quiet = $quiet;
return $this;
}
/**
* Control subprocess parallelism limit. Use `--limit` to set this.
*
* @param int Maximum number of subprocesses to run in parallel.
* @return this
*
* @task map
*/
public function setSubprocessLimit($limit) {
$this->subprocessLimit = $limit;
return $this;
}
/**
* Get the map of symbols in this library, analyzing the library to build it
* if necessary.
*
* @return map<string, wild> Information about symbols in this library.
*
* @task map
*/
public function buildMap() {
if ($this->librarySymbolMap === null) {
$this->analyzeLibrary();
}
return $this->librarySymbolMap;
}
/**
* Get the map of files in this library, analyzing the library to build it
* if necessary.
*
* Returns a map of file paths to information about symbols used and defined
* in the file.
*
* @return map<string, wild> Information about files in this library.
*
* @task map
*/
public function buildFileSymbolMap() {
if ($this->fileSymbolMap === null) {
$this->analyzeLibrary();
}
return $this->fileSymbolMap;
}
/**
* Build and update the library map.
*
* @return void
*
* @task map
*/
public function buildAndWriteMap() {
$library_map = $this->buildMap();
$this->log("Writing map...\n");
$this->writeLibraryMap($library_map);
}
/**
* Write a status message to the user, if not running in quiet mode.
*
* @param string Message to write.
* @return this
*
* @task map
*/
private function log($message) {
if (!$this->quiet) {
@fwrite(STDERR, $message);
}
return $this;
}
/* -( Path Management )---------------------------------------------------- */
/**
* Get the path to some file in the library.
*
* @param string A library-relative path. If omitted, returns the library
* root path.
* @return string An absolute path.
*
* @task path
*/
private function getPath($path = '') {
return $this->root.'/'.$path;
}
/**
* Get the path to the symbol cache file.
*
* @return string Absolute path to symbol cache.
*
* @task path
*/
private function getPathForSymbolCache() {
return $this->getPath('.phutil_module_cache');
}
/**
* Get the path to the map file.
*
* @return string Absolute path to the library map.
*
* @task path
*/
private function getPathForLibraryMap() {
return $this->getPath('__phutil_library_map__.php');
}
/**
* Get the path to the library init file.
*
* @return string Absolute path to the library init file
*
* @task path
*/
private function getPathForLibraryInit() {
return $this->getPath('__phutil_library_init__.php');
}
/* -( Symbol Analysis and Caching )---------------------------------------- */
/**
* Load the library symbol cache, if it exists and is readable and valid.
*
* @return dict Map of content hashes to cache of output from
* `phutil_symbols.php`.
*
* @task symbol
*/
private function loadSymbolCache() {
$cache_file = $this->getPathForSymbolCache();
try {
$cache = Filesystem::readFile($cache_file);
} catch (Exception $ex) {
$cache = null;
}
$symbol_cache = array();
if ($cache) {
$symbol_cache = json_decode($cache, true);
if (!is_array($symbol_cache)) {
$symbol_cache = array();
}
}
$version = idx($symbol_cache, self::SYMBOL_CACHE_VERSION_KEY);
if ($version != self::SYMBOL_CACHE_VERSION) {
// Throw away caches from a different version of the library.
$symbol_cache = array();
}
unset($symbol_cache[self::SYMBOL_CACHE_VERSION_KEY]);
return $symbol_cache;
}
/**
* Write a symbol map to disk cache.
*
* @param dict Symbol map of relative paths to symbols.
* @param dict Source map (like @{method:loadSourceFileMap}).
* @return void
*
* @task symbol
*/
private function writeSymbolCache(array $symbol_map, array $source_map) {
$cache_file = $this->getPathForSymbolCache();
$cache = array(
self::SYMBOL_CACHE_VERSION_KEY => self::SYMBOL_CACHE_VERSION,
);
foreach ($symbol_map as $file => $symbols) {
$cache[$source_map[$file]] = $symbols;
}
$json = json_encode($cache);
try {
Filesystem::writeFile($cache_file, $json);
} catch (FilesystemException $ex) {
$this->log("Unable to save the cache!\n");
}
}
/**
* Drop the symbol cache, forcing a clean rebuild.
*
* @return this
*
* @task symbol
*/
public function dropSymbolCache() {
$this->log("Dropping symbol cache...\n");
Filesystem::remove($this->getPathForSymbolCache());
}
/**
* Build a future which returns a `phutil_symbols.php` analysis of a source
* file.
*
* @param string Relative path to the source file to analyze.
* @return Future Analysis future.
*
* @task symbol
*/
private function buildSymbolAnalysisFuture($file) {
$absolute_file = $this->getPath($file);
$bin = dirname(__FILE__).'/../../scripts/phutil_symbols.php';
return new ExecFuture('php %s --ugly -- %s', $bin, $absolute_file);
}
/* -( Source Management )-------------------------------------------------- */
/**
* Build a map of all source files in a library to hashes of their content.
* Returns an array like this:
*
* array(
* 'src/parser/ExampleParser.php' => '60b725f10c9c85c70d97880dfe8191b3',
* // ...
* );
*
* @return dict Map of library-relative paths to content hashes.
* @task source
*/
private function loadSourceFileMap() {
$root = $this->getPath();
$init = $this->getPathForLibraryInit();
if (!Filesystem::pathExists($init)) {
throw new Exception("Provided path '{$root}' is not a phutil library.");
}
$files = id(new FileFinder($root))
->withType('f')
->withSuffix('php')
->excludePath('*/.*')
->setGenerateChecksums(true)
->find();
$map = array();
foreach ($files as $file => $hash) {
$file = Filesystem::readablePath($file, $root);
$file = ltrim($file, '/');
if (dirname($file) == '.') {
// We don't permit normal source files at the root level, so just ignore
// them; they're special library files.
continue;
}
if (dirname($file) == 'extensions') {
// Ignore files in the extensions/ directory.
continue;
}
// We include also filename in the hash to handle cases when the file is
// moved without modifying its content.
$map[$file] = md5($hash.$file);
}
return $map;
}
/**
* Convert the symbol analysis of all the source files in the library into
* a library map.
*
* @param dict Symbol analysis of all source files.
* @return dict Library map.
* @task source
*/
private function buildLibraryMap(array $symbol_map) {
$library_map = array(
'class' => array(),
'function' => array(),
'xmap' => array(),
);
// Detect duplicate symbols within the library.
foreach ($symbol_map as $file => $info) {
foreach ($info['have'] as $type => $symbols) {
foreach ($symbols as $symbol => $declaration) {
$lib_type = ($type == 'interface') ? 'class' : $type;
if (!empty($library_map[$lib_type][$symbol])) {
$prior = $library_map[$lib_type][$symbol];
throw new Exception(
"Definition of {$type} '{$symbol}' in file '{$file}' duplicates ".
"prior definition in file '{$prior}'. You can not declare the ".
"same symbol twice.");
}
$library_map[$lib_type][$symbol] = $file;
}
}
$library_map['xmap'] += $info['xmap'];
}
// Simplify the common case (one parent) to make the file a little easier
// to deal with.
foreach ($library_map['xmap'] as $class => $extends) {
if (count($extends) == 1) {
$library_map['xmap'][$class] = reset($extends);
}
}
// Sort the map so it is relatively stable across changes.
foreach ($library_map as $key => $symbols) {
ksort($symbols);
$library_map[$key] = $symbols;
}
ksort($library_map);
return $library_map;
}
/**
* Write a finalized library map.
*
* @param dict Library map structure to write.
* @return void
*
* @task source
*/
private function writeLibraryMap(array $library_map) {
$map_file = $this->getPathForLibraryMap();
$version = self::LIBRARY_MAP_VERSION;
$library_map = array(
self::LIBRARY_MAP_VERSION_KEY => $version,
) + $library_map;
$library_map = phutil_var_export($library_map);
$at = '@';
$source_file = <<<EOPHP
<?php
/**
* This file is automatically generated. Use 'arc liberate' to rebuild it.
*
* {$at}generated
* {$at}phutil-library-version {$version}
*/
phutil_register_library_map({$library_map});
EOPHP;
Filesystem::writeFile($map_file, $source_file);
}
/**
* Analyze the library, generating the file and symbol maps.
*
* @return void
*/
private function analyzeLibrary() {
// Identify all the ".php" source files in the library.
$this->log("Finding source files...\n");
$source_map = $this->loadSourceFileMap();
$this->log("Found ".number_format(count($source_map))." files.\n");
// Load the symbol cache with existing parsed symbols. This allows us
// to remap libraries quickly by analyzing only changed files.
$this->log("Loading symbol cache...\n");
$symbol_cache = $this->loadSymbolCache();
// Build out the symbol analysis for all the files in the library. For
// each file, check if it's in cache. If we miss in the cache, do a fresh
// analysis.
$symbol_map = array();
$futures = array();
foreach ($source_map as $file => $hash) {
if (!empty($symbol_cache[$hash])) {
$symbol_map[$file] = $symbol_cache[$hash];
continue;
}
$futures[$file] = $this->buildSymbolAnalysisFuture($file);
}
$this->log("Found ".number_format(count($symbol_map))." files in cache.\n");
// Run the analyzer on any files which need analysis.
if ($futures) {
$limit = $this->subprocessLimit;
$count = number_format(count($futures));
$this->log("Analyzing {$count} files with {$limit} subprocesses...\n");
$progress = new PhutilConsoleProgressBar();
if ($this->quiet) {
$progress->setQuiet(true);
}
$progress->setTotal(count($futures));
- foreach (Futures($futures)->limit($limit) as $file => $future) {
+ $futures = id(new FutureIterator($futures))
+ ->limit($limit);
+ foreach ($futures as $file => $future) {
$result = $future->resolveJSON();
if (empty($result['error'])) {
$symbol_map[$file] = $result;
} else {
$progress->done(false);
throw new XHPASTSyntaxErrorException(
$result['line'],
$file.': '.$result['error']);
}
$progress->update(1);
}
$progress->done();
}
$this->fileSymbolMap = $symbol_map;
// We're done building the cache, so write it out immediately. Note that
// we've only retained entries for files we found, so this implicitly cleans
// out old cache entries.
$this->writeSymbolCache($symbol_map, $source_map);
// Our map is up to date, so either show it on stdout or write it to disk.
$this->log("Building library map...\n");
$this->librarySymbolMap = $this->buildLibraryMap($symbol_map);
}
}