diff --git a/scripts/phutil_rebuild_map.php b/scripts/phutil_rebuild_map.php index 16b2a7b..0e420a9 100755 --- a/scripts/phutil_rebuild_map.php +++ b/scripts/phutil_rebuild_map.php @@ -1,70 +1,78 @@ #!/usr/bin/env php setTagline('rebuild the library map file'); $args->setSynopsis(<<parseStandardArguments(); $args->parse( array( array( 'name' => 'quiet', 'help' => 'Do not write status messages to stderr.', ), array( 'name' => 'drop-cache', 'help' => 'Drop the symbol cache and rebuild the entire map from '. 'scratch.', ), array( 'name' => 'limit', 'param' => 'N', 'default' => 8, 'help' => 'Controls the number of symbol mapper subprocesses run '. 'at once. Defaults to 8.', ), array( 'name' => 'show', 'help' => 'Print symbol map to stdout instead of writing it to the '. 'map file.', ), array( 'name' => 'ugly', 'help' => 'Use faster but less readable serialization for --show.', ), array( 'name' => 'root', 'wildcard' => true, - ) + ), )); $root = $args->getArg('root'); if (count($root) !== 1) { throw new Exception('Provide exactly one library root!'); } $root = Filesystem::resolvePath(head($root)); $builder = new PhutilLibraryMapBuilder($root); $builder->setQuiet($args->getArg('quiet')); $builder->setSubprocessLimit($args->getArg('limit')); if ($args->getArg('drop-cache')) { $builder->dropSymbolCache(); } if ($args->getArg('show')) { - $builder->setShowMap(true); - $builder->setUgly($args->getArg('ugly')); + $builder->setDryRun(true); +} + +$library_map = $builder->buildMap(); + +if ($args->getArg('show')) { + if ($args->getArg('ugly')) { + echo json_encode($library_map); + } else { + $json = new PhutilJSON(); + echo $json->encodeFormatted($library_map); + } } -$builder->buildMap(); exit(0); diff --git a/src/__tests__/PhutilInfrastructureTestCase.php b/src/__tests__/PhutilInfrastructureTestCase.php index da3171d..8739020 100644 --- a/src/__tests__/PhutilInfrastructureTestCase.php +++ b/src/__tests__/PhutilInfrastructureTestCase.php @@ -1,13 +1,38 @@ selectAndLoadSymbols(); $this->assertTrue(true); } + + /** + * This is more of an acceptance test case instead of a unit test. It verifies + * that all the library map is up-to-date. + */ + public function testLibraryMap() { + $root = phutil_get_library_root('phutil'); + + $new_library_map = id(new PhutilLibraryMapBuilder($root)) + ->setQuiet(true) + ->setDryRun(true) + ->buildMap(); + + $bootloader = PhutilBootloader::getInstance(); + $old_library_map = $bootloader->getLibraryMap('phutil'); + unset($old_library_map[PhutilLibraryMapBuilder::LIBRARY_MAP_VERSION_KEY]); + + $this->assertEqual( + $new_library_map, + $old_library_map, + 'The library map does not appear to be up-to-date. Try '. + 'rebuilding the map with `arc liberate`.'); + } + } diff --git a/src/moduleutils/PhutilLibraryMapBuilder.php b/src/moduleutils/PhutilLibraryMapBuilder.php index dab121b..d9bada8 100644 --- a/src/moduleutils/PhutilLibraryMapBuilder.php +++ b/src/moduleutils/PhutilLibraryMapBuilder.php @@ -1,511 +1,462 @@ root = $root; } - /** - * Control status output. Use --quiet to set this. + * Control status output. Use `--quiet` to set this. * - * @param bool If true, don't show status output. - * @return this + * @param bool If true, don't show status output. + * @return this * * @task map */ public function setQuiet($quiet) { $this->quiet = $quiet; return $this; } - /** - * Control subprocess parallelism limit. Use --limit to set this. + * Control subprocess parallelism limit. Use `--limit` to set this. * - * @param int Maximum number of subprocesses to run in parallel. - * @return this + * @param int Maximum number of subprocesses to run in parallel. + * @return this * * @task map */ public function setSubprocessLimit($limit) { $this->subprocessLimit = $limit; return $this; } - - /** - * Control whether the ugly (but fast) or pretty (but slower) JSON formatter - * is used. - * - * @param bool If true, use the fastest formatter. - * @return this - * - * @task map - */ - public function setUgly($ugly) { - $this->ugly = $ugly; - return $this; - } - - /** * Control whether the map should be rebuilt, or just shown (printed to * stdout in JSON). * - * @param bool If true, show map instead of updating. + * @param bool If true, show map instead of updating. * @return this * * @task map */ - public function setShowMap($show_map) { - $this->showMap = $show_map; + public function setDryRun($dry_run) { + $this->dryRun = $dry_run; return $this; } - /** * Build or rebuild the library map. * - * @return this + * @return dict * * @task map */ public function buildMap() { - // Identify all the ".php" source files in the library. $this->log("Finding source files...\n"); $source_map = $this->loadSourceFileMap(); $this->log("Found ".number_format(count($source_map))." files.\n"); - // Load the symbol cache with existing parsed symbols. This allows us // to remap libraries quickly by analyzing only changed files. $this->log("Loading symbol cache...\n"); $symbol_cache = $this->loadSymbolCache(); // Build out the symbol analysis for all the files in the library. For // each file, check if it's in cache. If we miss in the cache, do a fresh // analysis. $symbol_map = array(); $futures = array(); foreach ($source_map as $file => $hash) { if (!empty($symbol_cache[$hash])) { $symbol_map[$file] = $symbol_cache[$hash]; continue; } $futures[$file] = $this->buildSymbolAnalysisFuture($file); } $this->log("Found ".number_format(count($symbol_map))." files in cache.\n"); - // Run the analyzer on any files which need analysis. if ($futures) { $limit = $this->subprocessLimit; $count = number_format(count($futures)); $this->log("Analyzing {$count} files with {$limit} subprocesses...\n"); $progress = new PhutilConsoleProgressBar(); if ($this->quiet) { $progress->setQuiet(true); } $progress->setTotal(count($futures)); foreach (Futures($futures)->limit($limit) as $file => $future) { $result = $future->resolveJSON(); if (empty($result['error'])) { $symbol_map[$file] = $result; } else { $progress->done(false); echo phutil_console_format( "\n**SYNTAX ERROR!**\nFile: %s\nLine: %d\n\n%s\n", Filesystem::readablePath($result['file']), $result['line'], $result['error']); exit(1); } $progress->update(1); } $progress->done(); $this->log("\nDone.\n"); } - // We're done building the cache, so write it out immediately. Note that // we've only retained entries for files we found, so this implicitly cleans // out old cache entries. $this->writeSymbolCache($symbol_map, $source_map); - // Our map is up to date, so either show it on stdout or write it to disk. + $this->log("Building library map...\n"); + $library_map = $this->buildLibraryMap($symbol_map); - if ($this->showMap) { - $this->log("Showing map...\n"); - - if ($this->ugly) { - echo json_encode($symbol_map); - } else { - $json = new PhutilJSON(); - echo $json->encodeFormatted($symbol_map); - } - } else { - $this->log("Building library map...\n"); - $library_map = $this->buildLibraryMap($symbol_map); - + if (!$this->dryRun) { $this->log("Writing map...\n"); $this->writeLibraryMap($library_map); } $this->log("Done.\n"); - - return $this; + return $library_map; } - /** * Write a status message to the user, if not running in quiet mode. * - * @param string Message to write. - * @return this + * @param string Message to write. + * @return this * * @task map */ private function log($message) { if (!$this->quiet) { @fwrite(STDERR, $message); } return $this; } /* -( Path Management )---------------------------------------------------- */ - /** * Get the path to some file in the library. * - * @param string A library-relative path. If omitted, returns the library - * root path. - * @return string An absolute path. + * @param string A library-relative path. If omitted, returns the library + * root path. + * @return string An absolute path. * * @task path */ private function getPath($path = '') { return $this->root.'/'.$path; } - /** * Get the path to the symbol cache file. * * @return string Absolute path to symbol cache. * * @task path */ private function getPathForSymbolCache() { return $this->getPath('.phutil_module_cache'); } - /** * Get the path to the map file. * * @return string Absolute path to the library map. * * @task path */ private function getPathForLibraryMap() { return $this->getPath('__phutil_library_map__.php'); } - /** * Get the path to the library init file. * * @return string Absolute path to the library init file * * @task path */ private function getPathForLibraryInit() { return $this->getPath('__phutil_library_init__.php'); } /* -( Symbol Analysis and Caching )---------------------------------------- */ - /** * Load the library symbol cache, if it exists and is readable and valid. * - * @return dict Map of content hashes to cache of output from - * `phutil_symbols.php`. + * @return dict Map of content hashes to cache of output from + * `phutil_symbols.php`. * * @task symbol */ private function loadSymbolCache() { $cache_file = $this->getPathForSymbolCache(); try { $cache = Filesystem::readFile($cache_file); } catch (Exception $ex) { $cache = null; } $symbol_cache = array(); if ($cache) { $symbol_cache = json_decode($cache, true); if (!is_array($symbol_cache)) { $symbol_cache = array(); } } $version = idx($symbol_cache, self::SYMBOL_CACHE_VERSION_KEY); if ($version != self::SYMBOL_CACHE_VERSION) { // Throw away caches from a different version of the library. $symbol_cache = array(); } unset($symbol_cache[self::SYMBOL_CACHE_VERSION_KEY]); return $symbol_cache; } - /** * Write a symbol map to disk cache. * - * @param dict Symbol map of relative paths to symbols. - * @param dict Source map (like @{method:loadSourceFileMap}). + * @param dict Symbol map of relative paths to symbols. + * @param dict Source map (like @{method:loadSourceFileMap}). * @return void * * @task symbol */ private function writeSymbolCache(array $symbol_map, array $source_map) { $cache_file = $this->getPathForSymbolCache(); $cache = array( self::SYMBOL_CACHE_VERSION_KEY => self::SYMBOL_CACHE_VERSION, ); foreach ($symbol_map as $file => $symbols) { $cache[$source_map[$file]] = $symbols; } $json = json_encode($cache); try { Filesystem::writeFile($cache_file, $json); } catch (FilesystemException $ex) { $this->log("Unable to save the cache!\n"); } } - /** * Drop the symbol cache, forcing a clean rebuild. * * @return this * * @task symbol */ public function dropSymbolCache() { $this->log("Dropping symbol cache...\n"); Filesystem::remove($this->getPathForSymbolCache()); } - /** * Build a future which returns a `phutil_symbols.php` analysis of a source * file. * - * @param string Relative path to the source file to analyze. - * @return Future Analysis future. + * @param string Relative path to the source file to analyze. + * @return Future Analysis future. * * @task symbol */ private function buildSymbolAnalysisFuture($file) { $absolute_file = $this->getPath($file); $bin = dirname(__FILE__).'/../../scripts/phutil_symbols.php'; return new ExecFuture('php %s --ugly -- %s', $bin, $absolute_file); } /* -( Source Management )-------------------------------------------------- */ - /** * Build a map of all source files in a library to hashes of their content. * Returns an array like this: * * array( * 'src/parser/ExampleParser.php' => '60b725f10c9c85c70d97880dfe8191b3', * // ... * ); * - * @return dict Map of library-relative paths to content hashes. + * @return dict Map of library-relative paths to content hashes. * @task source */ private function loadSourceFileMap() { $root = $this->getPath(); $init = $this->getPathForLibraryInit(); if (!Filesystem::pathExists($init)) { throw new Exception("Provided path '{$root}' is not a phutil library."); } $files = id(new FileFinder($root)) ->withType('f') ->withSuffix('php') ->excludePath('*/.*') ->setGenerateChecksums(true) ->find(); $map = array(); foreach ($files as $file => $hash) { $file = Filesystem::readablePath($file, $root); $file = ltrim($file, '/'); if (dirname($file) == '.') { // We don't permit normal source files at the root level, so just ignore // them; they're special library files. continue; } if (dirname($file) == 'extensions') { // Ignore files in the extensions/ directory. continue; } // We include also filename in the hash to handle cases when the file is // moved without modifying its content. $map[$file] = md5($hash.$file); } return $map; } - /** * Convert the symbol analysis of all the source files in the library into * a library map. * * @param dict Symbol analysis of all source files. * @return dict Library map. * @task source */ private function buildLibraryMap(array $symbol_map) { $library_map = array( 'class' => array(), 'function' => array(), 'xmap' => array(), ); // Detect duplicate symbols within the library. foreach ($symbol_map as $file => $info) { foreach ($info['have'] as $type => $symbols) { foreach ($symbols as $symbol => $declaration) { $lib_type = ($type == 'interface') ? 'class' : $type; if (!empty($library_map[$lib_type][$symbol])) { $prior = $library_map[$lib_type][$symbol]; throw new Exception( "Definition of {$type} '{$symbol}' in file '{$file}' duplicates ". "prior definition in file '{$prior}'. You can not declare the ". "same symbol twice."); } $library_map[$lib_type][$symbol] = $file; } } $library_map['xmap'] += $info['xmap']; } // Simplify the common case (one parent) to make the file a little easier // to deal with. foreach ($library_map['xmap'] as $class => $extends) { if (count($extends) == 1) { $library_map['xmap'][$class] = reset($extends); } } // Sort the map so it is relatively stable across changes. foreach ($library_map as $key => $symbols) { ksort($symbols); $library_map[$key] = $symbols; } ksort($library_map); return $library_map; } - /** * Write a finalized library map. * * @param dict Library map structure to write. * @return void * * @task source */ private function writeLibraryMap(array $library_map) { $map_file = $this->getPathForLibraryMap(); $version = self::LIBRARY_MAP_VERSION; $library_map = array( self::LIBRARY_MAP_VERSION_KEY => $version, ) + $library_map; - $library_map = var_export($library_map, $return_string = true); + $library_map = var_export($library_map, true); $library_map = preg_replace('/\s+$/m', '', $library_map); $library_map = preg_replace('/array \(/', 'array(', $library_map); $at = '@'; $source_file = <<