diff --git a/scripts/__init_env__.php b/scripts/__init_env__.php index f58ac0a4e..892546493 100644 --- a/scripts/__init_env__.php +++ b/scripts/__init_env__.php @@ -1,39 +1,40 @@ \n"; + echo "usage: parse_one_commit.php [--herald]\n"; die(1); } $commit = isset($argv[1]) ? $argv[1] : null; if (!$commit) { throw new Exception("Provide a commit to parse!"); } $matches = null; if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $commit, $matches)) { throw new Exception("Can't parse commit identifier!"); } $repo = id(new PhabricatorRepository())->loadOneWhere( 'callsign = %s', $matches[1]); if (!$repo) { throw new Exception("Unknown repository!"); } $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere( 'repositoryID = %d AND commitIdentifier = %s', $repo->getID(), $matches[2]); if (!$commit) { throw new Exception('Unknown commit.'); } $workers = array(); $spec = array( 'commitID' => $commit->getID(), 'only' => true, ); switch ($repo->getVersionControlSystem()) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $workers[] = new PhabricatorRepositoryGitCommitMessageParserWorker( $spec); $workers[] = new PhabricatorRepositoryGitCommitChangeParserWorker( $spec); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: $workers[] = new PhabricatorRepositorySvnCommitMessageParserWorker( $spec); $workers[] = new PhabricatorRepositorySvnCommitChangeParserWorker( $spec); break; default: throw new Exception("Unknown repository type!"); } +if (isset($argv[2]) && $argv[2] == '--herald') { + $workers[] = new PhabricatorRepositoryCommitHeraldWorker($spec); +} + ExecFuture::pushEchoMode(true); foreach ($workers as $worker) { echo "Running ".get_class($worker)."...\n"; $worker->doWork(); } echo "Done.\n"; diff --git a/src/applications/herald/adapter/commit/HeraldCommitAdapter.php b/src/applications/herald/adapter/commit/HeraldCommitAdapter.php index 2939e2610..a897e3f6c 100644 --- a/src/applications/herald/adapter/commit/HeraldCommitAdapter.php +++ b/src/applications/herald/adapter/commit/HeraldCommitAdapter.php @@ -1,211 +1,211 @@ repository = $repository; $this->commit = $commit; $this->commitData = $commit_data; } public function getPHID() { return $this->commit->getPHID(); } public function getEmailPHIDs() { - return $this->emailPHIDs; + return array_keys($this->emailPHIDs); } public function getHeraldName() { return 'r'. $this->repository->getCallsign(). $this->commit->getCommitIdentifier(); } public function getHeraldTypeName() { return HeraldContentTypeConfig::CONTENT_TYPE_COMMIT; } public function loadAffectedPaths() { if ($this->affectedPaths === null) { $drequest = $this->buildDiffusionRequest(); $path_query = DiffusionPathChangeQuery::newFromDiffusionRequest( $drequest); $paths = $path_query->loadChanges(); $result = array(); foreach ($paths as $path) { $basic_path = '/'.$path->getPath(); if ($path->getFileType() == DifferentialChangeType::FILE_DIRECTORY) { $basic_path = rtrim($basic_path, '/').'/'; } $result[] = $basic_path; } $this->affectedPaths = $result; } return $this->affectedPaths; } public function loadAffectedPackages() { if ($this->affectedPackages === null) { $packages = PhabricatorOwnersPackage::loadAffectedPackages( $this->repository, $this->loadAffectedPaths()); $this->affectedPackages = $packages; } return $this->affectedPackages; } public function loadDifferentialRevision() { if ($this->affectedRevision === null) { $this->affectedRevision = false; $data = $this->commitData; $revision_id = $data->getCommitDetail('differential.revisionID'); if ($revision_id) { $revision = id(new DifferentialRevision())->load($revision_id); if ($revision) { $revision->loadRelationships(); $this->affectedRevision = $revision; } } } return $this->affectedRevision; } private function buildDiffusionRequest() { return DiffusionRequest::newFromAphrontRequestDictionary( array( 'callsign' => $this->repository->getCallsign(), 'commit' => $this->commit->getCommitIdentifier(), )); } public function getHeraldField($field) { $data = $this->commitData; switch ($field) { case HeraldFieldConfig::FIELD_BODY: return $data->getCommitMessage(); case HeraldFieldConfig::FIELD_AUTHOR: return $data->getCommitDetail('authorPHID'); case HeraldFieldConfig::FIELD_REVIEWER: return $data->getCommitDetail('reviewerPHID'); case HeraldFieldConfig::FIELD_DIFF_FILE: return $this->loadAffectedPaths(); case HeraldFieldConfig::FIELD_REPOSITORY: return $this->repository->getPHID(); case HeraldFieldConfig::FIELD_DIFF_CONTENT: // TODO! return null; /* try { $diff = $this->loadDiff(); } catch (Exception $ex) { // See rE280053 for an example. return array( '<<< Failed to load diff, this usually means the change committed '. 'a binary file as text. >>>', ); } $dict = array(); $changes = $diff->getChangesets(); $lines = array(); foreach ($changes as $change) { $lines = array(); foreach ($change->getHunks() as $hunk) { $lines[] = $hunk->makeChanges(); } $dict[$change->getTrueFilename()] = implode("\n", $lines); } return $dict; */ case HeraldFieldConfig::FIELD_AFFECTED_PACKAGE: $packages = $this->loadAffectedPackages(); return mpull($packages, 'getPHID'); case HeraldFieldConfig::FIELD_AFFECTED_PACKAGE_OWNER: $packages = $this->loadAffectedPackages(); $owners = PhabricatorOwnersOwner::loadAllForPackages($packages); return mpull($owners, 'getUserPHID'); case HeraldFieldConfig::FIELD_DIFFERENTIAL_REVISION: $revision = $this->loadDifferentialRevision(); if (!$revision) { return null; } return $revision->getID(); case HeraldFieldConfig::FIELD_DIFFERENTIAL_REVIEWERS: $revision = $this->loadDifferentialRevision(); if (!$revision) { return null; } return $revision->getReviewers(); case HeraldFieldConfig::FIELD_DIFFERENTIAL_CCS: $revision = $this->loadDifferentialRevision(); if (!$revision) { return null; } return $revision->getCCPHIDs(); default: throw new Exception("Invalid field '{$field}'."); } } public function applyHeraldEffects(array $effects) { $result = array(); foreach ($effects as $effect) { $action = $effect->getAction(); switch ($action) { case HeraldActionConfig::ACTION_NOTHING: $result[] = new HeraldApplyTranscript( $effect, true, 'Great success at doing nothing.'); break; case HeraldActionConfig::ACTION_EMAIL: foreach ($effect->getTarget() as $fbid) { - $this->emailPHIDs[] = $fbid; + $this->emailPHIDs[$fbid] = true; } $result[] = new HeraldApplyTranscript( $effect, true, 'Added address to email targets.'); break; default: throw new Exception("No rules to handle action '{$action}'."); } } return $result; } } diff --git a/src/applications/metamta/controller/send/PhabricatorMetaMTASendController.php b/src/applications/metamta/controller/send/PhabricatorMetaMTASendController.php index f347c09b3..716d35671 100644 --- a/src/applications/metamta/controller/send/PhabricatorMetaMTASendController.php +++ b/src/applications/metamta/controller/send/PhabricatorMetaMTASendController.php @@ -1,106 +1,107 @@ getRequest(); if ($request->isFormPost()) { $mail = new PhabricatorMetaMTAMail(); $mail->addTos($request->getArr('to')); $mail->addCCs($request->getArr('cc')); $mail->setSubject($request->getStr('subject')); $mail->setBody($request->getStr('body')); $mail->setFrom($request->getUser()->getPHID()); $mail->setSimulatedFailureCount($request->getInt('failures')); $mail->setIsHTML($request->getInt('html')); $mail->save(); - if ($request->getInt('immediately')) { + if ($request->getInt('immediately') && + !PhabricatorEnv::getEnvConfig('metamta.send-immediately')) { $mail->sendNow($force_send = true); } return id(new AphrontRedirectResponse()) ->setURI('/mail/view/'.$mail->getID().'/'); } $failure_caption = "Enter a number to simulate that many consecutive send failures before ". "really attempting to deliver via the underlying MTA."; $form = new AphrontFormView(); $form->setUser($request->getUser()); $form->setAction('/mail/send/'); $form ->appendChild( '

This form will send a normal '. 'email using MetaMTA as a transport mechanism.

') ->appendChild( id(new AphrontFormTokenizerControl()) ->setLabel('To') ->setName('to') ->setDatasource('/typeahead/common/mailable/')) ->appendChild( id(new AphrontFormTokenizerControl()) ->setLabel('CC') ->setName('cc') ->setDatasource('/typeahead/common/mailable/')) ->appendChild( id(new AphrontFormTextControl()) ->setLabel('Subject') ->setName('subject')) ->appendChild( id(new AphrontFormTextAreaControl()) ->setLabel('Body') ->setName('body')) ->appendChild( id(new AphrontFormTextControl()) ->setLabel('Simulate Failures') ->setName('failures') ->setCaption($failure_caption)) ->appendChild( id(new AphrontFormCheckboxControl()) ->setLabel('HTML') ->addCheckbox('html', '1', 'Send as HTML email.')) ->appendChild( id(new AphrontFormCheckboxControl()) ->setLabel('Send Now') ->addCheckbox( 'immediately', '1', 'Send immediately, not via MetaMTA background script.')) ->appendChild( id(new AphrontFormSubmitControl()) ->setValue('Send Mail')); $panel = new AphrontPanelView(); $panel->setHeader('Send Email'); $panel->appendChild($form); $panel->setWidth(AphrontPanelView::WIDTH_WIDE); return $this->buildStandardPageResponse( $panel, array( 'title' => 'Send Mail', )); } } diff --git a/src/applications/metamta/controller/send/__init__.php b/src/applications/metamta/controller/send/__init__.php index d606f64d0..7dcf0214e 100644 --- a/src/applications/metamta/controller/send/__init__.php +++ b/src/applications/metamta/controller/send/__init__.php @@ -1,19 +1,20 @@ establishConnection('w'); $result_map = $this->lookupPaths($paths); $missing_paths = array_fill_keys($paths, true); $missing_paths = array_diff_key($missing_paths, $result_map); $missing_paths = array_keys($missing_paths); if ($missing_paths) { foreach (array_chunk($missing_paths, 128) as $path_chunk) { $sql = array(); foreach ($path_chunk as $path) { $sql[] = qsprintf($conn_w, '(%s)', $path); } queryfx( $conn_w, 'INSERT IGNORE INTO %T (path) VALUES %Q', PhabricatorRepository::TABLE_PATH, implode(', ', $sql)); } $result_map += $this->lookupPaths($missing_paths); } return $result_map; } private function lookupPaths(array $paths) { $repository = new PhabricatorRepository(); $conn_w = $repository->establishConnection('w'); $result_map = array(); foreach (array_chunk($paths, 128) as $path_chunk) { $chunk_map = queryfx_all( $conn_w, 'SELECT path, id FROM %T WHERE path IN (%Ls)', PhabricatorRepository::TABLE_PATH, $path_chunk); foreach ($chunk_map as $row) { $result_map[$row['path']] = $row['id']; } } return $result_map; } + protected function finishParse() { + $commit = $this->commit; + if ($this->shouldQueueFollowupTasks()) { + $task = new PhabricatorWorkerTask(); + $task->setTaskClass('PhabricatorRepositoryCommitHeraldWorker'); + $task->setData( + array( + 'commitID' => $commit->getID(), + )); + $task->save(); + } + } } diff --git a/src/applications/repository/worker/commitchangeparser/base/__init__.php b/src/applications/repository/worker/commitchangeparser/base/__init__.php index 08e8c5125..b2279e18c 100644 --- a/src/applications/repository/worker/commitchangeparser/base/__init__.php +++ b/src/applications/repository/worker/commitchangeparser/base/__init__.php @@ -1,15 +1,16 @@ getCallsign().$commit->getCommitIdentifier(); echo "Parsing {$full_name}...\n"; if ($this->isBadCommit($full_name)) { echo "This commit is marked bad!\n"; return; } $local_path = $repository->getDetail('local-path'); list($raw) = execx( '(cd %s && git log -n1 -M -C -B --find-copies-harder --raw -t '. '--abbrev=40 --pretty=format: %s)', $local_path, $commit->getCommitIdentifier()); $changes = array(); $move_away = array(); $copy_away = array(); $lines = explode("\n", $raw); foreach ($lines as $line) { if (!strlen(trim($line))) { continue; } list($old_mode, $new_mode, $old_hash, $new_hash, $more_stuff) = preg_split('/ +/', $line); // We may only have two pieces here. list($action, $src_path, $dst_path) = array_merge( explode("\t", $more_stuff), array(null)); // Normalize the paths for consistency with the SVN workflow. $src_path = '/'.$src_path; if ($dst_path) { $dst_path = '/'.$dst_path; } $old_mode = intval($old_mode, 8); $new_mode = intval($new_mode, 8); $file_type = DifferentialChangeType::FILE_NORMAL; if ($new_mode & 040000) { $file_type = DifferentialChangeType::FILE_DIRECTORY; } else if ($new_mode & 0120000) { $file_type = DifferentialChangeType::FILE_SYMLINK; } // TODO: We can detect binary changes as git does, through a combination // of running 'git check-attr' for stuff like 'binary', 'merge' or 'diff', // and by falling back to inspecting the first 8,000 characters of the // buffer for null bytes (this is seriously git's algorithm, see // buffer_is_binary() in xdiff-interface.c). $change_type = null; $change_path = $src_path; $change_target = null; $is_direct = true; switch ($action[0]) { case 'A': $change_type = DifferentialChangeType::TYPE_ADD; break; case 'D': $change_type = DifferentialChangeType::TYPE_DELETE; break; case 'C': $change_type = DifferentialChangeType::TYPE_COPY_HERE; $change_path = $dst_path; $change_target = $src_path; $copy_away[$change_target][] = $change_path; break; case 'R': $change_type = DifferentialChangeType::TYPE_MOVE_HERE; $change_path = $dst_path; $change_target = $src_path; $move_away[$change_target][] = $change_path; break; case 'M': if ($file_type == DifferentialChangeType::FILE_DIRECTORY) { $change_type = DifferentialChangeType::TYPE_CHILD; $is_direct = false; } else { $change_type = DifferentialChangeType::TYPE_CHANGE; } break; default: throw new Exception("Failed to parse line '{$line}'."); } $changes[$change_path] = array( 'repositoryID' => $repository->getID(), 'commitID' => $commit->getID(), 'path' => $change_path, 'changeType' => $change_type, 'fileType' => $file_type, 'isDirect' => $is_direct, 'commitSequence' => $commit->getEpoch(), 'targetPath' => $change_target, 'targetCommitID' => $change_target ? $commit->getID() : null, ); } // Add a change to '/' since git doesn't mention it. $changes['/'] = array( 'repositoryID' => $repository->getID(), 'commitID' => $commit->getID(), 'path' => '/', 'changeType' => DifferentialChangeType::TYPE_CHILD, 'fileType' => DifferentialChangeType::FILE_DIRECTORY, 'isDirect' => false, 'commitSequence' => $commit->getEpoch(), 'targetPath' => null, 'targetCommitID' => null, ); foreach ($copy_away as $change_path => $destinations) { if (isset($move_away[$change_path])) { $change_type = DifferentialChangeType::TYPE_MULTICOPY; $is_direct = true; unset($move_away[$change_path]); } else { $change_type = DifferentialChangeType::TYPE_COPY_AWAY; $is_direct = false; } $reference = $changes[reset($destinations)]; $changes[$change_path] = array( 'repositoryID' => $repository->getID(), 'commitID' => $commit->getID(), 'path' => $change_path, 'changeType' => $change_type, 'fileType' => $reference['fileType'], 'isDirect' => $is_direct, 'commitSequence' => $commit->getEpoch(), 'targetPath' => null, 'targetCommitID' => null, ); } foreach ($move_away as $change_path => $destinations) { $reference = $changes[reset($destinations)]; $changes[$change_path] = array( 'repositoryID' => $repository->getID(), 'commitID' => $commit->getID(), 'path' => $change_path, 'changeType' => DifferentialChangeType::TYPE_MOVE_AWAY, 'fileType' => $reference['fileType'], 'isDirect' => true, 'commitSequence' => $commit->getEpoch(), 'targetPath' => null, 'targetCommitID' => null, ); } $paths = array(); foreach ($changes as $change) { $paths[$change['path']] = true; if ($change['targetPath']) { $paths[$change['targetPath']] = true; } } $path_map = $this->lookupOrCreatePaths(array_keys($paths)); foreach ($changes as $key => $change) { $changes[$key]['pathID'] = $path_map[$change['path']]; if ($change['targetPath']) { $changes[$key]['targetPathID'] = $path_map[$change['targetPath']]; } else { $changes[$key]['targetPathID'] = null; } } $conn_w = $repository->establishConnection('w'); $changes_sql = array(); foreach ($changes as $change) { $values = array( (int)$change['repositoryID'], (int)$change['pathID'], (int)$change['commitID'], $change['targetPathID'] ? (int)$change['targetPathID'] : 'null', $change['targetCommitID'] ? (int)$change['targetCommitID'] : 'null', (int)$change['changeType'], (int)$change['fileType'], (int)$change['isDirect'], (int)$change['commitSequence'], ); $changes_sql[] = '('.implode(', ', $values).')'; } queryfx( $conn_w, 'DELETE FROM %T WHERE commitID = %d', PhabricatorRepository::TABLE_PATHCHANGE, $commit->getID()); foreach (array_chunk($changes_sql, 256) as $sql_chunk) { queryfx( $conn_w, 'INSERT INTO %T (repositoryID, pathID, commitID, targetPathID, targetCommitID, changeType, fileType, isDirect, commitSequence) VALUES %Q', PhabricatorRepository::TABLE_PATHCHANGE, implode(', ', $sql_chunk)); } + + $this->finishParse(); } } diff --git a/src/applications/repository/worker/commitchangeparser/svn/PhabricatorRepositorySvnCommitChangeParserWorker.php b/src/applications/repository/worker/commitchangeparser/svn/PhabricatorRepositorySvnCommitChangeParserWorker.php index 3982a7b94..f36555247 100644 --- a/src/applications/repository/worker/commitchangeparser/svn/PhabricatorRepositorySvnCommitChangeParserWorker.php +++ b/src/applications/repository/worker/commitchangeparser/svn/PhabricatorRepositorySvnCommitChangeParserWorker.php @@ -1,734 +1,736 @@ getDetail('remote-uri'); $svn_commit = $commit->getCommitIdentifier(); $callsign = $repository->getCallsign(); $full_name = 'r'.$callsign.$svn_commit; echo "Parsing {$full_name}...\n"; if ($this->isBadCommit($full_name)) { echo "This commit is marked bad!\n"; return; } // Pull the top-level path changes out of "svn log". This is pretty // straightforward; just parse the XML log. $log = $this->getSVNLogXMLObject($uri, $svn_commit, $verbose = true); $entry = $log->logentry[0]; if (!$entry->paths) { // TODO: Explicitly mark this commit as broken elsewhere? This isn't // supposed to happen but we have some cases like rE27 and rG935 in the // Facebook repositories where things got all clowned up. return; } $raw_paths = array(); foreach ($entry->paths->path as $path) { $name = trim((string)$path); $raw_paths[$name] = array( 'rawPath' => $name, 'rawTargetPath' => (string)$path['copyfrom-path'], 'rawChangeType' => (string)$path['action'], 'rawTargetCommit' => (string)$path['copyfrom-rev'], ); } $copied_or_moved_map = array(); $deleted_paths = array(); $add_paths = array(); foreach ($raw_paths as $path => $raw_info) { if ($raw_info['rawTargetPath']) { $copied_or_moved_map[$raw_info['rawTargetPath']][] = $raw_info; } switch ($raw_info['rawChangeType']) { case 'D': $deleted_paths[$path] = $raw_info; break; case 'A': $add_paths[$path] = $raw_info; break; } } // If a path was deleted, we need to look in the repository history to // figure out where the former valid location for it is so we can figure out // if it was a directory or not, among other things. $lookup_here = array(); foreach ($raw_paths as $path => $raw_info) { if ($raw_info['rawChangeType'] != 'D') { continue; } // If a change copies a directory and then deletes something from it, // we need to look at the old location for information about the path, not // the new location. This workflow is pretty ridiculous -- so much so that // Trac gets it wrong. See Facebook rO6 for an example, if you happen to // work at Facebook. $parents = $this->expandAllParentPaths($path, $include_self = true); foreach ($parents as $parent) { if (isset($add_paths[$parent])) { $relative_path = substr($path, strlen($parent)); $lookup_here[$path] = array( 'rawPath' => $add_paths[$parent]['rawTargetPath'].$relative_path, 'rawCommit' => $add_paths[$parent]['rawTargetCommit'], ); continue 2; } } // Otherwise we can just look at the previous revision. $lookup_here[$path] = array( 'rawPath' => $path, 'rawCommit' => $svn_commit - 1, ); } $lookup = array(); foreach ($raw_paths as $path => $raw_info) { if ($raw_info['rawChangeType'] == 'D') { $lookup[$path] = $lookup_here[$path]; } else { // For everything that wasn't deleted, we can just look it up directly. $lookup[$path] = array( 'rawPath' => $path, 'rawCommit' => $svn_commit, ); } } $path_file_types = $this->lookupPathFileTypes($repository, $lookup); $effects = array(); $resolved_types = array(); $supplemental = array(); foreach ($raw_paths as $path => $raw_info) { if (isset($resolved_types[$path])) { $type = $resolved_types[$path]; } else { switch ($raw_info['rawChangeType']) { case 'D': if (isset($copied_or_moved_map[$path])) { if (count($copied_or_moved_map[$path]) > 1) { $type = DifferentialChangeType::TYPE_MULTICOPY; } else { $type = DifferentialChangeType::TYPE_MOVE_AWAY; } } else { $type = DifferentialChangeType::TYPE_DELETE; $file_type = $path_file_types[$path]; if ($file_type == DifferentialChangeType::FILE_DIRECTORY) { // Bad. Child paths aren't enumerated in "svn log" so we need // to go fishing. $list = $this->lookupRecursiveFileList( $repository, $lookup[$path]); foreach ($list as $deleted_path => $path_file_type) { $deleted_path = rtrim($path.'/'.$deleted_path, '/'); if (!empty($raw_paths[$deleted_path])) { // We somehow learned about this deletion explicitly? // TODO: Unclear how this is possible. continue; } $effects[$deleted_path] = array( 'rawPath' => $deleted_path, 'rawTargetPath' => null, 'rawTargetCommit' => null, 'rawDirect' => true, 'changeType' => $type, 'fileType' => $path_file_type, ); } } } break; case 'A': $copy_from = $raw_info['rawTargetPath']; $copy_rev = $raw_info['rawTargetCommit']; if (!strlen($copy_from)) { $type = DifferentialChangeType::TYPE_ADD; } else { if (isset($deleted_paths[$copy_from])) { $type = DifferentialChangeType::TYPE_MOVE_HERE; $other_type = DifferentialChangeType::TYPE_MOVE_AWAY; } else { $type = DifferentialChangeType::TYPE_COPY_HERE; $other_type = DifferentialChangeType::TYPE_COPY_AWAY; } $source_file_type = $this->lookupPathFileType( $repository, $copy_from, array( 'rawPath' => $copy_from, 'rawCommit' => $copy_rev, )); if ($source_file_type == DifferentialChangeType::FILE_DELETED) { throw new Exception( "Something is wrong; source of a copy must exist."); } if ($source_file_type != DifferentialChangeType::FILE_DIRECTORY) { if (isset($raw_paths[$copy_from])) { break; } $effects[$copy_from] = array( 'rawPath' => $copy_from, 'rawTargetPath' => null, 'rawTargetCommit' => null, 'rawDirect' => false, 'changeType' => $other_type, 'fileType' => $source_file_type, ); } else { // ULTRADISASTER. We've added a directory which was copied // or moved from somewhere else. This is the most complex and // ridiculous case. $list = $this->lookupRecursiveFileList( $repository, array( 'rawPath' => $copy_from, 'rawCommit' => $copy_rev, )); foreach ($list as $from_path => $from_file_type) { $full_from = rtrim($copy_from.'/'.$from_path, '/'); $full_to = rtrim($path.'/'.$from_path, '/'); if (empty($raw_paths[$full_to])) { $effects[$full_to] = array( 'rawPath' => $full_to, 'rawTargetPath' => $full_from, 'rawTargetCommit' => $copy_rev, 'rawDirect' => false, 'changeType' => $type, 'fileType' => $from_file_type, ); } else { // This means we picked the file up explicitly elsewhere. // If the file as modified, SVN will drop the copy // information. We need to restore it. $supplemental[$full_to]['rawTargetPath'] = $full_from; $supplemental[$full_to]['rawTargetCommit'] = $copy_rev; if ($raw_paths[$full_to]['rawChangeType'] == 'M') { $resolved_types[$full_to] = $type; } } if (empty($raw_paths[$full_from])) { if ($other_type == DifferentialChangeType::TYPE_COPY_AWAY) { $effects[$full_from] = array( 'rawPath' => $full_from, 'rawTargetPath' => null, 'rawTargetCommit' => null, 'rawDirect' => false, 'changeType' => $other_type, 'fileType' => $from_file_type, ); } } } } } break; // This is "replaced", caused by "svn rm"-ing a file, putting another // in its place, and then "svn add"-ing it. We do not distinguish // between this and "M". case 'R': case 'M': if (isset($copied_or_moved_map[$path])) { $type = DifferentialChangeType::TYPE_COPY_AWAY; } else { $type = DifferentialChangeType::TYPE_CHANGE; } break; } } $resolved_types[$path] = $type; } foreach ($raw_paths as $path => $raw_info) { $raw_paths[$path]['changeType'] = $resolved_types[$path]; if (isset($supplemental[$path])) { foreach ($supplemental[$path] as $key => $value) { $raw_paths[$path][$key] = $value; } } } foreach ($raw_paths as $path => $raw_info) { $effects[$path] = array( 'rawPath' => $path, 'rawTargetPath' => $raw_info['rawTargetPath'], 'rawTargetCommit' => $raw_info['rawTargetCommit'], 'rawDirect' => true, 'changeType' => $raw_info['changeType'], 'fileType' => $path_file_types[$path], ); } $parents = array(); foreach ($effects as $path => $effect) { foreach ($this->expandAllParentPaths($path) as $parent_path) { $parents[$parent_path] = true; } } $parents = array_keys($parents); foreach ($parents as $parent) { if (isset($effects[$parent])) { continue; } $effects[$parent] = array( 'rawPath' => $parent, 'rawTargetPath' => null, 'rawTargetCommit' => null, 'rawDirect' => false, 'changeType' => DifferentialChangeType::TYPE_CHILD, 'fileType' => DifferentialChangeType::FILE_DIRECTORY, ); } $lookup_paths = array(); foreach ($effects as $effect) { $lookup_paths[$effect['rawPath']] = true; if ($effect['rawTargetPath']) { $lookup_paths[$effect['rawTargetPath']] = true; } } $lookup_paths = array_keys($lookup_paths); $lookup_commits = array(); foreach ($effects as $effect) { if ($effect['rawTargetCommit']) { $lookup_commits[$effect['rawTargetCommit']] = true; } } $lookup_commits = array_keys($lookup_commits); $path_map = $this->lookupOrCreatePaths($lookup_paths); $commit_map = $this->lookupSvnCommits($repository, $lookup_commits); $this->writeChanges($repository, $commit, $effects, $path_map, $commit_map); $this->writeBrowse($repository, $commit, $effects, $path_map); + + $this->finishParse(); } private function writeChanges( PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit, array $effects, array $path_map, array $commit_map) { $conn_w = $repository->establishConnection('w'); $sql = array(); foreach ($effects as $effect) { $sql[] = qsprintf( $conn_w, '(%d, %d, %d, %nd, %nd, %d, %d, %d, %d)', $repository->getID(), $path_map[$effect['rawPath']], $commit->getID(), $effect['rawTargetPath'] ? $path_map[$effect['rawTargetPath']] : null, $effect['rawTargetCommit'] ? $commit_map[$effect['rawTargetCommit']] : null, $effect['changeType'], $effect['fileType'], $effect['rawDirect'] ? 1 : 0, $commit->getCommitIdentifier()); } queryfx( $conn_w, 'DELETE FROM %T WHERE commitID = %d', PhabricatorRepository::TABLE_PATHCHANGE, $commit->getID()); foreach (array_chunk($sql, 512) as $sql_chunk) { queryfx( $conn_w, 'INSERT INTO %T (repositoryID, pathID, commitID, targetPathID, targetCommitID, changeType, fileType, isDirect, commitSequence) VALUES %Q', PhabricatorRepository::TABLE_PATHCHANGE, implode(', ', $sql_chunk)); } } private function writeBrowse( PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit, array $effects, array $path_map) { $conn_w = $repository->establishConnection('w'); $sql = array(); foreach ($effects as $effect) { $type = $effect['changeType']; if (!$effect['rawDirect']) { if ($type == DifferentialChangeType::TYPE_COPY_AWAY) { // Don't write COPY_AWAY to the filesystem table if it isn't a direct // event. continue; } if ($type == DifferentialChangeType::TYPE_CHILD) { // Don't write CHILD to the filesystem table. Although doing these // writes has the nice property of letting you see when a directory's // contents were last changed, it explodes the table tremendously // and makes Diffusion far slower. continue; } } if ($effect['rawPath'] == '/') { // Don't write any events on '/' to the filesystem table; in // particular, it doesn't have a meaningful parentID. continue; } $existed = !DifferentialChangeType::isDeleteChangeType($type); $sql[] = qsprintf( $conn_w, '(%d, %d, %d, %d, %d, %d)', $repository->getID(), $path_map[$this->getParentPath($effect['rawPath'])], $commit->getCommitIdentifier(), $path_map[$effect['rawPath']], $existed ? 1 : 0, $effect['fileType']); } queryfx( $conn_w, 'DELETE FROM %T WHERE repositoryID = %d AND svnCommit = %d', PhabricatorRepository::TABLE_FILESYSTEM, $repository->getID(), $commit->getCommitIdentifier()); foreach (array_chunk($sql, 512) as $sql_chunk) { queryfx( $conn_w, 'INSERT INTO %T (repositoryID, parentID, svnCommit, pathID, existed, fileType) VALUES %Q', PhabricatorRepository::TABLE_FILESYSTEM, implode(', ', $sql_chunk)); } } private function lookupSvnCommits( PhabricatorRepository $repository, array $commits) { if (!$commits) { return array(); } $commit_table = new PhabricatorRepositoryCommit(); $commit_data = queryfx_all( $commit_table->establishConnection('w'), 'SELECT id, commitIdentifier FROM %T WHERE commitIdentifier in (%Ld)', $commit_table->getTableName(), $commits); return ipull($commit_data, 'id', 'commitIdentifier'); } private function lookupPathFileType( PhabricatorRepository $repository, $path, array $path_info) { $result = $this->lookupPathFileTypes( $repository, array( $path => $path_info, )); return $result[$path]; } private function lookupPathFileTypes( PhabricatorRepository $repository, array $paths) { $repository_uri = $repository->getDetail('remote-uri'); $parents = array(); $path_mapping = array(); foreach ($paths as $path => $lookup) { $parent = dirname($lookup['rawPath']); $parent = ltrim($parent, '/'); $parent = $this->encodeSVNPath($parent); $parent = $repository_uri.$parent.'@'.$lookup['rawCommit']; $parent = escapeshellarg($parent); $parents[$parent] = true; $path_mapping[$parent][] = dirname($path); } $result_map = array(); // Reverse this list so we can pop $path_mapping, as that's more efficient // than shifting it. We need to associate these maps positionally because // a change can copy the same source path from multiple revisions via // "svn cp path@1 a; svn cp path@2 b;" and the XML output gives us no way // to distinguish which revision we're looking at except based on its // position in the document. $all_paths = array_reverse(array_keys($parents)); foreach (array_chunk($all_paths, 64) as $path_chunk) { list($raw_xml) = execx( 'svn --non-interactive --xml ls %C', implode(' ', $path_chunk)); $xml = new SimpleXMLElement($raw_xml); foreach ($xml->list as $list) { $list_path = (string)$list['path']; // SVN is a big mess. See Facebook rG8 (a revision which adds files // with spaces in their names) for an example. $list_path = rawurldecode($list_path); if ($list_path == $repository_uri) { $base = '/'; } else { $base = substr($list_path, strlen($repository_uri)); } $mapping = array_pop($path_mapping); foreach ($list->entry as $entry) { $val = $this->getFileTypeFromSVNKind($entry['kind']); foreach ($mapping as $base_path) { // rtrim() causes us to handle top-level directories correctly. $key = rtrim($base_path, '/').'/'.$entry->name; $result_map[$key] = $val; } } } } foreach ($paths as $path => $lookup) { if (empty($result_map[$path])) { $result_map[$path] = DifferentialChangeType::FILE_DELETED; } } return $result_map; } private function encodeSVNPath($path) { $path = rawurlencode($path); $path = str_replace('%2F', '/', $path); return $path; } private function getFileTypeFromSVNKind($kind) { $kind = (string)$kind; switch ($kind) { case 'dir': return DifferentialChangeType::FILE_DIRECTORY; case 'file': return DifferentialChangeType::FILE_NORMAL; default: throw new Exception("Unknown SVN file kind '{$kind}'."); } } private function lookupRecursiveFileList( PhabricatorRepository $repository, array $info) { $path = $info['rawPath']; $rev = $info['rawCommit']; $path = $this->encodeSVNPath($path); $hashkey = md5($repository->getDetail('remote-uri').$path.'@'.$rev); // This method is quite horrible. The underlying challenge is that some // commits in the Facebook repository are enormous, taking multiple hours // to 'ls -R' out of the repository and producing XML files >1GB in size. // If we try to SimpleXML them, the object exhausts available memory on a // 64G machine. Instead, cache the XML output and then parse it line by line // to limit space requirements. $cache_loc = sys_get_temp_dir().'/diffusion.'.$hashkey.'.svnls'; if (!Filesystem::pathExists($cache_loc)) { $tmp = new TempFile(); execx( 'svn --non-interactive --xml ls -R %s%s@%d > %s', $repository->getDetail('remote-uri'), $path, $rev, $tmp); execx( 'mv %s %s', $tmp, $cache_loc); } $map = $this->parseRecursiveListFileData($cache_loc); Filesystem::remove($cache_loc); return $map; } private function parseRecursiveListFileData($file_path) { $map = array(); $mode = 'xml'; $done = false; $entry = null; foreach (new LinesOfALargeFile($file_path) as $lno => $line) { switch ($mode) { case 'entry': if ($line == '') { $entry = implode('', $entry); $pattern = '@^\s+kind="(file|dir)">'. '(.*?)'. '((.*?))?@'; $matches = null; if (!preg_match($pattern, $entry, $matches)) { throw new Exception("Unable to parse entry!"); } $map[html_entity_decode($matches[2])] = $this->getFileTypeFromSVNKind($matches[1]); $mode = 'entry-or-end'; } else { $entry[] = $line; } break; case 'entry-or-end': if ($line == '') { $done = true; break 2; } else if ($line == ' or = 1) { array_pop($parts); $parents[] = '/'.implode('/', $parts); } return $parents; } } diff --git a/src/applications/repository/worker/herald/PhabricatorRepositoryCommitHeraldWorker.php b/src/applications/repository/worker/herald/PhabricatorRepositoryCommitHeraldWorker.php index 8a0b32b2f..f5cf7b328 100644 --- a/src/applications/repository/worker/herald/PhabricatorRepositoryCommitHeraldWorker.php +++ b/src/applications/repository/worker/herald/PhabricatorRepositoryCommitHeraldWorker.php @@ -1,132 +1,133 @@ loadOneWhere( 'commitID = %d', $commit->getID()); $rules = HeraldRule::loadAllByContentTypeWithFullData( HeraldContentTypeConfig::CONTENT_TYPE_COMMIT); $adapter = new HeraldCommitAdapter( $repository, $commit, $data); $engine = new HeraldEngine(); - $effects = $engine->applyRules($this->rules, $adapter); + $effects = $engine->applyRules($rules, $adapter); $engine->applyEffects($effects, $adapter); - $phids = $adapter->getEmailPHIDs(); - if (!$phids) { + $email_phids = $adapter->getEmailPHIDs(); + if (!$email_phids) { return; } $xscript = $engine->getTranscript(); $commit_name = $adapter->getHeraldName(); $revision = $adapter->loadDifferentialRevision(); $name = null; if ($revision) { $name = $revision->getName(); } $author_phid = $data->getCommitDetail('authorPHID'); $reviewer_phid = $data->getCommitDetail('reviewerPHID'); $phids = array_filter(array($author_phid, $reviewer_phid)); $handles = array(); if ($phids) { $handles = id(new PhabricatorObjectHandleData($phids))->loadHandles(); } if ($author_phid) { $author_name = $handles[$author_phid]->getName(); } else { $author_name = $data->getAuthorName(); } if ($reviewer_phid) { $reviewer_name = $handles[$reviewer_phid]->getName(); } else { $reviewer_name = null; } $who = implode(', ', array_filter(array($author_name, $reviewer_name))); $description = $data->getCommitMessage(); $details = PhabricatorEnv::getURI('/'.$commit_name); $differential = $revision ? PhabricatorEnv::getURI('/D'.$revision->getID()) : 'No revision.'; $files = $adapter->loadAffectedPaths(); sort($files); $files = implode("\n ", $files); $xscript_id = $xscript->getID(); $manage_uri = PhabricatorEnv::getURI('/herald/view/commits/'); $why_uri = PhabricatorEnv::getURI('/herald/transcript/'.$xscript_id.'/'); $body = <<addTos($phids); + $mailer->setRelatedPHID($commit->getPHID()); + $mailer->addTos($email_phids); $mailer->setSubject($subject); $mailer->setBody($body); $mailer->addHeader('X-Herald-Rules', $xscript->getXHeraldRulesHeader()); if ($author_phid) { $mailer->setFrom($author_phid); } $mailer->save(); } } diff --git a/webroot/index.php b/webroot/index.php index 004b917d2..d3f6a5fd7 100644 --- a/webroot/index.php +++ b/webroot/index.php @@ -1,185 +1,181 @@ ', where '' ". "is one of 'development', 'production', or a custom environment."); } if (!function_exists('mysql_connect')) { phabricator_fatal_config_error( "The PHP MySQL extension is not installed. This extension is required."); } if (!isset($_REQUEST['__path__'])) { phabricator_fatal_config_error( "__path__ is not set. Your rewrite rules are not configured correctly."); } require_once dirname(dirname(__FILE__)).'/conf/__init_conf__.php'; $conf = phabricator_read_config_file($env); $conf['phabricator.env'] = $env; setup_aphront_basics(); phutil_require_module('phabricator', 'infrastructure/env'); PhabricatorEnv::setEnvConfig($conf); phutil_require_module('phabricator', 'aphront/console/plugin/xhprof/api'); DarkConsoleXHProfPluginAPI::hookProfiler(); phutil_require_module('phabricator', 'aphront/console/plugin/errorlog/api'); set_error_handler(array('DarkConsoleErrorLogPluginAPI', 'handleError')); set_exception_handler(array('DarkConsoleErrorLogPluginAPI', 'handleException')); $tz = PhabricatorEnv::getEnvConfig('phabricator.timezone'); if ($tz) { date_default_timezone_set($tz); } foreach (PhabricatorEnv::getEnvConfig('load-libraries') as $library) { phutil_load_library($library); } $host = $_SERVER['HTTP_HOST']; $path = $_REQUEST['__path__']; switch ($host) { default: $config_key = 'aphront.default-application-configuration-class'; $config_class = PhabricatorEnv::getEnvConfig($config_key); PhutilSymbolLoader::loadClass($config_class); $application = newv($config_class, array()); break; } $application->setHost($host); $application->setPath($path); $application->willBuildRequest(); $request = $application->buildRequest(); $application->setRequest($request); list($controller, $uri_data) = $application->buildController(); try { $response = $controller->willBeginExecution(); if (!$response) { $controller->willProcessRequest($uri_data); $response = $controller->processRequest(); } } catch (AphrontRedirectException $ex) { $response = id(new AphrontRedirectResponse()) ->setURI($ex->getURI()); } catch (Exception $ex) { $response = $application->handleException($ex); } $response = $application->willSendResponse($response); $response->setRequest($request); $response_string = $response->buildResponseString(); $code = $response->getHTTPResponseCode(); if ($code != 200) { header("HTTP/1.0 {$code}"); } $headers = $response->getCacheHeaders(); $headers = array_merge($headers, $response->getHeaders()); foreach ($headers as $header) { list($header, $value) = $header; header("{$header}: {$value}"); } // TODO: This shouldn't be possible in a production-configured environment. if (isset($_REQUEST['__profile__']) && ($_REQUEST['__profile__'] == 'all')) { $profile = DarkConsoleXHProfPluginAPI::stopProfiler(); $profile = ''; if (strpos($response_string, '') !== false) { $response_string = str_replace( '', ''.$profile, $response_string); } else { echo $profile; } } echo $response_string; /** * @group aphront */ function setup_aphront_basics() { $aphront_root = dirname(dirname(__FILE__)); $libraries_root = dirname($aphront_root); $root = null; if (!empty($_SERVER['PHUTIL_LIBRARY_ROOT'])) { $root = $_SERVER['PHUTIL_LIBRARY_ROOT']; } ini_set('include_path', $libraries_root.':'.ini_get('include_path')); @include_once $root.'libphutil/src/__phutil_library_init__.php'; if (!@constant('__LIBPHUTIL__')) { echo "ERROR: Unable to load libphutil. Update your PHP 'include_path' to ". "include the parent directory of libphutil/.\n"; exit(1); } // Load Phabricator itself using the absolute path, so we never end up doing // anything surprising (loading index.php and libraries from different // directories). phutil_load_library($aphront_root.'/src'); phutil_load_library('arcanist/src'); } -function __autoload($class_name) { - PhutilSymbolLoader::loadClass($class_name); -} - function phabricator_fatal_config_error($msg) { header('Content-Type: text/plain', $replace = true, $http_error = 500); $error = "CONFIG ERROR: ".$msg."\n"; error_log($error); echo $error; die(); }