PhabricatorRepositoryRefEngine.php
No OneTemporary
Actions

Subscribers

None

File Metadata

Created: Tue, Nov 19, 01:44

PhabricatorRepositoryRefEngine.php
View Options

	<?php

	/**
	* Update the ref cursors for a repository, which track the positions of
	* branches, bookmarks, and tags.
	*/
	final class PhabricatorRepositoryRefEngine
	extends PhabricatorRepositoryEngine {

	private $newPositions = array();
	private $deadPositions = array();
	private $closeCommits = array();
	private $hasNoCursors;

	public function updateRefs() {
	$this->newPositions = array();
	$this->deadPositions = array();
	$this->closeCommits = array();

	$repository = $this->getRepository();
	$viewer = $this->getViewer();

	$branches_may_close = false;

	$vcs = $repository->getVersionControlSystem();
	switch ($vcs) {
	case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
	// No meaningful refs of any type in Subversion.
	$maps = array();
	break;
	case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
	$branches = $this->loadMercurialBranchPositions($repository);
	$bookmarks = $this->loadMercurialBookmarkPositions($repository);
	$maps = array(
	PhabricatorRepositoryRefCursor::TYPE_BRANCH => $branches,
	PhabricatorRepositoryRefCursor::TYPE_BOOKMARK => $bookmarks,
	);

	$branches_may_close = true;
	break;
	case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
	$maps = $this->loadGitRefPositions($repository);
	break;
	default:
	throw new Exception(pht('Unknown VCS "%s"!', $vcs));
	}

	// Fill in any missing types with empty lists.
	$maps = $maps + array(
	PhabricatorRepositoryRefCursor::TYPE_BRANCH => array(),
	PhabricatorRepositoryRefCursor::TYPE_TAG => array(),
	PhabricatorRepositoryRefCursor::TYPE_BOOKMARK => array(),
	PhabricatorRepositoryRefCursor::TYPE_REF => array(),
	);

	$all_cursors = id(new PhabricatorRepositoryRefCursorQuery())
	->setViewer($viewer)
	->withRepositoryPHIDs(array($repository->getPHID()))
	->needPositions(true)
	->execute();
	$cursor_groups = mgroup($all_cursors, 'getRefType');

	$this->hasNoCursors = (!$all_cursors);

	// Find all the heads of closing refs.
	$all_closing_heads = array();
	foreach ($all_cursors as $cursor) {
	$should_close = $this->shouldCloseRef(
	$cursor->getRefType(),
	$cursor->getRefName());
	if (!$should_close) {
	continue;
	}

	foreach ($cursor->getPositionIdentifiers() as $identifier) {
	$all_closing_heads[] = $identifier;
	}
	}
	$all_closing_heads = array_unique($all_closing_heads);
	$all_closing_heads = $this->removeMissingCommits($all_closing_heads);

	foreach ($maps as $type => $refs) {
	$cursor_group = idx($cursor_groups, $type, array());
	$this->updateCursors($cursor_group, $refs, $type, $all_closing_heads);
	}

	if ($this->closeCommits) {
	$this->setCloseFlagOnCommits($this->closeCommits);
	}

	if ($this->newPositions \|\| $this->deadPositions) {
	$repository->openTransaction();

	$this->saveNewPositions();
	$this->deleteDeadPositions();

	$repository->saveTransaction();
	}

	$branches = $maps[PhabricatorRepositoryRefCursor::TYPE_BRANCH];
	if ($branches && $branches_may_close) {
	$this->updateBranchStates($repository, $branches);
	}
	}

	private function updateBranchStates(
	PhabricatorRepository $repository,
	array $branches) {

	assert_instances_of($branches, 'DiffusionRepositoryRef');
	$viewer = $this->getViewer();

	$all_cursors = id(new PhabricatorRepositoryRefCursorQuery())
	->setViewer($viewer)
	->withRepositoryPHIDs(array($repository->getPHID()))
	->needPositions(true)
	->execute();

	$state_map = array();
	$type_branch = PhabricatorRepositoryRefCursor::TYPE_BRANCH;
	foreach ($all_cursors as $cursor) {
	if ($cursor->getRefType() !== $type_branch) {
	continue;
	}
	$raw_name = $cursor->getRefNameRaw();

	foreach ($cursor->getPositions() as $position) {
	$hash = $position->getCommitIdentifier();
	$state_map[$raw_name][$hash] = $position;
	}
	}

	$updates = array();
	foreach ($branches as $branch) {
	$position = idx($state_map, $branch->getShortName(), array());
	$position = idx($position, $branch->getCommitIdentifier());
	if (!$position) {
	continue;
	}

	$fields = $branch->getRawFields();

	$position_state = (bool)$position->getIsClosed();
	$branch_state = (bool)idx($fields, 'closed');

	if ($position_state != $branch_state) {
	$updates[$position->getID()] = (int)$branch_state;
	}
	}

	if ($updates) {
	$position_table = id(new PhabricatorRepositoryRefPosition());
	$conn = $position_table->establishConnection('w');

	$position_table->openTransaction();
	foreach ($updates as $position_id => $branch_state) {
	queryfx(
	$conn,
	'UPDATE %T SET isClosed = %d WHERE id = %d',
	$position_table->getTableName(),
	$branch_state,
	$position_id);
	}
	$position_table->saveTransaction();
	}
	}

	private function markPositionNew(
	PhabricatorRepositoryRefPosition $position) {
	$this->newPositions[] = $position;
	return $this;
	}

	private function markPositionDead(
	PhabricatorRepositoryRefPosition $position) {
	$this->deadPositions[] = $position;
	return $this;
	}

	private function markCloseCommits(array $identifiers) {
	foreach ($identifiers as $identifier) {
	$this->closeCommits[$identifier] = $identifier;
	}
	return $this;
	}

	/**
	* Remove commits which no longer exist in the repository from a list.
	*
	* After a force push and garbage collection, we may have branch cursors which
	* point at commits which no longer exist. This can make commands issued later
	* fail. See T5839 for discussion.
	*
	* @param list<string> List of commit identifiers.
	* @return list<string> List with nonexistent identifiers removed.
	*/
	private function removeMissingCommits(array $identifiers) {
	if (!$identifiers) {
	return array();
	}

	$resolved = id(new DiffusionLowLevelResolveRefsQuery())
	->setRepository($this->getRepository())
	->withRefs($identifiers)
	->execute();

	foreach ($identifiers as $key => $identifier) {
	if (empty($resolved[$identifier])) {
	unset($identifiers[$key]);
	}
	}

	return $identifiers;
	}

	private function updateCursors(
	array $cursors,
	array $new_refs,
	$ref_type,
	array $all_closing_heads) {
	$repository = $this->getRepository();

	// NOTE: Mercurial branches may have multiple branch heads; this logic
	// is complex primarily to account for that.

	$cursors = mpull($cursors, null, 'getRefNameRaw');

	// Group all the new ref values by their name. As above, these groups may
	// have multiple members in Mercurial.
	$ref_groups = mgroup($new_refs, 'getShortName');

	foreach ($ref_groups as $name => $refs) {
	$new_commits = mpull($refs, 'getCommitIdentifier', 'getCommitIdentifier');

	$ref_cursor = idx($cursors, $name);
	if ($ref_cursor) {
	$old_positions = $ref_cursor->getPositions();
	} else {
	$old_positions = array();
	}

	// We're going to delete all the cursors pointing at commits which are
	// no longer associated with the refs. This primarily makes the Mercurial
	// multiple head case easier, and means that when we update a ref we
	// delete the old one and write a new one.
	foreach ($old_positions as $old_position) {
	$hash = $old_position->getCommitIdentifier();
	if (isset($new_commits[$hash])) {
	// This ref previously pointed at this commit, and still does.
	$this->log(
	pht(
	'Ref %s "%s" still points at %s.',
	$ref_type,
	$name,
	$hash));
	continue;
	}

	// This ref previously pointed at this commit, but no longer does.
	$this->log(
	pht(
	'Ref %s "%s" no longer points at %s.',
	$ref_type,
	$name,
	$hash));

	// Nuke the obsolete cursor.
	$this->markPositionDead($old_position);
	}

	// Now, we're going to insert new cursors for all the commits which are
	// associated with this ref that don't currently have cursors.
	$old_commits = mpull($old_positions, 'getCommitIdentifier');
	$old_commits = array_fuse($old_commits);

	$added_commits = array_diff_key($new_commits, $old_commits);
	foreach ($added_commits as $identifier) {
	$this->log(
	pht(
	'Ref %s "%s" now points at %s.',
	$ref_type,
	$name,
	$identifier));

	if (!$ref_cursor) {
	// If this is the first time we've seen a particular ref (for
	// example, a new branch) we need to insert a RefCursor record
	// for it before we can insert a RefPosition.

	$ref_cursor = $this->newRefCursor(
	$repository,
	$ref_type,
	$name);
	}

	$new_position = id(new PhabricatorRepositoryRefPosition())
	->setCursorID($ref_cursor->getID())
	->setCommitIdentifier($identifier)
	->setIsClosed(0);

	$this->markPositionNew($new_position);
	}

	if ($this->shouldCloseRef($ref_type, $name)) {
	foreach ($added_commits as $identifier) {
	$new_identifiers = $this->loadNewCommitIdentifiers(
	$identifier,
	$all_closing_heads);

	$this->markCloseCommits($new_identifiers);
	}
	}
	}

	// Find any cursors for refs which no longer exist. This happens when a
	// branch, tag or bookmark is deleted.

	foreach ($cursors as $name => $cursor) {
	if (!empty($ref_groups[$name])) {
	// This ref still has some positions, so we don't need to wipe it
	// out. Try the next one.
	continue;
	}

	foreach ($cursor->getPositions() as $position) {
	$this->log(
	pht(
	'Ref %s "%s" no longer exists.',
	$cursor->getRefType(),
	$cursor->getRefName()));

	$this->markPositionDead($position);
	}
	}
	}

	private function shouldCloseRef($ref_type, $ref_name) {
	if ($ref_type !== PhabricatorRepositoryRefCursor::TYPE_BRANCH) {
	return false;
	}

	if ($this->hasNoCursors) {
	// If we don't have any cursors, don't close things. Particularly, this
	// corresponds to the case where you've just updated to this code on an
	// existing repository: we don't want to requeue message steps for every
	// commit on a closeable ref.
	return false;
	}

	return $this->getRepository()->shouldAutocloseBranch($ref_name);
	}

	/**
	* Find all ancestors of a new closing branch head which are not ancestors
	* of any old closing branch head.
	*/
	private function loadNewCommitIdentifiers(
	$new_head,
	array $all_closing_heads) {

	$repository = $this->getRepository();
	$vcs = $repository->getVersionControlSystem();
	switch ($vcs) {
	case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
	if ($all_closing_heads) {
	$parts = array();
	foreach ($all_closing_heads as $head) {
	$parts[] = hgsprintf('%s', $head);
	}

	// See T5896. Mercurial can not parse an "X or Y or ..." rev list
	// with more than about 300 items, because it exceeds the maximum
	// allowed recursion depth. Split all the heads into chunks of
	// 256, and build a query like this:
	//
	// ((1 or 2 or ... or 255) or (256 or 257 or ... 511))
	//
	// If we have more than 65535 heads, we'll do that again:
	//
	// (((1 or ...) or ...) or ((65536 or ...) or ...))

	$chunk_size = 256;
	while (count($parts) > $chunk_size) {
	$chunks = array_chunk($parts, $chunk_size);
	foreach ($chunks as $key => $chunk) {
	$chunks[$key] = '('.implode(' or ', $chunk).')';
	}
	$parts = array_values($chunks);
	}
	$parts = '('.implode(' or ', $parts).')';

	list($stdout) = $this->getRepository()->execxLocalCommand(
	'log --template %s --rev %s',
	'{node}\n',
	hgsprintf('%s', $new_head).' - '.$parts);
	} else {
	list($stdout) = $this->getRepository()->execxLocalCommand(
	'log --template %s --rev %s',
	'{node}\n',
	hgsprintf('%s', $new_head));
	}

	$stdout = trim($stdout);
	if (!strlen($stdout)) {
	return array();
	}
	return phutil_split_lines($stdout, $retain_newlines = false);
	case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
	if ($all_closing_heads) {
	list($stdout) = $this->getRepository()->execxLocalCommand(
	'log --format=%s %s --not %Ls',
	'%H',
	$new_head,
	$all_closing_heads);
	} else {
	list($stdout) = $this->getRepository()->execxLocalCommand(
	'log --format=%s %s',
	'%H',
	$new_head);
	}

	$stdout = trim($stdout);
	if (!strlen($stdout)) {
	return array();
	}
	return phutil_split_lines($stdout, $retain_newlines = false);
	default:
	throw new Exception(pht('Unsupported VCS "%s"!', $vcs));
	}
	}

	/**
	* Mark a list of commits as closeable, and queue workers for those commits
	* which don't already have the flag.
	*/
	private function setCloseFlagOnCommits(array $identifiers) {
	$repository = $this->getRepository();
	$commit_table = new PhabricatorRepositoryCommit();
	$conn_w = $commit_table->establishConnection('w');

	$vcs = $repository->getVersionControlSystem();
	switch ($vcs) {
	case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
	$class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
	break;
	case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
	$class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
	break;
	case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
	$class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
	break;
	default:
	throw new Exception(pht("Unknown repository type '%s'!", $vcs));
	}

	$all_commits = queryfx_all(
	$conn_w,
	'SELECT id, commitIdentifier, importStatus FROM %T
	WHERE repositoryID = %d AND commitIdentifier IN (%Ls)',
	$commit_table->getTableName(),
	$repository->getID(),
	$identifiers);

	$closeable_flag = PhabricatorRepositoryCommit::IMPORTED_CLOSEABLE;

	$all_commits = ipull($all_commits, null, 'commitIdentifier');
	foreach ($identifiers as $identifier) {
	$row = idx($all_commits, $identifier);

	if (!$row) {
	throw new Exception(
	pht(
	'Commit "%s" has not been discovered yet! Run discovery before '.
	'updating refs.',
	$identifier));
	}

	if (!($row['importStatus'] & $closeable_flag)) {
	queryfx(
	$conn_w,
	'UPDATE %T SET importStatus = (importStatus \| %d) WHERE id = %d',
	$commit_table->getTableName(),
	$closeable_flag,
	$row['id']);

	$data = array(
	'commitID' => $row['id'],
	'only' => true,
	);

	PhabricatorWorker::scheduleTask($class, $data);
	}
	}

	return $this;
	}

	private function newRefCursor(
	PhabricatorRepository $repository,
	$ref_type,
	$ref_name) {

	$cursor = id(new PhabricatorRepositoryRefCursor())
	->setRepositoryPHID($repository->getPHID())
	->setRefType($ref_type)
	->setRefName($ref_name);

	try {
	return $cursor->save();
	} catch (AphrontDuplicateKeyQueryException $ex) {
	// If we raced another daemon to create this position and lost the race,
	// load the cursor the other daemon created instead.
	}

	$viewer = $this->getViewer();

	$cursor = id(new PhabricatorRepositoryRefCursorQuery())
	->setViewer($viewer)
	->withRepositoryPHIDs(array($repository->getPHID()))
	->withRefTypes(array($ref_type))
	->withRefNames(array($ref_name))
	->needPositions(true)
	->executeOne();
	if (!$cursor) {
	throw new Exception(
	pht(
	'Failed to create a new ref cursor (for "%s", of type "%s", in '.
	'repository "%s") because it collided with an existing cursor, '.
	'but then failed to load that cursor.',
	$ref_name,
	$ref_type,
	$repository->getDisplayName()));
	}

	return $cursor;
	}

	private function saveNewPositions() {
	$positions = $this->newPositions;

	foreach ($positions as $position) {
	try {
	$position->save();
	} catch (AphrontDuplicateKeyQueryException $ex) {
	// We may race another daemon to create this position. If we do, and
	// we lose the race, that's fine: the other daemon did our work for
	// us and we can continue.
	}
	}

	$this->newPositions = array();
	}

	private function deleteDeadPositions() {
	$positions = $this->deadPositions;
	$repository = $this->getRepository();

	foreach ($positions as $position) {
	// Shove this ref into the old refs table so the discovery engine
	// can check if any commits have been rendered unreachable.
	id(new PhabricatorRepositoryOldRef())
	->setRepositoryPHID($repository->getPHID())
	->setCommitIdentifier($position->getCommitIdentifier())
	->save();

	$position->delete();
	}

	$this->deadPositions = array();
	}



	/* -( Updating Git Refs )-------------------------------------------------- */


	/**
	* @task git
	*/
	private function loadGitRefPositions(PhabricatorRepository $repository) {
	$refs = id(new DiffusionLowLevelGitRefQuery())
	->setRepository($repository)
	->execute();

	return mgroup($refs, 'getRefType');
	}


	/* -( Updating Mercurial Refs )-------------------------------------------- */


	/**
	* @task hg
	*/
	private function loadMercurialBranchPositions(
	PhabricatorRepository $repository) {
	return id(new DiffusionLowLevelMercurialBranchesQuery())
	->setRepository($repository)
	->execute();
	}


	/**
	* @task hg
	*/
	private function loadMercurialBookmarkPositions(
	PhabricatorRepository $repository) {
	// TODO: Implement support for Mercurial bookmarks.
	return array();
	}

	}

PhabricatorRepositoryRefEngine.phpNo OneTemporaryActions

File Metadata

PhabricatorRepositoryRefEngine.phpView Options

Event Timeline

PhabricatorRepositoryRefEngine.php
No OneTemporary
Actions

PhabricatorRepositoryRefEngine.php
View Options