PhutilSearchStemmer.php
No OneTemporary
Actions

Subscribers

None

File Metadata

	<?php

	final class PhutilSearchStemmer
	extends Phobject {

	public function stemToken($token) {
	$token = $this->normalizeToken($token);
	return $this->applyStemmer($token);
	}

	public function stemCorpus($corpus) {
	$tokens = preg_split('/[^a-zA-Z0-9\x7F-\xFF._]+/', $corpus);

	$words = array();
	foreach ($tokens as $key => $token) {
	$token = trim($token, '._');

	if (strlen($token) < 3) {
	continue;
	}

	$normal_word = $this->normalizeToken($token);
	$words[$normal_word] = $normal_word;
	}

	$stems = array();
	foreach ($words as $normal_word) {
	$stems[] = $this->applyStemmer($normal_word);
	}

	return implode(' ', $stems);
	}

	private function normalizeToken($token) {
	return phutil_utf8_strtolower($token);
	}

	/**
	* @phutil-external-symbol class Porter
	*/
	private function applyStemmer($normalized_token) {
	// If the token has internal punctuation, handle it literally. This
	// deals with things like domain names, Conduit API methods, and other
	// sorts of informal tokens.
	if (preg_match('/[._]/', $normalized_token)) {
	return $normalized_token;
	}

	static $loaded;

	if ($loaded === null) {
	$root = dirname(phutil_get_library_root('phutil'));
	require_once $root.'/externals/porter-stemmer/src/Porter.php';
	$loaded = true;
	}


	$stem = Porter::stem($normalized_token);

	// If the stem is too short, it won't be a candidate for indexing. These
	// tokens are also likely to be acronyms (like "DNS") rather than real
	// English words.
	if (strlen($stem) < 3) {
	return $normalized_token;
	}

	return $stem;
	}

	}