PhutilSearchStemmer.php
82f71f62129bproduction

PhutilSearchStemmer.php
View Options

			<?php

			final class PhutilSearchStemmer
			extends Phobject {

			public function stemToken($token) {
			$token = $this->normalizeToken($token);
			return $this->applyStemmer($token);
			}

			public function stemCorpus($corpus) {
			$tokens = preg_split('/[^a-zA-Z0-9\x7F-\xFF._]+/', $corpus);

			$words = array();
			foreach ($tokens as $key => $token) {
			$token = trim($token, '._');

			if (strlen($token) < 3) {
			continue;
			}

			$normal_word = $this->normalizeToken($token);
			$words[$normal_word] = $normal_word;
			}

			$stems = array();
			foreach ($words as $normal_word) {
			$stems[] = $this->applyStemmer($normal_word);
			}

			return implode(' ', $stems);
			}

			private function normalizeToken($token) {
			return phutil_utf8_strtolower($token);
			}

			/**
			* @phutil-external-symbol class Porter
			*/
			private function applyStemmer($normalized_token) {
			// If the token has internal punctuation, handle it literally. This
			// deals with things like domain names, Conduit API methods, and other
			// sorts of informal tokens.
			if (preg_match('/[._]/', $normalized_token)) {
			return $normalized_token;
			}

			static $loaded;

			if ($loaded === null) {
			$root = dirname(phutil_get_library_root('phutil'));
			require_once $root.'/externals/porter-stemmer/src/Porter.php';
			$loaded = true;
			}


			$stem = Porter::stem($normalized_token);

			// If the stem is too short, it won't be a candidate for indexing. These
			// tokens are also likely to be acronyms (like "DNS") rather than real
			// English words.
			if (strlen($stem) < 3) {
			return $normalized_token;
			}

			return $stem;
			}

			}