Page Menu
Home
c4science
Search
Configure Global Search
Log In
Files
F80567164
PhutilSearchStemmer.php
No One
Temporary
Actions
Download File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Subscribers
None
File Metadata
Details
File Info
Storage
Attached
Created
Sun, Sep 1, 10:27
Size
1 KB
Mime Type
text/x-php
Expires
Tue, Sep 3, 10:27 (2 d)
Engine
blob
Format
Raw Data
Handle
20395337
Attached To
rPHU libphutil
PhutilSearchStemmer.php
View Options
<?php
final
class
PhutilSearchStemmer
extends
Phobject
{
public
function
stemToken
(
$token
)
{
$token
=
$this
->
normalizeToken
(
$token
);
return
$this
->
applyStemmer
(
$token
);
}
public
function
stemCorpus
(
$corpus
)
{
$tokens
=
preg_split
(
'/[^a-zA-Z0-9
\x
7F-
\x
FF._]+/'
,
$corpus
);
$words
=
array
();
foreach
(
$tokens
as
$key
=>
$token
)
{
$token
=
trim
(
$token
,
'._'
);
if
(
strlen
(
$token
)
<
3
)
{
continue
;
}
$normal_word
=
$this
->
normalizeToken
(
$token
);
$words
[
$normal_word
]
=
$normal_word
;
}
$stems
=
array
();
foreach
(
$words
as
$normal_word
)
{
$stems
[]
=
$this
->
applyStemmer
(
$normal_word
);
}
return
implode
(
' '
,
$stems
);
}
private
function
normalizeToken
(
$token
)
{
return
phutil_utf8_strtolower
(
$token
);
}
/**
* @phutil-external-symbol class Porter
*/
private
function
applyStemmer
(
$normalized_token
)
{
// If the token has internal punctuation, handle it literally. This
// deals with things like domain names, Conduit API methods, and other
// sorts of informal tokens.
if
(
preg_match
(
'/[._]/'
,
$normalized_token
))
{
return
$normalized_token
;
}
static
$loaded
;
if
(
$loaded
===
null
)
{
$root
=
dirname
(
phutil_get_library_root
(
'phutil'
));
require_once
$root
.
'/externals/porter-stemmer/src/Porter.php'
;
$loaded
=
true
;
}
$stem
=
Porter
::
stem
(
$normalized_token
);
// If the stem is too short, it won't be a candidate for indexing. These
// tokens are also likely to be acronyms (like "DNS") rather than real
// English words.
if
(
strlen
(
$stem
)
<
3
)
{
return
$normalized_token
;
}
return
$stem
;
}
}
Event Timeline
Log In to Comment