Homec4science
Diffusion invenio-infoscience (master) Browse

R3600/modules/bibauthorid/lib/bibauthorid_authorname_utils.py
df339938ec67master

Last Change
Raw File

bibauthorid_authorname_utils.py
View Options

# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
'''
bibauthorid_authornames_utils
Helper for accessing the author names data structure
'''
import bibauthorid_utils
from copy import deepcopy
from bibauthorid_utils import clean_name_string
from bibauthorid_utils import split_name_parts
import bibauthorid_structs as dat
import bibauthorid_config as bconfig
def get_bibrefs_by_authornames_id(authornames_id):
'''
Finds actual ids of the author name as it appears in bib10x or bib70x
@param authornames_id: id in aidAUTHORNAMES
@return: A list of sets.
- The first set in the list contains all ids in bib10x
- The second set in the list contains all ids in bib70x
@rtype: list of sets
'''
bibrefs = ''
bibref_str = [row['bibrefs'] for row in dat.AUTHOR_NAMES
if row['id'] == authornames_id]
if len(bibref_str) > 0:
bibrefs = bibref_str.split(",")
b100 = set()
b700 = set()
for bibref in bibrefs:
tag, refid = bibref.split(':')
if tag == "100":
b100.add(int(refid))
elif tag == "700":
b700.add(int(refid))
else:
bconfig.LOGGER.error("Wrong bibref Tag...how did you do that?")
return [b100, b700]
def name_matching(orig_name, target_name):
"""
Checks the compatibility of the given names.
@param orig_name: The original name String
@type orig_name: string
@param target_name: The target name string
@type target_name: string
@return: true or false in respect to the compatibility of the given names
@rtype: boolean
"""
orig = bibauthorid_utils.split_name_parts(orig_name)
targ = bibauthorid_utils.split_name_parts(target_name)
if (len(orig[1]) == 0) or (len(targ[1]) == 0):
return True
else:
initials_set = set(orig[1])
names_set = set(orig[2])
comp_initials_set = set(targ[1])
comp_names_set = set(targ[2])
names_intersection = names_set.intersection(comp_names_set)
initials_intersection = initials_set.intersection(comp_initials_set)
if len(initials_intersection) == 0:
if len(names_intersection) != 0:
bconfig.LOGGER.error("length of names intersection != 0..."
"This should never happen!")
if ((len(names_intersection) == 0) and (len(comp_names_set) > 0)
and (len(names_set) > 0)):
return False
if orig[1][0] == targ[1][0]:
return True
return False
def search_matching_names(authorname_string, match_function=name_matching,
consider_surname_only=True):
"""
search for matching names give a matching function.
@warning: searching for matching name with consider_surname_only=false
will be painfully slow! You've been warned.
@warning: for mental sanity purposes the surnames not ending with a comma
are being ignored;
if you're searching for a surname without comma or names, the comma is
being added automatically to the end of the string.
@param authorname_string: The author name string
@type authorname_string: string
@param match_function: The function to use for the name matching
@type match_function: function descriptor
@param consider_surname_only: Decides if only names with the same
surname shall be considered or _all_ other names.
@type consider_surname_only: boolean
@return: an array containing a tuple
@rtype: list of tuples
@note: example:
search_matching_names('einstein, albert')
Out[7]: [[(962L, 'Einstein, Albert'), ['Einstein', ['A'], ['Albert']]],
[(1128L, 'Einstein, A.'), ['Einstein', ['A'], []]]]
"""
possible_names = []
names = []
if authorname_string.count(',') == 0:
authorname_string += ','
authorname = bibauthorid_utils.split_name_parts(authorname_string)
if consider_surname_only:
names = [row for row in dat.AUTHOR_NAMES
if row['name'].startswith(authorname[0])]
else:
names = [row for row in dat.AUTHOR_NAMES]
for name in names:
if match_function(authorname_string, name['name']):
possible_names.append([(name['id'], name['name']),
bibauthorid_utils.split_name_parts(name['name'])])
return possible_names
def get_name_id(name_string):
"""
@return: the id associated to a given string in the authornames table.
Returns -1 if the string is not found.
@return: int
"""
name = [row['id'] for row in dat.AUTHOR_NAMES
if row['name'] == name_string]
try:
name_id = name[0]
except (IndexError, ValueError):
name_id = -1
return name_id
def get_name_string(authorname_id):
'''
Get name representation for an ID in authornames table
@return: the name string associated with a particular authorid in the
authornames table.
If the ID is not found returns an empty string.
@rtype: string
'''
name_string = ""
name = [row['name'] for row in dat.AUTHOR_NAMES
if row['id'] == authorname_id]
if len(name) > 0:
name_string = name[0]
return name_string
def get_db_name_string(authorname_id):
'''
Get name representation for an ID in authornames table
@return: the name string associated with a particular authorid in the
authornames table.
If the ID is not found returns an empty string.
@rtype: string
'''
name_string = ""
name = [row['db_name'] for row in dat.AUTHOR_NAMES
if row['id'] == authorname_id]
if len(name) > 0:
name_string = name[0]
return name_string
def get_name_and_db_name_strings(authorname_id):
'''
Get name representation for an ID in authornames table
@return: the name string and the db name string associated with a
particular authornameid in the authornames table.
If the ID is not found returns empty values for the dict keys.
@rtype: dict
'''
names_dict = {"name": "",
"db_name" : ""}
name = [row for row in dat.AUTHOR_NAMES
if row['id'] == authorname_id]
if len(name) > 0:
names_dict["name"] = name[0]['name']
names_dict["db_name"] = name[0]['db_name']
return names_dict
def get_name_bibrefs(authorname_id):
"""
Finds the bibrefID from authorname_id.
@param authorname_id: ID of the author name to look up the bibliographic
reference for
@type authorname_id: int
@return: the bibrefs associated with a particular authorid in the
authornames table. If the ID is not found, an empty string
shall be returned.
@rtype: string
"""
bibref_string = ""
bibrefs = [row['bibrefs'] for row in dat.AUTHOR_NAMES
if row['id'] == authorname_id]
if len(bibrefs) > 0:
bibref_string = bibrefs[0]
return bibref_string
def update_doclist(bibrec_id, authorname_id=""):
"""
Update doclist table given bibrec_id and processed author. (inserts a new
document in the doclist table)
@return: True if a new bibrecord has been added, false if this
bibrecord was previously processed
@rtype: boolean
"""
records = [row for row in dat.DOC_LIST
if row['bibrecid'] == bibrec_id]
if len(records) > 0:
# @note maybe it's better to have a comma-separated list in the
# 'authorname_id' column. That would keep the DB size
# lower. First steps for the implementation introduced; update
# procedure necessary. Descision might be harder.
# Performance tests might help.
for record in records:
if ((authorname_id) and
(authorname_id not in record['authornameids'])):
record['authornameids'] += [authorname_id]
else:
bconfig.LOGGER.warn("Bibrec has already been processed. That's"
+ " OK. Skipping record.")
return False
else:
if authorname_id:
dat.DOC_LIST.append({'bibrecid': bibrec_id,
'authornameids': [authorname_id]})
else:
dat.DOC_LIST.append({'bibrecid': bibrec_id,
'authornameids': []})
return True
def compare_names(origin_name, target_name):
"""
Compute an index of confidence that would like to indicate whether two
names might represent the same person.The computation is based on
similarities of name structure, in particular:
Initials:
We assign an high score if all the initials matches are in the
right order, much lower if they are in the wrong order
Names:
We assign a lower score for mismatching names and higher score for
fully matching names
If there is nothing to compare we are forced to assume a high score.
Example for splitting names:
In : bibauthorid.split_name_parts("Ellis, John R")
Out: ['Ellis', ['J', 'R'], ['John']]
Ellis, R. Keith => [ [Ellis], [R, K], [Keith] ]
Ellis, Richard Keith => [ [Ellis], [R, K], [Richard, Keith] ]
Since the initials are computed whether on the real initials present in the
name string and using the full name, if there is no initials match we are 1
00% confident that:
1. we have no names/initials at all, or
2. we have completely different names; hence if there is no initial
match we skip this step.
@param orig_name: The first author's last name, first name(s) and initial
@type orig_name: list of strings and lists of strings
@param targ_name: The second author's last name, first name(s) and initial
@type targ_name: list of strings and lists of strings
@return: a value that describes the likelihood of the names being the same
@rtype: float
"""
jaro_fctn = None
try:
from Levenshtein import jaro_winkler
jaro_fctn = jaro_winkler
except ImportError:
jaro_fctn = jaro_winkler_str_similarity
oname = deepcopy(origin_name)
tname = deepcopy(target_name)
orig_name = split_name_parts(oname.lower())
targ_name = split_name_parts(tname.lower())
bconfig.LOGGER.info("|--> Comparing Names: \"%s\" and \"%s\"" %
(origin_name, target_name))
lastname_modifier = 0.0
if not (orig_name[0] == targ_name[0]):
# last names are not equal before cleaning them. Assign entry penalty.
lastname_modifier = 0.15
orig_name[0] = clean_name_string(orig_name[0],
replacement="",
keep_whitespace=False)
targ_name[0] = clean_name_string(targ_name[0],
replacement="",
keep_whitespace=False)
if not (orig_name[0] == targ_name[0]):
if ((jaro_fctn(orig_name[0].lower(), targ_name[0].lower()) < .95)
or min(len(orig_name[0]), len(targ_name[0])) <= 4):
bconfig.LOGGER.warn(("Unequal lastnames(%s vs. %s)."
+ "Skipping Comparison")
% (orig_name[0], targ_name[0]))
return 0.0
else:
bconfig.LOGGER.log(25, "Last names are not equal; "
+ "but similar enough to continue the comparison")
# Let it go through...however, reduce the final result a little.
lastname_modifier = 0.24
else:
# last names are equal after cleaning them. Reduce penalty.
if lastname_modifier == 0.15:
lastname_modifier = 0.02
if orig_name[2] and targ_name[2]:
if len(orig_name[2]) > 1 or len(targ_name[2]) > 1:
variation_ps = []
oname_variations = _create_ntuples(orig_name[2])
tname_variations = _create_ntuples(targ_name[2])
for oname_variation in oname_variations:
for tname_variation in tname_variations:
oname_var = split_name_parts("%s, %s"
% (orig_name[0],
oname_variation))
tname_var = split_name_parts("%s, %s"
% (targ_name[0],
tname_variation))
variation_ps.append(_perform_matching(oname_var,
tname_var))
return max(variation_ps) - lastname_modifier
return _perform_matching(orig_name, targ_name) - lastname_modifier
def _perform_matching(orig_name, targ_name):
'''
@param orig_name:
@type orig_name:
@param targ_name:
@type targ_name:
'''
tname = deepcopy(targ_name)
oname = deepcopy(orig_name)
potential_name_matches = min(len(oname[2]), len(tname[2]))
names_p_weight = 0.0
initials_p_weight = _compare_initials(oname, tname)
if initials_p_weight > 0.0:
names_p_weight = _compare_first_names(oname, tname)
names_w = .5
ini_w = .5
if (names_p_weight > 0.6) and (potential_name_matches > 0):
names_w = .7
ini_w = .3
if (initials_p_weight == 1.0) and (len(oname[1]) != len(tname[1])):
initials_p_weight -= .1
if (names_p_weight == 1.0) and ((len(oname[2]) != len(tname[2]))
or not len(oname[2])) and (potential_name_matches < 2):
names_p_weight -= .1
if (initials_p_weight == 1.0) and (names_p_weight <= 0):
names_w = 0.
ini_w = 0.
res = names_w * names_p_weight + ini_w * initials_p_weight
# print "|--> Comparing Names: %s and %s" % (oname, tname)
bconfig.LOGGER.debug(("|---> iWeight (%s) * ip (%s) + nWeight " +
"(%s) * nP (%s) = %s") % (ini_w, initials_p_weight,
names_w, names_p_weight, res))
return (names_w * names_p_weight + ini_w * initials_p_weight)
def _compare_initials(orig_name, targ_name):
'''
Compares Author's initials and returns the assigned score.
@param orig_name: The first author's last name, first name(s) and initial
@type orig_name: list of strings and lists of strings
@param targ_name: The second author's last name, first name(s) and initial
@type targ_name: list of strings and lists of strings
@return: a value describing the likelihood of the initials being the same
@rtype: float
'''
# determine minimal number of initials and declare the
# count of max. possible matches
tname = deepcopy(targ_name)
oname = deepcopy(orig_name)
max_possible_matches = min(len(oname[1]), len(tname[1]))
initial_weight_denominator = (float(1 + max_possible_matches) /
2.0) * max_possible_matches
initials_p_weight = 0.0
if max_possible_matches > 0:
for index, item in enumerate(oname[1]):
# print "|---> Trying Initial: ", I
if index < max_possible_matches:
try:
targ_index = tname[1].index(item)
if index == targ_index:
initials_p_weight += (
float(index + 1) / initial_weight_denominator)
else:
initials_p_weight += 1. / (5 * max_possible_matches *
abs(index - targ_index))
tname[1][targ_index] = ''
except (IndexError, ValueError, ZeroDivisionError):
# initials_p_weight = 0.1
break
else:
initials_p_weight = 0.0
return initials_p_weight
def _compare_first_names(orig_name, targ_name):
'''
Compares Author's first names and returns the assigned score.
@param orig_name: The first author's last name, first name(s) and initial
@type orig_name: list of strings and lists of strings
@param targ_name: The second author's last name, first name(s) and initial
@type targ_name: list of strings and lists of strings
@return: a value that describes the likelihood of the names being the same
@rtype: float
'''
# determine minimal number of names and declare the
# count of max. possible matches
string_similarity = None
try:
from Levenshtein import jaro_winkler
string_similarity = jaro_winkler
except ImportError:
string_similarity = jaro_winkler_str_similarity
tname = deepcopy(targ_name)
oname = deepcopy(orig_name)
names_p_weight = 0.0
max_possible_matches = float(min(len(oname[2]), len(tname[2])))
name_weight_denominator = ((1.0 + max_possible_matches)
/ 2.0 * max_possible_matches)
equal_set = set(oname[2]).intersection(set(tname[2]))
equal_names = [i for i in oname[2] if i in equal_set]
if max_possible_matches < 1.:
return 1.0
if len(equal_names) == max_possible_matches:
for index, item in enumerate(equal_names):
if index <= max_possible_matches:
try:
targ_index = tname[2].index(item)
initial_index = oname[1].index(item[0].upper())
if (index == targ_index) or (initial_index == targ_index):
names_p_weight += (float(index + 1) /
float(name_weight_denominator))
else:
names_p_weight += 1. / (2 * max_possible_matches *
abs(index - targ_index))
tname[2][targ_index] = ''
except (IndexError, ValueError, ZeroDivisionError):
break
else:
fuzzy_matches = 0
wrong_position_modifier = 0
# for name1 in oname[2]:
# for name2 in tname[2]:
# similarity = string_similarity(name1, name2)
# if similarity > 0.91:
# fuzzy_matches += 1
# if oname[2].index(name1) != tname[2].index(name2):
# wrong_position_modifier += 0.05
for name1 in oname[2]:
for name2 in tname[2]:
fuzzy_matches += string_similarity(name1, name2)
if oname[2].index(name1) != tname[2].index(name2):
wrong_position_modifier += 0.05
if fuzzy_matches > 0:
num_comparisons = len(oname[2]) * len(tname[2])
names_p_weight = (fuzzy_matches / num_comparisons -
wrong_position_modifier)
else:
names_p_weight = -0.3
return names_p_weight
def _create_ntuples(names):
'''
Find name combinations, i.e. permutations of the names in different
positions of the name
@param names: a list of names
@type names: list of string
@return: the combinations of the names given
@rtype: list of lists of strings
'''
length = float(len(names))
max_tuples = int((length / 2) * (length - 1))
current_tuple = 1
pos = 0
off = 1
variants = [" ".join(names)]
for i in range(max_tuples):
variant = "%s %s %s" % (' '.join(names[0:pos]),
''.join(names[pos:off + 1]).capitalize(),
' '.join(names[off + 1::]))
variants.append(variant.strip())
pos += 1
off += 1
if off >= length:
pos = i * 0
off = current_tuple + 1
current_tuple += 1
return variants
def jaro_str_distance(str1, str2):
"""
The Jaro string similarity algorithm as described in
'Jaro, M.A. (1989): "Advances in record linkage methodology as applied
to the 1985 census of Tampa Florida'
@param str1: The first string
@type str1: string
@param str2: The second string
@type str2: string
@return: approximate string comparison measure (between 0.0 and 1.0)
@rtype: float
"""
if (not str1) or (not str2):
return 0.0
elif str1 == str2:
return 1.0
jaro_marker = chr(1)
len1 = len(str1)
len2 = len(str2)
halflen = max(len1, len2) / 2 + 1
assignments1 = ''
assignments2 = ''
workstr1 = str1
workstr2 = str2
common1 = common2 = 0
# Analyze the first string
for i in xrange(len1):
start = max(0, i - halflen)
end = min(i + halflen + 1, len2)
index = workstr2.find(str1[i], start, end)
if index > -1: # Found common character
common1 += 1
assignments1 = assignments1 + str1[i]
workstr2 = workstr2[:index] + jaro_marker + workstr2[index + 1:]
# Analyze the second string
for i in xrange(len2):
start = max(0, i - halflen)
end = min(i + halflen + 1, len1)
index = workstr1.find(str2[i], start, end)
if (index > -1): # Found common character
common2 += 1
assignments2 = assignments2 + str2[i]
workstr1 = workstr1[:index] + jaro_marker + workstr1[index + 1:]
common = float(common1 + common2) / 2.0
if (common == 0):
return 0.0
transpositions = 0
for i in xrange(len(assignments1)):
if (assignments1[i] != assignments2[i]):
transpositions += 1
transpositions /= 2.0
common = float(common)
len1 = float(len1)
len2 = float(len2)
jaro_constant = 1.0 / 3.0
jaro_transpositions = (common1 - transpositions) / common1
jaro_common_to_len_ratio = common1 / len1 + common1 / len2
dist = jaro_constant * (jaro_common_to_len_ratio + jaro_transpositions)
return dist
def _winkler_modifier(str1, str2, in_weight):
"""
Applies the winkler modifier to a score obtained by the Jaro string
similarity measure. This is described in Winkler, W.E. (1999) "The state
of record linkage and current research problems".
If the first characters of the two strings (up to first 4) are identical,
the similarity weight will be increased.
@param str1: The first string
@type str1: string
@param str2: The second string
@type str2: string
@param in_weight: Similarity score obtained by the Jaro algorithm
@type in_weight: float
@return: approximate string comparison measure (between 0.0 and 1.0)
@rtype: float
"""
if (not str1) or (not str2):
return 0.0
elif str1 == str2:
return 1.0
# Compute how many characters are common at beginning
minlen = min(len(str1), len(str2))
common_chars_num = 0
for common_chars_num in xrange(1, minlen + 1):
if str1[:common_chars_num] != str2[:common_chars_num]:
break
common_chars_num -= 1
if (common_chars_num > 4):
common_chars_num = 4
winkler_weight = in_weight + common_chars_num * 0.1 * (1.0 - in_weight)
final_result = 0.0
if winkler_weight >= 0.0 and winkler_weight <= 1.0:
final_result = winkler_weight
elif winkler_weight > 1.0:
final_result = 1.0
return final_result
def jaro_winkler_str_similarity(str1, str2):
"""
For backwards compatibility, call Jaro followed by Winkler modification.
@param str1: The first string
@type str1: string
@param str2: The second string
@type str2: string
@return: approximate string comparison measure (between 0.0 and 1.0)
@rtype: float
"""
jaro_weight = jaro_str_distance(str1, str2)
return _winkler_modifier(str1, str2, jaro_weight)
c4science · Help