Index: py2ttl/config.py =================================================================== --- py2ttl/config.py (revision 61) +++ py2ttl/config.py (revision 62) @@ -1,37 +1,37 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import getpass from os.path import isfile, isdir, exists import re PROJECT_NAME = 'tln' PROJECT_URL = 'http://www.knora.org/ontology/0068/nietzsche' ONTOLOGY_DIR = './ontologies' if getpass.getuser() == 'knister0' else './ontologies' # local onotology dir, script will read only KNORA_BASE_ONTOLOGY_FILE = '{}/knora-ontologies/knora-base.ttl'.format(ONTOLOGY_DIR) SHARED_ONTOLOGIES_DIR = '{}/Ontologies-shared'.format(ONTOLOGY_DIR) -PROJECT_ONTOLOGY_FILE = './nietzsche-ontology.ttl'.format(ONTOLOGY_DIR) +PROJECT_ONTOLOGY_FILE = './nietzsche-ontology.ttl' DATATYPES_DIR = './svgscripts/datatypes' # optional in config file, can be overwritten by passing a to py2ttl/py2ttl.py def check_config_files_exist(): """Checks whether all files exist that are specified in this file by uppercase variables ending in 'DIR' or 'FILE'. :return: exit code (int) """ for key in [ key for key in globals().keys() if re.match(r'^[A-Z_-]+(DIR|FILE)$', key) ]: if not exists(globals().get(key)): raise FileNotFoundError('Key {} does not specify an existing file or directory'.format(key)) if key.endswith('DIR') and not isdir(globals().get(key)): raise NotADirectoryError('Key {} does not specify an existing directory'.format(key)) return 0 def get_datatypes_dir(): """Returns value of DATATYPES_DIR if set, else None. """ if 'DATATYPES_DIR' in globals().keys(): return DATATYPES_DIR.replace('./','') else: None Index: py2ttl/py2ttl.py =================================================================== --- py2ttl/py2ttl.py (revision 61) +++ py2ttl/py2ttl.py (revision 62) @@ -1,392 +1,452 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ This program can be used to convert py classes that are subclasses of DATATYPES_DIR.class_spec.SemanticClass to a owl ontology in turtle format. """ # Copyright (C) University of Basel 2019 {{{1 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see 1}}} import getopt import importlib import importlib.util import inspect import lxml.etree as ET from os import sep, path, listdir from os.path import isfile, isdir, dirname, basename from progress.bar import Bar from rdflib import Graph, URIRef, Literal, BNode, OWL, RDF, RDFS, XSD import re import sys if dirname(__file__) not in sys.path: sys.path.append(dirname(__file__)) from config import check_config_files_exist, get_datatypes_dir, PROJECT_NAME, PROJECT_ONTOLOGY_FILE, PROJECT_URL, SHARED_ONTOLOGIES_DIR from knora_base import KNORA_BASE __author__ = "Christian Steiner" __maintainer__ = __author__ __copyright__ = 'University of Basel' __email__ = "christian.steiner@unibas.ch" __status__ = "Development" __license__ = "GPL v3" __version__ = "0.0.1" class Py2TTLConverter: """This class can be used convert semantic_dictionaries to a owl ontology in turtle format. """ UNITTESTING = False + ORDERED_LIST_KEYS = [ 'flag', 'part_cls_name', 'part_name', 'part_label'] + INFO_DICT_KEYS = [ 'cardinality_restriction', 'comment', 'label', 'name', 'xpath' ] + def __init__(self, project_ontology_file=None, create_super_cls_for_multi_property=True): self.list_value = -99 self.class_uri_dict = {} self.uri_xpath_mapping = {} self.create_super_cls_for_multi_property = create_super_cls_for_multi_property self.project_graph = Graph() self.base_uriref = URIRef(PROJECT_URL) self.project_name = PROJECT_NAME self.ns = { self.base_uriref + '#': self.project_name } if project_ontology_file is not None and isfile(project_ontology_file): self.project_graph.parse(project_ontology_file, format="turtle") if len(self.project_graph) > 0: self.base_uriref = self.project_graph.value(predicate=RDF.type, object=OWL.Ontology, any=False) self.ns = { uriref: ns for ns, uriref in self.project_graph.namespace_manager.namespaces() } self.project_name = self.ns.get(self.base_uriref + '#') self.project_graph.bind(self.project_name, self.base_uriref + '#') - def get_semantic_classes(self, datatypes_dir): - """Returns a list of all classes that are contained in datatypes_dir that are subclasses of DATATYPES_DIR.class_spec.SemanticClass. - - :return: a list of (str_name, class) + def addClass(self, cls_uri, comment='', label='', super_uri=KNORA_BASE.Resource): + """Add a class to project_graph. """ - base_dir = dirname(dirname(__file__)) - sys.path.append(base_dir) - root_modul_name = datatypes_dir.replace('/','.') - reference_cls = importlib.import_module('{}.{}'.format(root_modul_name, 'class_spec')) - try: - self.list_value = reference_cls.LIST - except AttributeError: - pass - files = [ file.replace('.py','') for file in listdir(datatypes_dir) if file.endswith('.py') and not file.startswith('test_') and not file.startswith('_')] - all_modules = [] - for name in files: - all_modules.append(importlib.import_module('{}.{}'.format(root_modul_name, name))) - all_classes = [] - for modul in all_modules: - all_classes += inspect.getmembers(modul, inspect.isclass) - all_classes = sorted(set(all_classes)) - semantic_classes = [ cls for name, cls in all_classes if issubclass(cls, reference_cls.SemanticClass) and not (cls == reference_cls.SemanticClass)] - return semantic_classes + if comment == '' and cls_uri.split('#')[1] in self.class_uri_dict: + comment, label = self.get_comment_label(self.class_uri_dict.get(cls_uri.split('#')[1])) + self.project_graph.add((cls_uri, RDF.type, OWL.Class)) + if comment != '': + self.project_graph.add((cls_uri, RDFS.comment, Literal(comment, lang='en'))) + if label != '': + self.project_graph.add((cls_uri, RDFS.label, Literal(label, lang='en'))) + self.project_graph.add((cls_uri, RDFS.subClassOf, super_uri)) + + def addProperty(self, property_uri, super_uri, subject_uri, object_uri, comment, label, cardinality, info_dict={}): + """Add a property to self.project_graph. + """ + self.project_graph.add((property_uri, RDF.type, OWL.ObjectProperty)) + self.project_graph.add((property_uri, RDFS.subPropertyOf, super_uri)) + self.project_graph.add((property_uri, KNORA_BASE.objectClassConstraint, object_uri)) + self.project_graph.add((property_uri, KNORA_BASE.subjectClassConstraint, subject_uri)) + self.project_graph.add((property_uri, RDFS.comment, Literal(comment, lang='en'))) + self.project_graph.add((property_uri, RDFS.label, Literal(label, lang='en'))) + self.addRestriction2Class(subject_uri, property_uri, cardinality=cardinality, info_dict=info_dict) def addRestriction2Class(self, cls_uri, property_uri, cardinality=0, comment="", label="", info_dict={}): """Adds restriction on property_uri to class cls_uri. """ if (cls_uri, None, None) not in self.project_graph: self.addClass(cls_uri, comment=comment, label=label) restriction = BNode() if 'cardinality_restriction' in info_dict.keys(): cardinality_restriction = URIRef(OWL + info_dict['cardinality_restriction']) else: cardinality_restriction = OWL.minCardinality if cardinality == 0 else OWL.cardinality self.project_graph.add((cls_uri, RDFS.subClassOf, restriction)) self.project_graph.add((restriction, RDF.type, OWL.Restriction)) self.project_graph.add((restriction, OWL.onProperty, property_uri)) self.project_graph.add((restriction, cardinality_restriction, Literal(str(cardinality), datatype=XSD.nonNegativeInteger))) - def createPropertyName(self, property_name=None, subject_uri=None, object_uri=None, connector='BelongsTo', prefix='has'): - """Returns a property name. + def convert_py2ttl(self, datatypes_dir, target_ontology_file): + """Convert all classes contained in datatypes_dir that are subclasses of DATATYPES_DIR.class_spec.SemanticClass to rdf. + + :return: exit code (int) """ - if property_name is not None: - property_name = ''.join([ property_name.split('_')[0].lower() ] + [ text.capitalize() for text in property_name.split('_')[1:] ]) - return prefix + property_name[0].upper() + property_name[1:] if property_name[0].islower()\ - else prefix + property_name - elif subject_uri is not None: - property_name = subject_uri.split('#')[1] + self.createPropertyName(object_uri=object_uri, prefix=connector) - return property_name[0].lower() + property_name[1:] - elif object_uri is not None: - return prefix + object_uri.split('#')[1] + if isdir(datatypes_dir): + semantic_classes = self.get_semantic_classes(datatypes_dir) + if not Py2TTLConverter.UNITTESTING: + bar = Bar('creating classes and properties', max=len(semantic_classes)) + for cls in semantic_classes: + self.createClassAndProperties(cls) + not bool(Py2TTLConverter.UNITTESTING) and bar.next() + not bool(Py2TTLConverter.UNITTESTING) and bar.finish() + f = open(target_ontology_file, 'wb+') + f.write(self.project_graph.serialize(format="turtle")) + f.close() + #print(self.uri_xpath_mapping.get(URIRef(self.base_uriref + '#TranskriptionPosition'))) else: - return prefix - - def createSuperClassForSubjectClassConstraint(self, property_uri, sub_uri): - """Creates a super class for classes that share a property. - """ - super_uri = URIRef(property_uri.replace('has', '') + 'Holder') - self.project_graph.add((sub_uri, RDFS.subClassOf, super_uri)) - self.project_graph.remove((sub_uri, RDFS.subClassOf, KNORA_BASE.Resource)) - if (super_uri, RDF.type, OWL.Class) not in self.project_graph: - label = 'holder of ' + property_uri.split('#')[1].replace('has', '') - comment = 'super class for classes that have a ' + property_uri.split('#')[1].replace('has', '') - self.addRestriction2Class(super_uri, property_uri, comment=comment, label=label) - for object_uri in self.project_graph.objects(subject=property_uri, predicate=KNORA_BASE.subjectClassConstraint): - self.project_graph.remove((property_uri, KNORA_BASE.subjectClassConstraint, object_uri)) - self.project_graph.add((object_uri, RDFS.subClassOf, super_uri)) - self.project_graph.remove((object_uri, RDFS.subClassOf, KNORA_BASE.Resource)) - self.project_graph.add((property_uri, KNORA_BASE.subjectClassConstraint, super_uri)) - objectClass = self.project_graph.value(subject=property_uri, predicate=KNORA_BASE.objectClassConstraint, any=False) - comment = 'connects {} with {}'.format(super_uri.split('#')[1], objectClass.split('#')[1].replace('has', '')) - self.project_graph.remove((property_uri, RDFS.comment, None)) - self.project_graph.add((property_uri, RDFS.comment, Literal(comment, lang='en'))) + print('Error: dir {} does not exist!'.format(datatypes_dir)) + usage + return 1 + return 0 - def addProperty(self, property_uri, super_uri, subject_uri, object_uri, comment, label, cardinality, info_dict={}): - """Add a property to self.project_graph. + def createClassAndProperties(self, cls): + """Creates a owl:Class and some owl:ObjectProperty from semantic_dictionary of a python class. """ - self.project_graph.add((property_uri, RDF.type, OWL.ObjectProperty)) - self.project_graph.add((property_uri, RDFS.subPropertyOf, super_uri)) - self.project_graph.add((property_uri, KNORA_BASE.objectClassConstraint, object_uri)) - self.project_graph.add((property_uri, KNORA_BASE.subjectClassConstraint, subject_uri)) - self.project_graph.add((property_uri, RDFS.comment, Literal(comment, lang='en'))) - self.project_graph.add((property_uri, RDFS.label, Literal(label, lang='en'))) - self.addRestriction2Class(subject_uri, property_uri, cardinality=cardinality, info_dict=info_dict) - - def createProperty(self, cls_uri, property_name, property_cls, cardinality, info_dict={}): + if not cls.__name__ in self.class_uri_dict: + self.class_uri_dict.update({cls.__name__: cls}) + semantic_dict = cls.get_semantic_dictionary() + super_uri = KNORA_BASE.Resource + super_cls = None + if bool(semantic_dict['class'].get('type')): + super_cls = semantic_dict['class'].get('type') + self.createClassAndProperties(super_cls) + super_uri = URIRef(self.base_uriref + '#' + super_cls.__name__) + cls_uri = URIRef(self.base_uriref + '#' + cls.__name__) + comment, label = self.get_comment_label(cls) + self.addClass(cls_uri, comment, label, super_uri) + for property_key in self._get_semantic_dictionary_keys_super_first(semantic_dict['properties']): + super_semantic_dict = {} if super_cls is None else super_cls.get_semantic_dictionary() + if len(super_semantic_dict) == 0 or not bool(super_semantic_dict['properties'].get(property_key))\ + or type(super_semantic_dict['properties'].get(property_key)) == dict\ + or super_semantic_dict['properties'].get(property_key)[1] != self.list_value\ + or semantic_dict['properties'].get(property_key)[0] != super_semantic_dict['properties'].get(property_key)[0]: + try: + if type(semantic_dict['properties'].get(property_key)) == dict: + property_dict4key = semantic_dict['properties'].get(property_key) + object_uri = None + property_cls = None + property_uri = None + cardinality = property_dict4key.get('cardinality') + xpath = property_dict4key.get('xpath') + if property_dict4key.get('flag') == 'ordered_list': + object_uri = self.createPartProperty(property_dict4key) + property_uri = URIRef(self.base_uriref + '#' + property_dict4key.get('name')) + else: + property_cls = property_dict4key.get('class') + subject_uri, property_uri = self.createProperty(cls_uri, property_key, property_cls=property_cls,\ + property_uri=property_uri, cardinality=cardinality, object_uri=object_uri, info_dict=property_dict4key) + else: + property_cls, cardinality, xpath = semantic_dict['properties'].get(property_key) + subject_uri, property_uri = self.createProperty(cls_uri, property_key, property_cls, cardinality) + if not subject_uri in self.uri_xpath_mapping: + self.uri_xpath_mapping.update({ subject_uri: {}}) + self.uri_xpath_mapping.get(subject_uri).update({property_uri: xpath}) + except ValueError: + raise Exception('Class {} does not have a xpath spec in its get_semantic_dictionary()'.format(cls)) + + def createPartProperty(self, info_dict): + """Creates a owl:ObjectProperty from a dictionary created by SemanticClass.get_cls_hasPart_objectCls_dictionaries(). + + :return: subject_uri (rdflib.URIRef) + """ + if info_dict.get('flag') == 'ordered_list' and type(info_dict.get('class')) == dict: + dictionary = info_dict.get('class') + subject_cls_name = dictionary.get('class_name') + subject_label = dictionary.get('label') + subject_comment = dictionary.get('comment') + subject_uri = URIRef(self.base_uriref + '#' + subject_cls_name) + self.addClass(subject_uri, comment=subject_comment, label=subject_label) + seqnum_dictionary = dictionary.get('has_seqnum') + seqnum_name = seqnum_dictionary.get('name') + seqnum_xpath= seqnum_dictionary.get('xpath') + seqnum_property_uri = URIRef(self.base_uriref + '#' + seqnum_name) + subject_uri, seqnum_property_uri = self.createProperty(subject_uri, property_uri=seqnum_property_uri, property_cls=int,\ + cardinality=seqnum_dictionary.get('cardinality'), super_uri=KNORA_BASE.seqnum, info_dict=seqnum_dictionary) + if not subject_uri in self.uri_xpath_mapping: + self.uri_xpath_mapping.update({ subject_uri: {}}) + self.uri_xpath_mapping.get(subject_uri).update({seqnum_property_uri: seqnum_xpath}) + part_dictionary = dictionary.get('has_part') + part_property_uri = URIRef(self.base_uriref + '#' + part_dictionary.get('name')) + part_xpath = part_dictionary.get('xpath') + object_uri = URIRef(self.base_uriref + '#' + part_dictionary.get('class').__name__) + subject_uri, property_uri = self.createProperty(subject_uri, property_uri=part_property_uri, object_uri=object_uri,\ + cardinality=part_dictionary.get('cardinality'), info_dict=part_dictionary) + self.uri_xpath_mapping.get(subject_uri).update({part_property_uri: part_xpath}) + return subject_uri + + def createProperty(self, cls_uri, property_name=None, property_cls=None, cardinality=0, property_uri=None, super_uri=None, object_uri=None, info_dict={}): """Creates a owl:ObjectProperty. :return: tuple of subject_uri (rdflib.URIRef) and property_uri (rdflib.URIRef) of created property """ inferredSubClass = RDFS.subClassOf * '*' - name = self.createPropertyName(property_name=property_name)\ + if property_uri is None: + name = self.createPropertyName(property_name=property_name)\ if 'name' not in info_dict.keys() else info_dict['name'] - property_uri = URIRef(self.base_uriref + '#' + name) + property_uri = URIRef(self.base_uriref + '#' + name) subject_uri = cls_uri label = 'has ' + name.replace('has','')\ if 'label' not in info_dict.keys() else info_dict['label'] - super_uri = KNORA_BASE.hasValue + if super_uri is None: + super_uri = KNORA_BASE.hasValue if (property_uri, None, None) not in self.project_graph: - if property_cls.__module__ == 'builtins': + if object_uri is None and property_cls.__module__ == 'builtins': datatype_mapping = { float: KNORA_BASE.DecimalValue, int: KNORA_BASE.IntValue,\ str: KNORA_BASE.TextValue, bool: KNORA_BASE.BooleanValue } + if property_cls == bool: + cardinality = 1 + info_dict.update({'cardinality_restriction': 'cardinality'}) object_uri = datatype_mapping.get(property_cls) if object_uri == KNORA_BASE.TextValue: if property_name == 'URL': object_uri = KNORA_BASE.UriValue elif property_name == 'file_name': object_uri = KNORA_BASE.FileValue else: - object_uri = URIRef(self.base_uriref + '#' + property_cls.__name__) + if object_uri is None: + object_uri = URIRef(self.base_uriref + '#' + property_cls.__name__) # if class X has a list of objects Y, we create a property YbelongsToX. if cardinality == self.list_value: subject_uri = object_uri object_uri = cls_uri result = self.project_graph.query(\ 'select ?p where {'\ + ' ?p <{0}> ?s .'.format(KNORA_BASE.subjectClassConstraint)\ + ' ?p <{0}> <{1}> .'.format(KNORA_BASE.objectClassConstraint, object_uri)\ + ' <{0}> ?s .'.format(subject_uri)\ + ' }') # if subject_uri is a subclass of a uri that is a subjectClassConstraint to a property_uri # that has object_uri as its objectClassConstraint, then we do not create a new property YbelongsToX, # instead we return subject_uri and this already existing property_uri. if len(result) > 0: return subject_uri, [ property_uri for property_uri in result ][0] name = self.createPropertyName(subject_uri=subject_uri, object_uri=object_uri) property_uri = URIRef(self.base_uriref + '#' + name) cardinality = 1 label = subject_uri.split('#')[1] + ' belongs to ' + object_uri.split('#')[1] super_uri = KNORA_BASE.hasLinkTo property_value_uri = URIRef(property_uri + 'Value') comment = 'Reification statement of relation between {} and {}'.format(subject_uri.split('#')[1], object_uri.split('#')[1]) reification_info_dict = {} if 'cardinality_restriction' in info_dict.keys(): reification_info_dict.update({'cardinality_restriction': info_dict['cardinality_restriction']}) self.addProperty(property_value_uri, KNORA_BASE.hasLinkToValue, subject_uri, KNORA_BASE.LinkValue,\ comment, label + ' - statement', cardinality, info_dict=reification_info_dict) comment = 'connects {} with {}'.format(subject_uri.split('#')[1], object_uri.split('#')[1])\ if 'comment' not in info_dict.keys() else info_dict['comment'] self.addProperty(property_uri, super_uri, subject_uri, object_uri, comment, label, cardinality, info_dict=info_dict) elif not True in [\ (cls_uri, inferredSubClass, o) in self.project_graph\ for o in self.project_graph.objects(property_uri, KNORA_BASE.subjectClassConstraint)\ ]: # if cls_uri is NOT a subclass of a cls specified by KNORA_BASE.subjectClassConstraint self.addRestriction2Class(subject_uri, property_uri, cardinality=cardinality, info_dict=info_dict) if self.create_super_cls_for_multi_property: self.createSuperClassForSubjectClassConstraint(property_uri, subject_uri) else: # not sure if Knora accepts this, i.e. several subject_uris specified by KNORA_BASE.subjectClassConstraint. self.project_graph.add((property_uri, KNORA_BASE.subjectClassConstraint, subject_uri)) return subject_uri, property_uri + def createPropertyName(self, property_name=None, subject_uri=None, object_uri=None, connector='BelongsTo', prefix='has'): + """Returns a property name. + """ + if property_name is not None: + property_name = ''.join([ property_name.split('_')[0].lower() ] + [ text.capitalize() for text in property_name.split('_')[1:] ]) + return prefix + property_name[0].upper() + property_name[1:] if property_name[0].islower()\ + else prefix + property_name + elif subject_uri is not None: + property_name = subject_uri.split('#')[1] + self.createPropertyName(object_uri=object_uri, prefix=connector) + return property_name[0].lower() + property_name[1:] + elif object_uri is not None: + return prefix + object_uri.split('#')[1] + else: + return prefix + + def createSuperClassForSubjectClassConstraint(self, property_uri, sub_uri): + """Creates a super class for classes that share a property. + """ + super_uri = URIRef(property_uri.replace('has', '') + 'Holder') + self.project_graph.add((sub_uri, RDFS.subClassOf, super_uri)) + self.project_graph.remove((sub_uri, RDFS.subClassOf, KNORA_BASE.Resource)) + if (super_uri, RDF.type, OWL.Class) not in self.project_graph: + label = 'holder of ' + property_uri.split('#')[1].replace('has', '') + comment = 'super class for classes that have a ' + property_uri.split('#')[1].replace('has', '') + self.addRestriction2Class(super_uri, property_uri, comment=comment, label=label) + for object_uri in self.project_graph.objects(subject=property_uri, predicate=KNORA_BASE.subjectClassConstraint): + self.project_graph.remove((property_uri, KNORA_BASE.subjectClassConstraint, object_uri)) + self.project_graph.add((object_uri, RDFS.subClassOf, super_uri)) + self.project_graph.remove((object_uri, RDFS.subClassOf, KNORA_BASE.Resource)) + self.project_graph.add((property_uri, KNORA_BASE.subjectClassConstraint, super_uri)) + objectClass = self.project_graph.value(subject=property_uri, predicate=KNORA_BASE.objectClassConstraint, any=False) + comment = 'connects {} with {}'.format(super_uri.split('#')[1], objectClass.split('#')[1].replace('has', '')) + self.project_graph.remove((property_uri, RDFS.comment, None)) + self.project_graph.add((property_uri, RDFS.comment, Literal(comment, lang='en'))) + def get_comment_label(self, cls): """Returns comment and label from cls __doc__. """ comment = cls.__doc__.replace('\n','').lstrip() label = cls.__name__ if '.' in cls.__doc__: comment = [ text for text in cls.__doc__.split('\n') if text != '' ][0].lstrip() if '@label' in cls.__doc__: m = re.search('(@label[:]*\s)(.*[\.]*)', cls.__doc__) label_tag, label = m.groups() elif re.search('([A-Z][a-z]+)', label): m = re.search('([A-Z]\w+)([A-Z]\w+)', label) label = ' '.join([ text.lower() for text in re.split(r'([A-Z][a-z]+)', label) if text != '' ]) return comment, label - def addClass(self, cls_uri, comment='', label='', super_uri=KNORA_BASE.Resource): - """Add a class to project_graph. + def get_semantic_classes(self, datatypes_dir): + """Returns a list of all classes that are contained in datatypes_dir that are subclasses of DATATYPES_DIR.class_spec.SemanticClass. + + :return: a list of (str_name, class) """ - if comment == '' and cls_uri.split('#')[1] in self.class_uri_dict: - comment, label = self.get_comment_label(self.class_uri_dict.get(cls_uri.split('#')[1])) - self.project_graph.add((cls_uri, RDF.type, OWL.Class)) - if comment != '': - self.project_graph.add((cls_uri, RDFS.comment, Literal(comment, lang='en'))) - if label != '': - self.project_graph.add((cls_uri, RDFS.label, Literal(label, lang='en'))) - self.project_graph.add((cls_uri, RDFS.subClassOf, super_uri)) + base_dir = dirname(dirname(__file__)) + sys.path.append(base_dir) + root_modul_name = datatypes_dir.replace('/','.') + reference_cls = importlib.import_module('{}.{}'.format(root_modul_name, 'class_spec')) + try: + self.list_value = reference_cls.LIST + except AttributeError: + pass + files = [ file.replace('.py','') for file in listdir(datatypes_dir) if file.endswith('.py') and not file.startswith('test_') and not file.startswith('_')] + all_modules = [] + for name in files: + all_modules.append(importlib.import_module('{}.{}'.format(root_modul_name, name))) + all_classes = [] + for modul in all_modules: + all_classes += inspect.getmembers(modul, inspect.isclass) + all_classes = sorted(set(all_classes)) + semantic_classes = [ cls for name, cls in all_classes if issubclass(cls, reference_cls.SemanticClass) and not (cls == reference_cls.SemanticClass)] + return semantic_classes def _get_builtin_cls_keys(self, property_dict): """Returns a list of keys for classes that are builtin. """ builtin_cls_keys = [] for key in property_dict.keys(): property_cls = property_dict.get(key).get('class')\ if type(property_dict.get(key)) is dict\ else property_dict.get(key)[0] - if property_cls.__module__ == 'builtins': + if type(property_cls) != dict\ + and property_cls.__module__ == 'builtins': builtin_cls_keys.append(key) return builtin_cls_keys def _get_semantic_dictionary_keys_super_first(self, property_dict): """Sorts the keys of the property part of a semantic dictionary and returns the keys for super classes before keys of subclasses. :return: a sorted list of keys. """ builtin_cls_keys = self._get_builtin_cls_keys(property_dict) complex_cls_keys = [] for key in [ key for key in property_dict.keys()\ if key not in builtin_cls_keys ]: current_cls = property_dict.get(key).get('class')\ if type(property_dict.get(key)) is dict\ else property_dict.get(key)[0] key_inserted = False for index, cls_key in enumerate(complex_cls_keys): potential_sub_cls = property_dict.get(cls_key).get('class')\ if type(property_dict.get(cls_key)) is dict\ else property_dict.get(cls_key)[0] - if issubclass(potential_sub_cls, current_cls): + if type(potential_sub_cls) != dict\ + and type(current_cls) != dict\ + and issubclass(potential_sub_cls, current_cls): complex_cls_keys.insert(index, key) key_inserted = True break if not key_inserted: complex_cls_keys.append(key) return builtin_cls_keys + complex_cls_keys - - def createClassAndProperties(self, cls): - """Creates a owl:Class and some owl:ObjectProperty from semantic_dictionary of a python class. - """ - if not cls.__name__ in self.class_uri_dict: - self.class_uri_dict.update({cls.__name__: cls}) - semantic_dict = cls.get_semantic_dictionary() - super_uri = KNORA_BASE.Resource - if bool(semantic_dict['class'].get('type')): - super_cls = semantic_dict['class'].get('type') - self.createClassAndProperties(super_cls) - super_uri = URIRef(self.base_uriref + '#' + super_cls.__name__) - cls_uri = URIRef(self.base_uriref + '#' + cls.__name__) - comment, label = self.get_comment_label(cls) - self.addClass(cls_uri, comment, label, super_uri) - for property_key in self._get_semantic_dictionary_keys_super_first(semantic_dict['properties']): - try: - if type(semantic_dict['properties'].get(property_key)) == dict: - property_dict4key = semantic_dict['properties'].get(property_key) - property_cls = property_dict4key.get('class') - cardinality = property_dict4key.get('cardinality') - xpath = property_dict4key.get('xpath') - info_dict = { key: value for key, value in property_dict4key.items() if key in\ - [ 'cardinality_restriction', 'comment', 'label', 'name', 'xpath' ]} - subject_uri, property_uri = self.createProperty(cls_uri, property_key, property_cls, cardinality, info_dict=info_dict) - else: - property_cls, cardinality, xpath = semantic_dict['properties'].get(property_key) - subject_uri, property_uri = self.createProperty(cls_uri, property_key, property_cls, cardinality) - if not subject_uri in self.uri_xpath_mapping: - self.uri_xpath_mapping.update({ subject_uri: {}}) - self.uri_xpath_mapping.get(subject_uri).update({property_uri: xpath}) - except ValueError: - raise Exception('Class {} does not have a xpath spec in its get_semantic_dictionary()'.format(cls)) - - def convert_py2ttl(self, datatypes_dir, target_ontology_file): - """Convert all classes contained in datatypes_dir that are subclasses of DATATYPES_DIR.class_spec.SemanticClass to rdf. - - :return: exit code (int) - """ - if isdir(datatypes_dir): - semantic_classes = self.get_semantic_classes(datatypes_dir) - if not Py2TTLConverter.UNITTESTING: - bar = Bar('creating classes and properties', max=len(semantic_classes)) - for cls in semantic_classes: - self.createClassAndProperties(cls) - not bool(Py2TTLConverter.UNITTESTING) and bar.next() - not bool(Py2TTLConverter.UNITTESTING) and bar.finish() - f = open(target_ontology_file, 'wb+') - f.write(self.project_graph.serialize(format="turtle")) - f.close() - #print(self.uri_xpath_mapping.get(URIRef(self.base_uriref + '#TranskriptionPosition'))) - else: - print('Error: dir {} does not exist!'.format(datatypes_dir)) - usage - return 1 - return 0 + +def create_dummy_cls(class_name): + """Return a dummy class for class_name (str). + """ + exec('class %s:pass' % class_name) + return eval('%s' % class_name) def usage(): """prints information on how to use the script """ print(main.__doc__) def main(argv): """This program can be used to convert py classes that are subclasses of .class_spec.SemanticClass to owl:Class. py2ttl/py2ttl.py [OPTIONS ] [optional] directory containing datatypes that are subclasses of .class_spec.SemanticClass. Overwrites DATATYPES_DIR in py2ttl/config.py. OPTIONS: -h|--help: show help -s|--source=source_ontology_file source ontology ttl file, option overwrites PROJECT_ONTOLOGY_FILE in py2ttl/config.py -t|--target=target_ontology_file target ontology ttl file, default: 'PROJECT_PREFIX-ontology_autogenerated.ttl' :return: exit code (int) """ check_config_files_exist() datatypes_dir = get_datatypes_dir() source_ontology_file = PROJECT_ONTOLOGY_FILE target_ontology_file = '.{0}{1}-ontology_autogenerated.ttl'.format(sep, PROJECT_NAME) try: opts, args = getopt.getopt(argv, "hs:t:", ["help","source=", "target="]) except getopt.GetoptError: usage() return 2 for opt, arg in opts: if opt in ('-h', '--help'): usage() return 0 elif opt in ('-t', '--target'): target_ontology_file = arg elif opt in ('-s', '--source'): source_ontology_file = arg converter = Py2TTLConverter(project_ontology_file=source_ontology_file) if len(args) < 1 and datatypes_dir is not None: return converter.convert_py2ttl(datatypes_dir, target_ontology_file) else: for datatypes_dir in args: if converter.convert_py2ttl(datatypes_dir, target_ontology_file) > 0: return 2 return 0 if len(args) > 1 else 2 if __name__ == "__main__": sys.exit(main(sys.argv[1:])) Index: tests_py2ttl/test_py2ttl.py =================================================================== --- tests_py2ttl/test_py2ttl.py (revision 61) +++ tests_py2ttl/test_py2ttl.py (revision 62) @@ -1,89 +1,129 @@ import unittest import lxml.etree as ET from os import sep, path, remove from os.path import isfile, dirname -from rdflib import Graph, URIRef +from rdflib import Graph, URIRef, Literal import sys sys.path.append('py2ttl') import py2ttl try: from py2ttl import Py2TTLConverter except ImportError: from py2ttl.py2ttl import Py2TTLConverter from config import PROJECT_NAME, PROJECT_ONTOLOGY_FILE +from knora_base import KNORA_BASE if dirname(dirname(__file__)) not in sys.path: sys.path.append(dirname(dirname(__file__))) from svgscripts.datatypes.word import Word from svgscripts.datatypes.word_position import WordPosition class TestPy2TTL(unittest.TestCase): """This is the unittest for py2ttl.py2ttl. @label unittest """ def setUp(self): self.ttl_target = __file__ + 'test.ttl' def test_main(self): Py2TTLConverter.UNITTESTING = True argv = ['-t', self.ttl_target ] try: self.assertEqual(py2ttl.main(argv), 0) except AttributeError: self.assertEqual(py2ttl.py2ttl.main(argv), 0) def test_init(self): converter = Py2TTLConverter(project_ontology_file=PROJECT_ONTOLOGY_FILE) self.assertEqual(converter.project_name, PROJECT_NAME) def test_get_semantic_classes(self): converter = Py2TTLConverter() classes = converter.get_semantic_classes('svgscripts/datatypes') self.assertEqual('FaksimileImage' in [ cls.__name__ for cls in classes ], True) self.assertEqual('Image' in [ cls.__name__ for cls in classes ], True) self.assertEqual('SemanticClass' in [ cls.__name__ for cls in classes ], False) - def test_createProperty(self): converter = Py2TTLConverter(project_ontology_file=PROJECT_ONTOLOGY_FILE) converter.createProperty(converter.base_uriref + "#Test", 'test', str, 1) name_uri = converter.base_uriref + '#hasTest' self.assertEqual((name_uri, None, None) in converter.project_graph, True) + def test_createClassAndProperties(self): + converter = Py2TTLConverter(project_ontology_file=PROJECT_ONTOLOGY_FILE) + part_uri = converter.base_uriref + '#WordPart' + #print([ property_key for property_key in converter._get_semantic_dictionary_keys_super_first(Word.get_semantic_dictionary().get('properties'))]) + converter.createClassAndProperties(Word) + subject_uri = converter.base_uriref + '#Word' + deleted_uri = converter.base_uriref + '#isWordDeleted' + self.assertEqual((subject_uri, None, None) in converter.project_graph, True) + self.assertEqual((part_uri, None, None) in converter.project_graph, True) + result = converter.project_graph.query(\ + 'select ?cardinality where {'\ + + ' ?s .'\ + + ' ?s <{0}> .'.format(deleted_uri)\ + + ' ?s ?cardinality .'\ + + ' }') + self.assertEqual(len(result), 1) + self.assertEqual(False not in [ uri[0].eq(1) for uri in result], True) + """ + for s, p, o in converter.project_graph.triples((subject_uri, None, None)): + print(s, p, o) + if not o.startswith(converter.base_uriref): + for a, b, c in converter.project_graph.triples((o, None, None)): + print(a, b, c) + """ + def test_createPropertyName(self): converter = Py2TTLConverter() name = converter.createPropertyName(property_name='test_asdf_asdf') self.assertEqual(name, 'hasTestAsdfAsdf') name = converter.createPropertyName(object_uri=converter.base_uriref + '#Asdf') self.assertEqual(name, 'hasAsdf') name = converter.createPropertyName(subject_uri=converter.base_uriref + '#Test',object_uri=converter.base_uriref + '#Asdf') self.assertEqual(name, 'testBelongsToAsdf') name = converter.createPropertyName(subject_uri=converter.base_uriref + '#Test') self.assertEqual(name, 'testBelongsTo') def test_get_comment_label(self): converter = Py2TTLConverter() comment, label = converter.get_comment_label(TestPy2TTL) self.assertEqual(label, 'unittest') self.assertEqual(comment, self.__doc__.split('\n')[0].lstrip()) def test_get_builtin_cls_keys(self): dictionary = WordPosition.get_semantic_dictionary() converter = Py2TTLConverter() builtin_cls_keys = converter._get_builtin_cls_keys(dictionary['properties']) self.assertEqual('width' in builtin_cls_keys, True) self.assertEqual('height' in builtin_cls_keys, True) def test_get_semantic_dictionary_keys_super_first(self): dict = Word.get_semantic_dictionary() converter = Py2TTLConverter() keys = converter._get_semantic_dictionary_keys_super_first(dict['properties']) self.assertEqual(keys.index('faksimile_positions') < keys.index('transkription_positions'), True) + def test_createPartProperty(self): + info_dict = Word.get_cls_hasPart_objectCls_dictionaries(Word, 'word/word') + converter = Py2TTLConverter() + converter.createPartProperty(info_dict) + name_uri = converter.base_uriref + '#' + info_dict['class'].get('class_name') + property_uri = converter.base_uriref + '#' + info_dict['class']['has_part'].get('name') + seqnum_property_uri = converter.base_uriref + '#' + info_dict['class']['has_seqnum'].get('name') + self.assertEqual((name_uri, None, None) in converter.project_graph, True) + converter.project_graph = Graph() + object_uri = converter.base_uriref + '#' + Word.__name__ + subject_uri = converter.createPartProperty(info_dict) + self.assertEqual(converter.uri_xpath_mapping.get(subject_uri).get(seqnum_property_uri), 'word/word/@id') + self.assertEqual((name_uri, None, None) in converter.project_graph, True) + self.assertEqual((property_uri, None, None) in converter.project_graph, True) + def tearDown(self): isfile(self.ttl_target) and remove(self.ttl_target) if __name__ == "__main__": unittest.main() Index: TODO.md =================================================================== --- TODO.md (revision 61) +++ TODO.md (revision 62) @@ -1,97 +1,97 @@ # Wortsuche: -- Die Wortsuche sollte über die Nähe der Wörter zueinander gewichtet werden. +- Die Wortsuche sollte über die topologische Nähe der Wörter zueinander gewichtet werden. - Wortpfade, d.h. Abfolgen der Wörter sollen vermieden werden, da dies nicht automatisch generiert werden kann und höchst fehleranfällig ist. - Daher sollen die Worteinfügungen auch nicht dafür verwendet werden, alternative Textverläufe aufzuzeichnen. # TODO ## Faksimile data input - word boxes on faksimile by drawing rects with inkscape [IN PROGRESS, see "Leitfaden.pdf"] - naming word boxes by using title of rects [IN PROGRESS, see "Leitfaden\_Kontrolle\_und\_Beschriftung\_der\_Wortrahmen.pdf"] - splitting word box if a word has parts by drawing a vertical path in rect [TODO] ## Processing ### faksimile data input, i.e. svg-file resulting from drawing boxes etc. with inkscape - process faksimile words: - join\_faksimileAndTranskription.py [DONE] - create a data input task for words that have parts: [TODO] - create pdf marking relevant words and their parts of transkription - create faksimile svg highlighting relevant rects - copy pdf and svg to designated folder for this task ### transkription, i.e. svg-file resulting from pdf-file ->created with InDesign - process text field: - Word [DONE] - SpecialWord - MarkForeignHands [DONE] - TextConnectionMark [DONE] - WordInsertionMark [DONE] - all paths -> page.categorize\_paths [TODO] - word-deletion -> Path [DONE] - make parts of word if only parts of a word are deleted, also introduce earlier version of word [TODO] - word-undeletion (e.g. N VII 1, 18,6 -> "mit") - text-area-deletion - text-connection-lines - underline - process footnotes: - TextConnectionMark [DONE] - TextConnection with uncertainty [TODO] - "Fortsetzung [0-9]+,[0-9]+?" - "Fortsetzung von [0-9]+,[0-9]+?" - concerning Word: - uncertain transcription: "?" - atypical writting: "¿" and bold word parts - clarification corrections ("Verdeutlichungskorrekturen"): "Vk" and bold word parts - correction: "word>" and ">?" (with uncertainty) - concerning word deletion: - atypical writting: "¿" and "Durchstreichung" (see N VII 1, 11,2) - process margins: - MarkForeignHands [DONE] - ForeignHandTextAreaDeletion [TODO] - boxes: make earlier version of a word [TODO] - TextConnection [TODO] - from: ([0-9]+,)*[0-9]+ -) - to: -) ([0-9]+,)*[0-9]+ ## Datatypes - make datatypes: - Page [ok] --> page orientation!!! - Word [ok] --> deal with non-horizontal text <<<< DONE! --> hyphenation --> add style info to word: font { German, Latin } --> pen color --> connect style with character glyph-id from svg path file --> handle word layers, i.e. later correction of words by insertion - --> has parts + --> has parts [TODO] --> versions: later version of earlier version - WritingProcess - correlates with font size: - biggest font to biggest-1 font: stage 0 - font in between: stage 1 - smallest font to smallest+1 font: stage 2 - Style - TODO: howto handle style_class in rdf? (as JSON?) - WordPosition [ok] - TranskriptionPosition [ok] - FaksimilePosition [ok] - LineNumber [reDo] - change to Line - Reference [TODO]+ - TextConnection - needs change of LineNumber to Line - ForeignHandTextAreaDeletion [TODO] - Freehand: - Deletion [DONE] - make parts of word if only parts of a word are deleted, also introduce earlier version of word [TODO] - MarkForeignHands ("Zeichen für Fremde Hand") [DONE] - isa SpecialWord - TextConnectionMark ("Anschlußzeichen") [DONE] - isa SpecialWord - has a Reference - WordInsertionMark [reDO] - Underline [TODO] Index: svgscripts/myxmlwriter.py =================================================================== --- svgscripts/myxmlwriter.py (revision 61) +++ svgscripts/myxmlwriter.py (revision 62) @@ -1,70 +1,72 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ This program can be used to pretty-write a xml string to a xml file. """ # Copyright (C) University of Basel 2019 {{{1 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see 1}}} import xml.dom.minidom as MD import xml.etree.ElementTree as ET import lxml.etree as LET from datetime import datetime +from datatypes.page import FILE_TYPE_SVG_WORD_POSITION, FILE_TYPE_XML_MANUSCRIPT + __author__ = "Christian Steiner" __maintainer__ = __author__ __copyright__ = 'University of Basel' __email__ = "christian.steiner@unibas.ch" __status__ = "Development" __license__ = "GPL v3" __version__ = "0.0.1" -FILE_TYPE_SVG_WORD_POSITION = 'svgWordPosition' -FILE_TYPE_XML_MANUSCRIPT = 'xmlManuscriptFile' +FILE_TYPE_SVG_WORD_POSITION = FILE_TYPE_SVG_WORD_POSITION +FILE_TYPE_XML_MANUSCRIPT = FILE_TYPE_XML_MANUSCRIPT def update_metadata(xml_element_tree, script_name, file_type=None): """Updates metadata of xml tree. """ if len(xml_element_tree.getroot().findall('./metadata')) > 0: if len(xml_element_tree.getroot().find('./metadata').findall('./modifiedBy[@script="{}"]'.format(script_name))) == 0: LET.SubElement(xml_element_tree.getroot().find('./metadata'), 'modifiedBy', attrib={'script': script_name}) xml_element_tree.getroot().find('./metadata').findall('./modifiedBy[@script="{}"]'.format(script_name))[0].text = \ datetime.now().strftime('%Y-%m-%d %H:%M:%S') else: metadata = LET.SubElement(xml_element_tree.getroot(), 'metadata') if file_type is not None: LET.SubElement(metadata, 'type').text = file_type createdBy = LET.SubElement(metadata, 'createdBy') LET.SubElement(createdBy, 'script').text = script_name LET.SubElement(createdBy, 'date').text = datetime.now().strftime('%Y-%m-%d %H:%M:%S') def write_pretty(xml_string=None, xml_element_tree=None, file_name=None, script_name=None, file_type=None): """Writes a xml string pretty to a file. """ if not bool(xml_string) and not bool(xml_element_tree): raise Exception("write_pretty needs a string or a xml.ElementTree!") if script_name is not None and xml_element_tree is not None: update_metadata(xml_element_tree, script_name, file_type=file_type) if file_name is None and xml_element_tree is not None\ and xml_element_tree.docinfo is not None and xml_element_tree.docinfo.URL is not None: file_name = xml_element_tree.docinfo.URL if file_name is None: raise Exception("write_pretty needs a file_name or a xml.ElementTree with a docinfo.URL!") dom = MD.parseString(xml_string) if(bool(xml_string)) else MD.parseString(ET.tostring(xml_element_tree.getroot())) f = open(file_name, "w") dom.writexml(f, addindent="\t", newl='\n', encoding='utf-8') f.close() Index: svgscripts/fix_missing_glyphs.py =================================================================== --- svgscripts/fix_missing_glyphs.py (revision 61) +++ svgscripts/fix_missing_glyphs.py (revision 62) @@ -1,190 +1,190 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ This program can be used to fix missing glyphs. """ # Copyright (C) University of Basel 2019 {{{1 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see 1}}} from colorama import Fore, Style import getopt import re import sys from os import listdir, sep, path from os.path import isfile, isdir, dirname import lxml.etree as ET if dirname(__file__) not in sys.path: sys.path.append(dirname(__file__)) -from datatypes.page import Page +from datatypes.page import Page, FILE_TYPE_SVG_WORD_POSITION, FILE_TYPE_XML_MANUSCRIPT from datatypes.positional_word_part import PositionalWordPart from datatypes.transkriptionField import TranskriptionField from datatypes.transkription_position import TranskriptionPosition -from myxmlwriter import write_pretty, FILE_TYPE_SVG_WORD_POSITION, FILE_TYPE_XML_MANUSCRIPT +from myxmlwriter import write_pretty from process_files import update_svgposfile_status __author__ = "Christian Steiner" __maintainer__ = __author__ __copyright__ = 'University of Basel' __email__ = "christian.steiner@unibas.ch" __status__ = "Development" __license__ = "GPL v3" __version__ = "0.0.1" UNITTESTING = False def find_missing_glyph_for_pwp(positional_word_part_node, svg_path_tree, namespaces, xmin=0.0, ymin=0.0): """Finds missing glyph for node of a PositionalWordPart. :return: list of PositionalWordPart """ THRESHOLD = 15.5 pwp = PositionalWordPart(node=positional_word_part_node) word_part_obj = { "x": pwp.left, "y": pwp.top, "text": pwp.text, "matrix": pwp.transform, "class": pwp.style_class } start_id = int(pwp.id) threshold = -0.5 positional_word_parts = [] while threshold < THRESHOLD and len(positional_word_parts) < 1: try: positional_word_parts = PositionalWordPart.CREATE_POSITIONAL_WORD_PART_LIST(word_part_obj, svg_path_tree, namespaces,\ start_id=start_id, xmin=xmin, ymin=ymin, threshold=threshold, throw_error_if_not_found=True) except Exception: threshold += 0.1 return positional_word_parts def update_word(page, positional_word_part_node, positional_word_parts): """Updates word according to new positional_word_parts. """ if len(positional_word_parts) > 0: debug_msg_string = 'update word from ' + __file__ positional_word_part_id = int(positional_word_part_node.get('id')) transkription_position_id = int(positional_word_part_node.getparent().get('id')) word_id = int(positional_word_part_node.getparent().getparent().get('id')) word = page.words[word_id] transkription_position = word.transkription_positions[transkription_position_id] transkription_position.positional_word_parts.pop(positional_word_part_id) positional_word_parts.reverse() for positional_word_part in positional_word_parts: transkription_position.positional_word_parts.insert(positional_word_part_id, positional_word_part) for index, positional_word_part in enumerate(transkription_position.positional_word_parts): positional_word_part.id = index transkription_positions = TranskriptionPosition.CREATE_TRANSKRIPTION_POSITION_LIST_FROM_PWPS(\ transkription_position.positional_word_parts, debug_msg_string=debug_msg_string, transkription_position_id=transkription_position_id) word.transkription_positions.pop(transkription_position_id) transkription_positions.reverse() for new_tp in transkription_positions: word.transkription_positions.insert(transkription_position_id, new_tp) text = '' for index, tp in enumerate(word.transkription_positions): tp.id = index tp.writing_process_id = transkription_position.writing_process_id for pwp in tp.positional_word_parts: text += pwp.text if word.text != text: word.text = text word.attach_word_to_tree(page.page_tree) def fix_missing_glyphs(svg_word_pos_file, manuscript_file=None): """Finds missing glyphs for xml file of type FILE_TYPE_SVG_WORD_POSITION. """ if isfile(svg_word_pos_file): if not UNITTESTING: print(Fore.LIGHTBLUE_EX + 'Fixing missing glyphs for file {} ... '.format(svg_word_pos_file), end='') #print(Style.RESET_ALL) page = Page(xml_source_file=svg_word_pos_file) transkription_field = TranskriptionField(page.svg_file) svg_path_tree = ET.parse(page.svg_file) namespaces = { k if k is not None else 'ns': v for k, v in svg_path_tree.getroot().nsmap.items() } number_of_missing_glyphs = len(page.page_tree.xpath('//' + PositionalWordPart.XML_TAG + '[not(@symbol-id)]')) for positional_word_part_node in page.page_tree.xpath('//' + PositionalWordPart.XML_TAG + '[not(@symbol-id)]'): pwps = find_missing_glyph_for_pwp(positional_word_part_node, svg_path_tree, namespaces, xmin=transkription_field.xmin, ymin=transkription_field.ymin) update_word(page, positional_word_part_node, pwps) write_pretty(xml_element_tree=page.page_tree, file_name=svg_word_pos_file, script_name=__file__, file_type=FILE_TYPE_SVG_WORD_POSITION) page = Page(xml_source_file=svg_word_pos_file) new_number_of_missing_glyphs = len(page.page_tree.xpath('//' + PositionalWordPart.XML_TAG + '[not(@symbol-id)]')) if not UNITTESTING: result_color = Fore.LIGHTBLUE_EX if new_number_of_missing_glyphs == 0 else Fore.MAGENTA print(result_color + ' {0}/{1}'.format(number_of_missing_glyphs-new_number_of_missing_glyphs, number_of_missing_glyphs), end='') print(Fore.LIGHTBLUE_EX + ' fixed.', end='') print(Style.RESET_ALL) if len(page.page_tree.xpath('//' + PositionalWordPart.XML_TAG + '[not(@symbol-id)]')) == 0: update_svgposfile_status(svg_word_pos_file, manuscript_file=manuscript_file, status='OK') def get_filelist_and_manuscript_file(file_a, file_b=None): """Returns a file list and a manuscript file (or None) """ file_list = [] manuscript_file = None source_tree = ET.parse(file_a) if source_tree.getroot().find('metadata/type').text == FILE_TYPE_SVG_WORD_POSITION\ and len([ word_part for word_part in source_tree.xpath('//' + PositionalWordPart.XML_TAG + '[not(@symbol-id)]')]) > 0: # if symbol_ids are missing ... file_list.append(file_a) if file_b is not None: manuscript_file = file_b elif source_tree.getroot().find('metadata/type').text == FILE_TYPE_XML_MANUSCRIPT: manuscript_file = file_a if file_b is not None: file_list.append(file_b) else: file_list = source_tree.xpath('//page[contains(@status, "{}")]/@output'.format(PositionalWordPart.WARN_NO_USE_NODE_FOUND.lower())) return file_list, manuscript_file def usage(): """prints information on how to use the script """ print(main.__doc__) def main(argv): """This program can be used to fix missing glyphs. svgscripts/fix_missing_glyphs.py [OPTIONS] -File [-File] a xml file about a manuscript, containing information about its pages. a xml file about a page, containing information about svg word positions. OPTIONS: -h|--help: show help :return: exit code (int) """ try: opts, args = getopt.getopt(argv, "h", ["help"]) except getopt.GetoptError: usage() return 2 for opt, arg in opts: if opt in ('-h', '--help'): usage() return 0 if len(args) < 1: usage() return 2 exit_status = 0 file_a = args[0] if isfile(file_a): file_b = None if len(args) > 1 and isfile(args[1]): file_b = args[1] file_list, manuscript_file = get_filelist_and_manuscript_file(file_a, file_b=file_b) for svg_word_pos_file in file_list: fix_missing_glyphs(svg_word_pos_file, manuscript_file=manuscript_file) else: raise FileNotFoundError('File {} does not exist!'.format(file_a)) return exit_status if __name__ == "__main__": sys.exit(main(sys.argv[1:])) Index: svgscripts/datatypes/word.py =================================================================== --- svgscripts/datatypes/word.py (revision 61) +++ svgscripts/datatypes/word.py (revision 62) @@ -1,252 +1,273 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ This class can be used to represent a word. """ # Copyright (C) University of Basel 2019 {{{1 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see 1}}} __author__ = "Christian Steiner" __maintainer__ = __author__ __copyright__ = 'University of Basel' __email__ = "christian.steiner@unibas.ch" __status__ = "Development" __license__ = "GPL v3" __version__ = "0.0.1" from lxml import etree as ET import warnings from .class_spec import SemanticClass from .lineNumber import LineNumber from .matrix import Matrix +from .simple_word import SimpleWord from .word_position import WordPosition from .transkription_position import TranskriptionPosition +from .writing_process import WritingProcess -class Word(SemanticClass): +class Word(SimpleWord): """ This class represents a word. """ DATA = 'debug-data' XML_TAG = 'word' - def __init__(self, id=0, text='', line_number=-1, deleted=False, transkription_positions=[], faksimile_positions=[], word_part_objs=[]): - self.id = id - self.text = text + def __init__(self, id=0, text='', line_number=-1, deleted=None, transkription_positions=None, faksimile_positions=None, word_part_objs=None, word_parts=None, writing_process_id=-1): + super(Word,self).__init__(id=id, text=text, line_number=line_number, transkription_positions=transkription_positions,\ + faksimile_positions=faksimile_positions) self.deleted = deleted - self.line_number = line_number - self.transkription_positions = transkription_positions - self.faksimile_positions = faksimile_positions - self.word_part_objs = word_part_objs + self.text = text if text != ''\ + else ''.join([ tp.get_text() for tp in self.transkription_positions ]) + self.word_part_objs = word_part_objs if word_part_objs is not None else [] self.is_head_of_inserted_words = False self.is_tail_of_inserted_words = False self.is_before_inserted_words = False self.is_after_inserted_words = False self.word_insertion_mark = None self.debug_msg = None - - @classmethod - def get_semantic_dictionary(cls): - """ Creates and returns a semantic dictionary as specified by SemanticClass. - """ - dictionary = {} - class_dict = cls.get_class_dictionary() - properties = {'text': (str, 1, 'word/@text'),\ - 'line_number': {'class': LineNumber, 'cardinality': 1,\ - 'name': 'wordHasLineNumber', 'xpath': 'word/@line-number',\ - 'label': 'word has a line number',\ - 'comment': 'Relating a word to a line number it has.'},\ - 'transkription_positions': (TranskriptionPosition, SemanticClass.LIST, 'word/@id'),\ - 'faksimile_positions': (WordPosition, SemanticClass.LIST, 'word/@id')} - dictionary.update({'class': class_dict}) - dictionary.update({'properties': properties}) - return dictionary - - def set_word_insertion_mark(self, word_insertion_mark): - """Sets word_insertion_mark - """ - self.word_insertion_mark = word_insertion_mark + self.writing_process_id = writing_process_id + self.word_parts = word_parts if word_parts is not None else [] def attach_word_to_tree(self, target_tree): """Attaches word to tree target_tree. """ - word_node = target_tree.getroot().xpath('//' + self.XML_TAG + '[@id="%s"]' % self.id)[0] \ - if(len(target_tree.getroot().xpath('//' + self.XML_TAG + '[@id="%s"]' % self.id)) > 0) \ - else ET.SubElement(target_tree.getroot(), self.XML_TAG, attrib={'id': str(self.id)}) - word_node.set('text', self.text) - word_node.set('deleted', str(self.deleted).lower()) - if self.line_number > -1: - word_node.set('line-number', str(self.line_number)) - for transkription_position in self.transkription_positions: - transkription_position.attach_object_to_tree(word_node) - for faksimile_position in self.faksimile_positions: - faksimile_position.attach_object_to_tree(word_node) + word_node = super(Word,self).attach_word_to_tree(target_tree) + if self.deleted is not None: + word_node.set('deleted', str(self.deleted).lower()) + if self.writing_process_id > -1: + word_node.set('writing-process-id', str(self.writing_process_id)) + for word_part in self.word_parts: + word_part.attach_word_to_tree(word_node) return word_node + + def belongs_to_multiple_writing_processes(self, include_parts=False): + """Returns true if transkription_positions belong to different WritingProcesses. + """ + if len(self.word_parts) > 0 and include_parts: + return len(set(word.writing_process_id for word in self.word_parts)) > 1 + return len(set(tp.writing_process_id for tp in self.transkription_positions )) > 1 + + def partition_according_to_writing_process_id(self): + """Partition a word according to its transkription_positions' writing_process_ids + ->split word and add partial words as its parts. + """ + if self.belongs_to_multiple_writing_processes(): + last_writing_process_id = -1 + transkription_positions = [] + for transkription_position in self.transkription_positions: + if transkription_position.writing_process_id != last_writing_process_id\ + and len(transkription_positions) > 0: + newWord = Word(id=len(self.word_parts), line_number=self.line_number,\ + transkription_positions=transkription_positions, writing_process_id=last_writing_process_id) + self.word_parts.append(newWord) + transkription_positions = [] + transkription_positions.append(transkription_position) + last_writing_process_id = transkription_position.writing_process_id + if len(transkription_positions) > 0: + newWord = Word(id=len(self.word_parts), line_number=self.line_number,\ + transkription_positions=transkription_positions, writing_process_id=last_writing_process_id) + self.word_parts.append(newWord) + self.transkription_positions = [] + self.line_number = -1 + self.deleted = None def split(self, split_string, start_id=0): """Splits the word and returns an 3-tuple of new words. """ previousString, currentString, nextString = self.text.partition(split_string) currentWord = None previousWord = None nextWord = None previousIndex = 0 current_id = start_id all_positional_word_parts = [] for position in self.transkription_positions: all_positional_word_parts += position.positional_word_parts if len(all_positional_word_parts) == 0: warnings.warn('ATTENTION: Word: {} {} with Strings "{}, {}, {}": there are no parts!'.format(self.id, self.text, previousString, currentString, nextString)) if len(previousString) > 0: previous_pwps = [] while previousIndex < len(all_positional_word_parts) and previousString != ''.join([ pwp.text for pwp in previous_pwps ]): previous_pwps.append(all_positional_word_parts[previousIndex]) previousIndex += 1 if previousString != ''.join([ pwp.text for pwp in previous_pwps ]): warnings.warn('ATTENTION: "{}" does not match a word_part_obj!'.format(previousString)) else: previous_transkription_positions = TranskriptionPosition.CREATE_TRANSKRIPTION_POSITION_LIST_FROM_PWPS(previous_pwps, debug_msg_string='word.split') previous_text = ''.join([ pwp.text for pwp in previous_pwps ]) previousWord = Word(text=previous_text, id=current_id, line_number=self.line_number, transkription_positions=previous_transkription_positions) current_id += 1 all_positional_word_parts = all_positional_word_parts[previousIndex:] if len(nextString) > 0: tmp_pwps = [] index = 0 while index < len(all_positional_word_parts) and currentString != ''.join([ pwp.text for pwp in tmp_pwps ]): tmp_pwps.append(all_positional_word_parts[index]) index += 1 if currentString != ''.join([ pwp.text for pwp in tmp_pwps ]): warnings.warn('ATTENTION: "{}" does not match a word_part_obj!'.format(currentString)) else: next_pwps = all_positional_word_parts[index:] next_transkription_positions = TranskriptionPosition.CREATE_TRANSKRIPTION_POSITION_LIST_FROM_PWPS(next_pwps, debug_msg_string='word.split') next_text = ''.join([ pwp.text for pwp in next_pwps ]) nextWord = Word(text=next_text, id=current_id+1, line_number=self.line_number, transkription_positions=next_transkription_positions) all_positional_word_parts = all_positional_word_parts[:index] current_transkription_positions = TranskriptionPosition.CREATE_TRANSKRIPTION_POSITION_LIST_FROM_PWPS(all_positional_word_parts, debug_msg_string='word.split') current_text = ''.join([ pwp.text for pwp in all_positional_word_parts ]) currentWord = Word(text=current_text, id=current_id, line_number=self.line_number, transkription_positions=current_transkription_positions) return previousWord, currentWord, nextWord def join(self, other_word, append_at_end_of_new_word=True): """Joins other_word with this word by changing the text of current word and adding other_word.transkription_positions. """ if append_at_end_of_new_word: self.text = self.text + other_word.text for position in other_word.transkription_positions: position.id = str(len(self.transkription_positions)) self.transkription_positions.append(position) else: self.text = other_word.text + self.text index = 0 for position in other_word.transkription_positions: self.transkription_positions.insert(index, position) index += 1 while index < len(self.transkription_positions): self.transkription_positions[index].id = str(index) index += 1 self.simplify_transkription_positions() + def set_word_insertion_mark(self, word_insertion_mark): + """Sets word_insertion_mark + """ + self.word_insertion_mark = word_insertion_mark + def simplify_transkription_positions(self): """Merge transkription_positions if possible. """ index = len(self.transkription_positions)-1 while index > 0\ and False not in [ 'positional_word_parts' in tp.__dict__.keys() for tp in self.transkription_positions ]: current_tp = self.transkription_positions[index] index -= 1 previous_tp = self.transkription_positions[index] if previous_tp.writing_process_id == current_tp.writing_process_id: positional_word_parts = previous_tp.positional_word_parts positional_word_parts += current_tp.positional_word_parts transkription_positions = TranskriptionPosition.CREATE_TRANSKRIPTION_POSITION_LIST_FROM_PWPS(\ positional_word_parts, debug_msg_string='simplifying transkription positions', transkription_position_id=previous_tp.id) if len(transkription_positions) == 1: transkription_positions[0].writing_process_id = previous_tp.writing_process_id self.transkription_positions.pop(index+1) self.transkription_positions[index] = transkription_positions[0] @classmethod def create_cls(cls, word_node): """Creates a word from a (lxml.Element) node. [:return:] Word """ - if word_node is not None: # init word from xml node - id = int(word_node.get('id')) - line_number = int(word_node.get('line-number')) if bool(word_node.get('line-number')) else -1 - text = word_node.get('text') - deleted = bool(word_node.get('deleted')) and word_node.get('deleted') == 'true' - transkription_positions = [ TranskriptionPosition(node=node) for node in word_node.findall('.//' + WordPosition.TRANSKRIPTION) ] - faksimile_positions = [ WordPosition(node=node) for node in word_node.findall('.//' + WordPosition.FAKSIMILE) ] - return cls(id=id, text=text, deleted=deleted, line_number=line_number, transkription_positions=transkription_positions,\ - faksimile_positions=faksimile_positions) - else: - error_msg = 'word_node has not been defined' - raise Exception('Error: {}'.format(error_msg)) + cls = super(Word,cls).create_cls(word_node) + cls.writing_process_id = int(word_node.get('writing-process-id')) if bool(word_node.get('writing-process-id')) else -1 + cls.deleted = word_node.get('deleted') == 'true'\ + if bool(word_node.get('deleted')) else None + cls.word_parts = [ cls.create_cls(node) for node in word_node.xpath('.//' + cls.XML_TAG) ] + return cls @staticmethod def CREATE_WORD(word_node=None, page=None, word_part_objs=[], id=0, height=0, endX=0, endSign=None, matrix=None, line_number=-1, debug_msg=None): """Creates a word from a (lxml.Element) node or word_part_objs. [:return:] Word """ if word_node is not None: # init word from xml node id = int(word_node.get('id')) line_number = int(word_node.get('line-number')) if bool(word_node.get('line-number')) else line_number text = word_node.get('text') deleted = bool(word_node.get('deleted')) and word_node.get('deleted') == 'true' transkription_positions = [ TranskriptionPosition(node=node) for node in word_node.findall('.//' + WordPosition.TRANSKRIPTION) ] faksimile_positions = [ WordPosition(node=node) for node in word_node.findall('.//' + WordPosition.FAKSIMILE) ] word_part_objs = [ item.attrib for item in word_node.findall('.//' + Word.DATA + '/part')]\ if len(word_node.findall('.//' + Word.DATA)) > 0\ else [ item.attrib for item in word_node.findall('.//part')] return Word(id=id, text=text, deleted=deleted, line_number=line_number, transkription_positions=transkription_positions,\ faksimile_positions=faksimile_positions, word_part_objs=word_part_objs) elif len(word_part_objs) > 0: # init word from word_part_obj that has been extracted from svg file WIDTH = 5 TOPCORRECTION = 2.0 FONTWIDTHFACTOR = 0.7 # factor that multiplies lastCharFontSize height = height x = round(float(word_part_objs[0]['x']), 3) if(page is not None and bool(page.style_dict)): HEIGHT_FACTOR = 1.1 # factor that multiplies biggest_font_size -> height style_set = set(' '.join(set( dict['class'] for dict in word_part_objs)).split(' ')) biggest_font_size = page.get_biggest_fontSize4styles(style_set=style_set) height = round(biggest_font_size * HEIGHT_FACTOR + HEIGHT_FACTOR / biggest_font_size, 3) TOPCORRECTION = 1 + HEIGHT_FACTOR / biggest_font_size if endSign is not None and '%' in endSign: lastCharFontSizeList = [ float(page.style_dict[key]['font-size'].replace('px',''))\ for key in word_part_objs[len(word_part_objs)-1]['class'].split(' ')\ if bool(page.style_dict[key].get('font-size'))] lastCharFontSize = lastCharFontSizeList[0] if len(lastCharFontSizeList) > 0 else 1 endX = float(endX) + lastCharFontSize * FONTWIDTHFACTOR elif endSign is not None and '%' in endSign: endX = float(endX) + WIDTH bottom = round(float(word_part_objs[0]['y']), 3) y = round(bottom - height + TOPCORRECTION, 3) width = round(float(endX) - x, 3) transkription_positions = [ WordPosition(height=height, width=width, x=x, y=y, matrix=matrix, tag=WordPosition.TRANSKRIPTION) ] text = ''.join([ dict['text'] for dict in word_part_objs]) line_number = page.get_line_number( (y + bottom)/2) if page is not None else line_number word = Word(id=id, text=text, line_number=line_number, transkription_positions=transkription_positions, word_part_objs=word_part_objs) word.debug_msg = debug_msg return word else: error_msg = 'word_node has not been defined' if (word_node is None) else 'word_part_objs is empty' raise Exception('Error: {}'.format(error_msg)) + @classmethod + def get_semantic_dictionary(cls): + """ Creates and returns a semantic dictionary as specified by SemanticClass. + """ + dictionary = super(Word,cls).get_semantic_dictionary() + dictionary['properties'].update({'deleted':\ + {'class': bool, 'cardinality': 1, 'xpath': '{0}/@deleted'.format(cls.XML_TAG),\ + 'name': 'isWordDeleted', 'label': 'has word been deleted'}}) + dictionary['properties'].update({'writing_process_id':\ + {'class': WritingProcess, 'cardinality': 0, 'xpath': '{0}/@writing-process-id'.format(cls.XML_TAG),\ + 'name': 'wordBelongsToWritingProcess', 'label': 'word has been written in a specific writing process'}}) + dictionary['properties'].update({'word_parts': Word.get_cls_hasPart_objectCls_dictionaries(Word, xpath='word/word')}) + return dictionary + Index: svgscripts/datatypes/special_word.py =================================================================== --- svgscripts/datatypes/special_word.py (revision 61) +++ svgscripts/datatypes/special_word.py (revision 62) @@ -1,85 +1,85 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ This super class can be used to represent a special word. """ # Copyright (C) University of Basel 2019 {{{1 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see 1}}} __author__ = "Christian Steiner" __maintainer__ = __author__ __copyright__ = 'University of Basel' __email__ = "christian.steiner@unibas.ch" __status__ = "Development" __license__ = "GPL v3" __version__ = "0.0.1" import abc from lxml import etree as ET -from .word import Word +from .simple_word import SimpleWord -class SpecialWord(Word, metaclass=abc.ABCMeta): +class SpecialWord(SimpleWord, metaclass=abc.ABCMeta): """ This class represents a special word. """ XML_TAG = 'special-word' XML_SUB_TAG = 'content' - def __init__(self, id=0, line_number=-1, text='', deleted=False, transkription_positions=[], faksimile_positions=[]): - super(SpecialWord, self).__init__(id=id, text=text, deleted=deleted, line_number=line_number,\ + def __init__(self, id=0, line_number=-1, text='', transkription_positions=[], faksimile_positions=[]): + super(SpecialWord, self).__init__(id=id, text=text, line_number=line_number,\ transkription_positions=transkription_positions, faksimile_positions=faksimile_positions) @abc.abstractmethod def add_content(self, node): """Adds content to a special word. node: a lxml.etree.Element """ pass @classmethod def create_cls(cls, word_node): """Creates a cls from a (lxml.Element) node. [:return:] cls """ cls = super(SpecialWord,cls).create_cls(word_node) sub_nodes = word_node.findall(cls.XML_SUB_TAG) for sub_node in sub_nodes: cls.add_content(sub_node) return cls @classmethod def create_cls_from_word(cls, word, id=0): """Creates a cls from a (lxml.Element) node. [:return:] cls """ for position in word.transkription_positions + word.faksimile_positions: position.writing_process_id = -1 return cls(id=id, line_number=word.line_number, text=word.text, transkription_positions=word.transkription_positions,\ faksimile_positions=word.faksimile_positions) @classmethod @abc.abstractmethod def get_special_char_list(cls): """Returns a list of the chars that define this special word. """ pass Index: svgscripts/datatypes/page.py =================================================================== --- svgscripts/datatypes/page.py (revision 61) +++ svgscripts/datatypes/page.py (revision 62) @@ -1,517 +1,546 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ This class can be used to represent a page. """ # Copyright (C) University of Basel 2019 {{{1 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see 1}}} __author__ = "Christian Steiner" __maintainer__ = __author__ __copyright__ = 'University of Basel' __email__ = "christian.steiner@unibas.ch" __status__ = "Development" __license__ = "GPL v3" __version__ = "0.0.1" from lxml import etree as ET from os.path import isfile from progress.bar import Bar from svgpathtools import svg2paths2, svg_to_paths from svgpathtools.parser import parse_path from .class_spec import SemanticClass from .image import Image, SVGImage from .faksimile_image import FaksimileImage from .lineNumber import LineNumber from .mark_foreign_hands import MarkForeignHands from .matrix import Matrix from .path import Path from .positional_word_part import PositionalWordPart from .text_connection_mark import TextConnectionMark from .transkriptionField import TranskriptionField from .writing_process import WritingProcess from .word import Word from .word_insertion_mark import WordInsertionMark +FILE_TYPE_SVG_WORD_POSITION = 'svgWordPosition' +FILE_TYPE_XML_MANUSCRIPT = 'xmlManuscriptFile' + class Page(SemanticClass): """ This class represents a page. Args: xml_source_file (str): name of the xml file to be instantiated. xml_target_file (str): name of the xml file to which page info will be written. """ UNITTESTING = False WARNING_MISSING_USE_NODE4PWP = PositionalWordPart.WARN_NO_USE_NODE_FOUND WARNING_MISSING_GLYPH_ID4WIM = WordInsertionMark.WARN_NO_GLYPH_ID PAGE_RECTO = 'recto' PAGE_VERSO = 'verso' def __init__(self, xml_source_file=None, xml_target_file=None, title=None, page_number=None, faksimile_image=None, faksimile_svgFile=None, pdfFile=None, svg_file=None, orientation='North', page_type=PAGE_VERSO, extract_transkription_field_only=False): self.title = title self.mark_foreign_hands = [] self.text_connection_marks = [] self.line_numbers = [] self.style_dict = {} self.sonderzeichen_list = [] self.svg_file = None self.svg_image = None self.pdfFile = None self.faksimile_svgFile = None self.source = None self.number = page_number if page_number is not None else -1 self.orientation = orientation self.page_type = page_type self.word_deletion_paths = [] self.faksimile_image = faksimile_image if xml_source_file is not None: if isfile(xml_source_file): parser = ET.XMLParser(remove_blank_text=True) self.page_tree = ET.parse(xml_source_file, parser) self.title = self.page_tree.getroot().get('title') self.number = self.page_tree.getroot().get('number') self.source = self.page_tree.getroot().get('source') self.orientation = self.page_tree.getroot().get('orientation') self.page_type = self.page_tree.getroot().get('pageType') self.init_words() self.add_style(style_node=self.page_tree.getroot().find('.//style')) self.pdfFile = self.page_tree.xpath('.//pdf/@file')[0]\ if len(self.page_tree.xpath('.//pdf/@file')) > 0 else None self.faksimile_svgFile = self.page_tree.xpath('.//faksimile-svg/@file')[0]\ if len(self.page_tree.xpath('.//faksimile-svg/@file')) > 0 else None self.svg_image = SVGImage(node=self.page_tree.xpath('.//' + SVGImage.XML_TAG)[0])\ if len(self.page_tree.xpath('.//' + SVGImage.XML_TAG)) > 0 else None self.faksimile_image = FaksimileImage(node=self.page_tree.xpath('.//' + FaksimileImage.XML_TAG)[0])\ if len(self.page_tree.xpath('.//' + FaksimileImage.XML_TAG)) > 0 else None self.svg_file = self.page_tree.xpath('.//svg/@file')[0]\ if len(self.page_tree.xpath('.//svg/@file')) > 0 else None self.width = float(self.page_tree.xpath('.//svg/@width')[0])\ if len(self.page_tree.xpath('.//svg/@width')) > 0 else 0.0 self.height = float(self.page_tree.xpath('.//svg/@height')[0])\ if len(self.page_tree.xpath('.//svg/@height')) > 0 else 0.0 if pdfFile is not None and self.pdfFile is None: self.pdfFile = pdfFile ET.SubElement(self.page_tree.getroot(), 'pdf', attrib={'file': self.pdfFile}) if faksimile_svgFile is not None and self.faksimile_svgFile is None: self.faksimile_svgFile = faksimile_svgFile ET.SubElement(self.page_tree.getroot(), 'faksimile-svg', attrib={'file': self.faksimile_svgFile}) if faksimile_image is not None: self.faksimile_image = faksimile_image self.faksimile_image.attach_object_to_tree(self.page_tree) if svg_file is not None and self.svg_file is None: self.svg_file = svg_file tf = TranskriptionField(svg_file) self.width = round(tf.documentWidth, 3) self.height = round(tf.documentHeight, 3) self.svg_image = SVGImage(file_name=self.svg_file, width=self.width, height=self.height) self.svg_image.attach_object_to_tree(self.page_tree) if self.svg_image is not None and self.svg_file is None: self.svg_file = self.svg_image.file_name if self.svg_image is not None and self.width == 0.0: self.width = self.svg_image.width if self.svg_image is not None and self.height == 0.0: self.height = self.svg_image.height else: raise Exception('File "{}" does not exist!'.format(xml_source_file)) elif xml_target_file is not None: self.word_insertion_marks = [] self.words = [] self.writing_processes = [] self.svg_file = svg_file self.pdfFile = pdfFile self.faksimile_svgFile = faksimile_svgFile if isfile(xml_target_file): parser = ET.XMLParser(remove_blank_text=True) self.page_tree = ET.parse(xml_target_file, parser) self.source = self.page_tree.getroot().get('source') if bool(self.page_tree.getroot().get('orientation')): self.orientation = self.page_tree.getroot().get('orientation') elif orientation is not None: self.page_tree.getroot().set('orientation', orientation) if bool(self.page_tree.getroot().get('title')): self.title = self.page_tree.getroot().get('title') elif title is not None: self.page_tree.getroot().set('title', title) if self.svg_file is None: self.svg_file = self.page_tree.xpath('.//svg/@file')[0]\ if len(self.page_tree.xpath('.//svg/@file')) > 0 else None self.width = float(self.page_tree.xpath('.//svg/@width')[0])\ if len(self.page_tree.xpath('.//svg/@width')) > 0 else 0.0 self.height = float(self.page_tree.xpath('.//svg/@height')[0])\ if len(self.page_tree.xpath('.//svg/@height')) > 0 else 0.0 elif len(self.page_tree.xpath('.//svg/@file')) == 0: tf = TranskriptionField(svg_file) self.width = round(tf.documentWidth, 3) self.height = round(tf.documentHeight, 3) self.svg_image = SVGImage(file_name=self.svg_file, width=self.width, height=self.height) self.svg_image.attach_object_to_tree(self.page_tree) #ET.SubElement(self.page_tree.getroot(), 'svg', attrib={'width': str(self.width), 'height': str(self.height), 'file': self.svg_file}) else: self.width = float(self.page_tree.xpath('.//svg/@width')[0])\ if len(self.page_tree.xpath('.//svg/@width')) > 0 else 0.0 self.height = float(self.page_tree.xpath('.//svg/@height')[0])\ if len(self.page_tree.xpath('.//svg/@height')) > 0 else 0.0 if self.pdfFile is None: self.pdfFile = self.page_tree.xpath('.//pdf/@file')[0]\ if len(self.page_tree.xpath('.//pdf/@file')) > 0 else None elif len(self.page_tree.xpath('.//pdf/@file')) == 0: ET.SubElement(self.page_tree.getroot(), 'pdf', attrib={'file': self.pdfFile}) for xpath2remove in [ 'word', 'style', 'freehand', LineNumber.XML_TAG, WordInsertionMark.XML_TAG,\ WritingProcess.XML_TAG, Path.WORD_DELETION_PATH_TAG ]: for node in self.page_tree.xpath('//' + xpath2remove): node.getparent().remove(node) else: self.page_tree = ET.ElementTree(ET.Element('page')) self.pdfFile = pdfFile self.svg_file = svg_file if title is not None: self.page_tree.getroot().set('title', title) if orientation is not None: self.page_tree.getroot().set('orientation', orientation) self.page_tree.getroot().set('transkription-field-only', str(extract_transkription_field_only).lower()) if page_number is not None: self.page_tree.getroot().set('number', str(page_number)) if self.pdfFile is not None: ET.SubElement(self.page_tree.getroot(), 'pdf', attrib={'file': self.pdfFile}) if self.svg_file is not None: tf = TranskriptionField(self.svg_file) self.width = round(tf.documentWidth, 3) self.height = round(tf.documentHeight, 3) self.svg_image = SVGImage(file_name=self.svg_file, width=self.width, height=self.height) self.svg_image.attach_object_to_tree(self.page_tree) #ET.SubElement(self.page_tree.getroot(), 'svg', attrib={'width': str(self.width), 'height': str(self.height), 'file': self.svg_file}) if self.svg_image is None and self.svg_file is not None: self.svg_image = SVGImage(file_name=self.svg_file, width=self.width, height=self.height) self.svg_image.attach_object_to_tree(self.page_tree) def add_style(self, sonderzeichen_list=[], letterspacing_list=[], style_dict={}, style_node=None): """Adds a list of classes that are sonderzeichen and a style dictionary to page. """ self.sonderzeichen_list = sonderzeichen_list self.letterspacing_list = letterspacing_list self.style_dict = style_dict if style_node is not None: self.style_dict = { item.get('name'): { key: value for key, value in item.attrib.items() if key != 'name' } for item in style_node.findall('.//class') } self.sonderzeichen_list = [ item.get('name') for item in style_node.findall('.//class')\ if bool(item.get('font-family')) and 'Sonderzeichen' in item.get('font-family') ] self.letterspacing_list = [ item.get('name') for item in style_node.findall('.//class')\ if bool(item.get('letterspacing-list')) ] elif bool(self.style_dict): style_node = ET.SubElement(self.page_tree.getroot(), 'style') if len(self.sonderzeichen_list) > 0: style_node.set('Sonderzeichen', ' '.join(self.sonderzeichen_list)) if len(self.letterspacing_list) > 0: style_node.set('letterspacing-list', ' '.join(self.letterspacing_list)) for key in self.style_dict.keys(): self.style_dict[key]['name'] = key ET.SubElement(style_node, 'class', attrib=self.style_dict[key]) fontsize_dict = { key: float(value.get('font-size').replace('px','')) for key, value in self.style_dict.items() if 'font-size' in value } fontsizes = sorted(fontsize_dict.values(), reverse=True) # create a mapping between fontsizes and word stages self.fontsizekey2stage_mapping = {} for fontsize_key, value in fontsize_dict.items(): if value >= fontsizes[0]-1: self.fontsizekey2stage_mapping.update({ fontsize_key: WritingProcess.FIRST_VERSION }) elif value <= fontsizes[len(fontsizes)-1]+1: self.fontsizekey2stage_mapping.update({ fontsize_key: WritingProcess.LATER_INSERTION_AND_ADDITION }) else: self.fontsizekey2stage_mapping.update({ fontsize_key: WritingProcess.INSERTION_AND_ADDITION }) def add_source(self, source): """Adds a source to page and attaches it to page_tree. """ self.source = source self.page_tree.getroot().set('source', self.source) def categorize_paths(self, transkription_field=None): """Categorize all paths that are part of the transkription field. """ if self.source is not None and isfile(self.source): MAX_HEIGHT_LINES = 1 max_line = sorted(\ [line_number.bottom-line_number.top for line_number in self.line_numbers if line_number.id % 2 == 0],\ reverse=True)[0] + 2 if len(self.line_numbers) > 0 else 17 tr_xmin = transkription_field.xmin if transkription_field is not None else 0.0 tr_ymin = transkription_field.ymin if transkription_field is not None else 0.0 paths, attributes = svg_to_paths.svg2paths(self.source) allpaths_on_tf = [] if transkription_field is not None: for index in range(0, len(paths)): path = paths[index] attribute = attributes[index] if len(path) > 0\ and path != transkription_field.path\ and path.start.real > tr_xmin\ and path.end.real < transkription_field.xmax: allpaths_on_tf.append(Path(id=index, path=path, style_class=attribute.get('class'))) text_area_deletion_paths = [] deletion_or_underline_paths = [] box_paths = [] dots_paths = [] word_connector_paths = [] uncategorized_paths = [] for mypath in allpaths_on_tf: xmin, xmax, ymin, ymax = mypath.path.bbox() start_line_number = self.get_line_number(mypath.path.start.imag-tr_ymin) if abs(xmax-xmin) < 1 and abs(ymax-ymin) < 1: dots_paths.append(mypath) elif abs(ymax-ymin) > MAX_HEIGHT_LINES and abs(ymax-ymin) < max_line and mypath.path.iscontinuous() and not mypath.path.isclosed(): deletion_or_underline_paths.append(mypath) elif abs(ymax-ymin) > MAX_HEIGHT_LINES and abs(ymax-ymin) < max_line and mypath.path.iscontinuous() and mypath.path.isclosed(): box_paths.append(mypath) elif abs(ymax-ymin) > MAX_HEIGHT_LINES and abs(ymax-ymin) > max_line and mypath.path.iscontinuous() and not mypath.path.isclosed(): word_connector_paths.append(mypath) elif abs(ymax-ymin) < MAX_HEIGHT_LINES: deletion_or_underline_paths.append(mypath) elif start_line_number != -1 and start_line_number != self.get_line_number(mypath.path.end.imag-tr_ymin): text_area_deletion_paths.append(mypath) else: uncategorized_paths.append(mypath) self.mark_words_intersecting_with_paths_as_deleted(deletion_or_underline_paths, tr_xmin, tr_ymin) elif not Page.UNITTESTING: error_msg = 'Svg source file {} does not exist!'.format(self.source)\ if self.source is not None else 'Page does not contain a source file!' raise FileNotFoundError(error_msg) def create_writing_processes_and_attach2tree(self): """Creates three stages of Nietzsche's process of writing. """ self.writing_processes = [ WritingProcess(version=WritingProcess.FIRST_VERSION),\ WritingProcess(version=WritingProcess.INSERTION_AND_ADDITION),\ WritingProcess(version=WritingProcess.LATER_INSERTION_AND_ADDITION) ] for writing_process in self.writing_processes: writing_process.attach_object_to_tree(self.page_tree) for word in self.words: for transkription_position in word.transkription_positions: for font_key in transkription_position.positional_word_parts[0].style_class.split(' '): if font_key in self.fontsizekey2stage_mapping.keys(): transkription_position.writing_process_id = self.fontsizekey2stage_mapping.get(font_key) def find_special_words(self, transkription_field=None): """Find special words, remove them from words, process their content. """ if self.source is None or not isfile(self.source): raise FileNotFoundError('Page does not have a source!') if transkription_field is None: transkription_field = TranskriptionField(self.source) special_char_list = MarkForeignHands.get_special_char_list() special_char_list += TextConnectionMark.get_special_char_list() single_char_words = [ word for word in self.words if len(word.text) == 1 and word.text in special_char_list ] for word in single_char_words: if word.text == MarkForeignHands.CLASS_MARK: id = len(self.mark_foreign_hands) self.mark_foreign_hands.append(MarkForeignHands.create_cls_from_word(word, id=id)) self.words.remove(word) elif word.text in TextConnectionMark.SPECIAL_CHAR_LIST[0]\ or (word.text in TextConnectionMark.SPECIAL_CHAR_LIST\ and any(style in self.sonderzeichen_list for style\ in word.transkription_positions[0].positional_word_parts[0].style_class.split(' '))): id = len(self.text_connection_marks) self.text_connection_marks.append(TextConnectionMark.create_cls_from_word(word, id=id)) self.words.remove(word) svg_tree = ET.parse(self.source) self.update_page_type(transkription_field=transkription_field) self.update_line_number_area(transkription_field, svg_tree=svg_tree) italic_classes = [ key for key in self.style_dict\ if bool(self.style_dict[key].get('font-family')) and self.style_dict[key]['font-family'].endswith('Italic') ] if len(self.mark_foreign_hands) > 0: MarkForeignHands.find_content(self.mark_foreign_hands, transkription_field, svg_tree, italic_classes=italic_classes,\ SonderzeichenList=self.sonderzeichen_list) if len(self.text_connection_marks) > 0: TextConnectionMark.find_content_in_footnotes(self.text_connection_marks, transkription_field, svg_tree,\ title=self.title, page_number=self.number) def get_biggest_fontSize4styles(self, style_set={}): """Returns biggest font size from style_dict for a set of style class names. [:returns:] (float) biggest font size OR 1 if style_dict is empty """ if bool(self.style_dict): sorted_font_sizes = sorted( (float(self.style_dict[key]['font-size'].replace('px','')) for key in style_set if bool(self.style_dict[key].get('font-size'))), reverse=True) return sorted_font_sizes[0] if len(sorted_font_sizes) > 0 else 1 else: return 1 def get_line_number(self, y): """Returns line number id for element at y. [:return:] (int) line number id or -1 """ if len(self.line_numbers) > 0: result_list = [ line_number.id for line_number in self.line_numbers if y >= line_number.top and y <= line_number.bottom ] return result_list[0] if len(result_list) > 0 else -1 else: return -1 @classmethod + def get_pages_from_xml_file(cls, xml_file, status_contains='', word_selection_function=None): + """Returns a list of Page instantiating a xml_file of type FILE_TYPE_SVG_WORD_POSITION + or xml_files contained in xml_file of type FILE_TYPE_XML_MANUSCRIPT. + [optional: instantiation depends on the fulfilment of a status_contains + and/or on the selection of some words by a word_selection_function]. + """ + source_tree = ET.parse(xml_file) + if source_tree.getroot().find('metadata/type').text == FILE_TYPE_SVG_WORD_POSITION: + page = cls(xml_source_file=xml_file) + if word_selection_function is None or len(word_selection_function(page.words)) > 0: + return [ page ] + else: + return [] + elif source_tree.getroot().find('metadata/type').text == FILE_TYPE_XML_MANUSCRIPT: + pages = [] + xpath = '//page/@output'\ + if status_contains == ''\ + else '//page[contains(@status, "{0}")]/@output'.format(status_contains) + for xml_source_file in source_tree.xpath(xpath): + if isfile(xml_source_file): + pages += cls.get_pages_from_xml_file(xml_source_file, word_selection_function=word_selection_function) + return pages + else: + return [] + + @classmethod def get_semantic_dictionary(cls): """ Creates a semantic dictionary as specified by SemanticClass. """ dictionary = {} class_dict = cls.get_class_dictionary() properties = { 'title': (str, 1, '/page/@title'), 'number': (str, 1, '/page/@number'),\ 'image': { 'class': Image, 'cardinality': 1, 'xpath': '/page/{}'.format(FaksimileImage.XML_TAG)},\ 'line_numbers': (LineNumber, SemanticClass.LIST, '/page/@number|/page/@title'),\ 'orientation': { 'class': str, 'cardinality': 1, 'xpath': '/page/@orientation'},\ 'words': (Word, SemanticClass.LIST, '/page/@number|/page/@title'),\ 'svg_image': { 'class': SVGImage, 'cardinality': 1, 'xpath': '/page/{}'.format(SVGImage.XML_TAG)},\ 'writing_processes': (WritingProcess, SemanticClass.LIST, '/page/@number|/page/@title'),\ 'word_deletion_paths': (Path, SemanticClass.LIST, '/page/@number|/page/@title'),\ 'word_insertion_marks': (WordInsertionMark, SemanticClass.LIST, '/page/@number|/page/@title')} dictionary.update({'class': class_dict}) dictionary.update({'properties': properties}) return dictionary def init_line_numbers(self, line_numbers, document_bottom): """Init line numbers. """ even_index = 0 MINABOVE = 1 self.line_numbers = [] if len(line_numbers) > 0: first_line_bottom = line_numbers[even_index].top - MINABOVE self.line_numbers.append(LineNumber(id=1, top=0, bottom=first_line_bottom)) self.line_numbers.append(line_numbers[even_index]) even_index += 1 while even_index < len(line_numbers): self.line_numbers.append(LineNumber(id=line_numbers[even_index].id-1,\ top=line_numbers[even_index-1].bottom+MINABOVE,\ bottom=line_numbers[even_index].top-MINABOVE)) self.line_numbers.append(line_numbers[even_index]) even_index += 1 self.line_numbers.append(LineNumber(id=line_numbers[even_index-1].id+1,\ top=line_numbers[even_index-1].bottom+MINABOVE,\ bottom=document_bottom)) for line_number in self.line_numbers: line_number.attach_object_to_tree(self.page_tree) def init_words(self): self.word_insertion_marks = [ WordInsertionMark(wim_node=wim_node) for wim_node in self.page_tree.getroot().xpath('//' + WordInsertionMark.XML_TAG) ] - self.words = [ Word.create_cls(word_node) for word_node in self.page_tree.getroot().xpath('//word') ] + self.words = [ Word.create_cls(word_node) for word_node in self.page_tree.getroot().xpath('.//word') ] self.mark_foreign_hands = [ MarkForeignHands.create_cls(node) for node in self.page_tree.getroot().xpath('//' + MarkForeignHands.XML_TAG) ] self.text_connection_marks = [ TextConnectionMark.create_cls(node) for node in self.page_tree.getroot().xpath('//' + TextConnectionMark.XML_TAG) ] self.line_numbers = [ LineNumber(xml_text_node=line_number_node) for line_number_node in self.page_tree.getroot().xpath('//' + LineNumber.XML_TAG) ] self.writing_processes = [ WritingProcess.create_writing_process_from_xml(node, self.words) for node in self.page_tree.xpath('//' + WritingProcess.XML_TAG) ] self.word_deletion_paths = [ Path(node=node) for node in self.page_tree.xpath('//' + Path.WORD_DELETION_PATH_TAG) ] def mark_words_intersecting_with_paths_as_deleted(self, deletion_paths, tr_xmin=0.0, tr_ymin=0.0): """Marks all words that intersect with deletion paths as deleted and adds these paths to word_deletion_paths. [:return:] list of .path.Path that might be word_underline_paths """ if not Page.UNITTESTING: bar = Bar('mark words that intersect with deletion paths', max=len(self.words)) for word in self.words: not bool(Page.UNITTESTING) and bar.next() for transkription_position in word.transkription_positions: first_pwp = transkription_position.positional_word_parts[0] last_pwp = transkription_position.positional_word_parts[len(transkription_position.positional_word_parts)-1] xmin = tr_xmin + first_pwp.left xmax = tr_xmin + last_pwp.left + last_pwp.width ymin = tr_ymin + sorted(pwp.top for pwp in transkription_position.positional_word_parts)[0] ymax = tr_ymin + sorted([pwp.bottom for pwp in transkription_position.positional_word_parts], reverse=True)[0] word_path = parse_path('M {}, {} L {}, {} L {}, {} L {}, {} z'.format(xmin, ymin, xmax, ymin, xmax, ymax, xmin, ymax)) intersecting_paths = [ deletion_path for deletion_path in deletion_paths\ if do_paths_intersect_saveMode(deletion_path.path, word_path) ] if len(intersecting_paths) > 0: word.deleted = True for deletion_path in intersecting_paths: if deletion_path not in self.word_deletion_paths: deletion_path.tag = Path.WORD_DELETION_PATH_TAG deletion_path.attach_object_to_tree(self.page_tree) self.word_deletion_paths.append(deletion_path) not bool(Page.UNITTESTING) and bar.finish() # return those paths in deletion_paths that are not in self.word_deletion_paths return [ word_underline_path for word_underline_path in set(deletion_paths) - set(self.word_deletion_paths) ] def update_and_attach_words2tree(self, update_function_on_word=None, include_special_words_of_type=[]): """Update word ids and attach them to page.page_tree. """ - for node in self.page_tree.xpath('//word|//' + MarkForeignHands.XML_TAG + '|//' + TextConnectionMark.XML_TAG): + for node in self.page_tree.xpath('.//word|.//' + MarkForeignHands.XML_TAG + '|.//' + TextConnectionMark.XML_TAG): node.getparent().remove(node) for index, word in enumerate(self.words): word.id = index if update_function_on_word is not None: update_function_on_word(word) word.attach_word_to_tree(self.page_tree) for index, mark_foreign_hands in enumerate(self.mark_foreign_hands): mark_foreign_hands.id = index if update_function_on_word is not None\ and MarkForeignHands in include_special_words_of_type: update_function_on_word(mark_foreign_hands) mark_foreign_hands.attach_word_to_tree(self.page_tree) for index, text_connection_mark in enumerate(self.text_connection_marks): text_connection_mark.id = index if update_function_on_word is not None\ and TextConnectionMark in include_special_words_of_type: update_function_on_word(text_connection_mark) text_connection_mark.attach_word_to_tree(self.page_tree) def update_line_number_area(self, transkription_field, svg_tree=None): """Determines the width of the area where the line numbers are written in the page.source file. """ THRESHOLD = 0.4 if svg_tree is None: svg_tree = ET.parse(self.source) if len(self.line_numbers) > 1: line_number = self.line_numbers[9]\ if transkription_field.is_page_verso() and len(self.line_numbers) > 8\ else self.line_numbers[1] ln_nodes = [ item for item in svg_tree.iterfind('//text', svg_tree.getroot().nsmap)\ if Matrix.IS_NEARX_TRANSKRIPTION_FIELD(item.get('transform'), transkription_field)\ and LineNumber.IS_A_LINE_NUMBER(item)\ and LineNumber(raw_text_node=item, transkription_field=transkription_field).id == line_number.id ] if len(ln_nodes) > 0: matrix = Matrix(transform_matrix_string=ln_nodes[0].get('transform')) if transkription_field.is_page_verso(): transkription_field.add_line_number_area_width(matrix.getX()) elif self.svg_file is not None and isfile(self.svg_file): svg_path_tree = ET.parse(self.svg_file) namespaces = { k if k is not None else 'ns': v for k, v in svg_path_tree.getroot().nsmap.items() } svg_x = matrix.getX() svg_y = self.line_numbers[1].bottom + transkription_field.ymin use_nodes = svg_path_tree.xpath('//ns:use[@x>="{0}" and @x<="{1}" and @y>="{2}" and @y<="{3}"]'\ .format(svg_x-THRESHOLD, svg_x+THRESHOLD,svg_y-THRESHOLD, svg_y+THRESHOLD), namespaces=namespaces) if len(use_nodes) > 0: symbol_id = use_nodes[0].get('{%s}href' % namespaces['xlink']).replace('#', '') d_strings = use_nodes[0].xpath('//ns:symbol[@id="{0}"]/ns:path/@d'.format(symbol_id), namespaces=namespaces) if len(d_strings) > 0 and d_strings[0] != '': path = parse_path(d_strings[0]) xmin, xmax, ymin, ymax = path.bbox() width = xmax - xmin transkription_field.add_line_number_area_width(matrix.getX() + width) def update_page_type(self, transkription_field=None): """Adds a source to page and attaches it to page_tree. """ if transkription_field is None: if self.source is None or not isfile(self.source): raise FileNotFoundError('Page does not have a source!') transkription_field = TranskriptionField(self.source) self.page_type = Page.PAGE_VERSO\ if transkription_field.is_page_verso()\ else Page.PAGE_RECTO self.page_tree.getroot().set('pageType', self.page_type) def do_paths_intersect_saveMode(path1, path2): """Returns true if paths intersect, false if not or if there was an exception. """ try: return path1.intersect(path2, justonemode=True) except AssertionError: return False Index: svgscripts/datatypes/text_connection_mark.py =================================================================== --- svgscripts/datatypes/text_connection_mark.py (revision 61) +++ svgscripts/datatypes/text_connection_mark.py (revision 62) @@ -1,95 +1,95 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ This class can be used to represent a text connection mark ("Anschlusszeichen"). """ # Copyright (C) University of Basel 2019 {{{1 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see 1}}} __author__ = "Christian Steiner" __maintainer__ = __author__ __copyright__ = 'University of Basel' __email__ = "christian.steiner@unibas.ch" __status__ = "Development" __license__ = "GPL v3" __version__ = "0.0.1" from lxml import etree as ET import sys from .footnotes import extract_footnotes_as_strings from .lineNumber import LineNumber from .reference import Reference from .special_word import SpecialWord class TextConnectionMark(SpecialWord): """ This class represents a text connection mark. """ XML_TAG = 'text-connection-mark' XML_SUB_TAG = Reference.XML_TAG SPECIAL_CHAR_LIST = [ '*', 'W' ] - def __init__(self, id=0, line_number=-1, text='*', deleted=False, transkription_positions=[], faksimile_positions=[], text_source=None): - super(TextConnectionMark, self).__init__(id=id, text=text, deleted=deleted, line_number=line_number, transkription_positions=transkription_positions,\ + def __init__(self, id=0, line_number=-1, text='*', transkription_positions=[], faksimile_positions=[], text_source=None): + super(TextConnectionMark, self).__init__(id=id, text=text, line_number=line_number, transkription_positions=transkription_positions,\ faksimile_positions=faksimile_positions) self.text_source = text_source def add_content(self, node): """Adds content to TextConnectionMark. """ self.text_source = Reference.create_cls(node=node) def attach_word_to_tree(self, target_tree): """Attaches TextConnectionMark to tree target_tree. """ node = super(TextConnectionMark,self).attach_word_to_tree(target_tree) if self.text_source is not None: self.text_source.attach_object_to_tree(node) @staticmethod def find_content_in_footnotes(list_of_text_connection_marks, transkription_field, svg_tree, title='', page_number=''): """Find content for the TextConnectionMark. """ footnotes = extract_footnotes_as_strings(transkription_field=transkription_field, svg_tree=svg_tree, contains_string='Anschlußzeichen') for text_connection_mark in list_of_text_connection_marks: relevant_footnotes = [ footnote_string for footnote_string in footnotes if footnote_string.strip().startswith(str(text_connection_mark.line_number)+ ':') ] if len(relevant_footnotes) > 0: footnote_string = relevant_footnotes[0].strip() line_number = int(footnote_string.split(':')[0]) is_uncertain = footnote_string.endswith('?') reference_string = footnote_string.replace('?', '').split('zu')[1].strip() text_connection_mark.text_source = Reference.create_cls(is_uncertain=is_uncertain,\ reference_string=reference_string, title=title, page_number=page_number) @classmethod def get_semantic_dictionary(cls): """ Creates a semantic dictionary as specified by SemanticClass. """ dictionary = super(TextConnectionMark,cls).get_semantic_dictionary() """TODO dictionary['properties'].update({'text_source': {'class': TextReference, 'cardinality': 1, 'xpath': '{}/@line-number'.format(cls.XML_TAG),\ 'name': 'textConnectionMarkHasLineNumber', 'label': 'text connection mark has a line number'}}) """ return dictionary @classmethod def get_special_char_list(cls): """Returns a list of the chars that define this special word. """ return cls.SPECIAL_CHAR_LIST Index: svgscripts/datatypes/class_spec.py =================================================================== --- svgscripts/datatypes/class_spec.py (revision 61) +++ svgscripts/datatypes/class_spec.py (revision 62) @@ -1,93 +1,118 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" This is an abstract class for all classes that are semantically interesting. +""" This is an abstract class for all classes that are semantically relevant. """ # Copyright (C) University of Basel 2019 {{{1 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see 1}}} __author__ = "Christian Steiner" __maintainer__ = __author__ __copyright__ = 'University of Basel' __email__ = "christian.steiner@unibas.ch" __status__ = "Development" __license__ = "GPL v3" __version__ = "0.0.1" import abc import inspect import warnings class SemanticClass(metaclass=abc.ABCMeta): """ - This is an abstract class for all classes that are semantically interesting. + This is an abstract class for all classes that are semantically relevant. """ SINGLE_VALUE = 1 LIST = -99 @classmethod def get_class_dictionary(cls): """Creates and returns a class_dictionary with the keys 'this' ['type']. """ class_dict = {'this': cls } if cls.__dict__.get('RDF_SUBCLASSES') and len(cls.RDF_SUBCLASSES) > 0: class_dict.update({'owl:equivalentClass': cls.RDF_SUBCLASSES }) else: direct_super_class = inspect.getclasstree([cls],unique=True)[0][0] if issubclass(direct_super_class, SemanticClass) and direct_super_class != SemanticClass: class_dict.update({'type': direct_super_class}) return class_dict def get_data_dictionary(self): # DEPRECIATED """Returns a data dictionary with the keys 'head' and 'body'. Key 'head' points to a dictionary with class information (key: 'class'). Key 'body' points to a dictionary with the data. """ warnings.warn("deprecated", DeprecationWarning) data_dict = {} semantic_dict = self.get_semantic_dictionary() data_dict.update({'head': {'class': semantic_dict['class'].get('this')}}) body = {} """ for key, (datatype, cardinality) in semantic_dict['properties'].items(): if self.__dict__.get(key) is not None: if issubclass(datatype, SemanticClass): if cardinality != SemanticClass.SINGLE_VALUE and len(self.__dict__.get(key)) > 1: items = [] for item in self.__dict__.get(key): items.append(item.get_data_dictionary().get('body')) body.update({ key: items}) else: body.update({ key: self.__dict__.get(key).get_data_dictionary().get('body')}) else: body.update({ key: self.__dict__.get(key) }) data_dict.update({'body': body}) """ return data_dict @classmethod + def get_cls_hasPart_objectCls_dictionaries(cls, object_cls, xpath, object_seqnum_xpath=None, cardinality=0, cardinality_restriction='minCardinality'): + """Return a dictionary containing the information for creating a class that can act + as an intermediary between cls and a number of object_cls if object_cls has + a position in a sequence of object_classes that belong to cls. + """ + part_name = object_cls.__name__ + 'Part' + has_part_name = object_cls.__name__.lower() + 'PartHas' + object_cls.__name__ + has_seqnum_name = object_cls.__name__.lower() + 'HasSeqNum' + if object_seqnum_xpath is None: + object_seqnum_xpath = xpath + '/@id' + object_part_dictionary = { 'class': object_cls, 'cardinality': 1, 'xpath': xpath,\ + 'name': has_part_name, 'label': '{0} has a {1}'.format(part_name, object_cls.__name__),\ + 'comment': '{0} has a part, that is a {1}'.format(part_name, object_cls.__name__)} + object_seqnum_dictionary = { 'class': int, 'cardinality': 1, 'xpath': object_seqnum_xpath,\ + 'name': has_seqnum_name, 'label': '{0} has a sequence number'.format(part_name),\ + 'comment': '{0} has a part, that stands in a sequence with this number'.format(part_name, object_cls.__name__)} + object_dictionary = { 'class_name': part_name, 'has_part': object_part_dictionary, 'has_seqnum': object_seqnum_dictionary,\ + 'label': '{0} part'.format(object_cls.__name__.lower()),\ + 'comment': 'This class servers as a intermediary between {0} and {1}. {0} has some {1} in a specific sequence.'.format(cls.__name__, object_cls.__name__)} + dictionary = { 'flag': 'ordered_list' , 'class': object_dictionary, 'cardinality': cardinality, 'cardinality_restriction': cardinality_restriction, 'xpath': xpath,\ + 'name': cls.__name__.lower() + 'Has' + part_name, 'label': '{0} has a part that connects it with a {1}'.format(cls.__name__, object_cls.__name__),\ + 'comment': '{0} has a part that connects it with a {1}, that has a position in a sequence of {1}'.format(cls.__name__, object_cls.__name__)} + return dictionary + + @classmethod @abc.abstractmethod def get_semantic_dictionary(cls): """Creates a semantic dictionary with 'class' and 'properties' as its keys. The class-key points to a class_dictionary with the keys: 'this' ['type', 'rdf:subClassOf'] The properties-key points to a properties_dictionary with semantically relevant keys of self.__dict__ as keys, and tuples of datatype (class), cardinality (int) as its values. - Cardinality can be SemanticClass.SINGLE_VALUE, 2, 3, ... SemanticClass.LIST. + Cardinality can be SemanticClass.SINGLE_VALUE, SemanticClass.LIST. """ pass Index: svgscripts/datatypes/word_insertion_mark.py =================================================================== --- svgscripts/datatypes/word_insertion_mark.py (revision 61) +++ svgscripts/datatypes/word_insertion_mark.py (revision 62) @@ -1,136 +1,136 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ This class can be used to represent a word insertion mark. """ # Copyright (C) University of Basel 2019 {{{1 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see 1}}} __author__ = "Christian Steiner" __maintainer__ = __author__ __copyright__ = 'University of Basel' __email__ = "christian.steiner@unibas.ch" __status__ = "Development" __license__ = "GPL v3" __version__ = "0.0.1" from lxml import etree as ET from svgpathtools.parser import parse_path import warnings from .lineNumber import LineNumber from .positional_object import PositionalObject from .word import Word class WordInsertionMark(PositionalObject): """ This class represents a word insertion mark. Args: wim_node (etree.Element): element that contains information about a word_insertion_mark. OR id (int): word id x (float) y (float) height (float) width (float) previous_word_id (int): id of the word to which word insertion mark is attached inserted_words: Array->Word of inserted words marked by the word insertion mark. """ WARN_NO_GLYPH_ID = 'No glyph_id found' XML_TAG = 'word-insertion-mark' extraStringKeys = [ 'mark_type', 'symbol_id' ] def __init__(self, wim_node=None, id=0, x=-1.0, y=-1.0, height=0, width=0, previous_word_id=-1, next_word_id=-1, line_number=-1, symbol_id=None, inserted_words=[], inserted_word_id=-1, mark_type='A'): super(WordInsertionMark, self).__init__(id=id, node=wim_node, height=height, width=width, x=x, y=y, tag=WordInsertionMark.XML_TAG) self.stringKeys += [ 'mark_type', 'symbol_id' ] self.intKeys += [ 'line_number', 'next_word_id', 'previous_word_id' ] self.symbol_id = symbol_id self.mark_type = mark_type self.line_number = line_number self.previous_word_id = previous_word_id self.next_word_id = next_word_id if wim_node is not None: self.mark_type = wim_node.get('mark-type') self.line_number = int(wim_node.get('line-number')) if bool(wim_node.get('line-number')) else -1 self.previous_word_id = int(wim_node.get('previous-word-id')) if bool(wim_node.get('previous-word-id')) else -1 self.next_word_id = int(wim_node.get('next-word-id')) if bool(wim_node.get('next-word-id')) else -1 def init_inserted_words(self, inserted_words=[], wim_node=None, inserted_word_id_string=None): if wim_node is not None and inserted_word_id_string is not None: ids = inserted_word_id_string.split(' ') - inserted_words = [ Word.CREATE_WORD(word_node=word_node) for word_node in wim_node.getroottree().getroot().xpath('//word[@id>="{0}" and @id<="{1}"]'.format(ids[0], ids[len(ids)-1])) ] + inserted_words = [ Word.CREATE_WORD(word_node=word_node) for word_node in wim_node.getroottree().getroot().xpath('.//word[@id>="{0}" and @id<="{1}"]'.format(ids[0], ids[len(ids)-1])) ] if len(inserted_words) > 0: inserted_words[0].is_head_of_inserted_words = True inserted_words[len(inserted_words)-1].is_tail_of_inserted_words = True for word in inserted_words: word.set_word_insertion_mark(self) return inserted_words def attach_and_update_word_if_involved(self, word): if word.id == self.previous_word_id: word.is_before_inserted_words = True word.word_insertion_mark = self elif word.id == self.next_word_id: word.is_after_inserted_words = True word.word_insertion_mark = self elif word.id in [ inserted.id for inserted in self.inserted_words ]: word = [ inserted for inserted in self.inserted_words if inserted.id == word.id ][0] return word @classmethod def get_semantic_dictionary(cls): """ Creates a semantic dictionary as specified by SemanticClass. """ dictionary = super(cls,cls).get_semantic_dictionary() word_dicts = { key: { 'class': Word, 'cardinality': 1, 'cardinality_restriction': 'maxCardinality',\ 'label': 'has {} word'.format(key.replace('_word_id','')),\ 'name': 'has{}'.format(key.title().replace('_Id','').replace('_','')),\ 'xpath': '{}/@{}'.format(cls.XML_TAG, key.replace('_','-')) } for key in [ 'previous_word_id', 'next_word_id' ] } dictionary['properties'].update(word_dicts) dictionary['properties'].update({'line_number': {'class': LineNumber, 'cardinality': 1, 'xpath': '{}/@line-number'.format(cls.XML_TAG),\ 'name': 'wordInsertionMarkHasLineNumber', 'label': 'word insertion mark has a line number'}}) dictionary['properties'].update(dict(zip(cls.extraStringKeys, [ (str, 1, '{}/@{}'.format(cls.XML_TAG, i.replace('_','-'))) for i in cls.extraStringKeys]))) return dictionary @staticmethod def CREATE_WORD_INSERTION_MARK(svg_path_tree, namespaces, id=0, x=0.0, y=0.0, xmin=0.0, ymin=0.0, line_number=-1, mark_type='A'): """Creates a (datatypes.word_insertion_mark) WordInsertionMark using a (lxml.ElementTree) svg_path_tree and the corresponding namespaces. """ THRESHOLD = 0.4 svg_x = x + xmin svg_y = y + ymin use_nodes = svg_path_tree.xpath('//ns:use[@x>="{0}" and @x<="{1}" and @y>="{2}" and @y<="{3}"]'\ .format(svg_x-THRESHOLD, svg_x+THRESHOLD,svg_y-THRESHOLD, svg_y+THRESHOLD), namespaces=namespaces) if len(use_nodes) > 0: symbol_id = use_nodes[0].get('{%s}href' % namespaces['xlink']).replace('#', '') d_strings = use_nodes[0].xpath('//ns:symbol[@id="{0}"]/ns:path/@d'.format(symbol_id), namespaces=namespaces) height = 0.0 width = 0.0 if len(d_strings) > 0 and d_strings[0] != '': path = parse_path(d_strings[0]) xmin, xmax, ymin, ymax = path.bbox() width = xmax - xmin height = ymax - ymin return WordInsertionMark(id=id, x=x, y=y-height, height=height, width=width, line_number=line_number,\ mark_type=mark_type, symbol_id=symbol_id) else: warnings.warn('{} for word insertion mark {} on line {}'.format(WordInsertionMark.WARN_NO_GLYPH_ID, id, line_number)) return WordInsertionMark(id=id, x=x, y=y, line_number=line_number, mark_type=mark_type) Index: svgscripts/datatypes/mark_foreign_hands.py =================================================================== --- svgscripts/datatypes/mark_foreign_hands.py (revision 61) +++ svgscripts/datatypes/mark_foreign_hands.py (revision 62) @@ -1,142 +1,143 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ This class can be used to represent the mark for text by some foreign hand. """ # Copyright (C) University of Basel 2019 {{{1 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see 1}}} __author__ = "Christian Steiner" __maintainer__ = __author__ __copyright__ = 'University of Basel' __email__ = "christian.steiner@unibas.ch" __status__ = "Development" __license__ = "GPL v3" __version__ = "0.0.1" from lxml import etree as ET from .matrix import Matrix from .special_word import SpecialWord class MarkForeignHands(SpecialWord): """ This class represents the mark for text by some foreign hand. """ XML_TAG = 'mark-foreign-hands' XML_SUB_TAG = 'text' CLASS_MARK = '$' REPLACE_DICT = { '+': 'x' } - def __init__(self, id=0, line_number=-1, text=CLASS_MARK, foreign_hands_text='', pen='', deleted=False, transkription_positions=[], faksimile_positions=[]): - super(MarkForeignHands, self).__init__(id=id, text=text, deleted=deleted, line_number=line_number,\ + def __init__(self, id=0, line_number=-1, text=CLASS_MARK, foreign_hands_text='', pen='', transkription_positions=[], faksimile_positions=[]): + super(MarkForeignHands, self).__init__(id=id, text=text, line_number=line_number,\ transkription_positions=transkription_positions, faksimile_positions=faksimile_positions) self.foreign_hands_text = foreign_hands_text self.pen = pen def add_content(self, node): """Adds content to MarkForeignHands. """ self.foreign_hands_text = node.text self.pen = node.get('pen') def attach_word_to_tree(self, target_tree): """Attaches MarkForeignHands to tree target_tree. """ node = super(MarkForeignHands,self).attach_word_to_tree(target_tree) if self.foreign_hands_text != '': content_node = ET.SubElement(node, MarkForeignHands.XML_SUB_TAG) content_node.text = self.foreign_hands_text if self.pen != '': content_node.set('pen', self.pen) @classmethod def get_semantic_dictionary(cls): """ Creates a semantic dictionary as specified by SemanticClass. """ dictionary = super(MarkForeignHands,cls).get_semantic_dictionary() dictionary['properties'].update({'foreign_hands_text':\ {'class': str, 'cardinality': 1, 'xpath': '{0}/{1}/text()'.format(cls.XML_TAG, MarkForeignHands.XML_SUB_TAG),\ 'name': 'textOfForeignHands', 'label': 'text traces of some foreign hand'}}) dictionary['properties'].update({'pen':\ - {'class': str, 'maxCardinality': 1, 'xpath': '{0}/{1}/@pen'.format(cls.XML_TAG, MarkForeignHands.XML_SUB_TAG),\ + {'class': str, 'cardinality': 1, 'cardinality_restriction': 'maxCardinality',\ + 'xpath': '{0}/{1}/@pen'.format(cls.XML_TAG, MarkForeignHands.XML_SUB_TAG),\ 'name': 'penOfForeignHands', 'label': 'pen used to write text by some foreign hand'}}) return dictionary @classmethod def get_special_char_list(cls): """Returns a list of the chars that define this special word. """ return [ cls.CLASS_MARK ] @staticmethod def find_content(list_of_special_words, transkription_field, svg_tree, style_dict={}, italic_classes=[], SonderzeichenList=[]): """Find content for the MarkForeignHands. """ if len(style_dict) > 0: if len(italic_classes) == 0: italic_classes = [ key for key in style_dict\ if bool(style_dict[key].get('font-family')) and style_dict[key]['font-family'].endswith('Italic') ] if len(SonderzeichenList) == 0: SonderzeichenList = [ key for key in style_dict\ if bool(style_dict[key].get('font-family')) and style_dict[key]['font-family'].startswith('Sonderzeichen') ] nodes_in_margin_field = [ item for item in filter(lambda x: Matrix.IS_IN_MARGIN_FIELD(x.get('transform'), transkription_field),\ svg_tree.getroot().iterfind('.//text', svg_tree.getroot().nsmap))] for mark_foreign_hands in list_of_special_words: relevant_nodes = [ node for node in nodes_in_margin_field\ if is_close((mark_foreign_hands.transkription_positions[0].bottom+mark_foreign_hands.transkription_positions[0].top)/2,\ node.get('transform'), transkription_field) ] relevant_nodes = sorted(relevant_nodes, key=lambda x: Matrix(transform_matrix_string=x.get('transform')).getX()) italic_found = False mark_foreign_hands_text = '' pen = '' for node in relevant_nodes: if len(node.getchildren()) == 0: if italic_found: pen += node.text elif any(style in italic_classes for style in node.get('class').split(' ')): italic_found = True pen = node.text else: mark_foreign_hands_text += get_text_from_node(node, SonderzeichenList) else: for tspan in node.getchildren(): if italic_found: pen += tspan.text elif any(style in italic_classes for style in tspan.get('class').split(' ')): italic_found = True pen = tspan.text else: mark_foreign_hands_text += get_text_from_node(tspan, SonderzeichenList) mark_foreign_hands.foreign_hands_text = mark_foreign_hands_text mark_foreign_hands.pen = pen def get_text_from_node(node, SonderzeichenList): """Returns the text of node. Replaces Sonderzeichen if node has a style class in SonderzeichenList. """ if any(style in SonderzeichenList for style in node.get('class').split(' '))\ and bool(MarkForeignHands.REPLACE_DICT.get(node.text)): return MarkForeignHands.REPLACE_DICT[node.text] else: return node.text def is_close(mark_foreign_hands_position, matrix_string, transkription_field): """Return true if mark_foreign_hands_position is == matrix.getY()+-THRESHOLD_Y """ THRESHOLD_Y = 4 matrix = Matrix(transform_matrix_string=matrix_string, transkription_field=transkription_field) return abs(mark_foreign_hands_position-matrix.getY()) < THRESHOLD_Y Index: svgscripts/datatypes/simple_word.py =================================================================== --- svgscripts/datatypes/simple_word.py (revision 0) +++ svgscripts/datatypes/simple_word.py (revision 62) @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" This super class can be used to represent a simple word. +""" +# Copyright (C) University of Basel 2019 {{{1 +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see 1}}} + +__author__ = "Christian Steiner" +__maintainer__ = __author__ +__copyright__ = 'University of Basel' +__email__ = "christian.steiner@unibas.ch" +__status__ = "Development" +__license__ = "GPL v3" +__version__ = "0.0.1" + +import abc +from lxml import etree as ET + +from .class_spec import SemanticClass +from .lineNumber import LineNumber +from .transkription_position import TranskriptionPosition +from .word_position import WordPosition + +class SimpleWord(SemanticClass, metaclass=abc.ABCMeta): + """ + This class represents a simple word. + + """ + XML_TAG = 'simple-word' + XML_SUB_TAG = 'content' + + def __init__(self, id=0, line_number=-1, text='', deleted=False, transkription_positions=None, faksimile_positions=None): + self.id = id + self.text = text + self.line_number = line_number + self.transkription_positions = transkription_positions if transkription_positions is not None else [] + self.faksimile_positions = faksimile_positions if faksimile_positions is not None else [] + + def attach_word_to_tree(self, target_tree): + """Attaches word to tree target_tree. + """ + if target_tree.__class__.__name__ == '_ElementTree': + target_tree = target_tree.getroot() + if len(target_tree.xpath('.//' + self.XML_TAG + '[@id="%s"]' % self.id)) > 0: + word_node = target_tree.xpath('.//' + self.XML_TAG + '[@id="%s"]' % self.id)[0] + word_node.getparent().remove(word_node) + word_node = ET.SubElement(target_tree, self.XML_TAG, attrib={'id': str(self.id)}) + word_node.set('text', self.text) + if self.line_number > -1: + word_node.set('line-number', str(self.line_number)) + for transkription_position in self.transkription_positions: + transkription_position.attach_object_to_tree(word_node) + for faksimile_position in self.faksimile_positions: + faksimile_position.attach_object_to_tree(word_node) + return word_node + + @classmethod + def create_cls(cls, word_node): + """Creates a cls from a (lxml.Element) node. + + [:return:] cls + """ + if word_node is not None: # init word from xml node + id = int(word_node.get('id')) + line_number = int(word_node.get('line-number')) if bool(word_node.get('line-number')) else -1 + text = word_node.get('text') + transkription_positions = [ TranskriptionPosition(node=node) for node in word_node.findall('.//' + WordPosition.TRANSKRIPTION) ] + faksimile_positions = [ WordPosition(node=node) for node in word_node.findall('.//' + WordPosition.FAKSIMILE) ] + return cls(id=id, text=text, line_number=line_number, transkription_positions=transkription_positions,\ + faksimile_positions=faksimile_positions) + else: + error_msg = 'word_node has not been defined' + raise Exception('Error: {}'.format(error_msg)) + + @classmethod + def get_semantic_dictionary(cls): + """ Creates and returns a semantic dictionary as specified by SemanticClass. + """ + dictionary = {} + class_dict = cls.get_class_dictionary() + properties = {'text': (str, 1, '{0}/@text'.format(cls.XML_TAG)),\ + 'line_number': {'class': LineNumber, 'cardinality': 0,\ + 'name': 'wordHasLineNumber', 'xpath': '{0}/@line-number'.format(cls.XML_TAG),\ + 'label': 'word has a line number',\ + 'comment': 'Relating a word to a line number it has.'},\ + 'transkription_positions': (TranskriptionPosition, SemanticClass.LIST, '{0}/@id'.format(cls.XML_TAG)),\ + 'faksimile_positions': (WordPosition, SemanticClass.LIST, '{0}/@id'.format(cls.XML_TAG))} + dictionary.update({'class': class_dict}) + dictionary.update({'properties': properties}) + return dictionary + Index: svgscripts/datatypes/transkription_position.py =================================================================== --- svgscripts/datatypes/transkription_position.py (revision 61) +++ svgscripts/datatypes/transkription_position.py (revision 62) @@ -1,146 +1,151 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ This class can be used to represent a transkription word position. """ # Copyright (C) University of Basel 2019 {{{1 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see 1}}} __author__ = "Christian Steiner" __maintainer__ = __author__ __copyright__ = 'University of Basel' __email__ = "christian.steiner@unibas.ch" __status__ = "Development" __license__ = "GPL v3" __version__ = "0.0.1" from lxml import etree as ET from os.path import isfile from .class_spec import SemanticClass from .debug_message import DebugMessage from .positional_word_part import PositionalWordPart from .word_position import WordPosition from .matrix import Matrix class TranskriptionPosition(WordPosition): """ This class represents a transkription word position. Args: id (int): word id matrix (datatypes.Matrix): matrix containing information about transformation. height (float): height of word width (float): width of word x (float): x position of word y (float): y position of word positional_word_parts a list of (datatypes.positional_word_part) PositionalWordPart debug_message a (datatypes.debug_message) DebugMessage """ ADD2X = 0.15 ADD2TOP = 1.0 ADD2BOTTOM = 0.2 HEIGHT_FACTOR = 1.1 # factor that multiplies biggest_font_size -> height XML_TAG = WordPosition.TRANSKRIPTION def __init__(self, id=0, node=None, height=0.0, width=0.0, x=0.0, y=0.0, matrix=None, positional_word_parts=[], debug_message=None): super(TranskriptionPosition, self).__init__(id=id, node=node, height=height, width=width, x=x, y=y, matrix=matrix, tag=WordPosition.TRANSKRIPTION) self.positional_word_parts = positional_word_parts self.debug_message = debug_message if node is not None: self.debug_message = DebugMessage(node=node.xpath('.//' + DebugMessage.XML_TAG)[0])\ if len(node.xpath('.//' + DebugMessage.XML_TAG)) > 0 else None self.positional_word_parts = [ PositionalWordPart(node=pwp_node) for pwp_node in node.xpath('.//' + PositionalWordPart.XML_TAG) ] self.attachable_objects += self.positional_word_parts if self.debug_message is not None: self.attachable_objects.append(self.debug_message) @classmethod def get_semantic_dictionary(cls): """ Creates a semantic dictionary as specified by SemanticClass. """ dictionary = super(TranskriptionPosition,cls).get_semantic_dictionary() dictionary['properties'].update({'positional_word_parts': (PositionalWordPart, SemanticClass.LIST, '{}/@id'.format(cls.XML_TAG))}) return dictionary + def get_text(self): + """Returns the concatenated text of all positional_word_parts. + """ + return ''.join([pwp.text for pwp in self.positional_word_parts]) + @staticmethod def CREATE_TRANSKRIPTION_POSITION_LIST_FROM_PWPS(positional_word_parts, debug_message=None, debug_msg_string=None, transkription_position_id=0): """Creates a list of TranskriptionPosition from a list of (datatypes.positional_word_part) PositionalWordPart. [:return:] a list of (datatypes.transkription_position) TranskriptionPosition """ TOPCORRECTION = 1 debug_message = DebugMessage(message=debug_msg_string)\ if debug_msg_string is not None else debug_message transkription_positions = [] if len(positional_word_parts) < 1: return [] matrix = positional_word_parts[0].transform index = 0 matrices_differ = False style_class = positional_word_parts[0].style_class styles_differ = False while index < len(positional_word_parts) and not matrices_differ and not styles_differ: if Matrix.DO_CONVERSION_FACTORS_DIFFER(matrix, positional_word_parts[index].transform): matrices_differ = True elif style_class != positional_word_parts[index].style_class: styles_differ = True else: index += 1 if (matrices_differ or styles_differ) and index < len(positional_word_parts): debug_msg_string = 'matrices differ' if matrices_differ else 'styles differ' transkription_positions += TranskriptionPosition.CREATE_TRANSKRIPTION_POSITION_LIST_FROM_PWPS(\ positional_word_parts[index:], debug_msg_string=debug_msg_string, transkription_position_id=int(transkription_position_id)+1) positional_word_parts = positional_word_parts[:index] height = [ pwp.height for pwp in sorted(positional_word_parts, key=lambda pwp: pwp.height, reverse=True)][0] + 2*TOPCORRECTION x = positional_word_parts[0].left - TranskriptionPosition.ADD2X y = [ pwp.top for pwp in sorted(positional_word_parts, key=lambda pwp: pwp.top)][0] - TOPCORRECTION width = positional_word_parts[len(positional_word_parts)-1].left - x\ + positional_word_parts[len(positional_word_parts)-1].width + TranskriptionPosition.ADD2X for pwp_index, pwp in enumerate(positional_word_parts): pwp.id = pwp_index transkription_positions.insert(0, TranskriptionPosition(id=transkription_position_id, height=height, width=width, x=x, y=y, matrix=matrix,\ positional_word_parts=positional_word_parts, debug_message=debug_message)) return transkription_positions @staticmethod def CREATE_TRANSKRIPTION_POSITION_LIST(page, word_part_objs, matrix=None, debug_msg_string=None, transkription_field=None): """Creates a list of TranskriptionPosition from word_part_objs (i.e. a list of dictionaries with the keys: text, x, y, matrix, class). [:return:] a list of (datatypes.transkription_position) TranskriptionPosition """ positional_word_parts = [] debug_message = DebugMessage(message=debug_msg_string)\ if debug_msg_string is not None else None if page.svg_file is not None and isfile(page.svg_file): svg_path_tree = ET.parse(page.svg_file) namespaces = { k if k is not None else 'ns': v for k, v in svg_path_tree.getroot().nsmap.items() } xmin = 0.0 ymin = 0.0 if transkription_field is not None: xmin = transkription_field.xmin ymin = transkription_field.ymin for part_obj in word_part_objs: positional_word_parts += PositionalWordPart.CREATE_POSITIONAL_WORD_PART_LIST(\ part_obj, svg_path_tree, namespaces, page, start_id=len(positional_word_parts),\ xmin=xmin, ymin=ymin) else: positional_word_parts = PositionalWordPart.CREATE_SIMPLE_POSITIONAL_WORD_PART_LIST(page, word_part_objs) if len(positional_word_parts) > 0: return TranskriptionPosition.CREATE_TRANSKRIPTION_POSITION_LIST_FROM_PWPS(positional_word_parts, debug_message=debug_message) else: return [ TranskriptionPosition(matrix=matrix, debug_message=debug_message) ] Index: svgscripts/util.py =================================================================== --- svgscripts/util.py (revision 61) +++ svgscripts/util.py (revision 62) @@ -1,227 +1,226 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ This program can be used to copy a faksimile svg file with the option of highlighting some word boxes. """ # Copyright (C) University of Basel 2019 {{{1 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see 1}}} from colorama import Fore, Style from functools import cmp_to_key import getopt import itertools import lxml.etree as ET import re import shutil import signal import string import subprocess from svgpathtools import svg_to_paths import sys import tempfile from operator import attrgetter import os from os import listdir, sep, path, setpgrp, devnull from os.path import exists, isfile, isdir, dirname, basename, splitext import warnings import xml.etree.ElementTree as XET if dirname(__file__) not in sys.path: sys.path.append(dirname(__file__)) -from convert_wordPositions import SVGConverter, create_pdf_with_highlighted_words from datatypes.faksimile import FaksimilePage, get_paths_inside_rect from datatypes.lineNumber import LineNumber from datatypes.mark_foreign_hands import MarkForeignHands from datatypes.page import Page from datatypes.transkriptionField import TranskriptionField from local_config import PDF_READER, SVG_EDITOR from myxmlwriter import write_pretty, FILE_TYPE_SVG_WORD_POSITION, FILE_TYPE_XML_MANUSCRIPT from process_files import update_svgposfile_status __author__ = "Christian Steiner" __maintainer__ = __author__ __copyright__ = 'University of Basel' __email__ = "christian.steiner@unibas.ch" __status__ = "Development" __license__ = "GPL v3" __version__ = "0.0.1" UNITTESTING = False HIGHLIGHT_COLOR = 'red' OPACITY = '0.5' class ExternalViewer: """This class can be used to show files with external viewers. """ file_format_viewer_dict = { '.pdf': PDF_READER, '.svg': SVG_EDITOR } @classmethod def show_files(cls, single_file=None, list_of_files=[]): """Opens file(s) with corresponding external viewer(s). """ DEVNULL = None if type(single_file) == list: list_of_files = single_file elif single_file is not None: list_of_files.append(single_file) if len(list_of_files) > 1: DEVNULL = open(devnull, 'wb') process_list = [] list_of_files.reverse() while len(list_of_files) > 0: file2open = list_of_files.pop() viewer = cls.file_format_viewer_dict.get(splitext(file2open)[1]) if viewer is not None: if len(list_of_files) > 0: process_list.append(\ subprocess.Popen([viewer, file2open], stdout=DEVNULL, stderr=DEVNULL, preexec_fn=os.setsid)) else: subprocess.run([viewer, file2open]) for process in process_list: os.killpg(os.getpgid(process.pid), signal.SIGTERM) if DEVNULL is not None: DEVNULL.close() def copy_faksimile_svg_file(target_file=None, faksimile_source_file=None, faksimile_tree=None, target_directory=None, local_image_path=None): """Copy a faksimile_svg_file to target_file. """ if faksimile_source_file is None and faksimile_tree is not None: faksimile_source_file = faksimile_tree.docinfo.URL elif faksimile_source_file is None: raise Exception('copy_faksimile_svg_file needs either a faksimile_tree (lxml.etree.ElementTree) or a faksimile_source_file') if target_file is not None and target_directory is not None: target_file = target_directory + sep + target_file elif target_file is None and target_directory is not None: target_file = target_directory + sep + basename(faksimile_source_file) elif target_file is None: raise Exception('copy_faksimile_svg_file needs either a target_file or a target_directory') paths, attributes, svg_attributes = svg_to_paths.svg2paths(faksimile_source_file, return_svg_attributes=True) if bool(svg_attributes.get('xmlns')): XET.register_namespace('', svg_attributes['xmlns']) for key in [ key for key in svg_attributes.keys() if key.startswith('xmlns:') ]: try: XET.register_namespace(key.replace('xmlns:', ''), svg_attributes[key]) except ValueError: pass namespaces = { 'ns': svg_attributes['xmlns'], 'xlink': svg_attributes['xmlns:xlink'] } if faksimile_tree is not None: element = XET.fromstring(ET.tostring(faksimile_tree))\ if type(faksimile_tree) == ET._ElementTree\ else XET.fromstring(XET.tostring(faksimile_tree.getroot())) target_tree = XET.ElementTree(element) else: target_tree = XET.parse(faksimile_source_file) if local_image_path is not None\ and len(target_tree.findall('.//ns:image', namespaces=namespaces)) > 0: image_node = target_tree.findall('.//ns:image', namespaces=namespaces)[0] image_node.set('{%s}href' % namespaces['xlink'], local_image_path) target_tree.write(target_file) def create_highlighted_svg_file(faksimile_tree, node_ids, target_file=None, target_directory=None, local_image_path=None, namespaces={}, highlight_color=HIGHLIGHT_COLOR, opacity=OPACITY): """Highlights the nodes of a faksimile_tree that are specified by the list of node_ids and writes the tree to a file. """ if len(namespaces) == 0: namespaces = { k if k is not None else 'ns': v for k, v in faksimile_tree.getroot().nsmap.items() } for node in itertools.chain(*[\ faksimile_tree.xpath('//ns:rect[@id="{0}"]|//ns:path[@id="{0}"]'.format(node_id), namespaces=namespaces)\ for node_id in node_ids\ ]): node.set('fill', highlight_color) node.set('opacity', opacity) node.set('style', '') copy_faksimile_svg_file(target_file=target_file, faksimile_tree=faksimile_tree, target_directory=target_directory,\ local_image_path=local_image_path) def get_empty_node_ids(faksimile_tree, x_min=0.0, x_max=0.0, y_min=0.0, y_max=0.0, text_field_id=None, faksimile_page=None, namespaces={}): """Returns a list of ids of rect and path nodes that do not have a title element. """ THRESHOLD_X = 10 if faksimile_page is not None: x_min = faksimile_page.text_field.xmin + faksimile_page.faksimile_image.x x_max = faksimile_page.text_field.xmax + faksimile_page.faksimile_image.x - THRESHOLD_X y_min = faksimile_page.text_field.ymin + faksimile_page.faksimile_image.y y_max = faksimile_page.text_field.ymax + faksimile_page.faksimile_image.y text_field_id = faksimile_page.text_field.id if len(namespaces) == 0: namespaces = { k if k is not None else 'ns': v for k, v in faksimile_tree.getroot().nsmap.items() } empyt_node_ids = [] nodes_without_title = faksimile_tree.xpath('//ns:rect[@x>"{0}" and @x<"{1}" and @y>"{2}" and @y<"{3}" and @id!="{4}" and not(./ns:title)]'.format(\ x_min, x_max, y_min, y_max, text_field_id), namespaces=namespaces) nodes_without_title += get_paths_inside_rect(faksimile_tree, '//ns:path[not(./ns:title)]', x_min, x_max, y_min, y_max, text_field_id, namespaces=namespaces) for node_without_title in nodes_without_title: empyt_node_ids.append(node_without_title.get('id')) return empyt_node_ids def record_changes(original_svg_file, changed_svg_file, node_ids, namespaces={}): """Copy changes made to changed_svg_file to original_svg_file. """ old_tree = ET.parse(original_svg_file) new_tree = ET.parse(changed_svg_file) if len(namespaces) == 0: namespaces = { k if k is not None else 'ns': v for k, v in new_tree.getroot().nsmap.items() } for node_id in node_ids: new_titles = new_tree.xpath('//ns:rect[@id="{0}"]/ns:title|//ns:path[@id="{0}"]/ns:title'.format(node_id), namespaces=namespaces) old_nodes = old_tree.xpath('//ns:rect[@id="{0}"]|//ns:path[@id="{0}"]'.format(node_id), namespaces=namespaces) if len(new_titles) > 0 and len(old_nodes) > 0: if old_nodes[0].find('ns:title', namespaces=namespaces) is not None: old_nodes[0].find('ns:title', namespaces=namespaces).text = new_titles[0].text else: old_title_id_string = new_titles[0].get('id') old_title = ET.SubElement(old_nodes[0], 'title', attrib={ 'id': old_title_id_string }) old_title.text = new_titles[0].text copy_faksimile_svg_file(target_file=original_svg_file, faksimile_tree=old_tree) def usage(): """prints information on how to use the script """ print(main.__doc__) def main(argv): """This program can be used to copy a faksimile svg file with the option of highlighting some word boxes. svgscripts/copy_faksimile_svg_file.py [OPTIONS] a svg file containing information about the word positions on the faksimile. the target directory. OPTIONS: -h|--help: show help :return: exit code (int) """ try: opts, args = getopt.getopt(argv, "h", ["help" ]) except getopt.GetoptError: usage() return 2 for opt, arg in opts: if opt in ('-h', '--help'): usage() return 0 if len(args) < 2: usage() return 2 exit_status = 0 if exists(args[0]) and exists(args[1]): faksimile_svg_file = args[0] if isfile(args[0]) else args[1] target_dir = args[1] if isdir(args[1]) else args[0] else: file_a = args[0] if not exists(args[0]) else args[1] raise FileNotFoundError('File {} does not exist!'.format(file_a)) return exit_status if __name__ == "__main__": sys.exit(main(sys.argv[1:])) Index: svgscripts/convert_wordPositions.py =================================================================== --- svgscripts/convert_wordPositions.py (revision 61) +++ svgscripts/convert_wordPositions.py (revision 62) @@ -1,348 +1,349 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ This program can be used to convert the word positions to HTML for testing purposes. """ # Copyright (C) University of Basel 2019 {{{1 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see 1}}} import cairosvg import getopt from lxml.html import builder as E from lxml.html import open_in_browser import lxml from os import sep, listdir, mkdir, path, remove from os.path import exists, isfile, isdir, dirname import re import sys from svgpathtools import svg_to_paths import xml.etree.ElementTree as ET if dirname(__file__) not in sys.path: sys.path.append(dirname(__file__)) from datatypes.matrix import Matrix from datatypes.page import Page from datatypes.transkriptionField import TranskriptionField from datatypes.writing_process import WritingProcess from datatypes.word import Word from myxmlwriter import write_pretty __author__ = "Christian Steiner" __maintainer__ = __author__ __copyright__ = 'University of Basel' __email__ = "christian.steiner@unibas.ch" __status__ = "Development" __license__ = "GPL v3" __version__ = "0.0.1" class Converter: """The converter super class. """ def __init__(self, page, non_testing=True, show_word_insertion_mark=False): self.page = page self.non_testing = non_testing self.show_word_insertion_mark = show_word_insertion_mark def _get_transkription_positions(self, transkription_positions, stage_version=''): """Returns the transkription_positions of the indicated stage_version. """ convertable_transkription_positions = transkription_positions if stage_version != '': convertable_transkription_positions = [] if re.match(r'^\d$', stage_version): writing_process_id = int(stage_version) for transkription_position in transkription_positions: if transkription_position.writing_process_id == writing_process_id: convertable_transkription_positions.append(transkription_position) elif re.match(r'^\d\+$', stage_version): version_range = [ *range(int(stage_version.replace('+','')), len(WritingProcess.VERSION_DESCRIPTION)) ] for transkription_position in transkription_positions: if transkription_position.writing_process_id in version_range: convertable_transkription_positions.append(transkription_position) elif re.match(r'^\d\-\d$', stage_version): start_stop = [ int(i) for i in re.split(r'-', stage_version) ] version_range = [ *range(start_stop[0], start_stop[1]+1) ] for transkription_position in transkription_positions: if transkription_position.writing_process_id in version_range: convertable_transkription_positions.append(transkription_position) return convertable_transkription_positions def _get_words(self, words, highlighted_words=[]): """Return the words that will be hightlighted. """ if len(highlighted_words) == 0: return words else: return highlighted_words def convert(self, output_file=None, stage_version='', highlighted_words=[]): """Prints all words. """ first_word_of_line = None out = sys.stdout if output_file is not None: out = open(output_file, 'w') for word in self.page.words: if first_word_of_line is None or first_word_of_line.line_number != word.line_number: out.write('\n') first_word_of_line = word if word.line_number % 2 == 0: out.write(str(word.line_number).zfill(2) + ' ') else: out.write(' ') if stage_version == '' or len(self._get_transkription_positions(word.transkription_positions, stage_version=stage_version)) > 0: if word.text is not None: out.write(word.text + ' ') out.close() @classmethod def CREATE_CONVERTER(cls, page, non_testing=True,converter_type='', show_word_insertion_mark=False): """Returns a converter of type converter_type. [:return:] SVGConverter for 'SVG', HTMLConverter for 'HTML', Converter for None """ cls_dict = { subclass.__name__: subclass for subclass in cls.__subclasses__() } cls_key = converter_type + 'Converter' if bool(cls_dict.get(cls_key)): return cls_dict.get(cls_key)(page, non_testing, show_word_insertion_mark) else: return Converter(page, non_testing, show_word_insertion_mark) class SVGConverter(Converter): """This class can be used to convert a 'svgWordPositions' xml file to a svg file that combines text as path and text-as-text. """ BG_COLOR = 'yellow' OPACITY = '0.2' def __init__(self, page, non_testing=True, show_word_insertion_mark=False, bg_color=BG_COLOR, opacity=OPACITY): Converter.__init__(self, page, non_testing, show_word_insertion_mark) self.bg_color = bg_color self.opacity = opacity def convert(self, output_file=None, stage_version='', highlighted_words=[]): """Converts Page to SVG """ title = self.page.title if(self.page.title is not None) else 'Test Page' title = '{}, S. {}'.format(title, self.page.number) if (self.page.number is not None) else title transkription_field = TranskriptionField(self.page.svg_file) if bool(transkription_field.get_svg_attributes('xmlns')): ET.register_namespace('', transkription_field.get_svg_attributes('xmlns')) if bool(transkription_field.get_svg_attributes('xmlns:xlink')): ET.register_namespace('xlink', transkription_field.get_svg_attributes('xmlns:xlink')) svg_tree = ET.parse(self.page.svg_file) transkription_node = ET.SubElement(svg_tree.getroot(), 'g', attrib={'id': 'Transkription'}) colors = [ 'yellow', 'orange' ] if self.bg_color == self.BG_COLOR else [ self.bg_color ] color_index = 0 for word in self._get_words(self.page.words, highlighted_words=highlighted_words): for transkription_position in self._get_transkription_positions(word.transkription_positions, stage_version=stage_version): rect_node = ET.SubElement(transkription_node, 'rect',\ attrib={'id': str(transkription_position.id), 'x': str(transkription_position.left + transkription_field.xmin),\ 'y': str(transkription_position.top + transkription_field.ymin), 'width': str(transkription_position.width),\ 'height': str(transkription_position.height), 'fill': colors[color_index], 'opacity': self.opacity}) if transkription_position.transform is not None: matrix = transkription_position.transform.clone_transformation_matrix() matrix.matrix[Matrix.XINDEX] = round(transkription_position.transform.matrix[Matrix.XINDEX] + transkription_field.xmin, 3) matrix.matrix[Matrix.YINDEX] = round(transkription_position.transform.matrix[Matrix.YINDEX] + transkription_field.ymin, 3) rect_node.set('transform', matrix.toString()) rect_node.set('x', str(round(transkription_position.left - transkription_position.transform.matrix[Matrix.XINDEX], 3))) rect_node.set('y', str(round((transkription_position.height-1.5)*-1, 3))) ET.SubElement(rect_node, 'title').text = word.text color_index = (color_index + 1) % len(colors) if output_file is not None: svg_tree.write(output_file) class HTMLConverter(Converter): """This class can be used to convert a 'svgWordPositions' xml file to a test HTML file. """ CSS = """ .highlight0 { background-color: yellow; opacity: 0.2; } .highlight1 { background-color: pink; opacity: 0.2; } .foreign { background-color: blue; opacity: 0.4; } .word-insertion-mark { background-color: orange; opacity: 0.2; } .deleted { background-color: grey; opacity: 0.2; } """ def __init__(self, page, non_testing=True, show_word_insertion_mark=False): Converter.__init__(self, page, non_testing, show_word_insertion_mark) def convert(self, output_file=None, stage_version='', highlighted_words=[]): """Converts Page to HTML """ title = self.page.title if(self.page.title is not None) else 'Test Page' title = '{}, S. {}'.format(title, self.page.number) if (self.page.number is not None) else title if stage_version != '': title = title + ', Schreibstufe: ' + stage_version width = self.page.width height = self.page.height style_content = ' position: relative; width: {}px; height: {}px; background-image: url({}); background-size: {}px {}px '\ .format(width, height, path.abspath(self.page.svg_file), width, height) style = E.STYLE('#transkription {' + style_content + '}', HTMLConverter.CSS) head = E.HEAD(E.TITLE(title),E.META(charset='UTF-8'), style) transkription = E.DIV(id="transkription") counter = 0 for word in self.page.words: highlight_class = 'highlight' + str(counter)\ if not word.deleted else 'deleted' word_title = 'id: {}/line: {}\n{}'.format(str(word.id), str(word.line_number), word.text) for transkription_position in self._get_transkription_positions(word.transkription_positions, stage_version=stage_version): self._append2transkription(transkription, highlight_class, word_title, transkription_position) counter = (counter + 1) % 2 word_insertion_mark_class = 'word-insertion-mark' counter = 0 for mark_foreign_hands in self.page.mark_foreign_hands: highlight_class = 'foreign' title = 'id: {}/line: {}\n{} {}'.format(str(mark_foreign_hands.id), str(word.line_number),\ mark_foreign_hands.foreign_hands_text, mark_foreign_hands.pen) for transkription_position in mark_foreign_hands.transkription_positions: self._append2transkription(transkription, highlight_class, title, transkription_position) if self.show_word_insertion_mark: for word_insertion_mark in self.page.word_insertion_marks: wim_title = 'id: {}/line: {}\nword insertion mark'.format(str(word_insertion_mark.id), str(word_insertion_mark.line_number)) style_content = 'position:absolute; top:{0}px; left:{1}px; width:{2}px; height:{3}px;'.format(\ word_insertion_mark.top, word_insertion_mark.left, word_insertion_mark.width, word_insertion_mark.height) link = E.A(' ', E.CLASS(word_insertion_mark_class), title=wim_title, style=style_content) transkription.append(link) html = E.HTML(head,E.BODY(transkription)) bool(self.non_testing) and open_in_browser(html) if output_file is not None: with open(output_file, 'wb') as f: f.write(lxml.html.tostring(html, pretty_print=True, include_meta_content_type=True, encoding='utf-8')) f.closed def _append2transkription(self, transkription, highlight_class, title, transkription_position): """Append content to transkription-div. """ style_content = 'position:absolute; top:{0}px; left:{1}px; width:{2}px; height:{3}px;'.format(\ transkription_position.top, transkription_position.left, transkription_position.width, transkription_position.height) if transkription_position.transform is not None: style_content = style_content + ' transform: {}; '.format(transkription_position.transform.toCSSTransformString()) transform_origin_x = (transkription_position.left-round(transkription_position.transform.getX(), 1))*-1\ if (transkription_position.left-round(transkription_position.transform.getX(), 1))*-1 < 0 else 0 style_content = style_content + ' transform-origin: {}px {}px; '.format(transform_origin_x, transkription_position.height) link = E.A(' ', E.CLASS(highlight_class), title=title, style=style_content) transkription.append(link) -def create_pdf_with_highlighted_words(xml_source_file, highlighted_words=[], pdf_file_name='output.pdf', bg_color=SVGConverter.BG_COLOR): +def create_pdf_with_highlighted_words(xml_source_file=None, page=None, highlighted_words=[], pdf_file_name='output.pdf', bg_color=SVGConverter.BG_COLOR): """Creates a pdf file highlighting some words. """ - page = Page(xml_source_file=xml_source_file) + if page is None and xml_source_file is not None: + page = Page(xml_source_file=xml_source_file) converter = SVGConverter(page, bg_color=bg_color) if not pdf_file_name.endswith('pdf'): pdf_file_name = pdf_file_name + '.pdf' tmp_svg_file = pdf_file_name.replace('.pdf', '.svg') converter.convert(output_file=tmp_svg_file, highlighted_words=highlighted_words) cairosvg.svg2pdf(url=tmp_svg_file, write_to=pdf_file_name) isfile(tmp_svg_file) and remove(tmp_svg_file) def usage(): """prints information on how to use the script """ print(main.__doc__) def main(argv): """This program can be used to convert the word positions to HTML, SVG or TEXT for testing purposes. svgscripts/convert_wordPositions.py OPTIONS OPTIONS: -h|--help: show help -H|--HTML [default] convert to HTML test file -o|--output=outputFile save output to file outputFile -P|--PDF convert to PDF test file -S|--SVG convert to SVG test file -s|--svg=svgFile: svg web file -T|--TEXT convert to TEXT output -t|--testing execute in test mode, do not write to file or open browser -w|--word-insertion-mark show word insertion mark on HTML -v|--version=VERSION show words that belong to writing process VERSION: { 0, 1, 2, 0-1, 0+, etc. } :return: exit code (int) """ convert_to_type = None svg_file = None output_file = None non_testing = True show_word_insertion_mark = False page = None stage_version = '' try: opts, args = getopt.getopt(argv, "htHPSTws:o:v:", ["help", "testing", "HTML", "PDF", "SVG", "TEXT", "word-insertion-mark", "svg=", "output=", "version="]) except getopt.GetoptError: usage() return 2 for opt, arg in opts: if opt in ('-h', '--help') or not args: usage() return 0 elif opt in ('-v', '--version'): if re.match(r'^(\d|\d\+|\d\-\d)$', arg): stage_version = arg else: raise ValueError('OPTION -v|--version=VERSION does not work with "{}" as value for VERSION!'.format(arg)) elif opt in ('-w', '--word-insertion-mark'): show_word_insertion_mark = True elif opt in ('-P', '--PDF'): convert_to_type = 'PDF' elif opt in ('-S', '--SVG'): convert_to_type = 'SVG' elif opt in ('-T', '--TEXT'): convert_to_type = 'TEXT' elif opt in ('-H', '--HTML'): convert_to_type = 'HTML' elif opt in ('-t', '--testing'): non_testing = False elif opt in ('-s', '--svg'): svg_file = arg elif opt in ('-o', '--output'): output_file = arg if len(args) < 1: usage() return 2 if convert_to_type is None: if output_file is not None and len(re.split(r'\.', output_file)) > 1: output_file_part_list = re.split(r'\.', output_file) convert_to_type = output_file_part_list[len(output_file_part_list)-1].upper() else: convert_to_type = 'HTML' for word_position_file in args: if not isfile(word_position_file): print("'{}' does not exist!".format(word_position_file)) return 2 if convert_to_type == 'PDF': if output_file is None: output_file = 'output.pdf' create_pdf_with_highlighted_words(word_position_file, pdf_file_name=output_file) else: if svg_file is not None: if isfile(svg_file): page = Page(xml_source_file=word_position_file, svg_file=svg_file) else: print("'{}' does not exist!".format(word_position_file)) return 2 else: page = Page(xml_source_file=word_position_file) if page.svg_file is None: print('Please specify a svg file!') usage() return 2 converter = Converter.CREATE_CONVERTER(page, non_testing=non_testing, converter_type=convert_to_type, show_word_insertion_mark=show_word_insertion_mark) converter.convert(output_file=output_file, stage_version=stage_version) return 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:])) Index: svgscripts/create_task.py =================================================================== --- svgscripts/create_task.py (revision 0) +++ svgscripts/create_task.py (revision 62) @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" This program can be used to create a task. +""" +# Copyright (C) University of Basel 2019 {{{1 +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see 1}}} + +import abc +from colorama import Fore, Style +import lxml.etree as ET +import re +import shutil +import sys +import os +from os import listdir, sep, makedirs +from os.path import exists, isfile, isdir, dirname, basename, splitext + +if dirname(__file__) not in sys.path: + sys.path.append(dirname(__file__)) + +from convert_wordPositions import create_pdf_with_highlighted_words +from datatypes.page import Page +from join_faksimileAndTranskription import STATUS_MERGED_OK +from util import ExternalViewer, create_highlighted_svg_file + +__author__ = "Christian Steiner" +__maintainer__ = __author__ +__copyright__ = 'University of Basel' +__email__ = "christian.steiner@unibas.ch" +__status__ = "Development" +__license__ = "GPL v3" +__version__ = "0.0.1" + +UNITTESTING = False +HIGHLIGHT_COLOR = 'red' +OPACITY = '0.5' + +class Task(metaclass=abc.ABCMeta): + """This abstract class can be used to create a task. + """ + finish_dir = 'Fertig' + + def __init__(self, xml_source_file, target_dir, dirname=None, description='', manual=None, status_contains='', bg_color=HIGHLIGHT_COLOR, opacity=OPACITY): + self.xml_source_file = xml_source_file + self.target_dir = target_dir + sep + dirname\ + if dirname is not None else target_dir + self.dirname = dirname + self.description = description + self.manual = manual + self.status_contains = status_contains + self.bg_color = bg_color + self.opacity = opacity + + def create(self): + makedirs(self.target_dir + sep + Task.finish_dir, exist_ok=True) + if self.manual is not None and isfile(self.manual): + shutil.copy(self.manual, self.target_dir) + for page in Page.get_pages_from_xml_file(self.xml_source_file,\ + status_contains=self.status_contains, word_selection_function=self.select_words): + words = self.select_words(page.words) + pdf_file_name = self.target_dir + sep + basename(page.page_tree.docinfo.URL).replace('.xml', '.pdf') + create_pdf_with_highlighted_words(page=page, highlighted_words=words,\ + pdf_file_name=pdf_file_name, bg_color=self.bg_color) + svg_file = self.target_dir + sep + basename(page.faksimile_svgFile) + if isfile(svg_file): + faksimile_tree = ET.parse(svg_file) + else: + faksimile_tree = ET.parse(page.faksimile_svgFile) + node_ids = [ word.faksimile_positions[0].id for word in words ] + create_highlighted_svg_file(faksimile_tree, node_ids, target_file=svg_file,\ + highlight_color=self.bg_color, opacity=self.opacity) + + @abc.abstractmethod + def select_words(words): + """Returns selected words. + """ + pass + +class SplitFaksimileWordBoxes(Task): + """This class creates the task to split faksimile word boxes according to how many boxes a word has on the transkription. + """ + description = 'Split faksimile word boxes according to how many boxes a word has on the transkription.' + + def __init__(self, xml_source_file, target_dir): + super(SplitFaksimileWordBoxes, self).__int__(xml_source_file, target_dir,\ + description=SplitFaksimileWordBoxes.description, status_contains=STATUS_MERGED_OK) + + def select_words(words): + """Returns selected words. TODO + """ + #TODO create those functions!!!! + return [ word for word in words if word.hasParts() and word.partsMissFaksimilePostion() ] + +def usage(): + """prints information on how to use the script + """ + print(main.__doc__) + +def main(argv): + """This program can be used to copy a faksimile svg file with the option of highlighting some word boxes. + + svgscripts/copy_faksimile_svg_file.py [OPTIONS] + + a svg file containing information about the word positions on the faksimile. + the target directory. + + OPTIONS: + -h|--help: show help + + :return: exit code (int) + """ + try: + opts, args = getopt.getopt(argv, "h", ["help" ]) + except getopt.GetoptError: + usage() + return 2 + for opt, arg in opts: + if opt in ('-h', '--help'): + usage() + return 0 + if len(args) < 2: + usage() + return 2 + exit_status = 0 + if exists(args[0]) and exists(args[1]): + faksimile_svg_file = args[0] if isfile(args[0]) else args[1] + target_dir = args[1] if isdir(args[1]) else args[0] + else: + file_a = args[0] if not exists(args[0]) else args[1] + raise FileNotFoundError('File {} does not exist!'.format(file_a)) + return exit_status + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) Index: svgscripts/join_faksimileAndTranskription.py =================================================================== --- svgscripts/join_faksimileAndTranskription.py (revision 61) +++ svgscripts/join_faksimileAndTranskription.py (revision 62) @@ -1,397 +1,397 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ This program can be used to join the data of a faksimile svg files with the data of xml files of type myxmlwriter.FILE_TYPE_SVG_WORD_POSITION. """ # Copyright (C) University of Basel 2019 {{{1 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see 1}}} from colorama import Fore, Style from functools import cmp_to_key import getopt import lxml.etree as ET import re import shutil import string import sys import tempfile from operator import attrgetter import os from os import listdir, sep, path, setpgrp, devnull from os.path import exists, isfile, isdir, dirname, basename import warnings if dirname(__file__) not in sys.path: sys.path.append(dirname(__file__)) from convert_wordPositions import create_pdf_with_highlighted_words from datatypes.faksimile import FaksimilePage, get_paths_inside_rect from datatypes.lineNumber import LineNumber from datatypes.page import Page from datatypes.transkriptionField import TranskriptionField from myxmlwriter import write_pretty, FILE_TYPE_SVG_WORD_POSITION, FILE_TYPE_XML_MANUSCRIPT from process_files import update_svgposfile_status from util import ExternalViewer, create_highlighted_svg_file, get_empty_node_ids, record_changes __author__ = "Christian Steiner" __maintainer__ = __author__ __copyright__ = 'University of Basel' __email__ = "christian.steiner@unibas.ch" __status__ = "Development" __license__ = "GPL v3" __version__ = "0.0.1" UNITTESTING = False PUNCTUATION_PATTERN = r"[{}]".format(string.punctuation) PUNCTUATION_EOW_PATTERN = r"\w+[{}]$".format('\"') SINGLE_PUNCTUATION_PATTERN = r"^[{}]$".format(string.punctuation) STATUS_MERGED_OK = 'faksimile merged' HIGHLIGHT_COLOR = 'red' OPACITY = '0.5' def fix_errors(faksimile_file, unmerged_faksimile_positions, unmerged_words, text_field_id=None, faksimile_page=None, xml_source_file=None, manuscript_file=None, namespaces={}): """Creates a faksimile svg file and a pdf file highlighting the positions of the word positions that could not been merged. After correction, results are inserted into origianl file and processed again. """ parser = ET.XMLParser(remove_blank_text=True) faksimile_tree = ET.parse(faksimile_file, parser) if len(namespaces) == 0: namespaces = { k if k is not None else 'ns': v for k, v in faksimile_tree.getroot().nsmap.items() } if faksimile_page is None: faksimile_pages = FaksimilePage.GET_FAKSIMILEPAGES(faksimile_tree, namespaces=namespaces) if text_field_id is not None\ and text_field_id in [ faksimile_page.text_field.id for faksimile_page in faksimile_pages ]: faksimile_page = [ faksimile_page for faksimile_page in faksimile_pages if faksimile_page.text_field.id == text_field_id ][0] else: faksimile_page = faksimile_pages[0] if xml_source_file is None or manuscript_file is None: xml_source_file, manuscript_file = get_svgPosFile_and_manuscriptFile(faksimile_page, manuscript_file=manuscript_file) tmp_dir = tempfile.mkdtemp() tmp_pdf_file = tmp_dir + sep + 'output.pdf' tmp_faksimile = tmp_dir + sep + 'faksimile.svg' empyt_node_ids = get_empty_node_ids(faksimile_tree, faksimile_page=faksimile_page, namespaces=namespaces)\ if len(unmerged_faksimile_positions) < len(unmerged_words) else [] highlight_node_ids = [ faksimile_position.id for faksimile_position in unmerged_faksimile_positions ] highlight_node_ids += empyt_node_ids create_highlighted_svg_file(faksimile_tree, highlight_node_ids, target_file=tmp_faksimile, local_image_path=faksimile_page.faksimile_image.local_path, namespaces=namespaces, highlight_color=HIGHLIGHT_COLOR) create_pdf_with_highlighted_words(xml_source_file, highlighted_words=unmerged_words, pdf_file_name=tmp_pdf_file, bg_color=HIGHLIGHT_COLOR) if isfile(tmp_pdf_file) and isfile(tmp_faksimile): ExternalViewer.show_files(list_of_files=[tmp_pdf_file, tmp_faksimile]) record_changes(faksimile_file, tmp_faksimile, highlight_node_ids, namespaces=namespaces) shutil.rmtree(tmp_dir) join_faksimileAndTranskription(faksimile_file, manuscript_file=manuscript_file, do_fix_errors=False) def get_filelist_and_manuscript_file(file_a, file_b=None): """Returns a file list and a manuscript file (or None) """ file_list = [] manuscript_file = None if isfile(file_a) and file_a.endswith('svg'): file_list.append(file_a) if file_b is not None and isfile(file_b): manuscript_file = file_b elif isfile(file_a) and file_a.endswith('xml'): manuscript_file = file_a if file_b is not None and isfile(file_b): file_list.append(file_b) elif isdir(file_b): file_list = [ svgfile for svgfile in listdir(file_b) if svgfile.endswith('svg') ] elif isdir(file_a): file_list = [ file_a + sep + svgfile for svgfile in listdir(file_a) if svgfile.endswith('svg') ] if file_b is not None and isfile(file_b): manuscript_file = file_b return file_list, manuscript_file def get_svgPosFile_and_manuscriptFile(faksimile_page, manuscript_file=None, redo_ok=False): """Return svg_pos_file and manuscript_file if they are ready for processing. """ svg_pos_file = None manuscript_tree = None if manuscript_file is not None\ and basename(manuscript_file).startswith(faksimile_page.title.replace(' ', '_')): manuscript_tree = ET.parse(manuscript_file) else: title_string = faksimile_page.title.replace(' ', '_') manuscript_file = '.{}xml'.format(sep) + sep + title_string + '.xml'\ if isdir('.{}xml'.format(sep)) else title_string + '.xml' if isfile(manuscript_file): manuscript_tree = ET.parse(manuscript_file) if manuscript_tree is not None: if redo_ok and len(manuscript_tree.getroot().xpath('//page[@number="%s" and contains(@status,"OK")]/@output' % faksimile_page.page_number)) > 0: svg_pos_file = manuscript_tree.getroot().xpath('//page[@number="%s"]/@output' % faksimile_page.page_number)[0] if len(manuscript_tree.getroot().xpath('//page[@number="%s" and @status="OK"]/@output' % faksimile_page.page_number)) > 0: svg_pos_file = manuscript_tree.getroot().xpath('//page[@number="%s"]/@output' % faksimile_page.page_number)[0] else: if not UNITTESTING: msg_color = Fore.CYAN if len(manuscript_tree.getroot().xpath('//page[@number="%s" and contains(@status,"OK")]/@output' % faksimile_page.page_number)) > 0\ else Fore.MAGENTA msg = 'Manuscript file {} does not contain a page number {} ready for joining ...'.format(manuscript_file, faksimile_page.page_number)\ if msg_color == Fore.MAGENTA\ else 'Faksimile already joined!' print(msg_color + msg, end='') print(Style.RESET_ALL) return svg_pos_file, manuscript_file def join_faksimileAndTranskription(faksimile_file, manuscript_file=None, test_word_text='', do_fix_errors=False, redo_ok=False): """Joins the data of a faksimile file with the data of svgposfile. """ if not UNITTESTING: print(Fore.LIGHTBLUE_EX + 'Processing file {} '.format(faksimile_file), end='') print(Style.RESET_ALL) faksimile_tree = ET.parse(faksimile_file) namespaces = { k if k is not None else 'ns': v for k, v in faksimile_tree.getroot().nsmap.items() } faksimile_pages = FaksimilePage.GET_FAKSIMILEPAGES(faksimile_tree, namespaces=namespaces) exit_status = 0 for faksimile_page in faksimile_pages: svg_pos_file, manuscript_file = get_svgPosFile_and_manuscriptFile(faksimile_page, manuscript_file=manuscript_file, redo_ok=redo_ok) if svg_pos_file is not None: if not UNITTESTING: print(Fore.CYAN + 'joining data with file {} ... '.format(svg_pos_file), end='') image4page = faksimile_page.faksimile_image.get_image_joined_with_text_field(faksimile_page.text_field) page = Page(xml_source_file=svg_pos_file, faksimile_image=image4page, faksimile_svgFile=faksimile_file) words = sort_words(page) faksimile_positions = sort_faksimile_positions(faksimile_page.word_positions) new_words = [] unique_faksimile_words = sorted(set(faksimile_position.text for faksimile_position in faksimile_positions),\ key=lambda text: len(text)) for word_text in unique_faksimile_words: process_word_text(new_words, word_text, faksimile_positions, words) if False not in [ word.joined for word in words ]\ and False not in [ position.joined for position in faksimile_positions]\ and not UNITTESTING: post_merging_processing_and_saving(svg_pos_file, new_words, page=page, manuscript_file=manuscript_file) print(Fore.GREEN + '[OK]') print(Style.RESET_ALL) elif not UNITTESTING: not_joined_fp = [ (position.id, position.text) for position in faksimile_positions if not position.joined ] plural_fp = '' if len(not_joined_fp) < 2 else 's' not_joined_tw = [ (word.id, word.text) for word in words if not word.joined ] plural_tw = '' if len(not_joined_tw) < 2 else 's' print(Fore.MAGENTA + '\n--->Not joined faksimile position{0}: {1}'.format(plural_fp, not_joined_fp)) print(Fore.MAGENTA + '--->Not joined word{0} from transkription: {1}'.format(plural_tw, not_joined_tw)) print(Style.RESET_ALL) if do_fix_errors: fix_errors(faksimile_file, [position for position in faksimile_positions if not position.joined],\ [ word for word in words if not word.joined ], text_field_id=faksimile_page.text_field.id,\ faksimile_page=faksimile_page, xml_source_file=svg_pos_file,\ manuscript_file=manuscript_file, namespaces=namespaces) exit_status = 2 elif False in [ word.joined for word in words ]: print([ (word.id, word.text) for word in words if not word.joined ]) exit_status = 2 elif test_word_text != '': print([ (word.id, word.text) for word in new_words if word.text == test_word_text ]) return exit_status def post_merging_processing_and_saving(svg_pos_file, new_words, page=None, manuscript_file=None, target_svg_pos_file=None): """Process words after merging with faksimile word positions. """ if page is None: page = Page(xml_source_file=svg_pos_file) page.words = sorted(new_words, key=attrgetter('id')) - for word_node in page.page_tree.xpath('//word'): + for word_node in page.page_tree.xpath('.//word'): word_node.getparent().remove(word_node) if page.source is None or not isfile(page.source): raise FileNotFoundError('Page instantiated from {} does not contain an existing source!'.format(svg_pos_file)) transkription_field = TranskriptionField(page.source) page.find_special_words(transkription_field=transkription_field) page.categorize_paths(transkription_field=transkription_field) page.update_and_attach_words2tree(update_function_on_word=update_writing_process,\ include_special_words_of_type=[]) if target_svg_pos_file is None: target_svg_pos_file = svg_pos_file update_svgposfile_status(svg_pos_file, manuscript_file=manuscript_file, status=STATUS_MERGED_OK) write_pretty(xml_element_tree=page.page_tree, file_name=target_svg_pos_file, script_name=__file__, file_type=FILE_TYPE_SVG_WORD_POSITION) def process_word_text(new_words, word_text, faksimile_positions, words, alt_word_text=''): """Joins faksimile_positions with text == word_text with words with text == word_text. """ text = word_text if alt_word_text == '' else alt_word_text fposition4word = [ faksimile_position for faksimile_position in faksimile_positions if faksimile_position.text == word_text ] words4word = [ word for word in words if word.text == word_text and not word.joined ] if alt_word_text != '': words4word += [ word for word in words if word.text == text and not word.joined ] words4word = sorted(words4word, key=attrgetter('id')) if len(fposition4word) == len(words4word): for index, faksimile_position in enumerate(fposition4word): faksimile_position.joined = True words4word[index].faksimile_positions = [ faksimile_position ] words[words4word[index].id].joined = True new_words.append(words4word[index]) elif len(words4word) < len(fposition4word): if re.match(r'(.*)ss(.*)', text): alt_word_text = re.sub(r'ss', 'ß', text) process_word_text(new_words, word_text, faksimile_positions, words, alt_word_text=alt_word_text) elif re.match(SINGLE_PUNCTUATION_PATTERN, text): if text == '-': alt_word_text = text.replace('-', '–') process_word_text(new_words, word_text, faksimile_positions, words, alt_word_text=alt_word_text) else: print('single', word_text, len(fposition4word), len(words4word)) elif re.match(PUNCTUATION_PATTERN, text) or re.match(PUNCTUATION_EOW_PATTERN, text): alt_word_text = re.sub(PUNCTUATION_PATTERN, "", text) if alt_word_text != '': pattern = r'(.*){0}(.*)'.format(alt_word_text) words4word += [ word for word in words if re.match(pattern, word.text) and not word.joined ] if len(words4word) < len(fposition4word): process_word_text(new_words, word_text, faksimile_positions, words, alt_word_text=alt_word_text) else: words4word = sorted(words4word, key=attrgetter('id')) for index, faksimile_position in enumerate(fposition4word): faksimile_position.joined = True words4word[index].faksimile_positions = [ faksimile_position ] if words4word[index].text[len(words4word[index].text)-1] != word_text[len(word_text)-1]\ and words4word[index].id+1 < len(words)\ and words[words4word[index].id+1].text == word_text[len(word_text)-1]: words4word[index].join(words[words4word[index].id+1]) words[words4word[index].id+1].joined = True words[words4word[index].id].joined = True words4word[index].text = word_text new_words.append(words4word[index]) else: if len(text) > 1: new_words4word = [ word for word in words if word.text == text[1:] and not word.joined ] if len(new_words4word) == 0: alt_word_text = text[1:] process_word_text(new_words, word_text, faksimile_positions, words, alt_word_text=alt_word_text) else: for new_word in new_words4word: collected_text = new_word.text current_word = new_word while collected_text != word_text and word_text.endswith(collected_text) and current_word.id > 0: previous_word = words[current_word.id-1] if word_text.endswith(previous_word.text + collected_text): words[current_word.id].joined = True previous_word.join(current_word) current_word = previous_word collected_text = current_word.text else: collected_text = previous_word.text + collected_text words4word.append(current_word) words4word = sorted(words4word, key=attrgetter('id')) for index, faksimile_position in enumerate(fposition4word): faksimile_position.joined = True words4word[index].faksimile_positions = [ faksimile_position ] words4word[index].text = word_text words[words4word[index].id].joined = True new_words.append(words4word[index]) else: print('<{0}> {1}/{2}, ids: {3}'.\ format(word_text, len(fposition4word), len(words4word), [ position.id for position in fposition4word ])) else: print(word_text, len(fposition4word), len(words4word)) def sort_words(page): """Returns sorted words (from top left to bottom right). """ if -1 in [ word.line_number for word in page.words ]: - warnings.warn('{} in page file {} for word with ids {}'.format(LineNumber.WARN_NO_LINE_NUMBER, page.page_tree.docinfo.URL, page.page_tree.xpath('//word[not(@line-number)]/@id'))) + warnings.warn('{} in page file {} for word with ids {}'.format(LineNumber.WARN_NO_LINE_NUMBER, page.page_tree.docinfo.URL, page.page_tree.xpath('.//word[not(@line-number)]/@id'))) words = [] for line_number in page.line_numbers: word_on_line = [ word for word in page.words if word.line_number == line_number.id ] if line_number.id % 2 == 0: words += sorted(word_on_line, key=lambda word: word.transkription_positions[0].left) else: words += sorted(word_on_line, key=cmp_to_key(\ lambda wordA, wordB: -1\ if wordA.transkription_positions[0].left < wordB.transkription_positions[0].left\ and abs(wordA.transkription_positions[0].bottom-wordB.transkription_positions[0].bottom) < wordA.transkription_positions[0].height/2\ else 1)) for index, word in enumerate(words): words[index].id = index words[index].joined = False return words def sort_faksimile_positions(faksimile_positions): """Returns sorted words (from top left to bottom right). """ for faksimile_position in faksimile_positions: faksimile_position.joined = False return sorted(faksimile_positions, key=cmp_to_key(\ lambda positionA, positionB: -1\ if positionA.left+positionA.width/2 <= positionB.left+positionB.width/2\ and positionA.top+positionA.height/2 <= positionB.top+positionB.height/2\ else 1\ )\ ) def update_writing_process(word): """Updates the writing process of the faksimile word position by synchronizing it with the corresponding transkription word position. If there are several transkription positions belonging to different writing processes but just one faksimile position, then we skip the update. We will fix these faksimile positions by manually adding more word positions and processing those additions in a later stage. """ writing_processes = [ writing_process_id for writing_process_id in set( tp.writing_process_id for tp in word.transkription_positions ) ] if len(writing_processes) == 1 and len(word.faksimile_positions) > 0: word.faksimile_positions[0].writing_process_id = writing_processes[0] def usage(): """prints information on how to use the script """ print(main.__doc__) def main(argv): """This program can be used to merge the data of some faksimile pages with the data of xml files that are of type myxmlwriter.FILE_TYPE_SVG_WORD_POSITION. svgscripts/join_faksimileAndTranskription.py [OPTIONS] [xmlManuscriptFile] a directory containing a svg file containing information about the word positions on the faksimile. a xml file about a manuscript, containing information about its pages. OPTIONS: -h|--help: show help -f|--fix-errors: open faksimilie svg file if there are errors -i|--ignore-status-ok ignore status "OK:faksimile merged" in manuscript file and redo merging. :return: exit code (int) """ fix_errors = False redo_ok = False try: opts, args = getopt.getopt(argv, "hfi", ["help", "fix-errors", "ignore-status-ok" ]) except getopt.GetoptError: usage() return 2 for opt, arg in opts: if opt in ('-h', '--help'): usage() return 0 elif opt in ('-f', '--fix-errors'): fix_errors = True elif opt in ('-i', '--ignore-status-ok '): redo_ok = True if len(args) < 1: usage() return 2 exit_status = 0 file_a = args[0] if exists(file_a): file_b = None if len(args) > 1 and exists(args[1]): file_b = args[1] file_list, manuscript_file = get_filelist_and_manuscript_file(file_a, file_b=file_b) for faksimile_file in file_list: join_faksimileAndTranskription(faksimile_file, manuscript_file=manuscript_file, do_fix_errors=fix_errors, redo_ok=redo_ok) else: raise FileNotFoundError('File {} does not exist!'.format(file_a)) return exit_status if __name__ == "__main__": sys.exit(main(sys.argv[1:])) Index: tests_svgscripts/test_data/N_VII_1_page005_web.svg =================================================================== --- tests_svgscripts/test_data/N_VII_1_page005_web.svg (revision 0) +++ tests_svgscripts/test_data/N_VII_1_page005_web.svg (revision 62) @@ -0,0 +1,1569 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file Index: tests_svgscripts/test_data/N_VII_1_page009.xml =================================================================== --- tests_svgscripts/test_data/N_VII_1_page009.xml (revision 0) +++ tests_svgscripts/test_data/N_VII_1_page009.xml (revision 62) @@ -0,0 +1,1727 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + svgWordPosition + + + 2019-08-02 15:17:42 + + 2019-08-02 15:17:43 + 2019-08-15 14:39:17 + 2019-08-12 11:53:36 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: tests_svgscripts/test_data/N_VII_1_page005_faksimile_merged.xml =================================================================== --- tests_svgscripts/test_data/N_VII_1_page005_faksimile_merged.xml (revision 0) +++ tests_svgscripts/test_data/N_VII_1_page005_faksimile_merged.xml (revision 62) @@ -0,0 +1,1688 @@ + + + + + svgWordPosition + + + 2019-08-02 15:17:40 + + 2019-08-02 15:17:40 + 2019-08-19 11:43:03 + 2019-08-19 11:43:03 + 2019-08-19 11:42:56 + 2019-08-19 11:43:27 + 2019-08-15 11:39:29 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + x + + Index: tests_svgscripts/test_data/N_VII_1.xml =================================================================== --- tests_svgscripts/test_data/N_VII_1.xml (revision 61) +++ tests_svgscripts/test_data/N_VII_1.xml (revision 62) @@ -1,170 +1,170 @@ xmlManuscriptFile 2019-08-02 15:28:57 2019-08-02 15:31:25 H. J. Mette In schwarzen Lederdeckel gebundenes Oktavheft (10,5x17), 194 durchweg beschriebene Seiten; Studien aus der Umwertungszeit, die zum Teil für das <i>Jenseits </i>verwandt worden sind. BAW 1, XCVI M. Montinari Oktavheft, 10,5x17. 194 Seiten. Schwarze und violette Tinte, sowie Bleistift. Überwiegend deutsche Schrift. Von hinten nach vorn beschrieben. Alte Signatur: N XLIII. KGW VII 4/2, 632 Oktavheft. Schwarzer Ledereinband mit Goldprägung (vorn und hinten senkrechte Linie, parallel zum Rücken; vorn rechts unten Initialen „F. N.“, Einzelstempel) und umlaufender Blindlinie. Am hinteren Deckel lederne Stifthülse. Buchblock stellenweise gelockert. Vorsätze aus Moiré-Papier. 194 Seiten, 10,8x17,3, unliniiert. April bis Juni 1885 von KGW VII Fragmentgruppe 34 16. April 1885 bis Anfang Juni 1885 der identifizierten Briefentwürfe - + KGW VII 34[1-256] M. Montinari (zu 34[257]): „dieses und die beiden folgenden Fragmente 34[258.259] wurden von N in einen Brief von Paul Lanzky von Anfang Juni 1885 (KGB III 4, S. 28, Nr. 281) eingetragen.“ KGW VII 4/2, 374. Vorderer Deckel Vorsatz Rekto Vorsatz Verso (kaschiert) 1 1 194 Lage, 6 Blatt Vorsatz - 11 1 11 Einzelblatt 12-13 12 13 Lage, 4 Blatt 14-21 14 21 Lage, 8 Blatt 22-37 22 37 Lage, 8 Blatt 38-53 38 53 Lage, 8 Blatt 54-69 54 69 Lage, 8 Blatt 70-85 70 85 Lage, 8 Blatt 86-101 86 101 Lage, 8 Blatt 102-117 102 117 Lage, 8 Blatt 118-133 118 133 Lage, 8 Blatt 134-149 134 149 Lage, 8 Blatt 150-165 150 165 Lage, 8 Blatt 166-181 166 181 Lage, 8 Blatt 182 - Vorsatz 182 194 Vorsatz Rekto (kaschiert) 194 1 194 Vorsatz Verso Hinterer Deckel 1885-4-1 1885-6-28 KGW IX 1 2001 Bearbeitet von Marie-Luise Haase, Michael Kohlenbach, Johannes Neininger, Wolfert von Rahden, Thomas Riebe und René Stockmar unter Mitarbeit von Dirk Setton. Marie-Luise Haase und Michael Kohlenbach 71/209 N XLIII Index: tests_svgscripts/test_join_faksimileAndTranskription.py =================================================================== --- tests_svgscripts/test_join_faksimileAndTranskription.py (revision 61) +++ tests_svgscripts/test_join_faksimileAndTranskription.py (revision 62) @@ -1,95 +1,96 @@ import unittest from os import sep, path, remove from os.path import isdir, isfile, dirname import shutil import sys import lxml.etree as ET import warnings import sys sys.path.append('svgscripts') import join_faksimileAndTranskription from datatypes.faksimile import FaksimilePage from datatypes.page import Page from datatypes.positional_word_part import PositionalWordPart from datatypes.transkriptionField import TranskriptionField from datatypes.word_position import WordPosition class TestJoin(unittest.TestCase): def setUp(self): join_faksimileAndTranskription.UNITTESTING = True DATADIR = path.dirname(__file__) + sep + 'test_data' self.faksimile_dir = DATADIR + sep + 'faksimile_svg' self.manuscript = DATADIR + sep + 'N_VII_1.xml' self.manuscript_copy = self.manuscript.replace('.', '_copy.') self.faksimile_file = self.faksimile_dir + sep + 'N-VII-1,5et6.svg' self.xml_file = DATADIR + sep + 'N_VII_1_page005.xml' self.Mp_XIV_1_mytest_421 = DATADIR + sep + 'Mp_XIV_1_mytest_421.xml' def test_sort_words(self): page = Page(xml_source_file=self.Mp_XIV_1_mytest_421) words_line7 = [ word for word in page.words if word.line_number == 7 ] page.words = words_line7 sorted_words = join_faksimileAndTranskription.sort_words(page) self.assertEqual(len(sorted_words), len(words_line7)) for index, word in enumerate(words_line7): self.assertEqual(sorted_words[index], word) def test_sort_faksimile_positions(self): faksimile_tree = ET.parse(self.faksimile_file) namespaces = { k if k is not None else 'ns': v for k, v in faksimile_tree.getroot().nsmap.items() } faksimile_pages = FaksimilePage.GET_FAKSIMILEPAGES(faksimile_tree, namespaces=namespaces) self.assertEqual(len(faksimile_pages), 2) - svg_pos_file, manuscript_file = join_faksimileAndTranskription.get_svgPosFile_and_manuscriptFile(faksimile_pages[0], manuscript_file=self.manuscript) + svg_pos_file, manuscript_file = join_faksimileAndTranskription.get_svgPosFile_and_manuscriptFile(faksimile_pages[0], manuscript_file=self.manuscript, redo_ok=True) sorted_positions = join_faksimileAndTranskription.sort_faksimile_positions(faksimile_pages[0].word_positions) page = Page(xml_source_file=svg_pos_file) for index in range(0, 10): id = sorted_positions[index].id if len(faksimile_tree.getroot().xpath('//ns:rect[@id="{0}"]/ns:title/text()|//ns:path[@id="{0}"]/ns:title/text()'\ .format(id), namespaces=namespaces)) > 0: word_text = faksimile_tree.getroot().xpath('//ns:rect[@id="{0}"]/ns:title/text()|//ns:path[@id="{0}"]/ns:title/text()'\ .format(id), namespaces=namespaces)[0] #print(sorted_positions[index].left, sorted_positions[index].top, word_text, page.words[index].text) self.assertEqual(word_text, page.words[index].text) def test_get_filelist_and_manuscript_file(self): file_list, manuscript_file = join_faksimileAndTranskription.get_filelist_and_manuscript_file(self.faksimile_dir, self.manuscript) self.assertEqual(len(file_list), 1) self.assertEqual(file_list[0], self.faksimile_file) self.assertEqual(manuscript_file, self.manuscript) file_list, manuscript_file = join_faksimileAndTranskription.get_filelist_and_manuscript_file(self.manuscript, self.faksimile_file) self.assertEqual(len(file_list), 1) self.assertEqual(file_list[0], self.faksimile_file) self.assertEqual(manuscript_file, self.manuscript) + @unittest.skipUnless(__name__ == "__main__", 'test uses path from within dir') def test_get_svgPosFile_and_manuscriptFile(self): faksimile_tree = ET.parse(self.faksimile_file) faksimile_pages = FaksimilePage.GET_FAKSIMILEPAGES(faksimile_tree) self.assertEqual(len(faksimile_pages), 2) - svg_pos_file, manuscript_file = join_faksimileAndTranskription.get_svgPosFile_and_manuscriptFile(faksimile_pages[0], manuscript_file=self.manuscript) - #self.assertEqual(svg_pos_file, self.manuscript.replace('.', '_page00{}.'.format(faksimile_pages[0].page_number))) + svg_pos_file, manuscript_file = join_faksimileAndTranskription.get_svgPosFile_and_manuscriptFile(faksimile_pages[0], manuscript_file=self.manuscript, redo_ok=True) + self.assertEqual(svg_pos_file, self.manuscript.replace('.', '_page00{}.'.format(faksimile_pages[0].page_number))) self.assertEqual(manuscript_file, self.manuscript) def test_join_faksimileAndTranskription(self): self.assertEqual(join_faksimileAndTranskription.join_faksimileAndTranskription(self.faksimile_file, self.manuscript), 0) #self.assertEqual(join_faksimileAndTranskription.join_faksimileAndTranskription(self.faksimile_file, self.manuscript, test_word_text='gar'), 0) def testupdate_writing_process(self): page = Page(xml_source_file=self.xml_file) word = page.words[12] self.assertEqual(len(word.faksimile_positions), 1) self.assertEqual(word.faksimile_positions[0].writing_process_id, -1) join_faksimileAndTranskription.update_writing_process(word) self.assertEqual(word.faksimile_positions[0].writing_process_id, 0) #@unittest.skipUnless(__name__ == "__main__", 'test takes too long, we do not run it with unittest discover') @unittest.skip('test takes too long, has been tested') def test_fix_errors(self): page = Page(xml_source_file=self.xml_file) word_position = WordPosition(id='rect945', text='Lenken') join_faksimileAndTranskription.fix_errors(self.faksimile_file, [ word_position], [page.words[12]], xml_source_file=self.xml_file, manuscript_file=self.manuscript ) if __name__ == "__main__": unittest.main() Index: tests_svgscripts/test_word.py =================================================================== --- tests_svgscripts/test_word.py (revision 61) +++ tests_svgscripts/test_word.py (revision 62) @@ -1,165 +1,186 @@ import unittest from os import sep, path import lxml.etree as ET import sys sys.path.append('svgscripts') from datatypes.matrix import Matrix +import datatypes.page from datatypes.positional_word_part import PositionalWordPart from datatypes.transkriptionField import TranskriptionField from datatypes.transkription_position import TranskriptionPosition from datatypes.word import Word from datatypes.word_position import WordPosition class Page: def __init__(self): self.svg_file = None def get_line_number(self, input=0): return -1 def get_biggest_fontSize4styles(self, style_set={}): return 7 class TestWord(unittest.TestCase): def setUp(self): + DATADIR = path.dirname(__file__) + sep + 'test_data' + self.test_file = DATADIR + sep + 'N_VII_1_page009.xml' self.word_part_objs = [{'text': 'a' }, {'text': 'b' }, {'text': 'c' }] x = 0 for dict in self.word_part_objs: dict['class'] = 'st22' dict['x'] = x dict['y'] = 11 x += 1 mylist = {'text': 'abc', 'id': '0', 'line-number': '2', 'deleted': 'true' } word_position = TranskriptionPosition(x=0, y=1, height=10, width=10, matrix=Matrix('matrix(0.94 0.342 -0.342 0.94 0 0)')) self.transkription_positions = [ word_position ] self.word_node = ET.Element('word', attrib=mylist) word_position.attach_object_to_tree(self.word_node) x = 0 for char in mylist['text']: ET.SubElement(self.word_node, 'part', attrib={'text': char, 'x': str(x), 'y': '11', 'class': 'st22' }) x += 1 def test_Word_with_word_part_objs(self): word = Word.CREATE_WORD(word_part_objs=self.word_part_objs, height=10, endX=10) self.assertEqual(word.id, 0) self.assertEqual(word.transkription_positions[0].bottom, 13) self.assertEqual(word.transkription_positions[0].height, 10) self.assertEqual(word.transkription_positions[0].top, 3) self.assertEqual(word.transkription_positions[0].left, 0) self.assertEqual(word.transkription_positions[0].width, 10) self.assertEqual(word.text, 'abc') def test_Word_with_word_node(self): word = Word.create_cls(self.word_node) self.assertEqual(word.id, 0) self.assertEqual(word.deleted, True) self.assertEqual(word.transkription_positions[0].bottom, 11) self.assertEqual(word.transkription_positions[0].height, 10) self.assertEqual(word.transkription_positions[0].top, 1) self.assertEqual(word.transkription_positions[0].left, 0) self.assertEqual(word.transkription_positions[0].width, 10) self.assertEqual(word.text, 'abc') self.assertEqual(word.line_number, 2) self.assertEqual(word.transkription_positions[0].transform.isRotationMatrix(), True) def test_attach_word_to_tree(self): newWord = Word.CREATE_WORD(word_part_objs=self.word_part_objs, height=10, endX=10) empty_tree = ET.ElementTree(ET.Element('page')) newWord.attach_word_to_tree(empty_tree) for word_node in empty_tree.getroot().xpath('//word'): word = Word.CREATE_WORD(word_node=word_node) self.assertEqual(word.id, 0) self.assertEqual(word.deleted, False) self.assertEqual(word.transkription_positions[0].bottom, 13) self.assertEqual(word.transkription_positions[0].height, 10) self.assertEqual(word.transkription_positions[0].top, 3) self.assertEqual(word.transkription_positions[0].left, 0) self.assertEqual(word.transkription_positions[0].width, 10) self.assertEqual(word.text, 'abc') def test_split(self): page = Page() pwps = PositionalWordPart.CREATE_SIMPLE_POSITIONAL_WORD_PART_LIST(page, self.word_part_objs) transkription_positions = TranskriptionPosition.CREATE_TRANSKRIPTION_POSITION_LIST_FROM_PWPS(pwps) word = Word(text=''.join([pwp.text for pwp in pwps]), transkription_positions=transkription_positions) previousWord, currentWord, nextWord = word.split('b') self.assertEqual(previousWord.id, 0) self.assertEqual(previousWord.text, 'a') self.assertEqual(currentWord.id, 1) self.assertEqual(nextWord.id, 2) word = Word(text=''.join([pwp.text for pwp in pwps]), transkription_positions=transkription_positions) previousWord, currentWord, nextWord = word.split('bc') self.assertEqual(previousWord.id, 0) self.assertEqual(previousWord.text, 'a') self.assertEqual(currentWord.id, 1) word = Word(text=''.join([pwp.text for pwp in pwps]), transkription_positions=transkription_positions) previousWord, currentWord, nextWord = word.split('ab', start_id=10) self.assertEqual(currentWord.id, 10) self.assertEqual(currentWord.text, 'ab') self.assertEqual(currentWord.transkription_positions[0].width, 2.1) self.assertEqual(nextWord.id, 11) self.assertEqual(nextWord.transkription_positions[0].width, 5.2) word_part_objs=[{'text': 'x', 'class':'st22', 'x': 0, 'y': 0},\ {'text': 'Insofern', 'class':'st22', 'x': 1, 'y': 0},\ {'text': 'x', 'class':'st22', 'x': 10, 'y': 0}] pwps = PositionalWordPart.CREATE_SIMPLE_POSITIONAL_WORD_PART_LIST(page, word_part_objs) transkription_positions = TranskriptionPosition.CREATE_TRANSKRIPTION_POSITION_LIST_FROM_PWPS(pwps) word = Word(text=''.join([pwp.text for pwp in pwps]), transkription_positions=transkription_positions) with self.assertWarns(Warning): previousWord, currentWord, nextWord = word.split('Insofer') word_part_objs=[{'text': 'xInsofern', 'class':'st22', 'x': 0, 'y': 0}] pwps = PositionalWordPart.CREATE_SIMPLE_POSITIONAL_WORD_PART_LIST(page, word_part_objs) transkription_positions = TranskriptionPosition.CREATE_TRANSKRIPTION_POSITION_LIST_FROM_PWPS(pwps) word = Word(text=''.join([pwp.text for pwp in pwps]), transkription_positions=transkription_positions) with self.assertWarns(Warning): previousWord, currentWord, nextWord = word.split('Insofern') def test_join(self): word = Word.CREATE_WORD(word_part_objs=self.word_part_objs, height=10, endX=10) other_word = Word.CREATE_WORD(word_part_objs=[{'text': '.', 'class':'st22', 'x': 3, 'y': 11}]) word.join(other_word) self.assertEqual(word.text, 'abc.') other_word = Word.CREATE_WORD(word_part_objs=[{'text': '.', 'class':'st22', 'x': 3, 'y': 11}]) word.join(other_word, append_at_end_of_new_word=False) self.assertEqual(word.text, '.abc.') """ tree = ET.ElementTree(ET.Element('page')) word.attach_word_to_tree(tree) print(ET.dump(tree.getroot())) """ def test_get_semanticAndDataDict(self): word = Word.CREATE_WORD(word_node=self.word_node) empty_tree = ET.ElementTree(ET.Element('page')) word.attach_word_to_tree(empty_tree) dictionary = Word.get_semantic_dictionary() - for key in dictionary['properties'].keys(): - xpath = dictionary['properties'].get(key).get('xpath')\ - if type(dictionary['properties'].get(key)) is dict\ - else dictionary['properties'].get(key)[2] - results = empty_tree.xpath(xpath) - self.assertEqual(len(results), 1) + print(dictionary) + #for key in dictionary['properties'].keys(): + # xpath = dictionary['properties'].get(key).get('xpath')\ + # if type(dictionary['properties'].get(key)) is dict\ + # else dictionary['properties'].get(key)[2] + # results = empty_tree.xpath(xpath) + # self.assertEqual(len(results), 1) #print('{}: {}'.format(key, results[0])) #self.assertEqual(word.get_data_dictionary()['body'].get('text'), 'abc') def test_simplify_transkription_positions(self): node_string = """ """ nodeA = ET.fromstring(node_string) node_string = """ """ nodeB = ET.fromstring(node_string) word = Word(text="Si", transkription_positions=[ TranskriptionPosition(node=nodeA), TranskriptionPosition(node=nodeB) ]) self.assertEqual(len(word.transkription_positions), 2) word.simplify_transkription_positions() self.assertEqual(len(word.transkription_positions), 1) """ tree = ET.ElementTree(ET.Element('page')) word.attach_word_to_tree(tree) print(ET.dump(tree.getroot())) """ + def test_partition(self): + page = datatypes.page.Page(xml_source_file=self.test_file) + word = page.words[67] + self.assertEqual(word.belongs_to_multiple_writing_processes(), True) + word.partition_according_to_writing_process_id() + self.assertEqual(len(word.word_parts), 3) + self.assertEqual(word.belongs_to_multiple_writing_processes(), False) + self.assertEqual(word.belongs_to_multiple_writing_processes(include_parts=True), True) + empty_tree = ET.ElementTree(ET.Element('page')) + word_node = word.attach_word_to_tree(empty_tree) + newWord = Word.create_cls(word_node) + self.assertEqual(len(newWord.word_parts), 3) + self.assertEqual(newWord.line_number, -1) + self.assertEqual(newWord.deleted, None) + #print(ET.dump(empty_tree.getroot())) + + if __name__ == "__main__": unittest.main() Index: tests_svgscripts/test_page.py =================================================================== --- tests_svgscripts/test_page.py (revision 61) +++ tests_svgscripts/test_page.py (revision 62) @@ -1,139 +1,150 @@ import unittest from os import sep, path from os.path import isdir, isfile, dirname, basename import lxml.etree as ET import sys import sys sys.path.append('svgscripts') dir_changed = False if not isdir('datatypes'): sys.path.append(dirname(sys.path[0])) dir_changed = True from datatypes.lineNumber import LineNumber from datatypes.mark_foreign_hands import MarkForeignHands from datatypes.page import Page from datatypes.text_connection_mark import TextConnectionMark from datatypes.transkriptionField import TranskriptionField from datatypes.writing_process import WritingProcess from datatypes.word import Word class TestPage(unittest.TestCase): def setUp(self): DATADIR = dirname(__file__) + sep + 'test_data' if not isdir(DATADIR): DATADIR = dirname(dirname(__file__)) + sep + 'test_data' self.test_file = DATADIR + sep + 'test.xml' self.test_svg_file = DATADIR + sep + 'test421.svg' self.pdf_xml = DATADIR + sep + 'W_I_8_page125.xml' self.xml_file = DATADIR + sep + 'N_VII_1_page005.xml' self.xml_fileB = DATADIR + sep + 'N_VII_1_page006.xml' self.pdf_xml_source = DATADIR + sep + 'W_I_8_neu_125-01.svg' self.test_tcm_xml = DATADIR + sep + 'N_VII_1_page001.xml' + self.test_manuscript = DATADIR + sep + 'N_VII_1.xml' def test_Page(self): page = Page(xml_source_file=self.test_file, svg_file=self.test_svg_file) self.assertEqual(page.title, 'Mp XIV 1') self.assertEqual(page.number, '421') self.assertEqual(len(page.sonderzeichen_list), 2) self.assertEqual('st21' in page.sonderzeichen_list, True) self.assertEqual('st23' in page.sonderzeichen_list, True) self.assertEqual(page.style_dict['st0']['fill'], '#F8F9F8') self.assertEqual(page.width, 493.23) stage0 = [ key for key, value in page.fontsizekey2stage_mapping.items() if value == 0 ] stage1 = [ key for key, value in page.fontsizekey2stage_mapping.items() if value == 1 ] stage2 = [ key for key, value in page.fontsizekey2stage_mapping.items() if value == 2 ] fontStage0 = float(page.style_dict.get(stage0[0]).get('font-size').replace('px','')) fontStage1 = float(page.style_dict.get(stage1[0]).get('font-size').replace('px','')) fontStage2 = float(page.style_dict.get(stage2[0]).get('font-size').replace('px','')) self.assertEqual(fontStage0 > fontStage1, True) self.assertEqual(fontStage1 > fontStage2, True) def test_get_biggest_fontSize4styles(self): page = Page(xml_source_file=self.test_file) style_set = { 'st12', 'st2', 'st14', 'st13' } self.assertEqual(page.get_biggest_fontSize4styles(style_set=style_set), 10) def test_get_words(self): page = Page(xml_source_file=self.test_file) words = page.words self.assertEqual(len(words), 440) self.assertEqual(words[0].text, '$') self.assertEqual(words[439].text, 'mußte!') def test_create_writing_process(self): page = Page(xml_source_file=self.test_file) page.create_writing_processes_and_attach2tree() self.assertEqual(page.words[97].transkription_positions[0].writing_process_id, WritingProcess.LATER_INSERTION_AND_ADDITION) self.assertEqual(page.words[129].transkription_positions[0].writing_process_id, WritingProcess.LATER_INSERTION_AND_ADDITION) def test_init_line_numbers(self): page = Page(xml_source_file=self.test_file) line_numbers = [ LineNumber(id=2, top=20, bottom=40), LineNumber(id=4, top=50, bottom=60), LineNumber(id=6, top=70, bottom=90) ] page.init_line_numbers(line_numbers, 122.345) self.assertEqual(len(page.line_numbers), 7) self.assertEqual(page.line_numbers[0].id, 1) self.assertEqual(page.line_numbers[6].id, 7) self.assertEqual(page.line_numbers[6].top, 91) self.assertEqual(page.line_numbers[6].bottom, 122.345) self.assertEqual(page.get_line_number(122), 7) self.assertEqual(page.get_line_number(92), 7) self.assertEqual(page.get_line_number(22), 2) def test_get_line_number(self): page = Page(xml_source_file=self.test_file) self.assertEqual(page.get_line_number( (page.words[0].transkription_positions[0].bottom+page.words[0].transkription_positions[0].top)/2), 1) self.assertEqual(page.get_line_number( (page.words[27].transkription_positions[0].bottom+page.words[27].transkription_positions[0].top)/2), 2) self.assertEqual(page.get_line_number( (page.words[105].transkription_positions[0].bottom+page.words[105].transkription_positions[0].top)/2), 7) def test_categorize_paths(self): Page.UNITTESTING = True page = Page(xml_source_file=self.pdf_xml) page.source = self.pdf_xml_source tr = TranskriptionField(page.source) page.words = [ word for word in page.words if word.line_number == 33 ] page.categorize_paths(tr) - self.assertEqual([ word.deleted for word in page.words if word.id == 269 ][0], False) + self.assertEqual(True in [ word.deleted for word in page.words if word.id == 269 ], False) def test_find_special_words(self): page = Page(xml_source_file=self.xml_file) page.find_special_words() self.assertEqual(len(page.mark_foreign_hands), 1) self.assertEqual(page.mark_foreign_hands[0].foreign_hands_text, 'x') page.update_and_attach_words2tree() nodes = page.page_tree.xpath('//' + MarkForeignHands.XML_TAG) page = Page(xml_source_file=self.test_tcm_xml) page.find_special_words() self.assertEqual(len(page.text_connection_marks), 1) self.assertEqual(page.text_connection_marks[0].text_source.first_line, 2) """ page.update_and_attach_words2tree() nodes = page.page_tree.xpath('//' + TextConnectionMark.XML_TAG) print(ET.dump(nodes[0])) """ def test_update_page_type(self): page = Page(xml_source_file=self.pdf_xml) tf = TranskriptionField(self.pdf_xml_source) page.update_page_type(transkription_field=tf) self.assertEqual(page.page_type, Page.PAGE_VERSO) #page = Page(xml_source_file=self.xml_fileB) #page.update_page_type() #self.assertEqual(page.page_type, Page.PAGE_RECTO) def test_update_line_number_area(self): page = Page(xml_source_file=self.xml_file) transkription_field = TranskriptionField(page.source) page.update_line_number_area(transkription_field) self.assertEqual(transkription_field.line_number_area_width > 0, True) self.assertEqual(transkription_field.line_number_area_width < 15, True) page = Page(xml_source_file=self.xml_fileB) transkription_field = TranskriptionField(page.source) page.update_line_number_area(transkription_field) self.assertEqual(transkription_field.line_number_area_width > 0, True) self.assertEqual(transkription_field.line_number_area_width < 15, True) + def test_get_pages_from_xml_file(self): + pages = Page.get_pages_from_xml_file(self.test_manuscript) + self.assertEqual(len(pages), 2) + self.assertEqual(pages[0].number, '5') + self.assertEqual(pages[1].number, '6') + pages = Page.get_pages_from_xml_file(self.test_manuscript, status_contains='faksimile merged') + self.assertEqual(len(pages), 1) + self.assertEqual(pages[0].number, '5') + + if __name__ == "__main__": unittest.main() Index: tests_svgscripts/test_simple_word.py =================================================================== --- tests_svgscripts/test_simple_word.py (revision 0) +++ tests_svgscripts/test_simple_word.py (revision 62) @@ -0,0 +1,28 @@ +import unittest +from os import sep, path +from os.path import dirname, isdir +import lxml.etree as ET +import sys + +sys.path.append('svgscripts') +from datatypes.matrix import Matrix +from datatypes.transkriptionField import TranskriptionField +from datatypes.transkription_position import TranskriptionPosition +from datatypes.simple_word import SimpleWord +from datatypes.mark_foreign_hands import MarkForeignHands +from datatypes.word import Word + +class TestSimpleWord(unittest.TestCase): + + def test_get_semanticAndDataDict(self): + dictionary = SimpleWord.get_semantic_dictionary() + #print(dictionary) + + def test_create_cls_from_word(self): + word = Word(text='test') + mark = MarkForeignHands.create_cls_from_word(word) + self.assertEqual(mark.text, word.text) + self.assertEqual(type(mark), MarkForeignHands) + +if __name__ == "__main__": + unittest.main() Index: tests_svgscripts/test_create_task.py =================================================================== --- tests_svgscripts/test_create_task.py (revision 0) +++ tests_svgscripts/test_create_task.py (revision 62) @@ -0,0 +1,41 @@ +import unittest +from os import sep, path, remove, listdir +from os.path import isdir, isfile, dirname, basename +import shutil +import sys +import lxml.etree as ET +import sys +import tempfile +import warnings + +sys.path.append('svgscripts') + +from create_task import Task +from datatypes.faksimile import FaksimilePage +from datatypes.page import Page +from datatypes.positional_word_part import PositionalWordPart +from datatypes.transkriptionField import TranskriptionField +from datatypes.word_position import WordPosition + +class TestTask(Task): + def select_words(self, words): + return words + +class TestCreateTask(unittest.TestCase): + def setUp(self): + DATADIR = path.dirname(__file__) + sep + 'test_data' + self.xml_source = DATADIR + sep + 'N_VII_1_page005_faksimile_merged.xml' + + def test_task(self): + tmp_dir = tempfile.mkdtemp() + task = TestTask(self.xml_source, tmp_dir) + task.create() + dir_elements = listdir(tmp_dir) + self.assertEqual(len(dir_elements), 3) + self.assertEqual(basename(self.xml_source).replace('.xml', '.pdf') in dir_elements, True) + shutil.rmtree(tmp_dir) + + + +if __name__ == "__main__": + unittest.main() Index: tests_svgscripts/test_class_spec.py =================================================================== --- tests_svgscripts/test_class_spec.py (revision 61) +++ tests_svgscripts/test_class_spec.py (revision 62) @@ -1,53 +1,57 @@ import unittest from os import sep, path import inspect import sys sys.path.append('svgscripts') from datatypes.class_spec import SemanticClass from datatypes.image import Image class TestSemanticClassFail(SemanticClass): def __init__(self): pass class TestSemanticClassOK(SemanticClass): def __init__(self): self.msg = 'Hello World!' @staticmethod def get_semantic_dictionary(): return {'class': { 'this': TestSemanticClassOK}, 'properties': { 'msg': (str, SemanticClass.SINGLE_VALUE) }} class TestSemanticClassB(SemanticClass): def __init__(self): self.data = [ 1, 2, 3, 4 ] self.test = [ TestSemanticClassOK(), TestSemanticClassOK() ] @staticmethod def get_semantic_dictionary(): return { 'class': {'this': TestSemanticClassB }, 'properties': { 'data': (int, SemanticClass.LIST), 'test': (TestSemanticClassOK, SemanticClass.LIST) }} def get_super(self): return inspect.getclasstree([self.__class__],unique=True)[0][0] class TestSemanticClassC(TestSemanticClassB): pass class TestSemanticClass(unittest.TestCase): def test_fail(self): with self.assertRaises(TypeError): TestSemanticClassFail() def test_success(self): test = TestSemanticClassOK() self.assertEqual(TestSemanticClassOK.get_semantic_dictionary()['properties'], { 'msg': (str, 1) }) #self.assertEqual(test.get_data_dictionary()['body'], { 'msg': 'Hello World!'}) test = TestSemanticClassB() self.assertEqual(TestSemanticClassB.get_semantic_dictionary()['properties'], { 'data': (int, SemanticClass.LIST), 'test': (TestSemanticClassOK, SemanticClass.LIST) }) self.assertEqual(test.get_semantic_dictionary()['class'].get('this'), TestSemanticClassB) def test_get_class_dictionary(self): test = TestSemanticClassC() self.assertEqual(test.get_class_dictionary().get('type') is not None, True) self.assertEqual(test.get_class_dictionary().get('type'), TestSemanticClassB) + def test_get_cls_hasPart_objectCls_dictionaries(self): + dictionary = SemanticClass.get_cls_hasPart_objectCls_dictionaries(SemanticClass, 'asdf/asdf') + #print(dictionary) + if __name__ == "__main__": unittest.main()