to
+ """
+ for greekWord in greekWords:
+ try:
+ del greekWord.attrib['class']
+ except KeyError:
+ pass
+ greekWord.tag = 'foreign'
+ greekWord.attrib[ET.QName(self.namespaces['xml'],'lang')] = 'grc' # see www.iana.org/assignments/language-subtag-registry/language-subtag-registry
+
+ def _get_faksimile_page(self, svg_file) ->FaksimilePage:
+ """Return the faksimile page of a faksimile image
+ """
+ faksimile_pages = FaksimilePage.get_faksimile_pages(svg_file, isBlank=True)
+ if len(faksimile_pages) == 1:
+ return faksimile_pages[0]
+ elif len(faksimile_pages):
+ return sorted(faksimile_pages, key=lambda f: f.text_field.height)[-1]
+ return None
+
+ def _get_element(self, xpath: str, firstElementOnly=True) ->ET.Element:
+ """Return element(s) and warn if xpath results in empty list.
+ """
+ elements = self.tei_tree.xpath(xpath, namespaces=self.namespaces)
+ if len(elements) == 0:
+ msg = f'There are no elements for xpath {xpath}!'
+ warnings.warn(msg)
+ return None
+ if firstElementOnly:
+ return elements[0]
+ return elements
+
+ def _set_idnos(self, idnos: list, contentDict: dict):
+ """Set idno of type URI, SpN, Blatt-ID
+ """
+ for idno in idnos:
+ if bool(contentDict.get(idno.get('type'))):
+ idno.text = contentDict[idno.get('type')]
+
+ def _write_file(self, target_file: str, faksimile_page: FaksimilePage, folioId: str, label: str, alias: str, url: str, content: str, contentFiles=None) ->int:
+ """Write data to target_file
+ """
+ title = self._get_element('/ns:TEI/ns:teiHeader/ns:fileDesc/ns:titleStmt/ns:title')
+ idnos = self._get_element('/ns:TEI/ns:teiHeader/ns:fileDesc/ns:sourceDesc/ns:msDesc/ns:msIdentifier/ns:idno', firstElementOnly=False)
+ objectDesc = self._get_element('/ns:TEI/ns:teiHeader/ns:fileDesc/ns:sourceDesc/ns:msDesc/ns:physDesc/ns:objectDesc')
+ faksimile = self._get_element('/ns:TEI/ns:facsimile')
+ body = self._get_element('/ns:TEI/ns:text/ns:body')
+ if title is not None and idnos is not None and objectDesc is not None and faksimile is not None and ',' in alias and body is not None:
+ xml_id = label.replace('.jpg','')
+ page_number = alias[alias.index(',')+1:].lstrip()
+ title.text = alias
+ objectDesc.text = content
+ self._set_idnos(idnos, { 'URI': faksimile_page.faksimile_image.nietzsche_source, 'SpN': alias, 'Blatt-ID': folioId })
+ faksimile_page.attach_to_tei_facsimile(faksimile, xml_id, page_number)
+ self._add_content(body, xml_id, page_number, contentFiles=contentFiles)
+ return write_tei(self.tei_tree, target_file, namespaces=self.namespaces)
+ return 2
+
+ def create_tei_file(self, folioId, label, alias, url, content) ->int:
+ """Create a svg file from csv input.
+ """
+ target_file = self.target_dir + sep + alias.replace(', ', '_page').replace(' ', '_') + '.xml'
+ if isfile(target_file):
+ return 1
+ svg_file = self.svg_dir + sep + label.replace('.jpg','.svg')
+ if not isfile(svg_file):
+ return 2
+ faksimile_page = self._get_faksimile_page(svg_file)
+ if faksimile_page is None:
+ msg = f'There is no faksimile_page in {svg_file} for {label}, {alias}!'
+ warnings.warn(msg)
+ return 2
+ contentFiles = []
+ if 'GM' in content:
+ try:
+ contentId = content[content.index('GM'):].replace(' ', '-')\
+ if ',' not in content\
+ else content[content.index('GM'):content.index(',')].replace(' ', '-')
+ if isfile(self.content_dir + sep + contentId + '.txt'):
+ contentFiles.append(self.content_dir + sep + contentId + '.txt')
+ elif len(contentId.split('-')) > 1 and re.match(r'\d', contentId.split('-')[-2]):
+ split = contentId.split('-')
+ for i in range(int(split[-2]), int(split[-1])+1):
+ contentFile = self.content_dir + sep + contentId[:contentId.index('-'+split[-2])] + '-' + str(i) + '.txt'
+ if isfile(contentFile):
+ contentFiles.append(contentFile)
+ except ValueError:
+ msg = f'There has been an error with content {content} of {label}, {alias}'
+ warnings.warn(msg)
+ self._write_file(target_file, faksimile_page, folioId, label, alias, url, content, contentFiles=contentFiles)
+ return 0
+
+def process_default(args) ->int:
+ """ Default process
+ """
+ if len(args) < 2:
+ usage()
+ return 2
+ exit_status = 0
+ template = args[0]
+ csv_file = args[1]
+ svg_dir = args[2] if len(args) > 2 else './svg'
+ target_dir = args[3] if len(args) > 3 else './tmp'
+ content_dir = args[4] if len(args) > 4 else './text'
+ not isdir(target_dir) and mkdir(target_dir)
+ if isfile(template) and isfile(csv_file) and isdir(svg_dir) and isdir(content_dir):
+ counter = 0
+ file_creater = CSVFileCreator(template, svg_dir=svg_dir, content_dir=content_dir, target_dir=target_dir)
+ with open(csv_file, newline='') as csvfile:
+ reader = csv.DictReader(csvfile)
+ for row in [ row for row in reader if not '_thumb' in row[CSVFileCreator.LABEL] ]:
+ status = file_creater.create_tei_file(row[CSVFileCreator.FOLIO], row[CSVFileCreator.LABEL],\
+ row[CSVFileCreator.ALIAS],row[CSVFileCreator.URL],row[CSVFileCreator.CONTENT])
+ if status > 1:
+ exit_status = 2
+ elif status == 0:
+ counter += 1
+ if not CSVFileCreator.UNITTESTING:
+ print(Style.RESET_ALL + f'[{counter} tei files created]')
+ else:
+ if not isdir(svg_dir):
+ raise FileNotFoundError(f'Directory {svg_dir} does not exist!')
+ if not isdir(content_dir):
+ raise FileNotFoundError(f'Directory {content_dir} does not exist!')
+ if not isfile(template):
+ raise FileNotFoundError(f'File {template} does not exist!')
+ raise FileNotFoundError(f'File {csv_file} does not exist!')
+ return 0
+
+def write_tei(tei_tree: ET.ElementTree, target_file: str, namespaces=None) ->int:
+ """Write to tei file
+ """
+ if namespaces is None:
+ namespaces = { k if k is not None else 'ns': v for k, v in tei_tree.getroot().nsmap.items() }
+ for key in namespaces.keys():
+ if key == 'ns':
+ XET.register_namespace('', namespaces[key])
+ else:
+ XET.register_namespace(key, namespaces[key])
+ target_tree = XET.ElementTree(XET.fromstring(ET.tostring(tei_tree, pretty_print=True)))
+ if not CSVFileCreator.UNITTESTING:
+ target_tree.write(target_file, encoding="UTF-8", xml_declaration=True)
+ return 0
+
+def usage():
+ """prints information on how to use the script
+ """
+ print(main.__doc__)
+
+def main(argv):
+ """This program can be used to create tei files.
+
+ fixes/create_tei_files.py [OPTIONS]
+
+ a tei file template.
+ a csv file with faksimile information.
+ a directory containing svg files (created with fixes/create_blank_svg_files.py)
+ the directory where the files should be saved to
+
+ OPTIONS:
+ -h|--help show help
+
+ :return: exit code (int)
+ """
+ try:
+ opts, args = getopt.getopt(argv, "h", ["help"])
+ except getopt.GetoptError:
+ usage()
+ return 2
+ for opt, arg in opts:
+ if opt in ('-h', '--help'):
+ usage()
+ return 0
+ return process_default(args)
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv[1:]))
Index: tests_svgscripts/test_manuscript_description.py
===================================================================
--- tests_svgscripts/test_manuscript_description.py (revision 0)
+++ tests_svgscripts/test_manuscript_description.py (revision 114)
@@ -0,0 +1,23 @@
+import unittest
+from os import sep, path
+from os.path import dirname, basename, isfile, isdir
+import lxml.etree as ET
+import sys
+
+sys.path.append('svgscripts')
+from datatypes.page import Page
+from datatypes.standoff_tag import StandoffTag
+from datatypes.text import Text
+from datatypes.manuscript_description import ManuscriptDescription
+
+class TestManuscriptDescription(unittest.TestCase):
+ def test_semantic(self):
+ pass
+ #print(ManuscriptDescription.get_semantic_dictionary())
+
+ def test_create(self):
+ text = "Heftblatt (restauriert), ca. 17,5+21,7, liniiert; Rißkante Rekto links. Verso vakat. Ursprünglich Doppelblatt einer Heftlage: Mp XVI, 64rv / Mp XVII, 118rv; ursprünglich gegenüberliegende Seiten: 124v / 118r (?)."
+ desc = ManuscriptDescription.create_cls_from_raw_text(text)
+
+if __name__ == "__main__":
+ unittest.main()
Index: tests_svgscripts/test_faksimile.py
===================================================================
--- tests_svgscripts/test_faksimile.py (revision 113)
+++ tests_svgscripts/test_faksimile.py (revision 114)
@@ -1,98 +1,107 @@
import unittest
from os import sep, path
from os.path import isdir, dirname, basename
import lxml.etree as ET
import sys
import sys
sys.path.append('svgscripts')
from datatypes.faksimile import FaksimilePage, get_paths_inside_rect
from datatypes.faksimile_image import FaksimileImage
from datatypes.text_field import TextField
class TestFaksimilePage(unittest.TestCase):
def setUp(self):
DATADIR = dirname(__file__) + sep + 'test_data'
if not isdir(DATADIR):
DATADIR = dirname(dirname(__file__)) + sep + 'test_data'
self.svg_file = DATADIR + sep + 'W-II-1,49et50.svg'
+ self.svg_file2 = DATADIR + sep + 'D-20b,61cv.svg'
+ self.tei_file = DATADIR + sep + 'D_20b_page001r.xml'
self.svg_testmatrix = DATADIR + sep + 'TESTMATRIX_1.svg'
self.faksimile_dir = DATADIR + sep + 'faksimile_svg'
self.faksimile_file = self.faksimile_dir + sep + 'N-VII-1,5et6.svg'
self.faksimile_rotate90 = self.faksimile_dir + sep + 'Mp-XV-2c,4.svg'
def test_init(self):
image = FaksimileImage(file_name='test.jpg', height=10, width=10)
text_field = TextField(width=10, height=10, x=10, y=10)
faksimile = FaksimilePage(title='test', page_number=1, faksimile_image=image, text_field=text_field)
self.assertEqual(faksimile.page_tree.getroot().get('title'), 'test')
self.assertEqual(faksimile.page_tree.getroot().get('page-number'), '1')
self.assertEqual(faksimile.faksimile_image.width, 10)
self.assertEqual(faksimile.text_field.width, 10)
def test_GET_TEXTFIELDS(self):
"""
svg_tree = ET.parse('/home/knister0/ownCloud/nietzscheDE/Bearbeitung_Faksimile/Myriam/Blank/Mp_XVI/Fertig/Mp_XVI_page52v.svg')
pages = FaksimilePage.GET_FAKSIMILEPAGES(svg_tree, page_number='52v', isBlank=True)
print(pages[0].text_field)
"""
svg_tree = ET.parse('/home/knister0/ownCloud/nietzscheDE/Bearbeitung_Faksimile/Eric/Mp_XVI/Fertig/Mp-XVI-1,1.svg')
pages = FaksimilePage.GET_FAKSIMILEPAGES(svg_tree, page_number='1r')
self.assertEqual(len(pages), 1)
svg_tree = ET.parse(self.svg_file)
pages = FaksimilePage.GET_FAKSIMILEPAGES(svg_tree)
self.assertEqual(len(pages), 2)
text_field = pages[0].text_field
self.assertEqual(text_field.width, 663.333)
result_dir = '.{}xml{}'.format(sep, sep) if isdir('xml') else ''
self.assertEqual(pages[0].xml_file, result_dir + 'W-II-1_49.xml')
self.assertEqual(pages[0].title, 'W II 1')
self.assertEqual(pages[0].page_number, '49')
pages = FaksimilePage.GET_FAKSIMILEPAGES(svg_tree, page_number='49')
self.assertEqual(len(pages), 1)
svg_tree = ET.parse(self.svg_testmatrix)
pages = FaksimilePage.GET_FAKSIMILEPAGES(svg_tree)
self.assertEqual(len(pages), 1)
self.assertEqual(len(pages[0].word_positions), 1)
self.assertEqual(pages[0].word_positions[0].transform.toCSSTransformString(), 'rotate(45deg)')
svg_tree = ET.parse(self.faksimile_file)
pages = FaksimilePage.GET_FAKSIMILEPAGES(svg_tree)
self.assertEqual(len(pages), 2)
textfield_id = pages[1].title.replace(' ', '-') + '_' + pages[1].page_number
#print([ position.id for position in pages[0].word_positions])
self.assertEqual(textfield_id not in [ position.id for position in pages[0].word_positions ], True)
self.assertEqual('path1237' in [ position.id for position in pages[0].word_positions ], True)
self.assertEqual('Vorgangs' in [ position.text for position in pages[0].word_positions ], False)
svg_tree = ET.parse(self.faksimile_file)
pages = FaksimilePage.GET_FAKSIMILEPAGES(svg_tree)
self.assertEqual(len(pages), 2)
self.assertEqual(pages[0].page_number, '5')
"""
svg_tree = ET.parse('/home/knister0/ownCloud/nietzscheDE/Bearbeitung_Faksimile/Eric/Kontrolle_und_Beschriftung_der_Wortrahmen/Fertig/W-II-1,141et142.svg')
pages = FaksimilePage.GET_FAKSIMILEPAGES(svg_tree)
self.assertEqual(len(pages), 1)
"""
svg_tree = ET.parse(self.faksimile_rotate90)
namespaces = { k if k is not None else 'ns': v for k, v in svg_tree.getroot().nsmap.items() }
pages = FaksimilePage.GET_FAKSIMILEPAGES(svg_tree)
self.assertEqual(len(pages), 1)
self.assertEqual(len(pages[0].word_positions), len(svg_tree.xpath('//ns:rect/ns:title', namespaces=namespaces)))
def test_get_paths_inside_rect(self):
svg_tree = ET.parse(self.faksimile_file)
paths = get_paths_inside_rect(svg_tree, '//ns:path', 360, 786, 92, 765, 'N-VII-1_5')
self.assertEqual(len(paths), 1)
svg_tree = ET.parse(self.svg_testmatrix)
paths = get_paths_inside_rect(svg_tree, '//ns:rect/ns:title', 0, 2038.72, 0, 974.08002, 'TESTMATRIX_1')
self.assertEqual(len(paths), 1)
svg_tree = ET.parse('/home/knister0/ownCloud/nietzscheDE/Bearbeitung_Faksimile/Myriam/Mp_XIV/Kontrolle_und_Beschriftung_der_Wortrahmen/Fertig/Mp-XIV-1,419a.svg')
namespaces = { k if k is not None else 'ns': v for k, v in svg_tree.getroot().nsmap.items() }
paths = get_paths_inside_rect(svg_tree, '//ns:rect', 52, 800, 58, 900, 'Mp-XIV-1_419a', namespaces=namespaces)
self.assertEqual(len([ path for path in paths if 'seinen' in path.xpath('./ns:title/text()', namespaces=namespaces)]), 1)
+ def test_attach_to_tei_facsimile(self):
+ tei_tree = ET.parse(self.tei_file)
+ namespaces = { k if k is not None else 'ns': v for k, v in tei_tree.getroot().nsmap.items() }
+ faksimile = tei_tree.xpath('//ns:facsimile', namespaces=namespaces)[0]
+ fp = FaksimilePage.get_faksimile_pages(self.svg_file2, isBlank=True)[1]
+ fp.attach_to_tei_facsimile(faksimile, basename(self.svg_file2).replace('.svg',''), '61cv')
+ #print(ET.dump(faksimile))
if __name__ == "__main__":
unittest.main()
Index: tests_svgscripts/test_data/D-20b,61cv.svg
===================================================================
--- tests_svgscripts/test_data/D-20b,61cv.svg (revision 0)
+++ tests_svgscripts/test_data/D-20b,61cv.svg (revision 114)
@@ -0,0 +1,72 @@
+
+
Index: tests_svgscripts/test_data/D_20b_page001r.xml
===================================================================
--- tests_svgscripts/test_data/D_20b_page001r.xml (revision 0)
+++ tests_svgscripts/test_data/D_20b_page001r.xml (revision 114)
@@ -0,0 +1,68 @@
+
+
+
+
+
+ D 20b, 1r
+
+
+
+
+ Weimar
+ Goethe- und Schiller-Archiv
+ http://www.nietzschesource.org/DFGA/D-20b,1
+ D 20b, 1r
+ GSA_71/27,2_001
+
+
+
+
+
+
+
+
+
+
+
+ ![]()
+
+
+
+
+
Was bedeuten asketische Ideale? — Bei Künstlern + Nichts oder zu Vielerlei; bei Philo=
sophen und Gelehrten Etwas + wie Witterung und Instinkt für die günstigsten Vorbedingungen + hoher Geistigkeit; bei Frauen, besten Falls, eine + Liebenswürdigkeit der Verführung
+
Index: tests_svgscripts/test_matrix.py
===================================================================
--- tests_svgscripts/test_matrix.py (revision 113)
+++ tests_svgscripts/test_matrix.py (revision 114)
@@ -1,245 +1,263 @@
import unittest
import lxml.etree as ET
from os import sep, path
from os.path import isdir, dirname
import sys
sys.path.append('svgscripts')
from datatypes.matrix import Matrix
from datatypes.transkriptionField import TranskriptionField
class FakeTF:
def __init__(self):
self.xmin = 297.6379999999997
self.xmax = 765.354
self.ymin = 157.328
self.ymax = 752.6040160033832
class TestMatrix(unittest.TestCase):
def setUp(self):
self.x = 219.4058
self.y = 106.4634
self.matrix_string = 'matrix(1 0 0 1 {} {})'.format(str(self.x), str(self.y))
self.test_data_dir = dirname(__file__) + sep + 'test_data'
if not isdir(self.test_data_dir):
self.test_data_dir = dirname(dirname(__file__)) + sep + 'test_data'
self.test_file = self.test_data_dir + sep + 'test_ai.svg'
self.rotation_angle = 20
self.rotation_matrix_string = 'matrix(0.94 0.342 -0.342 0.94 0 0)'
self.test_margin_field_file = self.test_data_dir + sep + 'W_I_8_neu_125-01.svg'
self.test_place_printing_verso = self.test_data_dir + sep + 'N_VII_1_xp5_4_page5.svg'
self.test_place_printing_recto = self.test_data_dir + sep + 'N_VII_1_xp5_4_page6.svg'
self.multipage = f'{self.test_data_dir}{sep}pdfsvg{sep}csv{sep}15.svg'
self.marginals_extra = f'{self.test_data_dir}{sep}pdfsvg{sep}csv{sep}45.svg'
self.marginals_extra_fn = f'{self.test_data_dir}{sep}pdfsvg{sep}csv{sep}44.svg'
def test_Matrix(self):
matrix = Matrix(self.matrix_string)
self.assertEqual(matrix.getX(), self.x)
self.assertEqual(matrix.add2X(1), self.x + 1)
self.assertEqual(matrix.getY(), self.y)
matrix = Matrix('matrix(0.98966578,0.1433933,-0.0913015,0.9958233,0,0)')
self.assertEqual(matrix.getX(), 0)
matrix = Matrix('matrix(1 2.998719e-04 -2.998719e-04 1 415.3643 476.7988)')
def test_Matrix_rotation(self):
rotation_string = 'rotate({})'.format(self.rotation_angle)
rotation_stringC = 'rotate(-{})'.format(self.rotation_angle)
matrixA = Matrix(rotation_string)
matrixB = Matrix(self.rotation_matrix_string)
matrixC = Matrix(rotation_stringC)
self.assertEqual(matrixA.matrix[Matrix.A], matrixB.matrix[Matrix.A])
self.assertEqual(matrixA.matrix[Matrix.B], matrixB.matrix[Matrix.B])
self.assertEqual(matrixA.matrix[Matrix.C], matrixB.matrix[Matrix.C])
self.assertEqual(matrixA.matrix[Matrix.D], matrixB.matrix[Matrix.D])
self.assertEqual(matrixA.matrix[Matrix.E], matrixB.matrix[Matrix.E])
self.assertEqual(matrixA.matrix[Matrix.F], matrixB.matrix[Matrix.F])
self.assertEqual(matrixA.toString(), self.rotation_matrix_string)
self.assertEqual(matrixC.toCSSTransformString(), 'rotate(-{}deg)'.format(self.rotation_angle))
def test_get_rotation_direction(self):
rotation_string = 'rotate(-{})'.format(self.rotation_angle)
matrixA = Matrix(rotation_string)
matrixB = Matrix(self.rotation_matrix_string)
matrixC = Matrix(self.matrix_string)
self.assertEqual(matrixA.get_rotation_direction(), Matrix.UP)
self.assertEqual(matrixB.get_rotation_direction(), Matrix.DOWN)
self.assertEqual(matrixC.get_rotation_direction(), Matrix.STRAIGHT)
def test_isRotationMatrix(self):
rotation_string = 'rotate({})'.format(self.rotation_angle)
matrixA = Matrix(rotation_string)
self.assertEqual(matrixA.isRotationMatrix(), True)
matrixB = Matrix(self.matrix_string)
self.assertEqual(matrixB.isRotationMatrix(), False)
-
+
+ def test_isRotationMatrix(self):
+ rotation_string = 'rotate(90deg)'
+ matrix = Matrix(rotation_string)
+ mindex = matrix.get90DegreeIndex()
+ self.assertEqual(mindex, 0)
+ rotation_string = 'rotate(180deg)'
+ matrix = Matrix(rotation_string)
+ mindex = matrix.get90DegreeIndex()
+ self.assertEqual(mindex, 1)
+ rotation_string = 'rotate(270deg)'
+ matrix = Matrix(rotation_string)
+ mindex = matrix.get90DegreeIndex()
+ self.assertEqual(mindex, 2)
+ rotation_string = 'rotate(70deg)'
+ matrix = Matrix(rotation_string)
+ mindex = matrix.get90DegreeIndex()
+ self.assertEqual(mindex, -1)
+
def test_toCSSTransformString(self):
rotation_string = 'rotate({})'.format(self.rotation_angle)
matrixA = Matrix(rotation_string)
self.assertEqual(matrixA.toCSSTransformString(), 'rotate({}deg)'.format(self.rotation_angle))
matrixB = Matrix(self.rotation_matrix_string)
self.assertEqual(matrixB.toCSSTransformString(), 'rotate({}deg)'.format(self.rotation_angle))
def test_Matrix_Exception(self):
with self.assertRaises(Exception):
Matrix('matrix({})'.format(' '.join([ '0.0' for i in range(5)])))
def test_Matrix_TranskriptionField(self):
tf = TranskriptionField(self.test_file)
matrix = Matrix(self.matrix_string, transkription_field=tf)
self.assertEqual(round(matrix.getX(), 3) , 28.706)
self.assertEqual(round(matrix.getY(), 3) , 31.563)
def test_get_transformed_positions(self):
# Test relies on the example from "https://developer.mozilla.org/en-US/docs/Web/SVG/Attribute/transform"
x = 10
y = 10
width = 30
height = 20
matrix = Matrix(transform_matrix_string='matrix(3 1 -1 3 30 40)')
new_x, new_y, new_width, new_height = matrix.get_transformed_positions(x=x, y=y, width=width, height=height)
self.assertEqual(new_x, 50)
self.assertEqual(new_y, 80)
self.assertEqual(new_width, 90)
self.assertEqual(new_height, 60)
def test_is_matrix_horizontal(self):
matrix = Matrix(transform_matrix_string='matrix(3 1 -1 3 30 40)')
self.assertEqual(matrix.is_matrix_horizontal(), False)
matrix = Matrix(transform_matrix_string='matrix(1 0 0 1 30 40)')
self.assertEqual(matrix.is_matrix_horizontal(), True)
def test_is_part_of_transkription_field(self):
tf = TranskriptionField(self.test_file)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 244.1211 91.7134)'})
self.assertEqual(Matrix.IS_PART_OF_TRANSKRIPTION_FIELD(tf, text_node=text_node), True)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 244.1211 51.7134)'})
self.assertEqual(Matrix.IS_PART_OF_TRANSKRIPTION_FIELD(tf, text_node=text_node), False)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 44.1211 91.7134)'})
self.assertEqual(Matrix.IS_PART_OF_TRANSKRIPTION_FIELD(tf, text_node=text_node), False)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 244.1211 891.7134)'})
self.assertEqual(Matrix.IS_PART_OF_TRANSKRIPTION_FIELD(tf, text_node=text_node), False)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 844.1211 91.7134)'})
self.assertEqual(Matrix.IS_PART_OF_TRANSKRIPTION_FIELD(tf, text_node=text_node), False)
text_node = ET.Element('text', attrib={'transform': 'matrix(0.866 -0.5 0.5 0.866 356.4303 753.4836)'})
tspan_node = ET.SubElement(text_node, 'tspan', attrib={'x': '41.82', 'y': '0'})
tspan_node.text = 'De'
fake_tf = FakeTF()
self.assertEqual(Matrix.IS_PART_OF_TRANSKRIPTION_FIELD(fake_tf, text_node=text_node), True)
"""
local_file = '/home/knister0/ownCloud/myNietzscheDE/KGW-IX_12/Bd_12_XIV-XVI_Druck_als_SVG/18.svg'
tf = TranskriptionField(local_file)
svg_tree = ET.parse(local_file)
namespaces = { k if k is not None else 'ns': v for k, v in svg_tree.getroot().nsmap.items() }
text_node = svg_tree.xpath('//ns:text[@transform="matrix(1 0 0 1 115.6299 719.3535)"]', namespaces=namespaces)[0]
self.assertTrue(Matrix.IS_PART_OF_TRANSKRIPTION_FIELD(tf, text_node=text_node))
"""
def test_is_nearx_tf(self):
tf = TranskriptionField(self.test_file)
matrix_string = 'matrix(1 0 0 1 180.8755 315.9131)'
self.assertEqual(Matrix.IS_NEARX_TRANSKRIPTION_FIELD(matrix_string, tf), True)
matrix_string = 'matrix(1 0 0 1 100.8755 315.9131)'
self.assertEqual(Matrix.IS_NEARX_TRANSKRIPTION_FIELD(matrix_string, tf), False)
def test_do_conversion_factors_differ(self):
self.assertEqual(Matrix.DO_CONVERSION_FACTORS_DIFFER(None, None), False)
matrix_a = Matrix('matrix(1 0 0 1 180.8755 315.9131)')
self.assertEqual(Matrix.DO_CONVERSION_FACTORS_DIFFER(matrix_a, None), True)
matrix_b = Matrix('matrix(1 0 0 1 100.8755 315.9131)')
self.assertEqual(Matrix.DO_CONVERSION_FACTORS_DIFFER(matrix_a, matrix_b), False)
matrix_b = Matrix('matrix(0 0 0 1 100.8755 315.9131)')
self.assertEqual(Matrix.DO_CONVERSION_FACTORS_DIFFER(matrix_a, matrix_b), True)
matrix_b = Matrix('matrix(1 1 0 1 100.8755 315.9131)')
self.assertEqual(Matrix.DO_CONVERSION_FACTORS_DIFFER(matrix_a, matrix_b), True)
matrix_b = Matrix('matrix(1 0 1 1 100.8755 315.9131)')
self.assertEqual(Matrix.DO_CONVERSION_FACTORS_DIFFER(matrix_a, matrix_b), True)
matrix_b = Matrix('matrix(1 0 0 0 100.8755 315.9131)')
self.assertEqual(Matrix.DO_CONVERSION_FACTORS_DIFFER(matrix_a, matrix_b), True)
def test_clone_transformation_matrix(self):
matrix_a = Matrix(matrix_list=[ 1, 0, 0, 1, 180.8755, 315.9131 ])
matrix_b = matrix_a.clone_transformation_matrix()
self.assertEqual(Matrix.DO_CONVERSION_FACTORS_DIFFER(matrix_a, matrix_b), False)
self.assertEqual(matrix_b.matrix[Matrix.E], 0)
self.assertEqual(matrix_b.matrix[Matrix.F], 0)
def test_toString(self):
matrix_string = 'matrix(1.0 0.0 0.0 1.0 180.8755 315.9131)'
matrix = Matrix(matrix_string)
self.assertEqual(matrix.toString(), matrix_string)
def test_get_semanticAndDataDict(self):
matrix = Matrix('rotate(20)')
#self.assertEqual(matrix.get_data_dictionary()['body'].get('matrix'), matrix.matrix)
def test_is_in_margin_field(self):
tf = TranskriptionField(self.test_margin_field_file)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 178.8916 182.0127)'})
self.assertEqual(Matrix.IS_IN_MARGIN_FIELD(text_node.get('transform'), tf), True)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 357.7339 818.3276)'})
self.assertEqual(Matrix.IS_IN_MARGIN_FIELD(text_node.get('transform'), tf), False)
tf = TranskriptionField(self.marginals_extra)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 778.519 407.1094)'})
self.assertEqual(Matrix.IS_IN_MARGIN_FIELD(text_node.get('transform'), tf, marginals_on_extra_page=True), True)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 51.8503 1056.1182)'})
self.assertEqual(Matrix.IS_IN_MARGIN_FIELD(text_node.get('transform'), tf, marginals_on_extra_page=True), False)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 552.9165 1072.1025)'})
self.assertEqual(Matrix.IS_IN_MARGIN_FIELD(text_node.get('transform'), tf, marginals_on_extra_page=True), False)
def test_is_in_place_of_printing_area(self):
tf = TranskriptionField(self.test_place_printing_verso)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 42.5195 575.8736)'})
self.assertEqual(Matrix.IS_IN_PLACE_OF_PRINTING_AREA(text_node.get('transform'), tf), True)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 109.145 575.8736)'})
self.assertEqual(Matrix.IS_IN_PLACE_OF_PRINTING_AREA(text_node.get('transform'), tf), True)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 191.0571 575.8736)'})
self.assertEqual(Matrix.IS_IN_PLACE_OF_PRINTING_AREA(text_node.get('transform'), tf), False)
tf = TranskriptionField(self.test_place_printing_recto)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 28.3462 575.8736)'})
self.assertEqual(Matrix.IS_IN_PLACE_OF_PRINTING_AREA(text_node.get('transform'), tf), True)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 28.3462 583.8736)'})
self.assertEqual(Matrix.IS_IN_PLACE_OF_PRINTING_AREA(text_node.get('transform'), tf), True)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 170.0791 575.8736)'})
self.assertEqual(Matrix.IS_IN_PLACE_OF_PRINTING_AREA(text_node.get('transform'), tf), False)
def test_is_in_footnote_area(self):
tf = TranskriptionField(self.test_place_printing_verso)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 42.5195 575.8736)'})
self.assertEqual(Matrix.IS_IN_FOOTNOTE_AREA(text_node.get('transform'), tf), False)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 109.145 575.8736)'})
self.assertEqual(Matrix.IS_IN_FOOTNOTE_AREA(text_node.get('transform'), tf), False)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 191.0571 575.8736)'})
self.assertEqual(Matrix.IS_IN_FOOTNOTE_AREA(text_node.get('transform'), tf), True)
tf = TranskriptionField(self.test_place_printing_recto)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 28.3462 575.8736)'})
self.assertEqual(Matrix.IS_IN_FOOTNOTE_AREA(text_node.get('transform'), tf), False)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 28.3462 583.8736)'})
self.assertEqual(Matrix.IS_IN_FOOTNOTE_AREA(text_node.get('transform'), tf), False)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 170.0791 575.8736)'})
self.assertEqual(Matrix.IS_IN_FOOTNOTE_AREA(text_node.get('transform'), tf), True)
tf = TranskriptionField(self.multipage, multipage_index=0)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 395.7141 463.6953)'})
self.assertEqual(Matrix.IS_IN_FOOTNOTE_AREA(text_node.get('transform'), tf), False)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 395.7141 453.6953)'})
self.assertEqual(Matrix.IS_IN_FOOTNOTE_AREA(text_node.get('transform'), tf), True)
tf = TranskriptionField(self.marginals_extra)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 552.9165 1072.1025)'})
self.assertEqual(Matrix.IS_IN_FOOTNOTE_AREA(text_node.get('transform'), tf, marginals_on_extra_page=True), True)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 51.8503 1056.1182)'})
self.assertEqual(Matrix.IS_IN_FOOTNOTE_AREA(text_node.get('transform'), tf, x=5.352, marginals_on_extra_page=True), False)
text_node = ET.Element('text', attrib={'transform': 'matrix(1 0 0 1 215.5483 1056.1182)'})
self.assertTrue(Matrix.IS_IN_FOOTNOTE_AREA(text_node.get('transform'), tf, x=24.732, marginals_on_extra_page=True))
svg_tree = ET.parse(self.marginals_extra_fn)
tf = TranskriptionField(self.marginals_extra)
namespaces = { k if k is not None else 'ns': v for k, v in svg_tree.getroot().nsmap.items() }
node = svg_tree.xpath('//ns:text[@transform="matrix(1 0 0 1.0101 698.1499 85.3594)"]', namespaces=namespaces)[0]
self.assertFalse(Matrix.NODE_HAS_CONTENT_IN_FOOTNOTE_AREA(node, tf, marginals_on_extra_page=True))
self.assertFalse(Matrix.IS_IN_FOOTNOTE_AREA(node.get('transform'), tf, marginals_on_extra_page=True))
node = svg_tree.xpath('//ns:text[@transform="matrix(1 0 0 1 215.5483 1056.1182)"]', namespaces=namespaces)[0]
self.assertTrue(Matrix.NODE_HAS_CONTENT_IN_FOOTNOTE_AREA(node, tf, marginals_on_extra_page=True))
if __name__ == "__main__":
unittest.main()
Index: tests_teiscripts/test_create_tei_files.py
===================================================================
--- tests_teiscripts/test_create_tei_files.py (revision 0)
+++ tests_teiscripts/test_create_tei_files.py (revision 114)
@@ -0,0 +1,29 @@
+import lxml.etree as ET
+from os import sep, path, remove
+from os.path import isdir, isfile, dirname, basename
+import shutil
+import sys
+import tempfile
+import unittest
+import warnings
+
+sys.path.append('teiscripts')
+from create_tei_files import CSVFileCreator, process_default
+
+
+sys.path.append('shared_util')
+from myxmlwriter import copy_to_bak_dir
+
+
+class TestCreateTEI(unittest.TestCase):
+ def setUp(self):
+ CSVFileCreator.UNITTESTING = True
+
+
+ def test_process_default(self):
+ csv_file = 'Druckmanuscripte/D_20.csv'
+ process_default(['Druckmanuscripte/D_20b_page001r.xml', csv_file, '/home/knister0/ownCloud/myNietzscheDE/Druckmanuscripte/svg', '/home/knister0/ownCloud/myNietzscheDE/Druckmanuscripte/tei','/home/knister0/ownCloud/myNietzscheDE/Druckmanuscripte/text'])
+
+
+if __name__ == "__main__":
+ unittest.main()
Index: fixes/test_get_text_field.py
===================================================================
--- fixes/test_get_text_field.py (revision 113)
+++ fixes/test_get_text_field.py (revision 114)
@@ -1,25 +1,31 @@
import lxml.etree as ET
from os import sep, path, remove
from os.path import isdir, isfile, dirname, basename
import shutil
import sys
import tempfile
import unittest
import warnings
import get_text_field
class TestGETIMAGEINFO(unittest.TestCase):
def setUp(self):
DATADIR = path.dirname(__file__) + sep + 'test_data'
self.xml_file = DATADIR + sep + 'N_VII_1_page138.xml'
self.fix_transkription_positions = DATADIR + sep + 'Mp_XIV_page419a.xml'
def test_main(self):
get_text_field.main(['/home/knister0/ownCloud/nietzscheDE/Bearbeitung_Faksimile/Mp_XVI/Mp-XVI-4,13.jpg','1260.3199','1339.36'])
+ get_text_field.main(['/home/knister0/ownCloud/myNietzscheDE/Druckmanuscripte/faksimiles/D-20a,1.jpg','1221.28','1446.08'])
+
+ def test_get_text_field_on_thumb(self):
+ pass
+ #tf, width, height = get_text_field.get_text_field_on_thumb('/home/knister0/ownCloud/Der_spaete_Nietzsche/DATA/faksimiles/Mp-XVII,243_thumb.jpg')
+ #print(tf, width, height)
if __name__ == "__main__":
unittest.main()
Index: fixes/get_text_field.py
===================================================================
--- fixes/get_text_field.py (revision 113)
+++ fixes/get_text_field.py (revision 114)
@@ -1,144 +1,184 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
""" This program can be used to create svg files with a rect for the text_field.
"""
# Copyright (C) University of Basel 2019 {{{1
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see width height
the directory where the files should be saved to
OPTIONS:
-h|--help show help
:return: exit code (int)
"""
try:
opts, args = getopt.getopt(argv, "h", ["help"])
except getopt.GetoptError:
usage()
return 2
for opt, arg in opts:
if opt in ('-h', '--help'):
usage()
return 0
if len(args) < 3:
usage()
return 2
exit_status = 0
image_file = args[0]
image_width = float(args[1])
image_height = float(args[2])
if isfile(image_file):
textfield = get_text_field_on_image(image_file, image_width, image_height)
print(textfield)
else:
raise FileNotFoundError(f'File {image_file} does not exist!')
return exit_status
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
Index: fixes/create_folio_name.py
===================================================================
--- fixes/create_folio_name.py (revision 0)
+++ fixes/create_folio_name.py (revision 114)
@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""This program can be used to add a name for a folio.
+"""
+# Copyright (C) University of Basel 2021 {{{1
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see
+
+ a csv-file containing a folio descriptions.
+
+ OPTIONS:
+ -h|--help show help
+ -f|--fix-url fix image url
+
+ :return: exit code (int)
+ """
+ fix_url = False
+ try:
+ opts, args = getopt.getopt(argv, "hf", ["help", "fix-url"])
+ except getopt.GetoptError:
+ usage()
+ return 2
+ for opt, arg in opts:
+ if opt in ('-h', '--help'):
+ usage()
+ return 0
+ elif opt in ('-f', '--fix-url'):
+ fix_url = True
+ exit_status = 0
+ if len(args) < 1:
+ usage()
+ return 2
+ csv_file = args[0]
+ if isfile(csv_file):
+ if fix_url:
+ process_fix_url(csv_file)
+ else:
+ process_csv_file(csv_file)
+ else:
+ raise FileNotFoundError('File {csv_file} does not exist!')
+ return exit_status
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv[1:]))
Index: fixes/checker_handler.py
===================================================================
--- fixes/checker_handler.py (revision 113)
+++ fixes/checker_handler.py (revision 114)
@@ -1,79 +1,79 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
""" This program can be used to handle the manual check status of xml files.
"""
# Copyright (C) University of Basel 2021 {{{1
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see
+
+ a xml file about a manuscript, containing information about its pages.
+ a xml file about a page, containing information about svg word positions.
+ a directory containing faksimile images
+
+ OPTIONS:
+ -h|--help show help
+ -d|--fix-doubles fix double pages
+
+ :return: exit code (int)
+ """
+ fix_doubles = False
+ try:
+ opts, args = getopt.getopt(argv, "hd", ["help", "fix-doubles"])
+ except getopt.GetoptError:
+ usage()
+ return 2
+ for opt, arg in opts:
+ if opt in ('-h', '--help'):
+ usage()
+ return 0
+ elif opt in ('-d', '--fix-doubles'):
+ fix_doubles = True
+ exit_status = 0
+ if len(args) < 2:
+ usage()
+ return 2
+ xml_file = args[0]
+ faksimile_dir = args[1]
+ if isfile(xml_file) and isdir(faksimile_dir):
+ counter = 0
+ xpath = f'//page'
+ manuscript_file = xml_file
+ if len(xml_file.split('_')) > 2: # svg_pos_file
+ manuscript_file = '_'.join(xml_file.split('_')[0:2]) + '.xml'
+ if isfile(manuscript_file):
+ source_tree = ET.parse(manuscript_file)
+ xpath = f'//page[contains(@output,"{xml_file}")]'
+ else:
+ raise FileNotFoundError(f'There is no manuscript file {manuscript_file} for svg_pos_file {xml_file}!')
+ else:
+ source_tree = ET.parse(xml_file)
+ image_manipulator = ImageManipulator(faksimile_dir, fix_doubles=fix_doubles)
+ title = basename(manuscript_file).replace('.xml','').replace('_',' ')
+ for page in source_tree.xpath(xpath):
+ if not ImageManipulator.UNITTESTING:
+ number = page.get('number')
+ print(Fore.CYAN + f'Processing thumb image of {title}, {number} ...' + Style.RESET_ALL)
+ if image_manipulator.process_thumb_of_page(page.get('output')) == 0:
+ counter += 1
+ if not ImageManipulator.UNITTESTING:
+ print(Style.RESET_ALL + f'[{counter} images changed]')
+ else:
+ if not isdir(faksimile_dir):
+ raise FileNotFoundError(f'Directory {faksimile_dir} does not exist!')
+ raise FileNotFoundError('File {} does not exist!'.format(xml_file))
+ return exit_status
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv[1:]))
Index: fixes/test_create_blank_svg_files.py
===================================================================
--- fixes/test_create_blank_svg_files.py (revision 113)
+++ fixes/test_create_blank_svg_files.py (revision 114)
@@ -1,38 +1,45 @@
import lxml.etree as ET
from os import sep, path, remove
from os.path import isdir, isfile, dirname, basename
import shutil
import sys
import tempfile
import unittest
import warnings
-from create_blank_svg_files import SVGFileCreator, OldSVGFileCreator
+from create_blank_svg_files import SVGFileCreator, OldSVGFileCreator, CSVSVGFileCreator, process_csv
sys.path.append('shared_util')
from myxmlwriter import copy_to_bak_dir
-
class TestCreateSVGFILES(unittest.TestCase):
def setUp(self):
SVGFileCreator.UNITTESTING = True
DATADIR = path.dirname(__file__) + sep + 'test_data'
self.xml_file = DATADIR + sep + 'N_VII_1_page138.xml'
self.fix_transkription_positions = DATADIR + sep + 'Mp_XIV_page419a.xml'
def test_init(self):
svg_creator = OldSVGFileCreator('Mp XVI', '/home/knister0/ownCloud/nietzscheDE/Bearbeitung_Faksimile/Mp_XVI')
self.assertTrue(svg_creator.avg_left != -1)
self.assertTrue(svg_creator.avg_top != -1)
#print(svg_creator.avg_left, svg_creator.avg_top, svg_creator.avg_width, svg_creator.avg_height)
def test_update_svg_file(self):
+ pass
+ """
svg_file = '/home/knister0/ownCloud/nietzscheDE/Bearbeitung_Faksimile/Myriam/Blank/Mp_XVI/Mp_XVI_page81v.svg'
copy_to_bak_dir(svg_file)
exit_status = SVGFileCreator.UPDATE_TEXTFIELD_OF_SVG_FILE(svg_file)
self.assertTrue(exit_status == 0)
+ """
+
+ def test_process_csv(self):
+ csv_file = 'Druckmanuscripte/D_20.csv'
+ process_csv(csv_file, '/home/knister0/ownCloud/myNietzscheDE/Druckmanuscripte/faksimiles', '/home/knister0/ownCloud/myNietzscheDE/Druckmanuscripte/svg')
+
if __name__ == "__main__":
unittest.main()
Index: fixes/create_blank_svg_files.py
===================================================================
--- fixes/create_blank_svg_files.py (revision 113)
+++ fixes/create_blank_svg_files.py (revision 114)
@@ -1,349 +1,554 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
""" This program can be used to create svg files with a rect for the text_field.
"""
# Copyright (C) University of Basel 2019 {{{1
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see
- fixes/create_blank_svg_files.py -u|--update
+def process_update(args) ->int:
+ """ Process option update
+ """
+ if len(args) < 0:
+ usage()
+ return 2
+ svg_dir = args[0]
+ if not isdir(svg_dir):
+ raise FileNotFoundError(f'Directory {svg_dir} does not exist!')
+ counter = 0
+ for svg_file in [ svg_dir + sep + svg_file for svg_file in listdir(svg_dir) if isfile(svg_dir + sep + svg_file) and svg_file.endswith('.svg') ]:
+ if not SVGFileCreator.UNITTESTING:
+ print(Fore.CYAN + f'Updating svg file {svg_file} ...' + Style.RESET_ALL)
+ copy_to_bak_dir(svg_file)
+ if SVGFileCreator.UPDATE_TEXTFIELD_OF_SVG_FILE(svg_file) == 0:
+ counter += 1
+ if not SVGFileCreator.UNITTESTING:
+ print(Style.RESET_ALL + f'[{counter} pages created]')
+ return 0
- a xml file about a manuscript, containing information about its pages.
- a xml file about a page, containing information about svg word positions.
- a directory containing faksimile images
- the directory where the files should be saved to
+def process_rotate(args) ->int:
+ """ Process option rotate
+ """
+ if len(args) < 2:
+ usage()
+ return 2
+ xml_file = args[0]
+ faksimile_dir = args[1]
+ if not isdir(faksimile_dir):
+ raise FileNotFoundError(f'Directory {faksimile_dir} does not exist!')
+ xpath = f'//page'
+ source_tree = ET.parse(xml_file)
+ if len(xml_file.split('_')) > 2: # svg_pos_file
+ manuscript_file = '_'.join(xml_file.split('_')[0:2]) + '.xml'
+ if isfile(manuscript_file):
+ source_tree = ET.parse(manuscript_file)
+ xpath = f'//page[contains(@output,"{xml_file}")]'
+ else:
+ raise FileNotFoundError(f'There is no manuscript file {manuscript_file} for svg_pos_file {xml_file}!')
+ counter = 0
+ svg_creator = SVGFileCreator('', faksimile_dir)
+ for page in [ page for page in source_tree.xpath(xpath) if len(ET.parse(page.get('output')).xpath('//faksimile-image/@transform')) > 0 ]:
+ if not SVGFileCreator.UNITTESTING:
+ number = page.get('number')
+ print(Fore.CYAN + f'Rotating svg file for page {number} ...' + Style.RESET_ALL)
+ if svg_creator.create_rotation_svg_file(page) == 0:
+ counter += 1
+ if not SVGFileCreator.UNITTESTING:
+ print(Style.RESET_ALL + f'[{counter} pages created]')
+ return 0
- OPTIONS:
- -h|--help show help
- -u|--update update svg_files: use image analysis in order to update the textfield of the svg_files
-
- :return: exit code (int)
+def process_thumb(args) ->int:
+ """ Process option thumb
"""
- update = False
- try:
- opts, args = getopt.getopt(argv, "hu", ["help","update"])
- except getopt.GetoptError:
+ if len(args) == 0:
usage()
return 2
- for opt, arg in opts:
- if opt in ('-h', '--help'):
- usage()
- return 0
- elif opt in ('-u', '--update'):
- update = True
- exit_status = 0
- if update:
- if len(args) == 0:
- usage()
- return 2
- svg_dir = args[0]
- if not isdir(svg_dir):
- raise FileNotFoundError(f'Directory {svg_dir} does not exist!')
- counter = 0
- for svg_file in [ svg_dir + sep + svg_file for svg_file in listdir(svg_dir) if isfile(svg_dir + sep + svg_file) and svg_file.endswith('.svg') ]:
- if not SVGFileCreator.UNITTESTING:
- print(Fore.CYAN + f'Updating svg file {svg_file} ...' + Style.RESET_ALL)
- copy_to_bak_dir(svg_file)
- if SVGFileCreator.UPDATE_TEXTFIELD_OF_SVG_FILE(svg_file) == 0:
- counter += 1
+ faksimile_dir = args[0]
+ title = args[1]\
+ if len(args) > 1\
+ else ''
+ svg_creator = SVGFileCreator('', faksimile_dir)
+ print(faksimile_dir, title)
+ if not isdir(faksimile_dir):
+ raise FileNotFoundError(f'Directory {faksimile_dir} does not exist!')
+ counter = 0
+ for faksimile_file in [ faksimile_dir + sep + faksimile_file for faksimile_file in listdir(faksimile_dir)\
+ if isfile(faksimile_dir + sep + faksimile_file)\
+ and basename(faksimile_file).startswith(title)\
+ and faksimile_file.endswith('_thumb.jpg') ]:
if not SVGFileCreator.UNITTESTING:
- print(Style.RESET_ALL + f'[{counter} pages created]')
- return exit_status
+ print(Fore.CYAN + f'Creating a svg for faksimile file {faksimile_file} ...' + Style.RESET_ALL)
+ if svg_creator.create_svg_thumb_file(faksimile_file) == 0:
+ counter += 1
+ if not SVGFileCreator.UNITTESTING:
+ print(Style.RESET_ALL + f'[{counter} thumb svg files created]')
+ return 0
+def process_csv(csv_file_name, faksimile_dir, target_dir, downloadOnly=False) ->int:
+ """ Default process
+ """
+ title = basename(csv_file_name).replace('.csv','').replace('_', ' ')
+ file_creator = CSVSVGFileCreator(title, faksimile_dir, target_dir)
+ exit_status = 0
+ with open(csv_file_name, newline='') as csvfile:
+ reader = csv.DictReader(csvfile)
+ for row in reader:
+ faksimile_name = row[CSVSVGFileCreator.LABEL]
+ if '_thumb' not in faksimile_name:
+ faksimile_url = row[CSVSVGFileCreator.URL]
+ page_id = row[CSVSVGFileCreator.ALIAS].replace(',', '').replace(' ', '_')
+ faksimile_file = faksimile_dir + sep + faksimile_name
+ if not isfile(faksimile_file):
+ wget.download(faksimile_url, out=faksimile_file)
+ if not downloadOnly:
+ if isfile(faksimile_file):
+ print(f'processing {faksimile_file} ...')
+ if file_creator.create_svg_file_from_csv_input(faksimile_file, page_id) > 0:
+ exit_status = 2
+ else:
+ exit_status = 2
+ print(f'There has been an error on downloading {faksimile_url}!', e)
+ return exit_status
+
+def process_default(args) ->int:
+ """ Default process
+ """
if len(args) < 3:
usage()
return 2
- xml_file = args[0]
faksimile_dir = args[1]
target_dir = args[2]
not isdir(target_dir) and mkdir(target_dir)
+ if args[0].endswith('.csv'):
+ return process_csv(args[0], faksimile_dir, target_dir)
+ xml_file = args[0]
if isfile(xml_file) and isdir(faksimile_dir):
counter = 0
xpath = f'//page[contains(@status, "{BLANK_STATUS}")]'
if len(xml_file.split('_')) > 2: # svg_pos_file
manuscript_file = '_'.join(xml_file.split('_')[0:2]) + '.xml'
if isfile(manuscript_file):
source_tree = ET.parse(manuscript_file)
xpath = f'//page[contains(@output,"{xml_file}")]'
else:
raise FileNotFoundError(f'There is no manuscript file {manuscript_file} for svg_pos_file {xml_file}!')
else:
source_tree = ET.parse(xml_file)
title = source_tree.getroot().get('title')
svg_creator = SVGFileCreator(title, faksimile_dir, target_dir=target_dir)
for page in source_tree.xpath(xpath):
if not SVGFileCreator.UNITTESTING:
number = page.get('number')
print(Fore.CYAN + f'Creating a svg file for {title}, {number} ...' + Style.RESET_ALL)
if svg_creator.create_svg_file(page) == 0:
counter += 1
if not SVGFileCreator.UNITTESTING:
print(Style.RESET_ALL + f'[{counter} pages created]')
else:
if not isdir(faksimile_dir):
raise FileNotFoundError(f'Directory {faksimile_dir} does not exist!')
raise FileNotFoundError('File {} does not exist!'.format(xml_file))
- return exit_status
+ return 0
+
+
+def usage():
+ """prints information on how to use the script
+ """
+ print(main.__doc__)
+
+def main(argv):
+ """This program can be used to create svg files with a rect for the text_field.
+
+ fixes/create_blank_svg_files.py [OPTIONS]
+ fixes/create_blank_svg_files.py -r|--rotate-thumb
+ fixes/create_blank_svg_files.py -u|--update
+ fixes/create_blank_svg_files.py -t|--thumb []
+
+ <xmlManuscriptFile> a xml file about a manuscript, containing information about its pages.
+ <svg_pos_file> a xml file about a page, containing information about svg word positions.
+ <csv_file> a csv file with faksimile information.
+ <faksimile-dir> a directory containing faksimile images
+ <target-dir> the directory where the files should be saved to
+
+ OPTIONS:
+ -h|--help show help
+ -r|--rotate-thumb use image analysis in order to create a svg file for the thumb faksimile files that need rotation.
+ -u|--update update svg_files: use image analysis in order to update the textfield of the svg_files
+ -t|--thumb use image analysis in order to create a svg file for the thumb faksimile files that need cropping.
+
+ :return: exit code (int)
+ """
+ update = False
+ thumb = False
+ rotate = True
+ try:
+ opts, args = getopt.getopt(argv, "hrut", ["help","rotate-thumb","update", "thumb"])
+ except getopt.GetoptError:
+ usage()
+ return 2
+ for opt, arg in opts:
+ if opt in ('-h', '--help'):
+ usage()
+ return 0
+ elif opt in ('-r', '--rotate-thumb'):
+ rotate = True
+ elif opt in ('-u', '--update'):
+ update = True
+ elif opt in ('-t', '--thumb'):
+ thumb = True
+ if update:
+ return process_update(args)
+ elif rotate:
+ return process_rotate(args)
+ elif thumb:
+ return process_thumb(args)
+ return process_default(args)
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
Index: fixes/test_add_faksimile_image.py
===================================================================
--- fixes/test_add_faksimile_image.py (revision 113)
+++ fixes/test_add_faksimile_image.py (revision 114)
@@ -1,32 +1,41 @@
import lxml.etree as ET
from os import sep, path, remove
from os.path import isdir, isfile, dirname, basename
import shutil
import sys
import tempfile
import unittest
import warnings
import add_faksimile_image
sys.path.append('shared_util')
from myxmlwriter import copy_to_bak_dir
sys.path.append('svgscripts')
from datatypes.page import Page
class TestAddFaksimileImage(unittest.TestCase):
+ def setUp(self):
+ add_faksimile_image.UNITTESTING = True
+ self.csv_file = 'tests_svgscripts/test_data/bd_12_Layout.csv'
def testAdd(self):
add_faksimile_image.UNITTESTING = True
page = Page.create_cls('xml/Mp_XVI_page52v.xml')
faksimile_dir = '/home/knister0/ownCloud/nietzscheDE/Bearbeitung_Faksimile/Myriam/Blank/Mp_XVI/Fertig'
exit_status = add_faksimile_image.add_faksimile_image(page, faksimile_dir)
self.assertEqual(exit_status, 0)
#print(ET.dump(page.page_tree.xpath('//faksimile-image')[0]))
+ def testCSV(self):
+ #csv_handler = add_faksimile_image.FlippedSidesCSVHandler(self.csv_file, 'Mp XVI')
+ csv_handler = add_faksimile_image.FlippedSidesCSVHandler('/home/knister0/ownCloud/myNietzscheDE/KGW-IX_13/bd_13.csv', 'Mp XVIII')
+ print(csv_handler.page_dictionary)
+ page = Page.create_cls('xml/Mp_XVIII_page12v.xml')
+ self.assertEqual(add_faksimile_image.update_faksimile_image(page, csv_handler), 0)
if __name__ == "__main__":
unittest.main()
Index: fixes/add_faksimile_image.py
===================================================================
--- fixes/add_faksimile_image.py (revision 113)
+++ fixes/add_faksimile_image.py (revision 114)
@@ -1,134 +1,205 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""This program can be used to add a faksimile image to a page xml file.
"""
# Copyright (C) University of Basel 2021 {{{1
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/> 1}}}
from colorama import Fore, Style
+import csv
import getopt
import json
import lxml.etree as ET
import shutil
import subprocess
import sys
import os
import wget
from os import listdir, sep, path, setpgrp, devnull, mkdir, remove
from os.path import exists, isfile, isdir, dirname, basename
from progress.bar import Bar
+import re
import warnings
from fix_old_data import save_page
from get_text_field import get_text_field_on_image
sys.path.append('svgscripts')
from datatypes.faksimile_image import FaksimileImage
from datatypes.faksimile import FaksimilePage
+from datatypes.matrix import Matrix
from datatypes.archival_manuscript import ArchivalManuscriptUnity
from datatypes.page import Page, STATUS_MERGED_OK, STATUS_POSTMERGED_OK
from util import back_up, back_up_svg_file, copy_faksimile_update_image_location, copy_faksimile_svg_file
from process_files import update_svgposfile_status
from process_words_post_merging import update_faksimile_line_positions, MERGED_DIR
sys.path.append('shared_util')
from myxmlwriter import copy_to_bak_dir, write_pretty, xml_has_type, FILE_TYPE_SVG_WORD_POSITION, FILE_TYPE_XML_MANUSCRIPT
from main_util import create_function_dictionary
__author__ = "Christian Steiner"
__maintainer__ = __author__
__copyright__ = 'University of Basel'
__email__ = "christian.steiner@unibas.ch"
__status__ = "Development"
__license__ = "GPL v3"
__version__ = "0.0.1"
MAX_SVG_XY_THRESHOLD = 10
BLANK_STATUS = 'blank'
UNITTESTING = False
+class FlippedSidesCSVHandler:
+ """This class can be used to handle csv files that contain information about the title and layout of the svg files.
+ """
+ ENTRY_KEY_PAGE = 'pdf_page_number'
+ ENTRY_KEY_FILE = 'svg_source_file'
+ ENTRY_KEY_TITLE = 'manuscript_title'
+ ENTRY_KEY_PAGE_NAMES = 'page_names'
+ ENTRY_KEY_MARG_PAGE = 'marginals_page_entry'
+ MANUSCRIPT_AE_REMOVAL = re.compile('[a-e]')
+ MANUSCRIPT_KEY = 'Ms'
+ MANUSCRIPT_PATTERN = re.compile(r'(\d+)(>\s)(.*)')
+ MANUSCRIPT_TITLE_EXTENSION = 'Mp'
+ MANUSCRIPT_TITLE_PARTS = re.compile(r'([I-X]+[a-e]*)(\s)(\d+\w*)(/\d+\w*)*')
+ MARGINALS_PAGE = re.compile(r'([I-X]+[a-e]*)(\s)(\d+\w*)(\s)(Marg)')
+ REMOVE_NONNUMERIC = re.compile('\D')
+
+ def __init__(self, csv_file_name: str, title: str):
+ self.page_dictionary = {}
+ self.title = title
+ self._init_csv_entries(csv_file_name)
+
+
+ def _init_csv_entries(self, csv_file_name):
+ """Init csv entries by reading the csv_file.
+ """
+ with open(csv_file_name, newline='') as csvfile:
+ reader = csv.DictReader(csvfile)
+ flipped_key = 'Gedrehte Seiten'
+ if len([ key for key in reader.fieldnames if flipped_key in key ]) > 0:
+ flipped_key = [ key for key in reader.fieldnames if flipped_key in key ][0]
+ for row in reader:
+ if bool(row[flipped_key]):
+ ms_string = row[self.MANUSCRIPT_KEY]
+ manuscript_match = re.match(self.MANUSCRIPT_PATTERN, ms_string)
+ for flipped_line in row[flipped_key].split('\n'):
+ flipped_msg = re.split(r'(\s*Drehung\s*|\s*\(.*)', flipped_line)
+ if len(flipped_msg) > 2 and manuscript_match is not None:
+ page_number = flipped_msg[0]
+ rotation = flipped_msg[2]
+ title_parts = re.match(self.MANUSCRIPT_TITLE_PARTS, manuscript_match.group(3))
+ if title_parts is not None\
+ and self.MANUSCRIPT_TITLE_EXTENSION + ' ' + self.MANUSCRIPT_AE_REMOVAL.sub('', title_parts.group(1)) == self.title:
+ self.page_dictionary.update({page_number: rotation})
+
def add_faksimile_image(page: Page, faksimile_dir: str) ->int:
"""Add faksimile image to page.
[:return:] exit_code
"""
svg_file = faksimile_dir + sep + basename(page.page_tree.docinfo.URL).replace('.xml','.svg')
if isfile(svg_file):
fp = FaksimilePage.get_faksimile_pages(svg_file, page_number=page.number, isBlank=True)
if len(fp) > 0 and fp[0].faksimile_image is not None:
fp[0].faksimile_image.text_field = fp[0].text_field
fp[0].faksimile_image.attach_object_to_tree(page.page_tree)
if not UNITTESTING:
save_page(page)
return 0
return 2
+def update_faksimile_image(page: Page, csv_handler: FlippedSidesCSVHandler) ->int:
+ """Update faksimile image of page: add rotation matrix.
+ [:return:] exit_code
+ """
+ rotation = csv_handler.page_dictionary.get(page.number)
+ if rotation is not None:
+ matrix = Matrix(f'rotate({rotation})')
+ page.faksimile_image.transform = matrix
+ page.faksimile_image.attach_object_to_tree(page.page_tree)
+ if not UNITTESTING:
+ save_page(page)
+ return 0
+ else:
+ print(page.number, csv_handler.page_dictionary.keys())
+ return 2
+
def usage():
"""prints information on how to use the script
"""
print(main.__doc__)
def main(argv):
- """This program can be used to add a faksimile image to a page xml file.
+ """This program can be used to add or update a faksimile image to a page xml file.
- fixes/add_faksimile_image.py [OPTIONS] <xmlManuscriptFile|svg_pos_file> <faksimile-dir>
+ fixes/add_faksimile_image.py [OPTIONS] <xmlManuscriptFile|svg_pos_file> <faksimile-dir|csv-file>
<xmlManuscriptFile> a xml file about a manuscript, containing information about its pages.
<svg_pos_file> a xml file about a page, containing information about svg word positions.
<faksimile-dir> a directory containing the blank faksimile svg files
+ <csv-file> a csv-file containing a column about the rotation of the transkription.
OPTIONS:
-h|--help show help
:return: exit code (int)
"""
try:
opts, args = getopt.getopt(argv, "h", ["help"])
except getopt.GetoptError:
usage()
return 2
for opt, arg in opts:
if opt in ('-h', '--help'):
usage()
return 0
exit_status = 0
if len(args) < 2:
usage()
return 2
xml_file = args[0]
manuscript_file = xml_file\
if len(xml_file.split('_')) < 3\
else '_'.join(xml_file.split('_')[0:2]) + '.xml'
- faksimile_dir = args[1]
- if isfile(xml_file) and isdir(faksimile_dir):
+ task = add_faksimile_image
+ argument = args[1]
+ if argument.endswith('.csv'):
+ if isfile(argument):
+ argument = FlippedSidesCSVHandler(argument, basename(manuscript_file).replace('_', ' ').replace('.xml',''))
+ task = update_faksimile_image
+ else:
+ raise FileNotFoundError(f'File {argument} does not exist!')
+ elif not isdir(argument):
+ raise FileNotFoundError(f'Directory {argument} does not exist!')
+ if isfile(xml_file):
counter = 0
for page in Page.get_pages_from_xml_file(xml_file, status_contains=BLANK_STATUS):
if not UNITTESTING:
- print(Fore.CYAN + f'Adding a faksimile image to {page.title}, {page.number} ...' + Style.RESET_ALL)
+ print(Fore.CYAN + f'{task.__name__}: {page.title}, {page.number} ...' + Style.RESET_ALL)
back_up(page, page.xml_file)
- if add_faksimile_image(page, faksimile_dir) == 0:
+ if task(page, argument) == 0:
counter += 1
if not UNITTESTING:
- print(Style.RESET_ALL + f'[{counter} pages created]')
+ print(Style.RESET_ALL + f'[{counter} pages processed]')
else:
- if not isdir(faksimile_dir):
- raise FileNotFoundError(f'Directory {faksimile_dir} does not exist!')
raise FileNotFoundError('File {} does not exist!'.format(xml_file))
return exit_status
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
Index: py2ttl/data_handler.py
===================================================================
--- py2ttl/data_handler.py (revision 113)
+++ py2ttl/data_handler.py (revision 114)
@@ -1,197 +1,294 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
""" This class can be used to add data to a rdf graph.
"""
# Copyright (C) University of Basel 2019 {{{1
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/> 1}}}
__author__ = "Christian Steiner"
__maintainer__ = __author__
__copyright__ = 'University of Basel'
__email__ = "christian.steiner@unibas.ch"
__status__ = "Development"
__license__ = "GPL v3"
__version__ = "0.0.1"
from rdflib import Graph, URIRef, Literal, BNode, OWL, RDF, RDFS, XSD
from rdflib import RDF as ns_rdf
from os.path import isfile
import random
import warnings
from class_spec import SemanticClass
from config import DATA_URL
class RDFDataHandler:
"""
This class can be used to add data to a rdf graph.
"""
UNITTESTING = False
SIMPLE_DATA_TYPE_MAPPING = { int: XSD.integer, float: XSD.float, str: XSD.string, bool: XSD.boolean, list: RDF.List }
def __init__(self, target_file, mapping_dictionary):
self.target_file = target_file
self.mapping_dictionary = mapping_dictionary
self.ontology_graph = Graph()
self.data_graph = Graph()
self.data_identifier_mapping = {}
if bool(self.mapping_dictionary.get('ontology')):
self.project_name = self.mapping_dictionary['ontology'].get('project_name')
self.project_uri = URIRef(self.mapping_dictionary['ontology'].get('project_uri'))
ontology_file = self.mapping_dictionary['ontology'].get('ontology_file')
if bool(ontology_file) and isfile(ontology_file):
self.ontology_graph.parse(ontology_file, format="turtle")
self.ns = { uriref: ns for ns, uriref in self.data_graph.namespace_manager.namespaces() }
self.data_graph.bind(self.project_name, self.project_uri)
self.data_graph.bind('data', DATA_URL + '#')
else:
raise Exception('Error: mapping_dictionary does not contain key "ontology"!')
def add_data(self, data_instance, identifier_prefix, parent_data_instance=None, skip_data_instance_list=None):
"""Add a data rdf instance of data_instance to the data_graph.
:return: (rdflib.URIRef) subject_uri of data instance
"""
skip_data_instance_list = [] if skip_data_instance_list is None else skip_data_instance_list
identifier_uri = self.create_identifier_uri(data_instance, identifier_prefix)
if type(data_instance).__name__ not in skip_data_instance_list and bool(self.mapping_dictionary['classes'].get(type(data_instance).__name__)):
class_uri = self.mapping_dictionary['classes'][type(data_instance).__name__]['class_uri']
self.data_identifier_mapping.update({data_instance: identifier_uri})
self.data_graph_add((identifier_uri, RDF.type, class_uri))
semantic_dict = data_instance.get_semantic_dictionary()
for key, content in semantic_dict['properties'].items():
if bool(self.mapping_dictionary['classes'][type(data_instance).__name__]['properties'].get(key)):
datatype = content.get('class')
cardinality = content.get('cardinality')\
if bool(content.get('cardinality')) else 0
if data_instance.__dict__.get(key) is not None\
and (type(data_instance.__dict__.get(key)) != int or data_instance.__dict__.get(key) != -1):
predicate_uri = self.mapping_dictionary['classes'][type(data_instance).__name__]['properties'][key]
child_data_instance = data_instance.__dict__.get(key)
new_identifier_prefix = identifier_uri[identifier_uri.index('#')+1:]
if datatype is list:
self.add_ordered_list(child_data_instance, identifier_uri, predicate_uri,\
new_identifier_prefix, data_instance, skip_data_instance_list=skip_data_instance_list)
elif issubclass(datatype, SemanticClass):
if type(child_data_instance) is not list:
if type(child_data_instance) != datatype\
and not issubclass(type(child_data_instance), datatype):
child_id = child_data_instance
child_data_instance = parent_data_instance.get_object_from_list_with_id(datatype,\
child_id)
if child_data_instance is None:
print(key, content)# parent_data_instance.number, child_id, type(child_id), datatype)
msg = 'No child_data_instance found for data_instance {0}: looking for {1} with id {2}'.format(\
type(parent_data_instance), datatype, child_id)
raise Exception(msg)
else:
new_list_name = 'list_of_' + datatype.__name__ + 's'
if new_list_name in data_instance.__dict__.keys():
data_instance.__dict__[new_list_name].append(child_data_instance)
else:
data_instance.__dict__.update({ new_list_name: [ child_data_instance ]})
if child_data_instance not in self.data_identifier_mapping.keys():
child_identifier_uri = self.add_data(child_data_instance, new_identifier_prefix,\
parent_data_instance=data_instance, skip_data_instance_list=skip_data_instance_list)
else:
child_identifier_uri = self.data_identifier_mapping[child_data_instance]
self.data_graph_add((identifier_uri, predicate_uri, child_identifier_uri))
else:
for child_item in child_data_instance:
if child_item not in self.data_identifier_mapping.keys():
child_identifier_uri = self.add_data(child_item, new_identifier_prefix,\
parent_data_instance=data_instance, skip_data_instance_list=skip_data_instance_list)
else:
child_identifier_uri = self.data_identifier_mapping[child_item]
self.data_graph_add((identifier_uri, predicate_uri, child_identifier_uri))
else:
literal_datatype = RDFDataHandler.SIMPLE_DATA_TYPE_MAPPING[datatype]
ontology_datatypes = [ o for o in self.ontology_graph.objects(subject=predicate_uri, predicate=RDFS.range) ]
if len(ontology_datatypes) > 0:
literal_datatype = ontology_datatypes[0]
if type(child_data_instance) is list:
for child_item in child_data_instance:
object_literal = Literal(str(child_item), datatype=literal_datatype)
self.data_graph_add((identifier_uri, predicate_uri, object_literal))
else:
object_literal = Literal(str(child_data_instance), datatype=literal_datatype)
self.data_graph_add((identifier_uri, predicate_uri, object_literal))
else:
msg = 'Mapping dictionary for {0} does not contain a entry for {1}!'.format(type(data_instance).__name__, key)
raise Exception(msg)
elif type(data_instance).__name__ not in skip_data_instance_list:
msg = 'Mapping dictionary does not contain a entry for {}!'.format(type(data_instance).__name__)
raise Exception(msg)
return identifier_uri
+ def add_data_instance2parent_uri(self, parent_uri, predicate, data_instance):
+ """Add a data rdf instance of data_instance to the data_graph.
+
+ :return: (rdflib.URIRef) subject_uri of data instance
+ """
+ predicate_uri = URIRef(self.project_uri + predicate)
+ if (parent_uri, predicate_uri, None) not in self.data_graph:
+ identifier_prefix = parent_uri[parent_uri.index('#')+1:]
+ identifier_uri = self.create_identifier_uri(data_instance, identifier_prefix)
+ if bool(self.mapping_dictionary['classes'].get(type(data_instance).__name__)):
+ class_uri = self.mapping_dictionary['classes'][type(data_instance).__name__]['class_uri']
+ self.data_identifier_mapping.update({data_instance: identifier_uri})
+ self.data_graph_add((identifier_uri, RDF.type, class_uri))
+ self.data_graph_add((parent_uri, predicate_uri, identifier_uri))
+ semantic_dict = data_instance.get_semantic_dictionary()
+ for key, content in semantic_dict['properties'].items():
+ if bool(self.mapping_dictionary['classes'][type(data_instance).__name__]['properties'].get(key)):
+ datatype = content.get('class')
+ cardinality = content.get('cardinality')\
+ if bool(content.get('cardinality')) else 0
+ if data_instance.__dict__.get(key) is not None\
+ and (type(data_instance.__dict__.get(key)) != int or data_instance.__dict__.get(key) != -1):
+ predicate_uri = self.mapping_dictionary['classes'][type(data_instance).__name__]['properties'][key]
+ child_data_instance = data_instance.__dict__.get(key)
+ new_identifier_prefix = identifier_uri[identifier_uri.index('#')+1:]
+ if datatype is list:
+ self.add_ordered_list(child_data_instance, identifier_uri, predicate_uri, new_identifier_prefix, data_instance)
+ elif issubclass(datatype, SemanticClass):
+ if type(child_data_instance) is not list:
+ if type(child_data_instance) != datatype\
+ and not issubclass(type(child_data_instance), datatype):
+ child_id = child_data_instance
+ child_data_instance = data_instance.get_object_from_list_with_id(datatype, child_id)
+ if child_data_instance is None:
+ print(key, content)# parent_data_instance.number, child_id, type(child_id), datatype)
+ msg = 'No child_data_instance found for data_instance {0}: looking for {1} with id {2}'.format(\
+ type(data_instance), datatype, child_id)
+ raise Exception(msg)
+ else:
+ new_list_name = 'list_of_' + datatype.__name__ + 's'
+ if new_list_name in data_instance.__dict__.keys():
+ data_instance.__dict__[new_list_name].append(child_data_instance)
+ else:
+ data_instance.__dict__.update({ new_list_name: [ child_data_instance ]})
+ if child_data_instance not in self.data_identifier_mapping.keys():
+ child_identifier_uri = self.add_data(child_data_instance, new_identifier_prefix,\
+ parent_data_instance=data_instance)
+ else:
+ child_identifier_uri = self.data_identifier_mapping[child_data_instance]
+ self.data_graph_add((identifier_uri, predicate_uri, child_identifier_uri))
+ else:
+ for child_item in child_data_instance:
+ if child_item not in self.data_identifier_mapping.keys():
+ child_identifier_uri = self.add_data(child_item, new_identifier_prefix, parent_data_instance=data_instance)
+ else:
+ child_identifier_uri = self.data_identifier_mapping[child_item]
+ self.data_graph_add((identifier_uri, predicate_uri, child_identifier_uri))
+ else:
+ literal_datatype = RDFDataHandler.SIMPLE_DATA_TYPE_MAPPING[datatype]
+ ontology_datatypes = [ o for o in self.ontology_graph.objects(subject=predicate_uri, predicate=RDFS.range) ]
+ if len(ontology_datatypes) > 0:
+ literal_datatype = ontology_datatypes[0]
+ if type(child_data_instance) is list:
+ for child_item in child_data_instance:
+ object_literal = Literal(str(child_item), datatype=literal_datatype)
+ self.data_graph_add((identifier_uri, predicate_uri, object_literal))
+ else:
+ object_literal = Literal(str(child_data_instance), datatype=literal_datatype)
+ self.data_graph_add((identifier_uri, predicate_uri, object_literal))
+ else:
+ msg = 'Mapping dictionary for {0} does not contain a entry for {1}!'.format(type(data_instance).__name__, key)
+ raise Exception(msg)
+ return identifier_uri
+
def add_ordered_list(self, data_instance_list, identifier_uri, predicate_uri, identifier_prefix, data_instance, skip_data_instance_list=None):
"""Add a data rdf instance of data_instance to the data_graph.
"""
if len(data_instance_list) > 0:
child_identifiers = []
for item in data_instance_list:
if item not in self.data_identifier_mapping.keys():
child_identifiers.append(self.add_data(item, identifier_prefix, data_instance, skip_data_instance_list=skip_data_instance_list))
else:
child_identifiers.append(self.data_identifier_mapping[item])
list_node = self.generate_RDF_collection(child_identifiers)
self.data_graph_add((identifier_uri, predicate_uri, list_node))
+ def add_external_class_item(self, data_type, identifier_prefix, id) ->URIRef:
+ """Add a externally defined item.
+ """
+ class_uri = URIRef(self.project_uri + data_type)
+ identifier_uri = self.create_simple_identifier_uri(data_type, identifier_prefix, id)
+ self.data_graph_add((identifier_uri, RDF.type, class_uri))
+ return identifier_uri
+
+ def add_external_property(self, identifier_uri, predicate, list_of_uris):
+ """Add a externally defined item.
+ """
+ predicate_uri = URIRef(self.project_uri + predicate)
+ for uri in [ uri.replace('"', '') for uri in list_of_uris ]:
+ self.data_graph_add((identifier_uri, predicate_uri, URIRef(DATA_URL + '#' + uri[uri.index('#')+1:])))
+
+ def create_simple_identifier_uri(self, data_type, identifier_prefix, id):
+ """Return a data identifier uri.
+
+ :return: (rdflib.URIRef) subject_uri of data instance
+ """
+ identifier_uri = URIRef(DATA_URL + '#' + identifier_prefix + '_' + data_type + str(id))
+ return identifier_uri
+
def create_identifier_uri(self, data_instance, identifier_prefix):
"""Return a data identifier uri.
:return: (rdflib.URIRef) subject_uri of data instance
"""
data_type, id = data_instance.get_name_and_id()
identifier_uri = URIRef(DATA_URL + '#' + identifier_prefix + '_' + data_type + str(id))
randombit_length = 5
while (identifier_uri, None, None) in self.data_graph:
identifier_uri = URIRef(DATA_URL + '#' + identifier_prefix + '_' + data_type + str(random.getrandbits(randombit_length)))
randombit_length += 1
return identifier_uri
def data_graph_add(self, rdf_triple):
"""Add a triple to the graph.
"""
#not RDFDataHandler.UNITTESTING and print(rdf_triple)
self.data_graph.add(rdf_triple)
def generate_RDF_collection(self, vals ) -> BNode:
"""
Generate an RDF List from vals, returns the head of the list
@URL: <https://github.com/MKLab-ITI/prophet/blob/master/rdflib/plugins/parsers/pyMicrodata/utils.py>
@organization: U{World Wide Web Consortium<http://www.w3.org>}
@author: U{Ivan Herman<a href="http://www.w3.org/People/Ivan/">}
@license:
U{W3C® SOFTWARE NOTICE AND LICENSE<href="http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231">}
@param graph: RDF graph
@type graph: RDFLib Graph
@param vals: array of RDF Resources
@return: head of the List (an RDF Resource)
"""
heads = [ BNode() for r in vals ] + [ ns_rdf["nil"] ]
for i in range(0, len(vals)) :
self.data_graph_add( (heads[i], ns_rdf["first"], vals[i]) )
self.data_graph_add( (heads[i], ns_rdf["rest"], heads[i+1]) )
return heads[0]
def write(self, output_format="turtle"):
"""Write graph.
"""
f = open(self.target_file, 'wb+')
f.write(self.data_graph.serialize(format=output_format))
f.close()
Index: py2ttl/create_manuscript_description.py
===================================================================
--- py2ttl/create_manuscript_description.py (revision 0)
+++ py2ttl/create_manuscript_description.py (revision 114)
@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+""" This program can be used to create a manuscript description in turtle format.
+"""
+# Copyright (C) University of Basel 2019 {{{1
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/> 1}}}
+
+from colorama import Fore, Style
+import csv
+import getopt
+import lxml.etree as ET
+from os import sep, path, listdir
+from os.path import isfile, isdir, dirname, basename
+from progress.bar import Bar
+import re
+import sys
+
+sys.path.append('svgscripts')
+from datatypes.manuscript_description import ManuscriptDescription
+
+if dirname(__file__) not in sys.path:
+ sys.path.append(dirname(__file__))
+
+from class_spec import SemanticClass
+from config import check_config_files_exist, get_datatypes_dir, PROJECT_NAME, PROJECT_ONTOLOGY_FILE, PROJECT_URL
+from py2ttl_data import Py2TTLDataConverter
+from py2ttl_ontology import Py2TTLOntologyConverter
+from data_handler import RDFDataHandler
+
+
+sys.path.append('shared_util')
+from myxmlwriter import xml2dict
+from main_util import get_manuscript_files_and_include_status
+
+__author__ = "Christian Steiner"
+__maintainer__ = __author__
+__copyright__ = 'University of Basel'
+__email__ = "christian.steiner@unibas.ch"
+__status__ = "Development"
+__license__ = "GPL v3"
+__version__ = "0.0.1"
+
+FILE_TYPE_XML_PROJECT = "xmlProjectFile"
+
+class Py2TTLCSVManuscriptDescriptionConverter:
+ """This class can be used convert manuscript description to rdf data in turtle format.
+ """
+ UNITTESTING = False
+ NAME = 'folio-name'
+ DESCRIPTION_KEY = 'description'
+ ALIAS = 'alias'
+ LABEL = 'label'
+ URL = 'nietzschesource-url'
+
+ def __init__(self, csv_file: str, faksimile_mapping: str, mapping_dictionary: str):
+ self.mapping_dictionary = mapping_dictionary
+ self.csv_file = csv_file
+ self.faksimile_mapping_dict = self._init_mapping(faksimile_mapping)
+
+ def _init_mapping(self, faksimile_mapping) ->dict:
+ """Init faksimile_mapping dictionary.
+ """
+ faksimile_mapping_dict = {}
+ with open(faksimile_mapping, newline='') as csvfile:
+ reader = csv.DictReader(csvfile)
+ fieldnames = reader.fieldnames
+ for row in reader:
+ if bool(faksimile_mapping_dict.get(row[fieldnames[1]])):
+ faksimile_mapping_dict[row[fieldnames[1]]].append(row[fieldnames[0]])
+ else:
+ faksimile_mapping_dict.update({ row[fieldnames[1]]: [row[fieldnames[0]]]})
+ return faksimile_mapping_dict
+
+ def convert(self, target_data_file):
+ """Convert manuscript decriptions from csv_file to rdf data and write to target_file.
+ """
+ not Py2TTLCSVManuscriptDescriptionConverter.UNITTESTING and print(Fore.CYAN + 'initializing python objects with file "{}" ...'.format(self.csv_file))
+ data_handler = RDFDataHandler(target_data_file, self.mapping_dictionary)
+ with open(self.csv_file, newline='') as csvfile:
+ reader = csv.DictReader(csvfile)
+ fieldnames = reader.fieldnames
+ leaf_id = ''
+ description = ''
+ images = []
+ counter = 0
+ for row in [ row for row in reader if '_thumb' not in row[self.LABEL]]:
+ if row[self.NAME] != leaf_id:
+ if leaf_id != '':
+ counter += 1
+ identifier_uri = data_handler.add_external_class_item('Leaf', '', leaf_id)
+ data_handler.add_external_property(identifier_uri, 'isDepictedBy', images)
+ manuscript_description = ManuscriptDescription.create_cls_from_raw_text(description)
+ data_handler.add_data_instance2parent_uri(identifier_uri, 'leafHasDescription', manuscript_description)
+ leaf_id = row[self.NAME]
+ description = row[self.DESCRIPTION_KEY]
+ images = []
+ if bool(self.faksimile_mapping_dict.get(row[self.LABEL])):
+ for image in self.faksimile_mapping_dict[row[self.LABEL]]:
+ images.append(image)
+ if len(images) > 0:
+ counter += 1
+ identifier_uri = data_handler.add_external_class_item('Leaf', '', leaf_id)
+ data_handler.add_external_property(identifier_uri, 'isDepictedBy', images)
+ manuscript_description = ManuscriptDescription.create_cls_from_raw_text(description)
+ data_handler.add_data_instance2parent_uri(identifier_uri, 'leafHasDescription', manuscript_description)
+ if not Py2TTLCSVManuscriptDescriptionConverter.UNITTESTING:
+ print(Fore.GREEN + f'[{counter} leafs added]')
+ if not Py2TTLCSVManuscriptDescriptionConverter.UNITTESTING:
+ print(Fore.CYAN + 'adding triples to rdf graph ... ')
+ if not Py2TTLCSVManuscriptDescriptionConverter.UNITTESTING:
+ print(Fore.GREEN + '[{} statements added]'.format(str(len(data_handler.data_graph))))
+ print(Fore.CYAN + 'writing graph to file "{}" ...'.format(target_data_file))
+ data_handler.write()
+ if not Py2TTLCSVManuscriptDescriptionConverter.UNITTESTING:
+ print(Fore.GREEN + '[OK]')
+ print(Style.RESET_ALL)
+
+def usage():
+ """prints information on how to use the script
+ """
+ print(main.__doc__)
+
+def main(argv):
+ """This program can be used to convert py objects to a owl:Ontology and rdf data in turtle format.
+
+ py2ttl/create_manuscript_description.py [OPTIONS] <csv-file> <mapping-csv>
+
+ <csv-file> csv file that contains the description of the manuscript.
+ <mapping-csv> a image to faksimile iri mapping csv file
+
+ OPTIONS:
+ -h|--help: show help
+ -o|--output=FILE specify output file
+
+ :return: exit code (int)
+ """
+ check_config_files_exist()
+ datatypes_dir = get_datatypes_dir()
+ source_ontology_file = PROJECT_ONTOLOGY_FILE
+ target_ontology_file = '.{0}{1}-ontology_autogenerated.ttl'.format(sep, PROJECT_NAME)
+ output_file = './include-ttl/descriptions.ttl'
+ try:
+ opts, args = getopt.getopt(argv, "ho:", ["help", "output="])
+ except getopt.GetoptError:
+ usage()
+ return 2
+ for opt, arg in opts:
+ if opt in ('-h', '--help'):
+ usage()
+ return 0
+ elif opt in ('-0', '--output'):
+ output_file = arg
+ if len(args) < 2 :
+ usage()
+ return 2
+ ontology_created = False
+ ontology_converter = Py2TTLOntologyConverter(project_ontology_file=source_ontology_file)
+ csv_file = args[0]
+ mapping_csv_file = args[1]
+ output = 2
+ print(Fore.CYAN + 'Create ontology ...')
+ if ontology_converter.create_ontology(datatypes_dir, target_ontology_file) == 0:
+ print(Fore.GREEN + '[Ontology file {0} created]'.format(target_ontology_file))
+ else:
+ return 2
+ print(Fore.CYAN + f'Create data from "{csv_file}" with ...')
+ data_converter = Py2TTLCSVManuscriptDescriptionConverter(csv_file, mapping_csv_file, mapping_dictionary=ontology_converter.uri_mapping4cls_and_properties)
+ output = data_converter.convert(output_file)
+ return output
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv[1:]))
Index: tests_py2ttl/test_data/mapping_dict.xml
===================================================================
--- tests_py2ttl/test_data/mapping_dict.xml (revision 113)
+++ tests_py2ttl/test_data/mapping_dict.xml (revision 114)
@@ -1,417 +1,421 @@
<?xml version="1.0" encoding="utf-8"?>
<root>
<dict>
<ontology>
<project_name type="str">tln</project_name>
<project_uri type="URIRef">http://www.nie.org/ontology/nietzsche#</project_uri>
<ontology_file type="str">./tln-ontology_autogenerated.ttl</ontology_file>
</ontology>
<classes>
<ManuscriptUnity>
<class_uri type="URIRef">http://www.nie.org/ontology/nietzsche#ManuscriptUnity</class_uri>
<properties>
<title type="URIRef">http://www.nie.org/ontology/nietzsche#hasTitle
http://www.nie.org/ontology/nietzsche#hasManuscriptType
http://www.nie.org/ontology/nietzsche#hasPages
http://www.nie.org/ontology/nietzsche#hasDescription
http://www.nie.org/ontology/nietzsche#ArchivalManuscriptUnity
http://www.nie.org/ontology/nietzsche#hasTitle
http://www.nie.org/ontology/nietzsche#hasManuscriptType
http://www.nie.org/ontology/nietzsche#hasPages
+ http://www.nie.org/ontology/nietzsche#archivalManuscriptUnityHasDataProcessingStatus
http://www.nie.org/ontology/nietzsche#hasStyles
http://www.nie.org/ontology/nietzsche#hasGsaSignature
http://www.nie.org/ontology/nietzsche#hasDescription
http://www.nie.org/ontology/nietzsche#partsBelongToReconstructedKonvolut
http://www.nie.org/ontology/nietzsche#hasEarlierDescriptions
http://www.nie.org/ontology/nietzsche#EditorComment
http://www.nie.org/ontology/nietzsche#isUncertain
http://www.nie.org/ontology/nietzsche#hasComment
http://www.nie.org/ontology/nietzsche#commentHasText
http://www.nie.org/ontology/nietzsche#AtypicalWriting
http://www.nie.org/ontology/nietzsche#isUncertain
http://www.nie.org/ontology/nietzsche#hasComment
http://www.nie.org/ontology/nietzsche#commentHasText
http://www.nie.org/ontology/nietzsche#Path
http://www.nie.org/ontology/nietzsche#hasDAttribute
http://www.nie.org/ontology/nietzsche#Box
http://www.nie.org/ontology/nietzsche#hasDAttribute
http://www.nie.org/ontology/nietzsche#hasEarlierText
http://www.nie.org/ontology/nietzsche#Clarification
http://www.nie.org/ontology/nietzsche#isUncertain
http://www.nie.org/ontology/nietzsche#hasComment
http://www.nie.org/ontology/nietzsche#commentHasText
http://www.nie.org/ontology/nietzsche#Color
http://www.nie.org/ontology/nietzsche#colorHasName
http://www.nie.org/ontology/nietzsche#hasHexadecimalValue
http://www.nie.org/ontology/nietzsche#Text
http://www.nie.org/ontology/nietzsche#textHasContent
http://www.nie.org/ontology/nietzsche#textHasMarkup
http://www.nie.org/ontology/nietzsche#Description
http://www.nie.org/ontology/nietzsche#textHasContent
http://www.nie.org/ontology/nietzsche#textHasMarkup
http://www.nie.org/ontology/nietzsche#EarlierDescription
http://www.nie.org/ontology/nietzsche#textHasContent
http://www.nie.org/ontology/nietzsche#hasAuthor
http://www.nie.org/ontology/nietzsche#hasCitation
http://www.nie.org/ontology/nietzsche#textHasMarkup
http://www.nie.org/ontology/nietzsche#EditorCorrection
http://www.nie.org/ontology/nietzsche#isUncertain
http://www.nie.org/ontology/nietzsche#hasComment
http://www.nie.org/ontology/nietzsche#hasCorrectionText
http://www.nie.org/ontology/nietzsche#commentHasText
http://www.nie.org/ontology/nietzsche#Image
http://www.nie.org/ontology/nietzsche#hasHeight
http://www.nie.org/ontology/nietzsche#hasWidth
http://www.nie.org/ontology/nietzsche#hasFileName
http://www.nie.org/ontology/nietzsche#hasTransform
http://www.nie.org/ontology/nietzsche#hasPrimaryurl
http://www.nie.org/ontology/nietzsche#hasSecondaryurl
http://www.nie.org/ontology/nietzsche#hasTextField
http://www.nie.org/ontology/nietzsche#FaksimileImage
http://www.nie.org/ontology/nietzsche#hasHeight
http://www.nie.org/ontology/nietzsche#hasWidth
http://www.nie.org/ontology/nietzsche#hasFileName
http://www.nie.org/ontology/nietzsche#hasTransform
http://www.nie.org/ontology/nietzsche#hasPrimaryurl
http://www.nie.org/ontology/nietzsche#hasSecondaryurl
http://www.nie.org/ontology/nietzsche#hasApiurl
http://www.nie.org/ontology/nietzsche#hasThumburl
http://www.nie.org/ontology/nietzsche#hasMediumurl
http://www.nie.org/ontology/nietzsche#hasTextField
http://www.nie.org/ontology/nietzsche#PositionalObject
http://www.nie.org/ontology/nietzsche#hasHeight
http://www.nie.org/ontology/nietzsche#hasWidth
http://www.nie.org/ontology/nietzsche#hasLeft
http://www.nie.org/ontology/nietzsche#hasTop
http://www.nie.org/ontology/nietzsche#hasBottom
http://www.nie.org/ontology/nietzsche#hasTransform
http://www.nie.org/ontology/nietzsche#WordPosition
http://www.nie.org/ontology/nietzsche#hasHeight
http://www.nie.org/ontology/nietzsche#hasWidth
http://www.nie.org/ontology/nietzsche#hasLeft
http://www.nie.org/ontology/nietzsche#hasTop
http://www.nie.org/ontology/nietzsche#hasBottom
http://www.nie.org/ontology/nietzsche#hasTransform
http://www.nie.org/ontology/nietzsche#FaksimilePosition
http://www.nie.org/ontology/nietzsche#hasHeight
http://www.nie.org/ontology/nietzsche#hasWidth
http://www.nie.org/ontology/nietzsche#hasLeft
http://www.nie.org/ontology/nietzsche#hasTop
http://www.nie.org/ontology/nietzsche#hasBottom
http://www.nie.org/ontology/nietzsche#hasTransform
http://www.nie.org/ontology/nietzsche#Imprint
http://www.nie.org/ontology/nietzsche#imprintHasReference
http://www.nie.org/ontology/nietzsche#imprintRefersToLines
http://www.nie.org/ontology/nietzsche#Line
http://www.nie.org/ontology/nietzsche#lineHasNumber
http://www.nie.org/ontology/nietzsche#lineHasBottomValueOnTranskription
http://www.nie.org/ontology/nietzsche#lineHasTopValueOnTranskription
http://www.nie.org/ontology/nietzsche#lineHasInnerBottomValueOnFaksimile
http://www.nie.org/ontology/nietzsche#lineHasInnerTopValueOnFaksimile
http://www.nie.org/ontology/nietzsche#lineHasOuterBottomValueOnFaksimile
http://www.nie.org/ontology/nietzsche#lineHasOuterTopValueOnFaksimile
http://www.nie.org/ontology/nietzsche#isMainLine
http://www.nie.org/ontology/nietzsche#lineHasEditorComment
http://www.nie.org/ontology/nietzsche#LineContinuation
http://www.nie.org/ontology/nietzsche#isUncertain
http://www.nie.org/ontology/nietzsche#hasComment
http://www.nie.org/ontology/nietzsche#isLineAContinuationTo
http://www.nie.org/ontology/nietzsche#commentHasText
http://www.nie.org/ontology/nietzsche#lineContinuationHasReference
http://www.nie.org/ontology/nietzsche#SimpleWord
http://www.nie.org/ontology/nietzsche#hasText
http://www.nie.org/ontology/nietzsche#hasCleanText
http://www.nie.org/ontology/nietzsche#wordBelongsToLine
http://www.nie.org/ontology/nietzsche#hasTranskriptionPosition
http://www.nie.org/ontology/nietzsche#hasFaksimilePosition
http://www.nie.org/ontology/nietzsche#SpecialWord
http://www.nie.org/ontology/nietzsche#hasText
http://www.nie.org/ontology/nietzsche#hasCleanText
http://www.nie.org/ontology/nietzsche#wordBelongsToLine
http://www.nie.org/ontology/nietzsche#hasTranskriptionPosition
http://www.nie.org/ontology/nietzsche#hasFaksimilePosition
http://www.nie.org/ontology/nietzsche#MarkForeignHands
http://www.nie.org/ontology/nietzsche#hasText
http://www.nie.org/ontology/nietzsche#hasCleanText
http://www.nie.org/ontology/nietzsche#penOfForeignHands
http://www.nie.org/ontology/nietzsche#resolutionOfAbbreviation
http://www.nie.org/ontology/nietzsche#foreignHandHasCommentByEditors
http://www.nie.org/ontology/nietzsche#wordBelongsToLine
http://www.nie.org/ontology/nietzsche#hasTranskriptionPosition
http://www.nie.org/ontology/nietzsche#hasFaksimilePosition
http://www.nie.org/ontology/nietzsche#textOfForeignHands
+ http://www.nie.org/ontology/nietzsche#foreignHandOverwritesForeignHand
http://www.nie.org/ontology/nietzsche#Page
http://www.nie.org/ontology/nietzsche#hasNumber
+ http://www.nie.org/ontology/nietzsche#pageHasDataProcessingStatus
http://www.nie.org/ontology/nietzsche#hasOrientation
http://www.nie.org/ontology/nietzsche#hasLines
http://www.nie.org/ontology/nietzsche#hasImprints
http://www.nie.org/ontology/nietzsche#hasMarkForeignHands
http://www.nie.org/ontology/nietzsche#hasWords
http://www.nie.org/ontology/nietzsche#hasWordDeletionPaths
http://www.nie.org/ontology/nietzsche#hasWordInsertionMarks
http://www.nie.org/ontology/nietzsche#hasEditorComments
http://www.nie.org/ontology/nietzsche#hasFaksimileImage
http://www.nie.org/ontology/nietzsche#pageIsOnSVGTextField
http://www.nie.org/ontology/nietzsche#pageIsOnFaksimileTextField
http://www.nie.org/ontology/nietzsche#hasSvgImage
http://www.nie.org/ontology/nietzsche#NonExistentPage
http://www.nie.org/ontology/nietzsche#hasNumber
+ http://www.nie.org/ontology/nietzsche#pageHasDataProcessingStatus
http://www.nie.org/ontology/nietzsche#hasOrientation
http://www.nie.org/ontology/nietzsche#hasLines
http://www.nie.org/ontology/nietzsche#hasImprints
http://www.nie.org/ontology/nietzsche#hasMarkForeignHands
http://www.nie.org/ontology/nietzsche#hasWords
http://www.nie.org/ontology/nietzsche#hasWordDeletionPaths
http://www.nie.org/ontology/nietzsche#hasWordInsertionMarks
http://www.nie.org/ontology/nietzsche#hasEditorComments
- http://www.nie.org/ontology/nietzsche#hasStatus
http://www.nie.org/ontology/nietzsche#hasFaksimileImage
http://www.nie.org/ontology/nietzsche#pageIsOnSVGTextField
http://www.nie.org/ontology/nietzsche#pageIsOnFaksimileTextField
http://www.nie.org/ontology/nietzsche#hasSvgImage
http://www.nie.org/ontology/nietzsche#ReconstructedKonvolut
http://www.nie.org/ontology/nietzsche#hasTitle
http://www.nie.org/ontology/nietzsche#hasManuscriptType
http://www.nie.org/ontology/nietzsche#hasPages
http://www.nie.org/ontology/nietzsche#hasDescription
http://www.nie.org/ontology/nietzsche#Reference
http://www.nie.org/ontology/nietzsche#firstLineOfReference
http://www.nie.org/ontology/nietzsche#lastLineOfReference
http://www.nie.org/ontology/nietzsche#wordReference
http://www.nie.org/ontology/nietzsche#IsUncertain
http://www.nie.org/ontology/nietzsche#hasTitle
http://www.nie.org/ontology/nietzsche#hasPageNumber
http://www.nie.org/ontology/nietzsche#SVGImage
http://www.nie.org/ontology/nietzsche#hasHeight
http://www.nie.org/ontology/nietzsche#hasWidth
http://www.nie.org/ontology/nietzsche#hasFileName
http://www.nie.org/ontology/nietzsche#hasTransform
http://www.nie.org/ontology/nietzsche#hasPrimaryurl
http://www.nie.org/ontology/nietzsche#hasSecondaryurl
http://www.nie.org/ontology/nietzsche#hasTextField
http://www.nie.org/ontology/nietzsche#StandoffTag
http://www.nie.org/ontology/nietzsche#standoffTagHasStartIndex
http://www.nie.org/ontology/nietzsche#standoffTagHasEndIndex
http://www.nie.org/ontology/nietzsche#standoffTagHasCSS
http://www.nie.org/ontology/nietzsche#TextConnectionMark
http://www.nie.org/ontology/nietzsche#hasText
http://www.nie.org/ontology/nietzsche#hasCleanText
http://www.nie.org/ontology/nietzsche#wordBelongsToLine
http://www.nie.org/ontology/nietzsche#hasTranskriptionPosition
http://www.nie.org/ontology/nietzsche#hasFaksimilePosition
http://www.nie.org/ontology/nietzsche#textConnectionMarkHasTextSource
http://www.nie.org/ontology/nietzsche#TextField
http://www.nie.org/ontology/nietzsche#hasHeight
http://www.nie.org/ontology/nietzsche#hasWidth
http://www.nie.org/ontology/nietzsche#hasLeft
http://www.nie.org/ontology/nietzsche#hasTop
http://www.nie.org/ontology/nietzsche#hasBottom
http://www.nie.org/ontology/nietzsche#hasTransform
http://www.nie.org/ontology/nietzsche#TranskriptionPosition
http://www.nie.org/ontology/nietzsche#hasHeight
http://www.nie.org/ontology/nietzsche#hasWidth
http://www.nie.org/ontology/nietzsche#hasLeft
http://www.nie.org/ontology/nietzsche#hasTop
http://www.nie.org/ontology/nietzsche#hasBottom
http://www.nie.org/ontology/nietzsche#hasTransform
http://www.nie.org/ontology/nietzsche#UncertainDecipherment
http://www.nie.org/ontology/nietzsche#isUncertain
http://www.nie.org/ontology/nietzsche#hasComment
http://www.nie.org/ontology/nietzsche#commentHasText
http://www.nie.org/ontology/nietzsche#Word
http://www.nie.org/ontology/nietzsche#hasText
http://www.nie.org/ontology/nietzsche#hasCleanText
http://www.nie.org/ontology/nietzsche#hasEditedText
http://www.nie.org/ontology/nietzsche#hasCleanEditedText
http://www.nie.org/ontology/nietzsche#wordHasWordParts
http://www.nie.org/ontology/nietzsche#wordBelongsToLine
http://www.nie.org/ontology/nietzsche#hasTranskriptionPosition
http://www.nie.org/ontology/nietzsche#hasFaksimilePosition
http://www.nie.org/ontology/nietzsche#wordHasStyle
http://www.nie.org/ontology/nietzsche#overwritesWord
+ http://www.nie.org/ontology/nietzsche#undosCorrectionOfWord
http://www.nie.org/ontology/nietzsche#isTransformationOfWord
http://www.nie.org/ontology/nietzsche#isExtensionOfWord
http://www.nie.org/ontology/nietzsche#isDeletionOfWord
http://www.nie.org/ontology/nietzsche#isClarificationOfWord
http://www.nie.org/ontology/nietzsche#wordHasEarlierVersion
http://www.nie.org/ontology/nietzsche#wordHasCorrection
http://www.nie.org/ontology/nietzsche#wordIsUndeletedFromPath
http://www.nie.org/ontology/nietzsche#wordIsDeletedByPath
http://www.nie.org/ontology/nietzsche#wordHasEditorComment
http://www.nie.org/ontology/nietzsche#hasStandoffMarkup4PartThatOverwritesWord
http://www.nie.org/ontology/nietzsche#WordDeletionPath
http://www.nie.org/ontology/nietzsche#hasDAttribute
http://www.nie.org/ontology/nietzsche#WordInsertionMark
http://www.nie.org/ontology/nietzsche#hasHeight
http://www.nie.org/ontology/nietzsche#hasWidth
http://www.nie.org/ontology/nietzsche#hasLeft
http://www.nie.org/ontology/nietzsche#hasTop
http://www.nie.org/ontology/nietzsche#hasBottom
http://www.nie.org/ontology/nietzsche#hasTransform
http://www.nie.org/ontology/nietzsche#hasMarkType
http://www.nie.org/ontology/nietzsche#hasSymbolId
http://www.nie.org/ontology/nietzsche#hasNextWord
http://www.nie.org/ontology/nietzsche#hasPreviousWord
http://www.nie.org/ontology/nietzsche#wordInsertionMarkBelongsToLine
xml-dictionary
- 2021-09-30 15:34:35
+ 2021-12-14 11:26:46
Index: tests_py2ttl/test_data/descriptions.csv
===================================================================
--- tests_py2ttl/test_data/descriptions.csv (revision 0)
+++ tests_py2ttl/test_data/descriptions.csv (revision 114)
@@ -0,0 +1,10 @@
+folio-name,label,alias,nietzschesource-url,license,description
+"Mp-XIV,416_417","Mp-XIV-1,416.jpg","Mp XIV, 416","http://www.nietzschesource.org/DFGAapi/api/page/download/Mp-XIV-1,416",CC BY-NC-ND 4.0,"Heftblatt, 17,4x22, liniiert; Rekto links beschnitten. Ursprünglich gegenüberliegende Seiten: 418 / 416, 417 / Mp XVI, 21ar."
+"Mp-XIV,416_417","Mp-XIV-1,416_thumb.jpg","Mp XIV, 416","http://www.nietzschesource.org/DFGAapi/images/DFGA/Mp-XIV-1/mini/Mp-XIV-1,416.jpg",CC BY-NC-ND 4.0,"Heftblatt, 17,4x22, liniiert; Rekto links beschnitten. Ursprünglich gegenüberliegende Seiten: 418 / 416, 417 / Mp XVI, 21ar."
+"Mp-XIV,416_417","Mp-XIV-1,417.jpg","Mp XIV, 417","http://www.nietzschesource.org/DFGAapi/api/page/download/Mp-XIV-1,417",CC BY-NC-ND 4.0,"Heftblatt, 17,4x22, liniiert; Rekto links beschnitten. Ursprünglich gegenüberliegende Seiten: 418 / 416, 417 / Mp XVI, 21ar."
+"Mp-XIV,416_417","Mp-XIV-1,417_thumb.jpg","Mp XIV, 417","http://www.nietzschesource.org/DFGAapi/images/DFGA/Mp-XIV-1/mini/Mp-XIV-1,417.jpg",CC BY-NC-ND 4.0,"Heftblatt, 17,4x22, liniiert; Rekto links beschnitten. Ursprünglich gegenüberliegende Seiten: 418 / 416, 417 / Mp XVI, 21ar."
+"Mp-XIV,418_419","Mp-XIV-1,418.jpg","Mp XIV, 418","http://www.nietzschesource.org/DFGAapi/api/page/download/Mp-XIV-1,418",CC BY-NC-ND 4.0,"Heftblatt, 17,4x22, liniiert; Rekto rechts beschnitten. Ursprünglich gegenüberliegende Seiten: 418 / 416, 419a / 419."
+"Mp-XIV,418_419","Mp-XIV-1,418_thumb.jpg","Mp XIV, 418","http://www.nietzschesource.org/DFGAapi/images/DFGA/Mp-XIV-1/mini/Mp-XIV-1,418.jpg",CC BY-NC-ND 4.0,"Heftblatt, 17,4x22, liniiert; Rekto rechts beschnitten. Ursprünglich gegenüberliegende Seiten: 418 / 416, 419a / 419."
+"Mp-XIV,419a_419b","Mp-XIV-1,419a.jpg","Mp XIV, 419a","http://www.nietzschesource.org/DFGAapi/api/page/download/Mp-XIV-1,419a",CC BY-NC-ND 4.0,"Heftblatt, 17,4x22, liniiert; Rekto rechts beschnitten. Ursprünglich gegenüberliegende Seiten: 419a / 419."
+"Mp-XIV,419a_419b","Mp-XIV-1,419a_thumb.jpg","Mp XIV, 419a","http://www.nietzschesource.org/DFGAapi/images/DFGA/Mp-XIV-1/mini/Mp-XIV-1,419a.jpg",CC BY-NC-ND 4.0,"Heftblatt, 17,4x22, liniiert; Rekto rechts beschnitten. Ursprünglich gegenüberliegende Seiten: 419a / 419."
+"Mp-XIV,419a_419b","Mp-XIV-1,419b.jpg","Mp XIV, 419b","http://www.nietzschesource.org/DFGAapi/api/page/download/Mp-XIV-1,419b",CC BY-NC-ND 4.0,"Heftblatt, 17,4x22, liniiert; Rekto rechts beschnitten. Ursprünglich gegenüberliegende Seiten: 419a / 419."
Index: tests_py2ttl/test_data_handler.py
===================================================================
--- tests_py2ttl/test_data_handler.py (revision 113)
+++ tests_py2ttl/test_data_handler.py (revision 114)
@@ -1,55 +1,57 @@
import unittest
from os import sep, path
from os.path import dirname, isfile
import inspect
from rdflib import Graph, URIRef, Literal, BNode, OWL, RDF, RDFS, XSD
import rdflib
import sys
sys.path.append('shared_util')
from myxmlwriter import xml2dict
sys.path.append('svgscripts')
from datatypes.image import Image
from datatypes.page import Page
from datatypes.archival_manuscript import ArchivalManuscriptUnity
from datatypes.text_field import TextField
sys.path.append('py2ttl')
from data_handler import RDFDataHandler
class TestRDFDataHandler(unittest.TestCase):
def setUp(self):
RDFDataHandler.UNITTESTING = True
DATADIR = dirname(__file__) + sep + 'test_data'
self.mapping_dictionary = xml2dict(DATADIR + sep + 'mapping_dict.xml')
self.xml_file = DATADIR + sep + 'N_VII_1_page001.xml'
self.manuscript_file = DATADIR + sep + 'N_VII_1.xml'
def test_add_data(self):
data_handler = RDFDataHandler('test.ttl', self.mapping_dictionary)
page = Page(self.xml_file)
data_handler.add_data(page, page.title.replace(' ', '_'))
manuscript = ArchivalManuscriptUnity.create_cls(self.manuscript_file)
data_handler.add_data(page, '', skip_data_instance_list=['Page'])
#print(data_handler.data_graph.serialize(format="turtle"))
#data_handler.write()
+ #manuscript = ArchivalManuscriptUnity.create_cls('xml/Mp_XVIII.xml')
+ #data_handler.add_data(manuscript, '')
def test_init(self):
with self.assertRaises(Exception):
RDFDataHandler(None, {})
mapping_dictionary = { 'ontology': { 'project_name': 'test', 'project_uri': 'test' }}
data_handler = RDFDataHandler('test.ttl', mapping_dictionary)
self.assertEqual(data_handler.project_name, 'test')
def test_create_identifier_uri(self):
tf = TextField()
mapping_dictionary = { 'ontology': { 'project_name': 'test', 'project_uri': 'test' }}
data_handler = RDFDataHandler('test.ttl', mapping_dictionary)
identifier_uri = data_handler.create_identifier_uri(tf, 'asdf')
data_handler.data_graph.add((identifier_uri, RDF.type, OWL.Class))
next_identifier_uri = data_handler.create_identifier_uri(tf, 'asdf')
self.assertEqual(identifier_uri != next_identifier_uri, True)
if __name__ == "__main__":
unittest.main()
Index: tests_py2ttl/test_create_manuscript_description.py
===================================================================
--- tests_py2ttl/test_create_manuscript_description.py (revision 0)
+++ tests_py2ttl/test_create_manuscript_description.py (revision 114)
@@ -0,0 +1,30 @@
+import unittest
+from os import sep, path
+import lxml.etree as ET
+import sys
+
+sys.path.append('py2ttl')
+import config
+
+class TestConfig(unittest.TestCase):
+ def test_check_config_files_exist(self):
+ self.assertEqual(config.check_config_files_exist(), 0)
+ dir_saved = config.__dict__.get('ONTOLOGY_DIR')
+ config.__dict__['ONTOLOGY_DIR'] = __file__
+ with self.assertRaises(NotADirectoryError):
+ config.check_config_files_exist()
+ config.__dict__['ONTOLOGY_DIR'] = 'asf'
+ with self.assertRaises(FileNotFoundError):
+ config.check_config_files_exist()
+ config.__dict__['ONTOLOGY_DIR'] = dir_saved
+
+ def test_get_datatypes_dir(self):
+ self.assertEqual(config.get_datatypes_dir(), config.DATATYPES_DIR.replace('./', ''))
+ dir_saved = config.__dict__.get('DATATYPES_DIR')
+ del config.__dict__['DATATYPES_DIR']
+ self.assertEqual(config.get_datatypes_dir(), None)
+ config.__dict__['DATATYPES_DIR'] = dir_saved
+
+
+if __name__ == "__main__":
+ unittest.main()
Druckvorlage GM III 1
+
+
drittenAbhandlung. +
+
Was bedeuten asketische Ideale? — Bei Künstlern + Nichts oder zu Vielerlei; bei Philo=
sophen und Gelehrten Etwas + wie Witterung und Instinkt für die günstigsten Vorbedingungen + hoher Geistigkeit; bei Frauen, besten Falls, eine + Liebenswürdigkeit der Verführung
mehr, + ein wenig morbidezza auf + schönem Fleische, die Engelhaftigkeit eines hübschen fetten + Thiers; bei physiologisch Verunglückten und Verstimmten (bei + der
Mehrzahlder Sterblichen) einen + Versuch, sich „zu gut“ für diese Welt vorzukommen, eine + heilige Form der Ausschweifung, ihr Hauptmittel im Kampf mit + dem langsamen Schmerz und der Langenweile; bei Priestern den + eigentlichen Priesterglauben, ihr bestes Werkzeug der Macht, + auch die „allerhöchste“ Erlaubniss zur Macht; bei Heiligen + endlich einen Vorwand zum Winterschlaf, ihre novissima + gloriae cupido, ihre Ruhe im Nichts („Gott“), ihre Form des + Irrsinns.
Dassaber überhaupt das + asketische Ideal dem Menschen so viel bedeutet hat, darin + drückt sich die Grundthatsache des menschlichen Willens aus, + sein horror vacui:
er braucht ein Ziel, + — und eher will er noch
das Nichts+ wollen, als
nichtwollen. — Versteht man mich?… + Hat man mich verstanden?… +
„Schlechterdings nicht! mein Herr!“+ — Fangen wir also von vorne an. + + + +