batch_engine.py
No OneTemporary
Actions

Subscribers

None

File Metadata

Created: Mon, Nov 18, 22:14

batch_engine.py
View Options

	# -- coding: utf-8 --
	#
	# This file is part of Invenio.
	# Copyright (C) 2011 CERN.
	#
	# Invenio is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public License as
	# published by the Free Software Foundation; either version 2 of the
	# License, or (at your option) any later version.
	#
	# Invenio is distributed in the hope that it will be useful, but
	# WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	# General Public License for more details.
	#
	# You should have received a copy of the GNU General Public License
	# along with Invenio; if not, write to the Free Software Foundation, Inc.,
	# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

	"""Bibencode batch processing submodule"""

	from string import Template
	from pprint import pprint
	import os
	import shutil
	import uuid
	from pprint import pformat

	from invenio.legacy.bibsched.bibtask import (
	task_update_progress,
	write_message,
	task_low_level_submission
	)
	from invenio.legacy.bibdocfile.api import BibRecDocs, compose_file, compose_format, decompose_file
	from invenio.legacy.search_engine import (
	record_exists,
	get_collection_reclist,
	search_pattern,
	get_fieldvalues
	)
	from invenio.modules.encoder.encode import encode_video, assure_quality
	from invenio.modules.encoder.extract import extract_frames
	from invenio.modules.encoder.profiles import (
	get_encoding_profile,
	get_extract_profile
	)
	from invenio.legacy.bibdocfile.cli import cli_fix_marc
	from invenio.modules.encoder.utils import chose2
	from invenio.modules.encoder.metadata import (
	pbcore_metadata
	)
	from invenio.modules.encoder.utils import getval, chose2, generate_timestamp
	from invenio.modules.encoder.config import (
	CFG_BIBENCODE_DAEMON_DIR_NEWJOBS,
	CFG_BIBENCODE_PBCORE_MARC_XSLT,
	CFG_BIBENCODE_ASPECT_RATIO_MARC_FIELD
	)
	from invenio.ext.email import send_email
	from invenio.base.i18n import gettext_set_language
	from invenio.legacy.webuser import emailUnique, get_user_preferences
	from invenio.modules.formatter.engines.xslt import format
	from invenio.utils.json import json, json_decode_file
	import invenio.config

	# Stored messages for email notifications
	global _BATCH_STEP, _BATCH_STEPS
	_BATCH_STEP = 1
	_BATCH_STEPS = 1
	global _MSG_HISTORY, _UPD_HISTORY
	_MSG_HISTORY = []
	_UPD_HISTORY = []

	def _notify_error_admin(batch_job,
	email_admin=invenio.config.CFG_SITE_ADMIN_EMAIL):
	"""Sends a notification email to the specified address, containing
	admin-only information. Is called by process_batch_job() if an error
	occured during the processing.
	@param email_admin: email address of the admin
	@type email_admin: string
	"""
	if not email_admin:
	return
	template = ("BibEncode batch processing has reported an error during the"
	"execution of a job within the batch description <br/><br/>"
	"This is the batch description: <br/><br/>"
	"%(batch_description)s <br/><br/>"
	"This is the message log: <br/><br/>"
	"%(message_log)s")
	html_text = template % {"batch_description": pformat(batch_job).replace("\n", "<br/>"),
	"message_log": "\n".join(_MSG_HISTORY)}
	text = html_text.replace("<br/>", "\n")
	send_email(fromaddr=invenio.config.CFG_SITE_ADMIN_EMAIL,
	toaddr=email_admin,
	subject="Error during BibEncode batch processing",
	content=text,
	html_content=html_text)

	def _notify_error_user(email_user, original_filename, recid, submission_title, ln=invenio.config.CFG_SITE_LANG):
	"""Sends an error notification to the specified address of the user.
	Is called by process_batch_job() if an error occured during the processing.
	@param email_user: email address of the user
	@type email_user: string
	@param email_admin: email address of the admin
	@type email_admin: string
	"""
	if not email_user:
	return
	uid = emailUnique(email_user)
	if uid != -1 and uid != 0:
	language = getval(get_user_preferences(uid), "language")
	if language:
	ln = language
	_ = gettext_set_language(ln)
	rec_url = invenio.config.CFG_SITE_URL + "/record/" + str(recid)
	template = ("<br/>" +
	_("We are sorry, a problem has occured during the processing of"
	" your video upload%(submission_title)s.") +
	"<br/><br/>" +
	_("The file you uploaded was %(input_filename)s.") +
	"<br/><br/>" +
	_("Your video might not be fully available until intervention.") +
	"<br/>" +
	_("You can check the status of your video here: %(record_url)s.") +
	"<br/>" +
	_("You might want to take a look at "
	" %(guidelines_url)s"
	" and modify or redo your submission."))
	text = template % {"input_filename": "%s" % original_filename,
	"submission_title": " %s" % submission_title,
	"record_url": "%s" % rec_url,
	"guidelines_url": "localhost"}
	text = text.replace("<br/>", "\n")
	html_text = template % {"input_filename": "<strong>%s</strong>" % original_filename,
	"submission_title": " <strong>%s</strong>" % submission_title,
	"record_url": "<a href=\"%s\">%s</a>" % (rec_url, rec_url),
	"guidelines_url": "<a href=\"locahost\">%s</a>" % _("the video guidelines")}
	send_email(fromaddr=invenio.config.CFG_SITE_ADMIN_EMAIL,
	toaddr=email_user,
	subject="Problem during the processing of your video",
	content=text,
	html_content=html_text
	)

	def _notify_success_user(email_user, original_filename, recid, submission_title, ln=invenio.config.CFG_SITE_LANG):
	"""Sends an success notification to the specified address of the user.
	Is called by process_batch_job() if the processing was successful.
	@param email_user: email address of the user
	@type email_user: string
	@param email_admin: email address of the admin
	@type email_admin: string
	"""
	uid = emailUnique(email_user)
	if uid != -1 and uid != 0:
	language = getval(get_user_preferences(uid), "language")
	if language:
	ln = language
	_ = gettext_set_language(ln)
	rec_url = invenio.config.CFG_SITE_URL + "/record/" + str(recid)
	template = ("<br/>" +
	_("Your video submission%(submission_title)s was successfully processed.") +
	"<br/><br/>" +
	_("The file you uploaded was %(input_filename)s.") +
	"<br/><br/>" +
	_("Your video is now available here: %(record_url)s.") +
	"<br/>" +
	_("If the videos quality is not as expected, you might want to take "
	"a look at %(guidelines_url)s"
	" and modify or redo your submission."))
	text = template % {"input_filename": "%s" % original_filename,
	"submission_title": " %s" % submission_title,
	"record_url": "%s" % rec_url,
	"guidelines_url": "localhost"}
	text = text.replace("<br/>", "\n")
	html_text = template % {"input_filename": "<strong>%s</strong>" % original_filename,
	"submission_title": " <strong>%s</strong>" % submission_title,
	"record_url": "<a href=\"%s\">%s</a>" % (rec_url, rec_url),
	"guidelines_url": "<a href=\"locahost\">%s</a>" % _("the video guidelines")}
	send_email(fromaddr=invenio.config.CFG_SITE_ADMIN_EMAIL,
	toaddr=email_user,
	subject="Your video submission is now complete",
	content=text,
	html_content=html_text
	)

	def _task_update_overall_status(message):
	""" Generates an overall update message for the BibEncode task.
	Stores the messages in a global list for notifications
	@param message: the message that should be printed as task status
	@type message: string
	"""
	message = "[%d/%d]%s" % (_BATCH_STEP, _BATCH_STEPS, message)
	task_update_progress(message)
	global _UPD_HISTORY
	_UPD_HISTORY.append(message)

	def _task_write_message(message):
	""" Stores the messages in a global list for notifications
	@param message: the message that should be printed as task status
	@type message: string
	"""
	write_message(message)
	global _MSG_HISTORY
	_MSG_HISTORY.append(message)

	def clean_job_for_quality(batch_job_dict, fallback=True):
	"""
	Removes jobs from the batch description that are not suitable for the master
	video's quality. It applies only for encoding jobs!
	@param batch_job_dict: the dict containing the batch description
	@type batch_job_dict: dict
	@param
	@return: the cleaned dict
	@rtype: dict
	"""
	survived_jobs = []
	fallback_jobs = []
	other_jobs = []
	for job in batch_job_dict['jobs']:
	if job['mode'] == 'encode':
	if getval(job, 'fallback') and fallback:
	fallback_jobs.append(job)
	if getval(job, 'enforce'):
	survived_jobs.append(job)
	else:
	profile = None
	if getval(job, 'profile'):
	profile = get_encoding_profile(job['profile'])
	if assure_quality(input_file=batch_job_dict['input'],
	aspect=chose2('aspect', job, profile),
	target_width=chose2('width', job, profile),
	target_height=chose2('height', job, profile),
	target_bitrate=chose2('videobitrate', job, profile)):
	survived_jobs.append(job)
	else:
	other_jobs.append(job)
	if survived_jobs:
	survived_jobs.extend(other_jobs)
	new_jobs = survived_jobs
	else:
	fallback_jobs.extend(other_jobs)
	new_jobs = fallback_jobs
	pprint(locals())
	batch_job_dict['jobs'] = new_jobs
	return batch_job_dict

	def create_update_jobs_by_collection(
	batch_template_file,
	collection,
	job_directory=CFG_BIBENCODE_DAEMON_DIR_NEWJOBS):
	""" Creates the job description files to update a whole collection
	@param batch_template_file: fullpath to the template for the update
	@type batch_tempalte_file: string
	@param collection: name of the collection that should be updated
	@type collection: string
	@param job_directory: fullpath to the directory storing the job files
	@type job_directory: string
	"""
	recids = get_collection_reclist(collection)
	return create_update_jobs_by_recids(recids, batch_template_file,
	job_directory)

	def create_update_jobs_by_search(pattern,
	batch_template_file,
	job_directory=CFG_BIBENCODE_DAEMON_DIR_NEWJOBS
	):
	""" Creates the job description files to update all records that fit a
	search pattern. Be aware of the search limitations!
	@param search_pattern: The pattern to search for
	@type search_pattern: string
	@param batch_template_file: fullpath to the template for the update
	@type batch_tempalte_file: string
	@param job_directory: fullpath to the directory storing the job files
	@type job_directory: string
	"""
	recids = search_pattern(p=pattern)
	return create_update_jobs_by_recids(recids, batch_template_file,
	job_directory)

	def create_update_jobs_by_recids(recids,
	batch_template_file,
	job_directory=CFG_BIBENCODE_DAEMON_DIR_NEWJOBS
	):
	""" Creates the job description files to update all given recids
	@param recids: Iterable set of recids
	@type recids: iterable
	@param batch_template_file: fullpath to the template for the update
	@type batch_tempalte_file: string
	@param job_directory: fullpath to the directory storing the job files
	@type job_directory: string
	"""
	batch_template = json_decode_file(batch_template_file)
	for recid in recids:
	task_update_progress("Creating Update Job for %d" % recid)
	write_message("Creating Update Job for %d" % recid)
	job = dict(batch_template)
	job['recid'] = recid
	timestamp = generate_timestamp()
	job_filename = "update_%d_%s.job" % (recid, timestamp)
	create_job_from_dictionary(job, job_filename, job_directory)
	return 1

	def create_job_from_dictionary(
	job_dict,
	job_filename=None,
	job_directory=CFG_BIBENCODE_DAEMON_DIR_NEWJOBS
	):
	""" Creates a job from a given dictionary
	@param job_dict: Dictionary that contains the job description
	@type job_dict: job_dict
	@param job_filename: Filename for the job
	@type job_filename: string
	@param job_directory: fullpath to the directory storing the job files
	@type job_directory: string
	"""
	if not job_filename:
	job_filename = str(uuid.uuid4())
	if not job_filename.endswith(".job"):
	job_filename += ".job"
	job_fullpath = os.path.join(job_directory, job_filename)
	job_string = json.dumps(job_dict, sort_keys=False, indent=4)
	file = open(job_fullpath, "w")
	file.write(job_string)
	file.close()

	def sanitise_batch_job(batch_job):
	""" Checks the correctness of the batch job dictionary and additionally
	sanitises some values.
	@param batch_job: The batch description dictionary
	@type batch_job: dictionary
	"""
	def san_bitrate(bitrate):
	""" Sanitizes bitrates
	"""
	if type(str()) == type(bitrate):
	if bitrate.endswith('k'):
	try:
	bitrate = int(bitrate[:-1])
	bitrate *= 1000
	return bitrate
	except ValueError:
	raise Exception("Could not parse bitrate")
	elif type(int) == type(bitrate):
	return bitrate
	else:
	raise Exception("Could not parse bitrate")

	if not getval(batch_job, 'update_from_master'):
	if not getval(batch_job, 'input'):
	raise Exception("No input file in batch description")

	if not getval(batch_job, 'recid'):
	raise Exception("No recid in batch description")

	if not getval(batch_job, 'jobs'):
	raise Exception("No job list in batch description")

	if getval(batch_job, 'update_from_master'):
	if (not getval(batch_job, 'bibdoc_master_comment') and
	not getval(batch_job, 'bibdoc_master_description') and
	not getval(batch_job, 'bibdoc_master_subformat')):
	raise Exception("If update_from_master ist set, a comment or"
	" description or subformat for matching must be given")

	if getval(batch_job, 'marc_snippet'):
	if not os.path.exists(getval(batch_job, 'marc_snippet')):
	raise Exception("The marc snipped file %s was not found" %
	getval(batch_job, 'marc_snippet'))

	for job in batch_job['jobs']:
	if job['mode'] == 'encode':
	if getval(job, 'videobitrate'):
	job['videobitrate'] = san_bitrate(getval(job, 'videobitrate'))
	if getval(job, 'audiobitrate'):
	job['audiobitrate'] = san_bitrate(getval(job, 'audiobitrate'))

	return batch_job

	def process_batch_job(batch_job_file):
	""" Processes a batch job description dictionary

	@param batch_job_file: a fullpath to a batch job file
	@type batch_job_file: string
	@return: 1 if the process was successful, 0 if not
	@rtype; int
	"""

	def upload_marcxml_file(marcxml):
	""" Creates a temporary marcxml file and sends it to bibupload
	"""
	xml_filename = 'bibencode_'+ str(batch_job['recid']) + '_' + str(uuid.uuid4()) + '.xml'
	xml_filename = os.path.join(invenio.config.CFG_TMPSHAREDDIR, xml_filename)
	xml_file = file(xml_filename, 'w')
	xml_file.write(marcxml)
	xml_file.close()
	targs = ['-c', xml_filename]
	task_low_level_submission('bibupload', 'bibencode', *targs)

	#---------#
	# GENERAL #
	#---------#

	_task_write_message("----------- Handling Master -----------")

	## Check the validity of the batch file here
	batch_job = json_decode_file(batch_job_file)

	## Sanitise batch description and raise errrors
	batch_job = sanitise_batch_job(batch_job)

	## Check if the record exists
	if record_exists(batch_job['recid']) < 1:
	raise Exception("Record not found")

	recdoc = BibRecDocs(batch_job['recid'])

	#--------------------#
	# UPDATE FROM MASTER #
	#--------------------#

	## We want to add new stuff to the video's record, using the master as input
	if getval(batch_job, 'update_from_master'):
	found_master = False
	bibdocs = recdoc.list_bibdocs()
	for bibdoc in bibdocs:
	bibdocfiles = bibdoc.list_all_files()
	for bibdocfile in bibdocfiles:
	comment = bibdocfile.get_comment()
	description = bibdocfile.get_description()
	subformat = bibdocfile.get_subformat()
	m_comment = getval(batch_job, 'bibdoc_master_comment', comment)
	m_description = getval(batch_job, 'bibdoc_master_description', description)
	m_subformat = getval(batch_job, 'bibdoc_master_subformat', subformat)
	if (comment == m_comment and
	description == m_description and
	subformat == m_subformat):
	found_master = True
	batch_job['input'] = bibdocfile.get_full_path()
	## Get the aspect of the from the record
	try:
	## Assumes pbcore metadata mapping
	batch_job['aspect'] = get_fieldvalues(124, CFG_BIBENCODE_ASPECT_RATIO_MARC_FIELD)[0]
	except IndexError:
	pass
	break
	if found_master:
	break
	if not found_master:
	_task_write_message("Video master for record %d not found"
	% batch_job['recid'])
	task_update_progress("Video master for record %d not found"
	% batch_job['recid'])
	## Maybe send an email?
	return 1

	## Clean the job to do no upscaling etc
	if getval(batch_job, 'assure_quality'):
	batch_job = clean_job_for_quality(batch_job)

	global _BATCH_STEPS
	_BATCH_STEPS = len(batch_job['jobs'])

	## Generate the docname from the input filename's name or given name
	bibdoc_video_docname, bibdoc_video_extension = decompose_file(batch_job['input'])[1:]
	if not bibdoc_video_extension or getval(batch_job, 'bibdoc_master_extension'):
	bibdoc_video_extension = getval(batch_job, 'bibdoc_master_extension')
	if getval(batch_job, 'bibdoc_master_docname'):
	bibdoc_video_docname = getval(batch_job, 'bibdoc_master_docname')

	write_message("Creating BibDoc for %s" % bibdoc_video_docname)
	## If the bibdoc exists, receive it
	if bibdoc_video_docname in recdoc.get_bibdoc_names():
	bibdoc_video = recdoc.get_bibdoc(bibdoc_video_docname)
	## Create a new bibdoc if it does not exist
	else:
	bibdoc_video = recdoc.add_bibdoc(docname=bibdoc_video_docname)

	## Get the directory auf the newly created bibdoc to copy stuff there
	bibdoc_video_directory = bibdoc_video.get_base_dir()

	#--------#
	# MASTER #
	#--------#
	if not getval(batch_job, 'update_from_master'):
	if getval(batch_job, 'add_master'):
	## Generate the right name for the master
	## The master should be hidden first an then renamed
	## when it is really available
	## !!! FIX !!!
	_task_write_message("Adding %s master to the BibDoc"
	% bibdoc_video_docname)
	master_format = compose_format(
	bibdoc_video_extension,
	getval(batch_job, 'bibdoc_master_subformat', 'master')
	)
	## If a file of the same format is there, something is wrong, remove it!
	## it might be caused by a previous corrupted submission etc.
	if bibdoc_video.format_already_exists_p(master_format):
	bibdoc_video.delete_file(master_format, 1)
	bibdoc_video.add_file_new_format(
	batch_job['input'],
	version=1,
	description=getval(batch_job, 'bibdoc_master_description'),
	comment=getval(batch_job, 'bibdoc_master_comment'),
	docformat=master_format
	)

	#-----------#
	# JOBS LOOP #
	#-----------#

	return_code = 1
	global _BATCH_STEP

	for job in batch_job['jobs']:

	_task_write_message("----------- Job %s of %s -----------"
	% (_BATCH_STEP, _BATCH_STEPS))

	## Try to substitute docname with master docname
	if getval(job, 'bibdoc_docname'):
	job['bibdoc_docname'] = Template(job['bibdoc_docname']).safe_substitute({'bibdoc_master_docname': bibdoc_video_docname})

	#-------------#
	# TRANSCODING #
	#-------------#

	if job['mode'] == 'encode':

	## Skip the job if assure_quality is not set and marked as fallback
	if not getval(batch_job, 'assure_quality') and getval(job, 'fallback'):
	continue

	if getval(job, 'profile'):
	profile = get_encoding_profile(job['profile'])
	else:
	profile = None
	## We need an extension defined fot the video container
	bibdoc_video_extension = getval(job, 'extension',
	getval(profile, 'extension'))
	if not bibdoc_video_extension:
	raise Exception("No container/extension defined")
	## Get the docname and subformat
	bibdoc_video_subformat = getval(job, 'bibdoc_subformat')
	bibdoc_slave_video_docname = getval(job, 'bibdoc_docname', bibdoc_video_docname)
	## The subformat is incompatible with ffmpegs name convention
	## We do the encoding without and rename it afterwards
	bibdoc_video_fullpath = compose_file(
	bibdoc_video_directory,
	bibdoc_slave_video_docname,
	bibdoc_video_extension
	)
	_task_write_message("Transcoding %s to %s;%s" % (bibdoc_slave_video_docname,
	bibdoc_video_extension,
	bibdoc_video_subformat))
	## We encode now directly into the bibdocs directory
	encoding_result = encode_video(
	input_file=batch_job['input'],
	output_file=bibdoc_video_fullpath,
	acodec=getval(job, 'audiocodec'),
	vcodec=getval(job, 'videocodec'),
	abitrate=getval(job, 'videobitrate'),
	vbitrate=getval(job, 'audiobitrate'),
	resolution=getval(job, 'resolution'),
	passes=getval(job, 'passes', 1),
	special=getval(job, 'special'),
	specialfirst=getval(job, 'specialfirst'),
	specialsecond=getval(job, 'specialsecond'),
	metadata=getval(job, 'metadata'),
	width=getval(job, 'width'),
	height=getval(job, 'height'),
	aspect=getval(batch_job, 'aspect'), # Aspect for every job
	profile=getval(job, 'profile'),
	update_fnc=_task_update_overall_status,
	message_fnc=_task_write_message
	)
	return_code &= encoding_result
	## only on success
	if encoding_result:
	## Rename it, adding the subformat
	os.rename(bibdoc_video_fullpath,
	compose_file(bibdoc_video_directory,
	bibdoc_video_extension,
	bibdoc_video_subformat,
	1,
	bibdoc_slave_video_docname)
	)
	#bibdoc_video._build_file_list()
	bibdoc_video.touch()
	bibdoc_video._sync_to_db()
	bibdoc_video_format = compose_format(bibdoc_video_extension,
	bibdoc_video_subformat)
	if getval(job, 'bibdoc_comment'):
	bibdoc_video.set_comment(getval(job, 'bibdoc_comment'),
	bibdoc_video_format)
	if getval(job, 'bibdoc_description'):
	bibdoc_video.set_description(getval(job, 'bibdoc_description'),
	bibdoc_video_format)

	#------------#
	# EXTRACTION #
	#------------#

	# if there are multiple extraction jobs, all the produced files
	# with the same name will be in the same bibdoc! Make sure that
	# you use different subformats or docname templates to avoid
	# conflicts.

	if job['mode'] == 'extract':
	if getval(job, 'profile'):
	profile = get_extract_profile(job['profile'])
	else:
	profile = {}
	bibdoc_frame_subformat = getval(job, 'bibdoc_subformat')
	_task_write_message("Extracting frames to temporary directory")
	tmpdir = invenio.config.CFG_TMPDIR + "/" + str(uuid.uuid4())
	os.mkdir(tmpdir)
	#Move this to the batch description
	bibdoc_frame_docname = getval(job, 'bibdoc_docname', bibdoc_video_docname)
	tmpfname = (tmpdir + "/" + bibdoc_frame_docname + '.'
	+ getval(profile, 'extension',
	getval(job, 'extension', 'jpg')))
	extraction_result = extract_frames(input_file=batch_job['input'],
	output_file=tmpfname,
	size=getval(job, 'size'),
	positions=getval(job, 'positions'),
	numberof=getval(job, 'numberof'),
	width=getval(job, 'width'),
	height=getval(job, 'height'),
	aspect=getval(batch_job, 'aspect'),
	profile=getval(job, 'profile'),
	update_fnc=_task_update_overall_status,
	)
	return_code &= extraction_result

	## only on success:
	if extraction_result:
	## for every filename in the directorys, create a bibdoc that contains
	## all sizes of the frame from the two directories
	files = os.listdir(tmpdir)
	for filename in files:
	## The docname was altered by BibEncode extract through substitution
	## Retrieve it from the filename again
	bibdoc_frame_docname, bibdoc_frame_extension = os.path.splitext(filename)
	_task_write_message("Creating new bibdoc for %s" % bibdoc_frame_docname)
	## If the bibdoc exists, receive it
	if bibdoc_frame_docname in recdoc.get_bibdoc_names():
	bibdoc_frame = recdoc.get_bibdoc(bibdoc_frame_docname)
	## Create a new bibdoc if it does not exist
	else:
	bibdoc_frame = recdoc.add_bibdoc(docname=bibdoc_frame_docname)

	## The filename including path from tmpdir
	fname = os.path.join(tmpdir, filename)

	bibdoc_frame_format = compose_format(bibdoc_frame_extension, bibdoc_frame_subformat)
	## Same as with the master, if the format allready exists,
	## override it, because something went wrong before
	if bibdoc_frame.format_already_exists_p(bibdoc_frame_format):
	bibdoc_frame.delete_file(bibdoc_frame_format, 1)
	_task_write_message("Adding %s jpg;%s to BibDoc"
	% (bibdoc_frame_docname,
	getval(job, 'bibdoc_subformat')))
	bibdoc_frame.add_file_new_format(
	fname,
	version=1,
	description=getval(job, 'bibdoc_description'),
	comment=getval(job, 'bibdoc_comment'),
	docformat=bibdoc_frame_format)
	## Remove the temporary folders
	_task_write_message("Removing temporary directory")
	shutil.rmtree(tmpdir)

	_BATCH_STEP = _BATCH_STEP + 1

	#-----------------#
	# FIX BIBDOC/MARC #
	#-----------------#

	_task_write_message("----------- Handling MARCXML -----------")

	## Fix the BibDoc for all the videos previously created
	_task_write_message("Updating BibDoc of %s" % bibdoc_video_docname)
	bibdoc_video._build_file_list()

	## Fix the MARC
	_task_write_message("Fixing MARC")
	cli_fix_marc({}, [batch_job['recid']], False)

	if getval(batch_job, 'collection'):
	## Make the record visible by moving in from the collection
	marcxml = ("<record><controlfield tag=\"001\">%d</controlfield>"
	"<datafield tag=\"980\" ind1=\" \" ind2=\" \">"
	"<subfield code=\"a\">%s</subfield></datafield></record>"
	) % (batch_job['recid'], batch_job['collection'])
	upload_marcxml_file(marcxml)

	#---------------------#
	# ADD MASTER METADATA #
	#---------------------#

	if getval(batch_job, 'add_master_metadata'):
	_task_write_message("Adding master metadata")
	pbcore = pbcore_metadata(input_file = getval(batch_job, 'input'),
	pbcoreIdentifier = batch_job['recid'],
	aspect_override = getval(batch_job, 'aspect'))
	marcxml = format(pbcore, CFG_BIBENCODE_PBCORE_MARC_XSLT)
	upload_marcxml_file(marcxml)

	#------------------#
	# ADD MARC SNIPPET #
	#------------------#

	if getval(batch_job, 'marc_snippet'):
	marc_snippet = open(getval(batch_job, 'marc_snippet'))
	marcxml = marc_snippet.read()
	marc_snippet.close()
	upload_marcxml_file(marcxml)

	#--------------#
	# DELETE INPUT #
	#--------------#

	if getval(batch_job, 'delete_input'):
	_task_write_message("Deleting input file")
	# only if successfull
	if not return_code:
	# only if input matches pattern
	if getval(batch_job, 'delete_input_pattern', '') in getval(batch_job, 'input'):
	try:
	os.remove(getval(batch_job, 'input'))
	except OSError:
	pass

	#--------------#
	# NOTIFICATION #
	#--------------#

	## Send Notification emails on errors
	if not return_code:
	if getval(batch_job, 'notify_user'):
	_notify_error_user(getval(batch_job, 'notify_user'),
	getval(batch_job, 'submission_filename', batch_job['input']),
	getval(batch_job, 'recid'),
	getval(batch_job, 'submission_title', ""))
	_task_write_message("Notify user because of an error")
	if getval(batch_job, 'notify_admin'):
	_task_write_message("Notify admin because of an error")
	if type(getval(batch_job, 'notify_admin') == type(str()) ):
	_notify_error_admin(batch_job,
	getval(batch_job, 'notify_admin'))

	else:
	_notify_error_admin(batch_job)
	else:
	if getval(batch_job, 'notify_user'):
	_task_write_message("Notify user because of success")
	_notify_success_user(getval(batch_job, 'notify_user'),
	getval(batch_job, 'submission_filename', batch_job['input']),
	getval(batch_job, 'recid'),
	getval(batch_job, 'submission_title', ""))
	return 1

batch_engine.pyNo OneTemporaryActions

File Metadata

batch_engine.pyView Options

Event Timeline

batch_engine.py
No OneTemporary
Actions

batch_engine.py
View Options