From f2ce19a3979b0a20ceb4653795d610ef5f209f78 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sat, 8 Mar 2014 22:09:04 +0100 Subject: [PATCH] rewrite to use only a single rendering routine and document all code --- 2014/make.py | 413 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 252 insertions(+), 161 deletions(-) diff --git a/2014/make.py b/2014/make.py index 0365b4a..276ca1b 100755 --- a/2014/make.py +++ b/2014/make.py @@ -17,39 +17,55 @@ import multiprocessing from threading import Thread, Lock from Queue import Queue -# Debug rendert einen Vor- und einen Abspann +# Frames per second. Increasing this renders more frames, the avconf-statements would still need modifications fps = 25 + +# using --debug skips the threading, the network fetching of the schedule and +# just renders one type of video debug = ('--debug' in sys.argv) +# using --offline only skips the network fetching and use a local schedule.de.xml +offline = ('--offline' in sys.argv) + +# set charset of output-terminal reload(sys) sys.setdefaultencoding('utf-8') # t: current time, b: begInnIng value, c: change In value, d: duration +# copied from jqueryui def easeOutCubic(t, b, c, d): t=float(t)/d-1 return c*((t)*t*t + 1) + b -def ensure_path_exists(path): +# try to create all folders needed and skip, they already exist +def ensurePathExists(path): try: os.makedirs(path) except OSError as exception: if exception.errno != errno.EEXIST: raise -def ensure_files_removed(files): - for f in glob.glob(files): +# remove the files matched by the pattern +def ensureFilesRemoved(pattern): + for f in glob.glob(pattern): os.unlink(f) +# Normalizes string, converts to lowercase, removes non-alpha characters, + #and converts spaces to hyphens. def slugify(value): - """ - Normalizes string, converts to lowercase, removes non-alpha characters, - and converts spaces to hyphens. - """ value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore') value = unicode(re.sub('[^\w\s-]', '', value).strip().lower()) value = unicode(re.sub('[-\s]+', '-', value)) return value +# create a filename from the events' id and a slugified version of the title +def vorspannFilename(id, title): + return u'{0:04d}-{1}.mp4'.format(id, slugify(unicode(title))) + +# svg does not have a method for automatic line breaking, that rsvg is capable of +# so we do it in python as good as we can +def vorspannTitle(title): + return ''.join(textwrap.wrap(title, 35)) @@ -57,291 +73,366 @@ def abspannFrames(): # 5 Sekunden # 2 Sekunden Fadein Text - frame = 0 frames = 2*fps for i in range(0, frames): - yield (frame+i, easeOutCubic(i, 0, 1, frames), 0) + yield { + '%opacity': easeOutCubic(i, 0, 1, frames), + '%opacityLizenz': 0 + } # 2 Sekunde Fadein Lizenz-Logo - frame = frame+i+1 frames = 2*fps for i in range(0, frames): - yield (frame+i, 1, float(i)/frames) + yield { + '%opacity': 1, + '%opacityLizenz': float(i)/frames + } # 1 Sekunde stehen bleiben - frame = frame+i+1 frames = 1*fps for i in range(0, frames): - yield (frame+i, 1, 1) + yield { + '%opacity': 1, + '%opacityLizenz': 1 + } def vorspannFrames(): # 7 Sekunden # 2 Sekunden Text 1 - frame = 0 frames = 2*fps for i in range(0, frames): - yield (frame+i, easeOutCubic(i, 0, 1, frames), easeOutCubic(i, 0, 1, frames), 0) + yield { + '%opacityBox': easeOutCubic(i, 0, 1, frames), + '%opacity1': easeOutCubic(i, 0, 1, frames), + '%opacity2': 0 + } # 1 Sekunde Fadeout Text 1 - frame = frame+i+1 frames = 1*fps for i in range(0, frames): - yield (frame+i, 1, 1-(float(i)/frames), 0) + yield { + '%opacityBox': 1, + '%opacity1': 1-(float(i)/frames), + '%opacity2': 0 + } # 2 Sekunden Text 2 - frame = frame+i+1 frames = 2*fps for i in range(0, frames): - yield (frame+i, 1, 0, easeOutCubic(i, 0, 1, frames)) + yield { + '%opacityBox': 1, + '%opacity1': 0, + '%opacity2': easeOutCubic(i, 0, 1, frames) + } # 2 Sekunden stehen bleiben - frame = frame+i+1 frames = 2*fps for i in range(0, frames): - yield (frame+i, 1, 0, 1) + yield { + '%opacityBox': 1, + '%opacity1': 0, + '%opacity2': 1 + } def pauseFrames(): # 12 Sekunden # 2 Sekunden Text1 stehen - frame = 0 frames = 2*fps for i in range(0, frames): - yield (frame+i, 1, 0) + yield { + '%opacity1': 1, + '%opacity2': 0 + } # 2 Sekunden Fadeout Text1 - frame = frame+i+1 frames = 2*fps for i in range(0, frames): - yield (frame+i, 1-easeOutCubic(i, 0, 1, frames), 0) + yield { + '%opacity1': 1-easeOutCubic(i, 0, 1, frames), + '%opacity2': 0 + } # 2 Sekunden Fadein Text2 - frame = frame+i+1 frames = 2*fps for i in range(0, frames): - yield (frame+i, 0, easeOutCubic(i, 0, 1, frames)) + yield { + '%opacity1': 0, + '%opacity2': easeOutCubic(i, 0, 1, frames) + } # 2 Sekunden Text2 stehen - frame = frame+i+1 frames = 2*fps for i in range(0, frames): - yield (frame+i, 0, 1) + yield { + '%opacity1': 0, + '%opacity2': 1 + } # 2 Sekunden Fadeout Text2 - frame = frame+i+1 frames = 2*fps for i in range(0, frames): - yield (frame+i, 0, 1-easeOutCubic(i, 0, 1, frames)) + yield { + '%opacity1': 0, + '%opacity2': 1-easeOutCubic(i, 0, 1, frames) + } # 2 Sekunden Fadein Text1 - frame = frame+i+1 frames = 2*fps for i in range(0, frames): - yield (frame+i, easeOutCubic(i, 0, 1, frames), 0) + yield { + '%opacity1': easeOutCubic(i, 0, 1, frames), + '%opacity2': 0 + } -def abspann(lizenz, workdir='artwork', outdir='..'): +def render(infile, outfile, sequence, parameters={}, workdir='artwork'): + # in debug mode we have no thread-worker which prints its progress if debug: - print "erzeuge Abspann" + print "generating {0} from {1}".format(outfile, infile) - filename = os.path.join(outdir, 'abspann-{0}.mp4'.format(lizenz)) + # make sure a .frames-directory exists in out workdir + ensurePathExists(os.path.join(workdir, '.frames')) - ensure_path_exists(os.path.join(workdir, '.frames')) + # open and parse the input file + with open(os.path.join(workdir, infile), 'r') as fp: + svg = fp.read() - with open(os.path.join(workdir, 'abspann.svg'), 'r') as abspann_file: - abspann = abspann_file.read() + # frame-number counter + frameNr = 0 - for (frameNr, opacity, opacityLizenz) in abspannFrames(): + # iterate through the animation seqence frame by frame + # frame is a dictionary with key/value-pairs ("replace-pairs"), where the key + # is searched for in the source svg-file and every occurence is replaced by + # its companion the value + for frame in sequence(): + # print a line for each and every frame generated if debug: - print "frameNr {0:2d} => opacity {1:0.2f}, opacityLizenz {2:0.2f}".format(frameNr, opacity, opacityLizenz) + print "frameNr {0:2d} => {1}".format(frameNr, frame) - pairs = \ - ('%opacityLizenz', str(opacityLizenz)), \ - ('%opacity', str(opacity)), \ - ('%lizenz', lizenz), \ - ('%workdir', os.path.realpath(workdir) ) + # extend the frame-dictionary with additional replace-pairs from the arguments + frame.update(parameters) - with open(os.path.join(workdir, '.gen.svg'), 'w') as gen_file: - gen_abspann = reduce(lambda a, kv: a.replace(*kv), pairs, abspann) - gen_file.write( gen_abspann ) + # add some more useful replace-pairs + frame['%workdir'] = os.path.realpath(workdir) + # open the output-file (named ".gen.svg" in the workdir) + with open(os.path.join(workdir, '.gen.svg'), 'w') as fp: + # apply the replace-pairs to the input text, by replacing each key with its companion value + gen_svg = reduce(lambda x, y: x.replace(y, str(frame[y])), frame, svg) + + # write the generated svg-text into the output-file + fp.write( gen_svg ) + + # invoke rsvg to convert the generated svg-file into a png inside the .frames-directory os.system('cd {0} && rsvg-convert .gen.svg > .frames/{1:04d}.png'.format(workdir, frameNr)) - ensure_files_removed(filename) - os.system('cd {0} && avconv -f image2 -i .frames/%04d.png -c:v libx264 -preset veryslow -qp 0 "{1}"'.format(workdir, filename) + ('' if debug else '>/dev/null 2>&1')) + # incrwement frame-number + frameNr += 1 + # remove the mp4 we are about to (re-)generate + ensureFilesRemoved(os.path.join(workdir, outfile)) + + # invoke avconv aka ffmpeg and renerate a lossles-mp4 from the frames + # if we're not in debug-mode, suppress all output + os.system('cd {0} && avconv -f image2 -i .frames/%04d.png -c:v libx264 -preset veryslow -qp 0 "{1}"'.format(workdir, outfile) + ('' if debug else '>/dev/null 2>&1')) + + # as before, in non-debug-mode the thread-worker does all progress messages if debug: - print "aufräumen" + print "cleanup" + + # remove the .frames-dir with all frames in it shutil.rmtree(os.path.join(workdir, '.frames')) - ensure_files_removed(os.path.join(workdir, '.gen.svg')) -def vorspann(id, title, personnames, workdir='artwork', outdir='..'): - if debug: - print u'erzeuge Vorspann für {0:4d} ("{1}")'.format(id, title) - - filename = os.path.join( outdir, u'{0:04d}-{1}.mp4'.format(id, slugify(unicode(title))) ) - - ensure_path_exists(os.path.join(workdir, '.frames')) - - with open(os.path.join(workdir, 'vorspann.svg'), 'r') as vorspann_file: - vorspann = vorspann_file.read() - - # svg does not have a method for automatic line breaking, that rsvg is capable of - # so we do it in python as good as we can - breaktitle = ''.join(textwrap.wrap(title, 35)) - - for (frameNr, opacityBox, opacity1, opacity2) in vorspannFrames(): - if debug: - print "frameNr {0:2d} => opacityBox {1:0.2f}, opacity1 {2:0.2f}, opacity2 {3:0.2f}".format(frameNr, opacityBox, opacity1, opacity2) - - pairs = \ - ('%opacity1', str(opacity1)), \ - ('%opacity2', str(opacity2)), \ - ('%opacityBox', str(opacityBox)), \ - ('%id', str(id)), \ - ('%title', breaktitle), \ - ('%personnames', personnames), \ - ('%workdir', os.path.realpath(workdir) ) - - with open(os.path.join(workdir, '.gen.svg'), 'w') as gen_file: - gen_vorspann = reduce(lambda a, kv: a.replace(*kv), pairs, vorspann) - gen_file.write( gen_vorspann ) - - os.system('cd {0} && rsvg-convert .gen.svg > .frames/{1:04d}.png'.format(workdir, frameNr)) - - ensure_files_removed(filename) - os.system(u'cd {0} && avconv -f image2 -i .frames/%04d.png -c:v libx264 -preset veryslow -qp 0 "{1}"'.format(workdir, filename) + ('' if debug else '>/dev/null 2>&1')) - - if debug: - print "aufräumen" - shutil.rmtree(os.path.join(workdir, '.frames')) - ensure_files_removed(os.path.join(workdir, '.gen.svg')) - -def pause(workdir='artwork', outdir='..'): - if debug: - print "erzeuge Pause-Loop" - - filename = os.path.join(outdir, 'pause.mp4') - dvfilename = os.path.join(outdir, 'pause.dv') - - ensure_path_exists(os.path.join(workdir, '.frames')) - - with open(os.path.join(workdir, 'pause.svg'), 'r') as pause_file: - pause = pause_file.read() - - for (frameNr, opacity1, opacity2) in pauseFrames(): - if debug: - print "frameNr {0:2d} => opacity1 {1:0.2f}, opacity2 {2:0.2f}".format(frameNr, opacity1, opacity2) - - pairs = \ - ('%opacity1', str(opacity1)), \ - ('%opacity2', str(opacity2)), \ - ('%workdir', os.path.realpath(workdir) ) - - with open(os.path.join(workdir, '.gen.svg'), 'w') as gen_file: - gen_pause = reduce(lambda a, kv: a.replace(*kv), pairs, pause) - gen_file.write( gen_pause ) - - os.system('cd {0} && rsvg-convert .gen.svg > .frames/{1:04d}.png'.format(workdir, frameNr)) - - ensure_files_removed(filename) - os.system('cd {0} && avconv -f image2 -i .frames/%04d.png -c:v libx264 -preset veryslow -qp 0 "{1}"'.format(workdir, filename) + ('' if debug else '>/dev/null 2>&1')) - - ensure_files_removed(dvfilename) - os.system('cd {0} && avconv -f image2 -i .frames/%04d.png -target pal-dv "{1}"'.format(workdir, dvfilename) + ('' if debug else '>/dev/null 2>&1')) - - if debug: - print "aufräumen" - shutil.rmtree(os.path.join(workdir, '.frames')) - ensure_files_removed(os.path.join(workdir, '.gen.svg')) + # remove the generated svg + ensureFilesRemoved(os.path.join(workdir, '.gen.svg')) + +# Download the Events-Schedule ans parse all Events out of it. Yield a tupel for each Event def events(): print "downloading pentabarf schedule" - response = urllib2.urlopen('http://www.fossgis.de/konferenz/2014/programm/schedule.de.xml') - xml = response.read() - schedule = ET.fromstring(xml) - #schedule = ET.parse('schedule.de.xml') + # use --offline to skip networking + if offline: + # parse the offline-version + schedule = ET.parse('schedule.de.xml').getroot() + + else: + # download the schedule + response = urllib2.urlopen('http://www.fossgis.de/konferenz/2014/programm/schedule.de.xml') + + # read xml-source + xml = response.read() + + # parse into ElementTree + schedule = ET.fromstring(xml) + + # iterate all days for day in schedule.iter('day'): - date = day.get('date') + # iterate all rooms for room in day.iter('room'): + # iterate events on that day in this room for event in room.iter('event'): + # aggregate names of the persons holding this talk personnames = [] for person in event.find('persons').iter('person'): personnames.append(person.text) + # yield a tupel with the event-id, event-title and person-names yield ( int(event.get('id')), event.find('title').text, ', '.join(personnames) ) - +# debug-mode selected by --debug switch if debug: print "!!! DEBUG MODE !!!" - vorspann(667, 'OpenJUMP - Überblick, Neuigkeiten, Zusammenarbeit/Schnittstellen mit proprietärer Software', 'Matthias Scholz') - abspann('by-sa') - pause() + title = 'OpenJUMP - Überblick, Neuigkeiten, Zusammenarbeit/Schnittstellen mit proprietärer Software' + + render( + 'vorspann.svg', + os.path.join('..', vorspannFilename(667, title)), + vorspannFrames, + {'%id': 664, '%title': vorspannTitle(title), '%personnames': 'Matthias Scholz' } + ) + + render( + 'abspann.svg', + '../abspann-by-sa.mp4', + abspannFrames, + {'%lizenz': 'by-sa'} + ) + + render('pause.svg', + '../pause.mp4', + pauseFrames + ) + sys.exit(0) - +# threaded task queue tasks = Queue() +# iterate over all events extracted from the schedule xml-export for (id, title, personnames) in events(): - tasks.put( ('vorspann', id, title, personnames) ) + # generate a task description and put them into the queue + tasks.put(( + 'vorspann.svg', + vorspannFilename(id, title), + vorspannFrames, + {'%id':id, '%title':vorspannTitle(title), '%personnames':personnames } + )) -tasks.put( ('abspann', 'by-sa') ) -tasks.put( ('abspann', 'by-nc-sa') ) -tasks.put( ('abspann', 'cc-zero') ) -tasks.put( ('pause') ) +# iterate over the licences and place a task into the queue +for lizenz in ('by-sa', 'by-nc-sa', 'cc-zero'): + tasks.put(( + 'abspann.svg', + 'abspann-{0}.mp4'.format(lizenz), + abspannFrames, + {'%lizenz':lizenz} + )) +# place the pause-sequence into the queue +tasks.put(( + 'pause.svg', + 'pause.mp4', + pauseFrames +)) + +# one working thread per cpu num_worker_threads = multiprocessing.cpu_count() -print "{0} tasks in queue, starting {1} worker threads".format( tasks.qsize() - num_worker_threads, num_worker_threads ) +print "{0} tasks in queue, starting {1} worker threads".format(tasks.qsize(), num_worker_threads) +# put a sentinel for each thread into the queue to signal the end for _ in range(num_worker_threads): - tasks.put(None) # put sentinel to signal the end - + tasks.put(None) +# this lock ensures, that only one thread at a time is writing to stdout +# and avoids output from multiple threads intermixing printLock = Lock() def tprint(str): + # aquire lock printLock.acquire() + + # print thread-name and message print threading.current_thread().name+': '+str + + # release lock printLock.release() - +# thread worker def worker(): - tempdir = os.path.join(tempfile.mkdtemp(), 'artwork') + # generate a tempdir for this worker-thread and use the artwork-subdir as temporary folder + tempdir = tempfile.mkdtemp() + workdir = os.path.join(tempdir, 'artwork') + + # save the current working dir as output-dir outdir = os.getcwd() + # print a message that we're about to initialize our environment tprint("initializing worker in {0}, writing result to {1}".format(tempdir, outdir)) - shutil.copytree('artwork', tempdir) + # copy the artwork-dir into the tempdir + shutil.copytree('artwork', workdir) + + # loop until all tasks are done (when the thread fetches a sentinal from the queue) while True: + # fetch a task from the queue task = tasks.get() + + # if it is a stop-sentinal break out of the loop if task == None: break - tprint( 'processing {0}'.format(task) ) - fnname = task[0] - fn = globals()[fnname] - opts = task[1:] + (tempdir, outdir) + # print that we're about to render a task + tprint('rendering {0}'.format(task[1])) - fn(*opts) - tprint( 'finished {0}, {1} tasks left'.format(task, tasks.qsize() - num_worker_threads) ) + # render options + opts = ( + # argument 0 is the input file. prepend the workdir + os.path.join(workdir, task[0]), + + # argument 1 is the output file. prepend the outdir + os.path.join(outdir, task[1]), + + # argument 2 is the frame generator, nothing to do here + task[2], + + # argument 3 are the extra parameters + task[3] if len(task) > 3 else {}, + + # argument 4 is the workdir path + workdir + ) + + # render with these arguments + render(*opts) + + # print that we're finished + tprint('finished {0}, {1} tasks left'.format(task[1], max(0, tasks.qsize() - num_worker_threads))) + + # mark the task as finished tasks.task_done() + # all tasks from the queue done, clean up tprint("cleaning up worker") + + # remove the tempdir shutil.rmtree(tempdir) + + # mark the sentinal as done tasks.task_done() - +# generate and start the threads for i in range(num_worker_threads): t = Thread(target=worker) t.daemon = True t.start() +# wait until they finished doing the work tasks.join() print "all worker threads ended"