From 772327fe568eaf5f3dcb0fcdbc2c82a0887cce25 Mon Sep 17 00:00:00 2001 From: Anthony Scemama Date: Sat, 2 Oct 2010 19:41:03 +0200 Subject: [PATCH] Parallelization of preprocessed_text Version:1.1.48 --- src/preprocessed_text.py | 10 +++------- src/util.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/preprocessed_text.py b/src/preprocessed_text.py index 9e96a7e..b1d9702 100644 --- a/src/preprocessed_text.py +++ b/src/preprocessed_text.py @@ -24,7 +24,6 @@ # 31062 Toulouse Cedex 4 # scemama@irsamc.ups-tlse.fr - from irpf90_t import * from regexps import * import error @@ -549,7 +548,7 @@ def irp_simple_statements(text): i = line.i f = line.filename result = [ Begin_provider(i,line.text, (f,varname)), - Declaration(i," character*(%d) :: irp_here = '%s'"%(length,varname), filename) ] + Declaration(i," character*(%d) :: irp_here = '%s'"%(length,varname), line.filename) ] if command_line.do_assert or command_line.do_debug: result += [ Simple_line(i," call irp_enter(irp_here)", f), @@ -826,14 +825,11 @@ def create_preprocessed_text(filename): result = change_single_line_ifs(result) result = process_old_style_do(result) result = irp_simple_statements(result) + check_begin_end(result) return result ###################################################################### -preprocessed_text = [] -for filename in irpf90_files: - result = create_preprocessed_text(filename) - check_begin_end(result) - preprocessed_text.append( (filename, result) ) +preprocessed_text = parallel_loop( create_preprocessed_text, irpf90_files ) ###################################################################### def debug(): diff --git a/src/util.py b/src/util.py index fed2104..c6db0c0 100644 --- a/src/util.py +++ b/src/util.py @@ -126,6 +126,35 @@ def put_info(text,filename): line.text = format%(line.text.ljust(lenmax),line.filename,str(line.i)) return text +import cPickle as pickle +import os, sys +def fork_and_pickle(f,filename): + fork = os.fork() + if fork == 0: + result = f(filename) + file = open('%s.pickle'%filename,'w') + pickle.dump(result,file,-1) + file.close() + sys.exit(0) + else: + return fork + +def parallel_loop(f,files): + pidlist = {} + for filename in files: + pidlist[filename] = fork_and_pickle( f, filename ) + + result = [] + for filename in files: + os.waitpid(pidlist[filename],0) + file = open('%s.pickle'%filename,'r') + data = pickle.load(file) + file.close() + os.remove('%s.pickle'%filename) + result.append( (filename, data) ) + + return result + if __name__ == '__main__': print "10",dimsize("10") #-> "10" print "0:10",dimsize("0:10") # -> "11"