From 0da7952476d5837459d7bd1d7eeef2e3c1a4b8fb Mon Sep 17 00:00:00 2001 From: Anthony Scemama Date: Sat, 2 Oct 2010 21:19:04 +0200 Subject: [PATCH] Version:1.1.52 --- src/util.py | 44 ++++++++++++++++++++++++++++++-------------- src/version.py | 2 +- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/util.py b/src/util.py index abb41e3..fa86659 100644 --- a/src/util.py +++ b/src/util.py @@ -24,6 +24,7 @@ # 31062 Toulouse Cedex 4 # scemama@irsamc.ups-tlse.fr +NTHREADS=3 def strip(x): return x.strip() @@ -128,25 +129,40 @@ def put_info(text,filename): import cPickle as pickle import os, sys -def fork_and_pickle(f,filename,text): - fork = os.fork() - if fork == 0: - result = f(filename,text) - file = open('%s.pickle'%filename,'w') - pickle.dump(result,file,-1) - file.close() - sys.exit(0) - else: - return fork - def parallel_loop(f,source): pidlist = {} - for filename, text in source: - pidlist[filename] = fork_and_pickle( f, filename, text ) + + src = [ [] for i in xrange(NTHREADS) ] + index = 0 + for i in source: + index = index+1 + if index == NTHREADS: + index = 0 + src[index].append(i) + + for thread_id in xrange(1,NTHREADS): + fork = os.fork() + if fork == 0: + break + else: + import time + time.sleep(1) + pidlist[i] = fork + thread_id = 0 + + print "fork : ", fork, thread_id + for filename, text in src[thread_id]: + result = f(filename,text) + file = open('%s.pickle'%filename,'w') + pickle.dump(result,file,-1) + file.close() + sys.exit(0) + + for i in xrange(NTHREADS): + os.waitpid(pidlist[i],0) result = [] for filename,text in source: - os.waitpid(pidlist[filename],0) file = open('%s.pickle'%filename,'r') data = pickle.load(file) file.close() diff --git a/src/version.py b/src/version.py index 29e3edb..1cf2e84 100644 --- a/src/version.py +++ b/src/version.py @@ -1 +1 @@ -version = "1.1.51" +version = "1.1.52"