diff --git a/EMSL_api.py b/EMSL_api.py index 664397a..28a47e4 100755 --- a/EMSL_api.py +++ b/EMSL_api.py @@ -4,18 +4,18 @@ """EMSL Api. Usage: - EMSL_api.py list_basis [--atom=atom_name...] - [--db_path=db_path] - EMSL_api.py list_atoms --basis=basis_name - [--db_path=db_path] - EMSL_api.py get_basis_data --basis=basis_name - [--atom=atom_name...] - [--db_path=db_path] + EMSL_api.py list_basis [--atom=...] + [--db_path=] + EMSL_api.py list_atoms --basis= + [--db_path=] + EMSL_api.py get_basis_data --basis= + [--atom=...] + [--db_path=] [--with_l] - [(--save [--path=path])] + [(--save [--path=])] EMSL_api.py list_formats - EMSL_api.py create_db --db_path=db_path - --format=format + EMSL_api.py create_db --db_path= + --format= [--no-contraction] EMSL_api.py (-h | --help) EMSL_api.py --version @@ -59,13 +59,13 @@ if __name__ == '__main__': except: sys.exit(1) - # _ _ _ ______ _ - #| | (_) | | | ___ \ (_) - #| | _ ___| |_ | |_/ / __ _ ___ _ ___ - #| | | / __| __| | ___ \/ _` / __| / __| - #| |___| \__ \ |_ | |_/ / (_| \__ \ \__ \ - #\_____/_|___/\__| \____/ \__,_|___/_|___/ - # + # _ _ _ ______ _ + # | | (_) | | | ___ \ (_) + # | | _ ___| |_ | |_/ / __ _ ___ _ ___ + # | | | / __| __| | ___ \/ _` / __| / __| + # | |___| \__ \ |_ | |_/ / (_| \__ \ \__ \ + # \_____/_|___/\__| \____/ \__,_|___/_|___/ + if arguments["list_basis"]: e = EMSL_local(db_path=db_path) @@ -75,12 +75,12 @@ if __name__ == '__main__': for name, des in l: print name, "|", des - # _ _ _ _____ _ _ - #| | (_) | | | ___| | | | - #| | _ ___| |_ | |__ | | ___ _ __ ___ ___ _ __ | |_ ___ - #| | | / __| __| | __|| |/ _ \ '_ ` _ \ / _ \ '_ \| __/ __| - #| |___| \__ \ |_ | |___| | __/ | | | | | __/ | | | |_\__ \ - #\_____/_|___/\__| \____/|_|\___|_| |_| |_|\___|_| |_|\__|___/ + # _ _ _ _____ _ _ + # | | (_) | | | ___| | | | + # | | _ ___| |_ | |__ | | ___ _ __ ___ ___ _ __ | |_ ___ + # | | | / __| __| | __|| |/ _ \ '_ ` _ \ / _ \ '_ \| __/ __| + # | |___| \__ \ |_ | |___| | __/ | | | | | __/ | | | |_\__ \ + # \_____/_|___/\__| \____/|_|\___|_| |_| |_|\___|_| |_|\__|___/ if arguments["list_atoms"]: e = EMSL_local(db_path=db_path) @@ -88,12 +88,12 @@ if __name__ == '__main__': l = e.get_list_element_available(basis_name) print ", ".join(l) - #______ _ _ _ - #| ___ \ (_) | | | | - #| |_/ / __ _ ___ _ ___ __| | __ _| |_ __ _ - #| ___ \/ _` / __| / __| / _` |/ _` | __/ _` | - #| |_/ / (_| \__ \ \__ \ | (_| | (_| | || (_| | - #\____/ \__,_|___/_|___/ \__,_|\__,_|\__\__,_| + # ______ _ _ _ + # | ___ \ (_) | | | | + # | |_/ / __ _ ___ _ ___ __| | __ _| |_ __ _ + # | ___ \/ _` / __| / __| / _` |/ _` | __/ _` | + # | |_/ / (_| \__ \ \__ \ | (_| | (_| | || (_| | + # \____/ \__,_|___/_|___/ \__,_|\__,_|\__\__,_| if arguments["get_basis_data"]: e = EMSL_local(db_path=db_path) basis_name = arguments["--basis"] @@ -116,23 +116,23 @@ if __name__ == '__main__': else: print str_ - # _ _ _ __ _ - #| | (_) | | / _| | | - #| | _ ___| |_ | |_ ___ _ __ _ __ ___ __ _| |_ ___ - #| | | / __| __| | _/ _ \| '__| '_ ` _ \ / _` | __/ __| - #| |___| \__ \ |_ | || (_) | | | | | | | | (_| | |_\__ \ - #\_____/_|___/\__| |_| \___/|_| |_| |_| |_|\__,_|\__|___/ + # _ _ _ __ _ + # | | (_) | | / _| | | + # | | _ ___| |_ | |_ ___ _ __ _ __ ___ __ _| |_ ___ + # | | | / __| __| | _/ _ \| '__| '_ ` _ \ / _` | __/ __| + # | |___| \__ \ |_ | || (_) | | | | | | | | (_| | |_\__ \ + # \_____/_|___/\__| |_| \___/|_| |_| |_| |_|\__,_|\__|___/ if arguments["list_formats"]: e = EMSL_dump() for i in e.get_list_format(): print i - # _____ _ _ _ - #/ __ \ | | | | | - #| / \/_ __ ___ __ _| |_ ___ __| | |__ - #| | | '__/ _ \/ _` | __/ _ \ / _` | '_ \ - #| \__/\ | | __/ (_| | || __/ | (_| | |_) | - # \____/_| \___|\__,_|\__\___| \__,_|_.__/ + # _____ _ _ _ + # / __ \ | | | | | + # | / \/_ __ ___ __ _| |_ ___ __| | |__ + # | | | '__/ _ \/ _` | __/ _ \ / _` | '_ \ + # | \__/\ | | __/ (_| | || __/ | (_| | |_) | + # \____/_| \___|\__,_|\__\___| \__,_|_.__/ if arguments["create_db"]: db_path = arguments["--db_path"] format = arguments["--format"] @@ -149,11 +149,11 @@ if __name__ == '__main__': contraction=contraction) e.new_db() - # _ - # / | _ _. ._ o ._ _ - # \_ | (/_ (_| | | | | | (_| - # _| + # _ + # / | _ _. ._ o ._ _ + # \_ | (/_ (_| | | | | | (_| + # _| # Clean up on exit - if db_path_changed: + if not(arguments['create_db']) and db_path_changed: os.system("rm -f /dev/shm/%d.db" % (os.getpid())) diff --git a/db/Gamess-us.db b/db/Gamess-us.db index fe60097..fdfc227 100644 Binary files a/db/Gamess-us.db and b/db/Gamess-us.db differ diff --git a/src/EMSL_utility.py b/src/EMSL_utility.py index 6c3b060..044e95a 100755 --- a/src/EMSL_utility.py +++ b/src/EMSL_utility.py @@ -6,7 +6,7 @@ import sys import os import time -debug = False +debug = True def checkSQLite3(db_path): @@ -189,7 +189,7 @@ class EMSL_dump: elts = re.sub('[["\ \]]', '', tup[3]).split(',') - des = tup[-1] + des = re.sub('\s+', ' ', tup[-1]) if "-ecp" in xml_path.lower(): continue @@ -229,18 +229,20 @@ class EMSL_dump: elt_long_exp = data_elt.split()[0].lower() if "$" in data_elt: - print "Eror", + if debug: + print "Eror", raise Exception("WARNING not bad split") if elt_long_th == elt_long_exp: - d.append((name, des, elt, data_elt.strip())) + d.append([elt, data_elt.strip()]) else: - print "th", elt_long_th - print "exp", elt_long_exp - print "abv", elt + if debug: + print "th", elt_long_th + print "exp", elt_long_exp + print "abv", elt raise Exception("WARNING not good ELEMENT") - return d + return [name, des, d] def create_sql(self, list_basis_array): """Create the sql from the list of basis available data""" @@ -248,9 +250,32 @@ class EMSL_dump: conn = sqlite3.connect(self.db_path) c = conn.cursor() - # Create table - c.execute('''CREATE TABLE all_value - (name text, description text, elt text, data text)''') + c.execute('''CREATE TABLE basis_tab( + basis_id INTEGER PRIMARY KEY AUTOINCREMENT, + name text, + description text, + UNIQUE(name) + );''') + + c.execute('''CREATE TABLE data_tab( + basis_id INTEGER, + elt TEXT, + data TEXT, + FOREIGN KEY(basis_id) + REFERENCES basis_tab(basis_id) + );''') + + c.execute(''' CREATE VIEW output_tab AS + SELECT basis_id, + name, + description, + elt, + data + FROM basis_tab + NATURAL JOIN data_tab + ''') + + conn.commit() import Queue import threading @@ -264,7 +289,7 @@ class EMSL_dump: def worker(): """get a Job from the q_in, do stuff, when finish put it in the q_out""" while True: - [name, path_xml, des, elts] = q_in.get() + name, path_xml, des, elts = q_in.get() url = "https://bse.pnl.gov:443/bse/portal/user/anon/js_peid/11535052407933/action/portlets.BasisSetAction/template/courier_content/panel/Main/" url += "/eventSubmit_doDownload/true" @@ -286,16 +311,16 @@ class EMSL_dump: attemps += 1 try: - q_out.put(([name, path_xml, des, elts], basis_data)) + q_out.put(basis_data) q_in.task_done() except: if debug: - print "Fail on q_out.put", name, path_xml, des + print "Fail on q_out.put", basis_data raise def enqueue(): for [name, path_xml, des, elts] in list_basis_array: - q_in.put(([name, path_xml, des, elts])) + q_in.put([name, path_xml, des, elts]) return 0 @@ -311,17 +336,30 @@ class EMSL_dump: nb_basis = len(list_basis_array) for i in range(nb_basis): - [name, path_xml, des, elts], basis_data = q_out.get() + name, des, d = q_out.get() + q_out.task_done() + try: + c.execute( + "INSERT INTO basis_tab(name,description) VALUES (?,?)", [ + name, des]) + conn.commit() + except sqlite3.IntegrityError: + print '{:>3}'.format(i + 1), "/", nb_basis, name, "fail" + + id_ = c.lastrowid try: c.executemany( - "INSERT INTO all_value VALUES (?,?,?,?)", basis_data) + "INSERT INTO data_tab VALUES (?,?,?)", [ + [id_] + k for k in d]) conn.commit() print '{:>3}'.format(i + 1), "/", nb_basis, name + except: print '{:>3}'.format(i + 1), "/", nb_basis, name, "fail" raise + conn.close() q_in.join() @@ -348,12 +386,12 @@ class EMSL_local: if not elts: - c.execute("SELECT DISTINCT name,description from all_value") + c.execute("SELECT DISTINCT name,description from basis_tab") data = c.fetchall() else: cmd = [ - "SELECT name,description FROM all_value WHERE elt=?"] * len(elts) + "SELECT name,description FROM output_tab WHERE elt=?"] * len(elts) cmd = " INTERSECT ".join(cmd) + ";" c.execute(cmd, elts) @@ -371,7 +409,7 @@ class EMSL_local: c = conn.cursor() c.execute( - "SELECT DISTINCT elt from all_value WHERE name=:name_us COLLATE NOCASE", { + "SELECT DISTINCT elt from output_tab WHERE name=:name_us COLLATE NOCASE", { "name_us": basis_name}) data = c.fetchall() @@ -383,6 +421,8 @@ class EMSL_local: def get_basis(self, basis_name, elts=None, with_l=False): + import re + def get_list_type(l_line): l = [] for i, line in enumerate(l_line): @@ -398,8 +438,6 @@ class EMSL_local: l[-1].append(i + 1) return l - import re - # __ _ # /__ _ _|_ _|_ ._ _ ._ _ _ _. | # \_| (/_ |_ | | (_) | | | _> (_| | @@ -412,7 +450,7 @@ class EMSL_local: else: cmd_ele = "" - c.execute('''SELECT DISTINCT data from all_value + c.execute('''SELECT DISTINCT data from output_tab WHERE name="{basis_name}" COLLATE NOCASE {cmd_ele}'''.format(basis_name=basis_name, cmd_ele=cmd_ele))