Merge pull request #4 from TApplencourt/master

Restor many ele ; and Case Insentivity for basis in get_basis_data

EMSL_api.py

@@ -44,7 +44,7 @@ import os
 
 from src.misc.docopt import docopt
 from src.EMSL_dump import EMSL_dump
-from src.EMSL_local import EMSL_local, checkSQLite3
+from src.EMSL_local import EMSL_local
 
 if __name__ == '__main__':
 
@@ -63,6 +63,7 @@ if __name__ == '__main__':
     # Check the db
     try:
         if not(arguments['create_db']):
+            from src.EMSL_local import checkSQLite3
             db_path, db_path_changed = checkSQLite3(db_path)
     except:
         raise
@@ -83,10 +84,12 @@ if __name__ == '__main__':
 
         if arguments["--average_mo_number"]:
             for name, des, avg in l:
-                print "- '{}' ({}) || {:<50}".format(name, avg, des)
+                des_str = "{:<50}".format(des)
+                print "- '{0}' ({1}) || {2}".format(name, avg, des_str)
         else:
             for name, des in l:
-                print "- '{}' || {:<50}".format(name, des)
+                des_str = "{:<50}".format(des)
+                print "- '{0}' || {1}".format(name, des_str)
 
     #  _     _     _     _____ _                           _
     # | |   (_)   | |   |  ___| |                         | |
@@ -140,8 +143,7 @@ if __name__ == '__main__':
     # | |___| \__ \ |_  | || (_) | |  | | | | | | (_| | |_\__ \
     # \_____/_|___/\__| |_| \___/|_|  |_| |_| |_|\__,_|\__|___/
     elif arguments["list_formats"]:
-        e = EMSL_dump()
+        for i in EMSL_dump.get_list_format():
-        for i in e.get_list_format():
             print i
 
     #  _____                _             _ _

src/EMSL_dump.py

@@ -2,7 +2,13 @@ import os
 import sys
 import re
 import time
+
+try:
     import sqlite3
+except ImportError:
+    print "you maybe need libsqlite3-dev from the package manager"
+    print "and the recompile Python"
+    raise
 
 from collections import OrderedDict
 
@@ -32,9 +38,8 @@ def install_with_pip(name):
 
 
 class EMSL_dump:
-
     """
-    This call implement all you need for download the EMSL and save it localy
+    This class implement all you need for download the EMSL and save it localy
     """
 
     def __init__(self, db_path=None, format="GAMESS-US", contraction="True"):
@@ -45,14 +50,16 @@ class EMSL_dump:
         self.format = check_format(format)
         self.parser = get_parser_function(self.format)
 
+        """Define the database path"""
         if db_path:
             self.db_path = db_path
         else:
             head_path = os.path.dirname(__file__)
-            self.db_path = "{0}/../db/{1}.db".format(head_path, self.format)
+            db_path = "{0}/../db/{1}.db".format(head_path, self.format)
+            self.db_path = os.path.abspath(db_path)
 
         self.contraction = str(contraction)
-        self.debug = False
+        self.debug = True
 
         try:
             import requests
@@ -62,15 +69,12 @@ class EMSL_dump:
         finally:
             self.requests = requests
 
-    def get_list_format(self):
+    @staticmethod
+    def get_list_format():
         """List all the format available in EMSL"""
         from src.parser_handler import parser_dict
         return parser_dict.keys()
 
-    def set_db_path(self, path):
-        """Define the database path"""
-        self.db_path = path
-
     def dwl_basis_list_raw(self):
         """Return the source code of the iframe
            who contains the list of the basis set available"""
@@ -85,13 +89,9 @@ class EMSL_dump:
             dbcache = 'db/cache'
             if not os.path.isfile(dbcache):
                 page = self.requests.get(url).text
-                file = open(dbcache, 'w')
+                pickle.dump(page, open(dbcache, 'wb'))
-                pickle.dump(page, file)
             else:
-                file = open(dbcache, 'r')
+                page = pickle.load(open(dbcache, 'rb'))
-                page = pickle.load(file)
-            file.close()
-
         else:
             page = self.requests.get(url).text
 
@@ -142,6 +142,7 @@ class EMSL_dump:
 
                 name = tup[1]
                 elts = re.sub('[["\ \]]', '', tup[3]).split(',')
+
                 des = re.sub('\s+', ' ', tup[-1])
 
                 d[name] = [name, xml_path, des, elts]
@@ -155,7 +156,7 @@ class EMSL_dump:
     # | \__/\ | |  __/ (_| | ||  __/
     #  \____/_|  \___|\__,_|\__\___|
     #
-    def create_sql(self, dict_basis_list):
+    def create_and_populate_sql(self, dict_basis_list):
         """Create the sql from strach.
             Take the list of basis available data,
             download her, put her in sql"""
@@ -204,7 +205,14 @@ class EMSL_dump:
         num_worker_threads = 7
         attemps_max = 20
 
-        q_in = Queue.Queue(num_worker_threads)
+        # All the task need to be executed
+        nb_basis = len(dict_basis_list)
+        q_in = Queue.Queue(nb_basis)
+        # Populate the  q_in list
+        for [name, path_xml, des, elts] in dict_basis_list.itervalues():
+                q_in.put([name, path_xml, des, elts])
+
+        # All the queue who have been executed
         q_out = Queue.Queue(num_worker_threads)
 
         def worker():
@@ -216,7 +224,8 @@ class EMSL_dump:
                 url = "https://bse.pnl.gov:443/bse/portal/user/anon/js_peid/11535052407933/action/portlets.BasisSetAction/template/courier_content/panel/Main/"
                 url += "/eventSubmit_doDownload/true"
 
-                params = {'bsurl': path_xml, 'bsname': name,
+                params = {'bsurl': path_xml,
+                          'bsname': name,
                           'elts': " ".join(elts),
                           'format': self.format,
                           'minimize': self.contraction}
@@ -239,26 +248,15 @@ class EMSL_dump:
                     if self.debug:
                         print "Fail on q_out.put", basis_data
                     raise
-                else:
-                    q_in.task_done()
-
-        def enqueue():
-            for [name, path_xml, des, elts] in dict_basis_list.itervalues():
-                q_in.put([name, path_xml, des, elts])
-
-            return 0
-
-        t = threading.Thread(target=enqueue)
-        t.daemon = True
-        t.start()
 
+        # Create all the worker (q_in |> worker |> q_out)
         for i in range(num_worker_threads):
             t = threading.Thread(target=worker)
             t.daemon = True
             t.start()
 
-        nb_basis = len(dict_basis_list)
+        # Take the result from the out queue (populate by the worker)
-
+        # and put in in the SQL database
         for i in range(nb_basis):
             name, des, basis_data = q_out.get()
             q_out.task_done()
@@ -300,4 +298,4 @@ class EMSL_dump:
         _data = self.dwl_basis_list_raw()
         array_basis = self.basis_list_raw_to_array(_data)
 
-        self.create_sql(array_basis)
+        self.create_and_populate_sql(array_basis)

src/EMSL_local.py

@@ -63,13 +63,14 @@ def checkSQLite3(db_path):
 
 
 def cond_sql_or(table_name, l_value, glob=False):
-    """Take a table_name, a list of value and create the sql or combande"""
+    """Take a table_name, a list of value and create the sql 'or' commande
+       for example : (elt = "Na" OR elt = "Mg"')"""
 
     opr = "GLOB" if glob else "="
 
-    return [" OR ".join(['{} {} "{}"'.format(table_name,
+    l_cmd = ['{} {} "{}"'.format(table_name, opr, val) for val in l_value]
-                                             opr,
+
-                                             val) for val in l_value])]
+    return "({0})".format(" OR ".join(l_cmd))
 
 
 def string_to_nb_mo(str_type):
@@ -93,7 +94,7 @@ def string_to_nb_mo(str_type):
 # |_ |\/| (_  |    |   _   _  _. |
 # |_ |  | __) |_   |_ (_) (_ (_| |
 #
-class EMSL_local:
+class EMSL_local(object):
 
     """
     All the method for using the EMSL db localy
@@ -128,7 +129,7 @@ class EMSL_local:
         # ~#~#~#~#~#~ #
 
         if basis:
-            cmd_filter_basis = " ".join(cond_sql_or("name", basis, glob=True))
+            cmd_filter_basis = cond_sql_or("name", basis, glob=True)
         else:
             cmd_filter_basis = "(1)"
 
@@ -164,8 +165,8 @@ class EMSL_local:
             # C r e a t e _ t h e _ c m d #
             # ~#~#~#~#~#~#~#~#~#~#~#~#~#~ #
 
-            cmd_filter_basis = " ".join(cond_sql_or("basis_id", l_basis_id))
+            cmd_filter_basis = cond_sql_or("basis_id", l_basis_id)
-            cmd_filter_ele = " ".join(cond_sql_or("elt", elts))
+            cmd_filter_ele = cond_sql_or("elt", elts)
 
             column_to_fech = "name, description"
             if average_mo_number:
@@ -251,17 +252,23 @@ class EMSL_local:
                   handle_l_format=False, check_format=None):
         """
         Return the data from the basis set
+        basis_name : The value of 'name'raw from output_tab in the SQL database
+        elts : List of element avalaible in 'elt'raw
+        handle_l_format : If you want to use special treatement for SP function
+                        (see src.parser_handler.get_handle_l_function)
+        check_format : If you want to verify some condition for special program
+                       (see src.parser.check_validity)
         """
 
         # ~#~#~#~#~#~ #
         # F i l t e r #
         # ~#~#~#~#~#~ #
 
-        cmd_filter_ele = " ".join(cond_sql_or("elt", elts)) if elts else "(1)"
+        cmd_filter_ele = cond_sql_or("elt", elts) if elts else "(1)"
 
         self.c.execute('''SELECT DISTINCT data from output_tab
-                     WHERE name="{0}"
+                     WHERE name LIKE "{0}"
-                     AND  {1}'''.format(basis_name, cmd_filter_ele))
+                     AND  ({1})'''.format(basis_name, cmd_filter_ele))
 
         # We need to take i[0] because fetchall return a tuple [(value),...]
         l_atom_basis = [i[0].strip() for i in self.c.fetchall()]
@@ -281,7 +288,7 @@ class EMSL_local:
         if check_format:
 
                 from src.parser_handler import get_symmetry_function
-                from src.parser.check_validity import get_check_function
+                from src.parser_handler import get_check_function
 
                 f = get_check_function(check_format)
                 f_symmetry = get_symmetry_function(self.format)
@@ -302,16 +309,3 @@ class EMSL_local:
         # R e t u r n #
         # ~#~#~#~#~#~ #
         return l_atom_basis
-if __name__ == "__main__":
-
-    e = EMSL_local(db_path="EMSL.db")
-    l = e.get_list_basis_available()
-    for i in l:
-        print i
-
-    l = e.get_list_element_available("pc-0")
-    print l
-
-    l = e.get_basis("cc-pVTZ", ["H", "He"])
-    for i in l:
-        print i

src/parser/check_validity.py

@@ -1,52 +0,0 @@
-#  _
-# /  |_   _   _ |        _. | o  _| o _|_
-# \_ | | (/_ (_ |<   \/ (_| | | (_| |  |_ \/
-#                                         /
-# Do this After the L special case traitement.
-
-import sys
-
-
-def check_gamess(str_type):
-    """Check is the orbital type is handle by gamess"""
-
-    assert len(str_type) == 1
-
-    if str_type in "S P D".split():
-        return True
-    elif str_type == "SP":
-        raise BaseException
-    else:
-        return True
-
-
-def check_NWChem(str_type):
-    """Check is the orbital type is handle by gamess"""
-
-    assert len(str_type) == 1
-
-    if str_type in "S P D".split():
-        return True
-    elif str_type > "I" or str_type in "K L M".split():
-        raise BaseException
-    else:
-        return True
-
-
-d_check = {"GAMESS-US": check_gamess,
-           "NWChem": check_NWChem}
-
-
-def get_check_function(name_program):
-    """
-    Tranforme SP special function (create using get_symmetry_function)
-    into S and P
-    """
-    try:
-        f = d_check[name_program]
-    except KeyError:
-        str_ = "You need to add a check funtion for your program {0}"
-        print >> sys.stderr, str_.format(name_program)
-        print >> sys.stderr, "This one are avalaible {0}".format(d_check.keys())
-        sys.exit(1)
-    return f

src/parser/gamess_us.py

@@ -136,3 +136,16 @@ def handle_l_gamess_us(l_atom_basis):
         l_data.append("\n".join(l_line))
 
     return l_data
+
+
+def check_gamess(str_type):
+    """Check is the orbital type is handle by gamess"""
+
+    assert len(str_type) == 1
+
+    if str_type in "S P D".split():
+        return True
+    elif str_type == "SP":
+        raise BaseException
+    else:
+        return True

src/parser/nwchem.py

@@ -226,3 +226,16 @@ def parse_basis_data_nwchem(data, name, description, elements, debug=True):
         serialized = json.dumps(chunk)
         pairs.append([symbol, serialized])
     return [name, description, pairs]
+
+
+def check_NWChem(str_type):
+    """Check is the orbital type is handle by gamess"""
+
+    assert len(str_type) == 1
+
+    if str_type in "S P D".split():
+        return True
+    elif str_type > "I" or str_type in "K L M".split():
+        raise BaseException
+    else:
+        return True

src/parser_handler.py

@@ -1,6 +1,5 @@
 import sys
 import os
-import re
 
 
 def get_dict_ele():
@@ -129,10 +128,38 @@ def get_handle_l_function(format):
     into S and P
     """
     try:
-        f = handle_l_dict[format]
+        return handle_l_dict[format]
     except KeyError:
         print >> sys.stderr, "You need to add a function in handle_l_dict"
         print >> sys.stderr, "for your format ({0})".format(format)
         sys.exit(1)
-    else:
+
-        return f
+
+#  _   _       _ _     _       _   _
+# | | | |     | (_)   | |     | | (_)
+# | | | | __ _| |_  __| | __ _| |_ _  ___  _ __
+# | | | |/ _` | | |/ _` |/ _` | __| |/ _ \| '_ \
+# \ \_/ / (_| | | | (_| | (_| | |_| | (_) | | | |
+#  \___/ \__,_|_|_|\__,_|\__,_|\__|_|\___/|_| |_|
+#
+from src.parser.gamess_us import check_gamess
+from src.parser.nwchem import check_NWChem
+
+d_check = {"GAMESS-US": check_gamess,
+           "NWChem": check_NWChem}
+
+
+def get_check_function(name_program):
+    """
+    Tranforme SP special function (create using get_symmetry_function)
+    into S and P
+    """
+
+    try:
+        return d_check[name_program]
+    except KeyError:
+        str_ = "You need to add a check funtion for your program {0}"
+        print >> sys.stderr, str_.format(name_program)
+        print >> sys.stderr, "This one are avalaible {0}".format(
+            d_check.keys())
+        sys.exit(1)