Add dump, remove db

EMSL_api.py

@@ -6,19 +6,19 @@
 Usage:
   EMSL_api.py list_basis [--basis=<basis_name>...]
                          [--atom=<atom_name>...]
-                         [--db_path=<db_path>]
+                         [--db_path=<db_path> |--db_dump_path=<db_dump_path>]
                          [--average_mo_number]
   EMSL_api.py list_atoms --basis=<basis_name>
-                         [--db_path=<db_path>]
+                         [--db_path=<db_path> |--db_dump_path=<db_dump_path>]
   EMSL_api.py get_basis_data --basis=<basis_name>
                              [--atom=<atom_name>...]
-                             [--db_path=<db_path>]
+                             [--db_path=<db_path> |--db_dump_path=<db_dump_path>]
                              [(--save [--path=<path>])]
                              [--check=<program_name>]
                              [--treat_l]
   EMSL_api.py list_formats
   EMSL_api.py create_db --format=<format>
-                        [--db_path=<db_path>]
+                        [--db_path=<db_path> |--db_dump_path=<db_dump_path>]
                         [--no-contraction]
   EMSL_api.py (-h | --help)
   EMSL_api.py --version
@@ -49,7 +49,6 @@ from src.EMSL_local import EMSL_local
 if __name__ == '__main__':
 
     arguments = docopt(__doc__, version='EMSL Api ' + version)
-
     # ___
     #  |  ._  o _|_
     # _|_ | | |  |_
@@ -57,17 +56,24 @@ if __name__ == '__main__':
 
     if arguments["--db_path"]:
         db_path = arguments["--db_path"]
+        db_dump_path = None
+    elif arguments["--db_dump_path"]:
+    	db_path = None
+    	db_dump_path = arguments["--db_dump_path"]
     else:
-        db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+    	db_dump_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
-                               "db/GAMESS-US.db")
+                               "db/GAMESS-US.dump")
+        db_path = None
+#        db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+#                               "db/GAMESS-US.db")
 
     # Check the db
-    try:
+#    try:
-        if not(arguments['create_db']):
+#        if not(arguments['create_db']):
-            from src.EMSL_local import checkSQLite3
+#            from src.EMSL_local import checkSQLite3
-            db_path, db_path_changed = checkSQLite3(db_path)
+#            db_path, db_path_changed = checkSQLite3(db_path)
-    except:
+#    except:
-        raise
+#        raise
 
     #  _     _     _    ______           _
     # | |   (_)   | |   | ___ \         (_)
@@ -77,7 +83,7 @@ if __name__ == '__main__':
     # \_____/_|___/\__| \____/ \__,_|___/_|___/
 
     if arguments["list_basis"]:
-        e = EMSL_local(db_path=db_path)
+        e = EMSL_local(db_path=db_path, db_dump_path=db_dump_path)
 
         l = e.list_basis_available(arguments["--atom"],
                                    arguments["--basis"],
@@ -99,7 +105,7 @@ if __name__ == '__main__':
     # | |___| \__ \ |_  | |___| |  __/ | | | | |  __/ | | | |_\__ \
     # \_____/_|___/\__| \____/|_|\___|_| |_| |_|\___|_| |_|\__|___/
     elif arguments["list_atoms"]:
-        e = EMSL_local(db_path=db_path)
+        e = EMSL_local(db_path=db_path, db_dump_path=db_dump_path)
 
         basis_name = arguments["--basis"]
         l = e.get_list_element_available(basis_name)
@@ -112,7 +118,7 @@ if __name__ == '__main__':
     # | |_/ / (_| \__ \ \__ \ | (_| | (_| | || (_| |
     # \____/ \__,_|___/_|___/  \__,_|\__,_|\__\__,_|
     elif arguments["get_basis_data"]:
-        e = EMSL_local(db_path=db_path)
+        e = EMSL_local(db_path=db_path, db_dump_path=db_dump_path)
         basis_name = arguments["--basis"][0]
         elts = arguments["--atom"]
 
@@ -170,5 +176,5 @@ if __name__ == '__main__':
     #                         _|
 
     # Clean up on exit
-    if not(arguments['create_db']) and db_path_changed:
+#    if not(arguments['create_db']) and db_path_changed:
-        os.system("rm -f /dev/shm/%d.db" % (os.getpid()))
+#        os.system("rm -f /dev/shm/%d.db" % (os.getpid()))

src/EMSL_local.py

@@ -5,6 +5,7 @@ import re
 import sys
 import os
 
+from misc.sqlit import connect4git
 
 def checkSQLite3(db_path):
     """Check if the db_path is a good one"""
@@ -100,10 +101,16 @@ class EMSL_local(object):
     All the method for using the EMSL db localy
     """
 
-    def __init__(self, db_path=None):
+    def __init__(self, db_path=None, db_dump_path=None):
-        self.db_path = db_path
+
+#        print db_path
+#        print db_dump_path
+
+        if db_path:
+            self.conn = sqlite3.connect(db_path)
+        if db_dump_path:
+            self.conn = connect4git(db_dump_path)
 
-        self.conn = sqlite3.connect(self.db_path)
         self.c = self.conn.cursor()
 
         self.c.execute("SELECT * from format_tab")

src/misc/sqlit.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+import sys
+
+try:
+    import sqlite3
+except:
+    print "Sorry, you need sqlite3"
+    sys.exit(1)
+
+
+#
+#  _____         _  _  _
+# /  ___|       | |(_)| |
+# \ `--.   __ _ | | _ | |_
+#  `--. \ / _` || || || __|
+# /\__/ /| (_| || || || |_
+# \____/  \__, ||_||_| \__|
+#            | |
+#            |_|
+# You cannot Diff / Merge sqlite binary file. But you can dump it into
+# a plain text file with contain the list of command needed to recreate
+# the db_file.
+#   (`sqlite3 db_file .dump > dump_file` For a example of dump file, see below
+#    in the main section)
+#
+# This dump_file can be added to your git.
+#
+# This module ensure the coherencies between the db_file and the dump_file
+
+class ConnectionForGit(sqlite3.Connection):
+
+    """
+    A sqlite3 connection for Git. It will always dumps the db when needed.
+        (`sqlite3 db_file .dump > dump_file`)
+    so now you can add dump_path to your git and work with the db like always.
+    The main idea is:
+        - When you create a sqlite3 connection: create or update the .db file
+        - When you commit: update the .dump file
+    /!\ If you create a table, you -maybe- need to make a dummy commit
+            for update the dump
+    """
+
+    def __init__(self, db_path, dump_path, *args, **kwargs):
+        super(ConnectionForGit, self).__init__(db_path, *args, **kwargs)
+        self.db_path = db_path
+        self.dump_path = dump_path
+
+    def commit(self):
+        '''
+        0/ Update the DB if needed
+             (aka if the dump is more recent than the db)
+        1/ Commit in the DB
+        2/ Dump the DB
+             (`sqlite3 db_file .dump > dump_file`)
+        '''
+        try:
+            ConnectionForGit.dump_to_sqlite(self.dump_path, self.db_path)
+            sqlite3.Connection.commit(self)
+        except sqlite3.Error:
+            raise
+        else:
+            ConnectionForGit.sqlite_to_dump(self.db_path, self.dump_path)
+
+    @staticmethod
+    def isSQLite3(filename):
+        """
+        Verify is filename is a SQLite3 format db
+        """
+        from os.path import isfile, getsize
+
+        if not isfile(filename):
+            return False
+        # SQLite database file header is 100 bytes
+        if getsize(filename) < 100:
+            return False
+
+        with open(filename, 'rb') as fd:
+            header = fd.read(100)
+
+        if header[:16] == 'SQLite format 3\x00':
+            return True
+        else:
+            return False
+
+    @staticmethod
+    def dump_to_sqlite(dump_path, db_path):
+        """
+        Take a dump and populate the db if the dump is the most recent
+        0/ If no .dump file touch it
+        1/ If no .db  file populate it
+        2/ Check the date a modify accordingly
+        """
+        if not os.path.isfile(dump_path):
+            os.system('touch {0}'.format(dump_path))
+        if not os.path.isfile(db_path):
+            os.system("sqlite3 {0} < {1}".format(db_path, dump_path))
+        else:
+            dump_time = os.path.getmtime(dump_path)
+            db_time = os.path.getmtime(db_path)
+            if dump_time > db_time:
+                os.system("rm {0}".format(db_path))
+                os.system("sqlite3 {0} < {1}".format(db_path, dump_path))
+
+    @staticmethod
+    def sqlite_to_dump(db_path, dump_path):
+        """
+        Take a db and dump it if the db is the most recent
+        """
+
+        if not ConnectionForGit.isSQLite3(db_path):
+            raise sqlite3.Error
+
+        dump_time = os.path.getmtime(dump_path)
+        db_time = os.path.getmtime(db_path)
+        if db_time > dump_time:
+            os.system("sqlite3 {0} .dump > {1}".format(db_path, dump_path))
+            os.system("touch {0}".format(db_path))
+
+
+def connect4git(dump_path, db_path=None, *args, **kwargs):
+    '''
+    dump_path :  Is the sqlite dump file you will monitor with git.
+    db_path   :  Is the <<dummy>> sqlite3 file.
+
+    0/ Update and create the db if needed
+            (aka if the dump is more recent than the db)
+    1/ Return a connection the db (a ConnectionForGit instance)
+    '''
+
+    if not db_path:
+        db_path = "{0}.db".format(os.path.splitext(dump_path)[0])
+
+    try:
+        ConnectionForGit.dump_to_sqlite(dump_path, db_path)
+    except:
+        raise
+    else:
+        return ConnectionForGit(db_path, dump_path, *args, **kwargs)
+
+
+# ___  ___      _
+# |  \/  |     (_)
+# | .  . | __ _ _ _ __
+# | |\/| |/ _` | | '_ \
+# | |  | | (_| | | | | |
+# \_|  |_/\__,_|_|_| |_|
+#
+if __name__ == "__main__":
+    # From the famous http://zetcode.com/db/sqlitepythontutorial/
+
+    # It will create a test.dump and a test.db
+    # You can git add the test.dump and do your work like always in the db.
+    con = connect4git('test.dump')
+
+    with con:
+        cur = con.cursor()
+        cur.execute("CREATE TABLE Cars(Id INT, Name TEXT, Price INT)")
+        cur.execute("INSERT INTO Cars VALUES(1,'Audi',52642)")
+        cur.execute("INSERT INTO Cars VALUES(2,'Mercedes',57127)")
+        cur.execute("INSERT INTO Cars VALUES(3,'Skoda',9000)")
+        cur.execute("INSERT INTO Cars VALUES(4,'Volvo',29000)")
+        cur.execute("INSERT INTO Cars VALUES(5,'Bentley',350000)")
+        cur.execute("INSERT INTO Cars VALUES(6,'Citroen',21000)")
+        cur.execute("INSERT INTO Cars VALUES(7,'Hummer',41400)")
+        cur.execute("INSERT INTO Cars VALUES(8,'Volkswagen',21600)")
+
+    #  Test.db :
+    #
+    #  PRAGMA foreign_keys=OFF;
+    #  BEGIN TRANSACTION;
+    #  CREATE TABLE Cars(Id INT, Name TEXT, Price INT);
+    #  INSERT INTO "Cars" VALUES(1,'Audi',52642);
+    #  INSERT INTO "Cars" VALUES(2,'Mercedes',57127);
+    #  INSERT INTO "Cars" VALUES(3,'Skoda',9000);
+    #  INSERT INTO "Cars" VALUES(4,'Volvo',29000);
+    #  INSERT INTO "Cars" VALUES(5,'Bentley',350000);
+    #  INSERT INTO "Cars" VALUES(6,'Citroen',21000);
+    #  INSERT INTO "Cars" VALUES(7,'Hummer',41400);
+    #  INSERT INTO "Cars" VALUES(8,'Volkswagen',21600);
+    #  COMMIT;