[mirrorbrain-commits] r7970 - in /trunk/mirrordoctor/mb: conn.py hashes.py

Author: poeml
Date: Wed Mar 10 01:58:32 2010
New Revision: 7970

URL: http://svn.mirrorbrain.org/viewvc/mirrorbrain?rev=7970&view=rev
Log:
mb:
- Automatically install the newly added hash table in the database if it's not there yet.
  (This is crude. There should be a better place for this code. Maybe a separate module for upgrades.)
mb makehashes:
- Now, hashes are also stored for files which exists only locally, and not on any mirror
  (and which weren't present in the filearr table yet, therefore).
- Minor cosmetic fixes.

Modified:
    trunk/mirrordoctor/mb/conn.py
    trunk/mirrordoctor/mb/hashes.py

Modified: trunk/mirrordoctor/mb/conn.py
URL: http://svn.mirrorbrain.org/viewvc/mirrorbrain/trunk/mirrordoctor/mb/conn.py?rev=7970&r1=7969&r2=7970&view=diff
==============================================================================
--- trunk/mirrordoctor/mb/conn.py (original)
+++ trunk/mirrordoctor/mb/conn.py Wed Mar 10 01:58:32 2010
_at_@ -170,8 +170,48 @@
                     idName = 'file_id'
             self.Hash = Hash
         except psycopg2.ProgrammingError:
-            # this is raised if the table hasn't been installed yet
-            pass
+            # This is what's raised if the table hasn't been installed yet
+            # Which is the case when coming from a 2.12.0 or earlier install
+            # XXX This feels like being totally the wrong place for a database migration.
+            #     maybe a separate module with upgrade procedures to be run would be better.
+            #     The main point is that this is a migration that we want to happen fully automatically.
+            print >>sys.stderr
+            print >>sys.stderr, '>>> A database table for hashes does not exit. Creating...'
+            query = """
+            CREATE TABLE "hash" (
+                    "file_id" INTEGER REFERENCES filearr PRIMARY KEY,
+                    "mtime" INTEGER NOT NULL,
+                    "size"  INTEGER NOT NULL,
+                    "md5"    BYTEA NOT NULL,
+                    "sha1"   BYTEA NOT NULL,
+                    "sha256" BYTEA NOT NULL,
+                    "sha1piecesize" INTEGER NOT NULL,
+                    "sha1pieces" BYTEA,
+                    "pgp" TEXT NOT NULL
+            );
+            """
+            Filearr._connection.query(query)
+            query = """
+            CREATE VIEW hexhash AS
+              SELECT file_id, mtime, size,
+                     encode(md5, 'hex') AS md5,
+                     encode(sha1, 'hex') AS sha1,
+                     encode(sha256, 'hex') AS sha256,
+                     sha1piecesize,
+                     encode(sha1pieces, 'hex') AS sha1pieces,
+                     pgp
+              FROM hash;
+            """
+            Filearr._connection.query(query)
+            print >>sys.stderr, '>>> Done.'
+            print >>sys.stderr 
+            # now try again
+            class Hash(SQLObject):
+                """the hashes table"""
+                class sqlmeta:
+                    fromDatabase = True
+                    idName = 'file_id'
+            self.Hash = Hash
 
         if debug:
             self.Server._connection.debug = True

Modified: trunk/mirrordoctor/mb/hashes.py
URL: http://svn.mirrorbrain.org/viewvc/mirrorbrain/trunk/mirrordoctor/mb/hashes.py?rev=7970&r1=7969&r2=7970&view=diff
==============================================================================
--- trunk/mirrordoctor/mb/hashes.py (original)
+++ trunk/mirrordoctor/mb/hashes.py Wed Mar 10 01:58:32 2010
_at_@ -72,7 +72,7 @@
             return 
 
         if dry_run: 
-            print 'Would make hashes for: ', self.src
+            print 'Would make hash file', self.dst
             return
 
         if self.hb.empty:
_at_@ -107,17 +107,26 @@
         c.execute("SELECT id FROM filearr WHERE path = %s LIMIT 1",
                   [self.src_rel])
         res = c.fetchone()
-        if not res:
-            print 'file %r not found (no mirror has it?)' % self.src_rel
-            ### XXX we'd need to insert it, if we want to support hashes for files that are not on any mirror...
-            return
-        file_id = res[0]
+        if res:
+            file_id = res[0]
+        else:
+            print 'File %r not found. Not on mirrors yet? Inserting.' % self.src_rel
+            c.execute("INSERT INTO filearr (path, mirrors) VALUES (%s, '{}')",
+                      [self.src_rel])
+            c.execute("SELECT currval('filearr_id_seq')")
+            file_id =  c.fetchone()[0]
+            c.execute("commit")
+            
 
         c.execute("SELECT file_id, mtime, size FROM hash WHERE file_id = %s LIMIT 1",
                   [file_id])
         res = c.fetchone()
 
         if not res:
+
+            if dry_run: 
+                print 'Would create hashes in db for: ', self.src_rel
+                return
 
             if self.hb.empty:
                 self.hb.fill(verbose=verbose)
_at_@ -136,7 +145,8 @@
                        PIECESIZE,
                        ''.join(self.hb.pieceshex),
                        self.hb.pgp or ''])
-            print 'hash was not present yet in database - inserted'
+            if verbose:
+                print 'Hash was not present yet in database - inserted'
         else:
             mtime, size = res[1], res[2]
             if int(self.mtime) == mtime and self.size == size and not force:




_______________________________________________
mirrorbrain-commits mailing list
Archive: http://mirrorbrain.org/archive/mirrorbrain-commits/

Note: To remove yourself from this list, send a mail with the content
 	unsubscribe
to the address mirrorbrain-commits-request_at_mirrorbrain.org