[mirrorbrain-commits] r8065 - /trunk/mb/mb/hashes.py

Author: poeml
Date: Fri May 28 18:37:28 2010
New Revision: 8065

URL: http://svn.mirrorbrain.org/viewvc/mirrorbrain?rev=8065&view=rev
Log:
mb makehashes:
- fixing issue 54, non-readable files are now ignored again.

Modified:
    trunk/mb/mb/hashes.py

Modified: trunk/mb/mb/hashes.py
URL: http://svn.mirrorbrain.org/viewvc/mirrorbrain/trunk/mb/mb/hashes.py?rev=8065&r1=8064&r2=8065&view=diff
==============================================================================
--- trunk/mb/mb/hashes.py (original)
+++ trunk/mb/mb/hashes.py Fri May 28 18:37:28 2010
_at_@ -84,6 +84,9 @@
 
         if self.hb.empty:
             self.hb.fill(verbose=verbose)
+        if self.hb.empty:
+            sys.stderr.write('skipping hash (file) generation\n')
+            return
 
         d = open(self.dst, 'wb')
         d.write(self.hb.dump_2_12_template())
_at_@ -114,30 +117,35 @@
 
         c.execute("SELECT id FROM filearr WHERE path = %s LIMIT 1",
                   [self.src_rel])
-        res = c.fetchone()
-        if res:
-            file_id = res[0]
-        else:
-            print 'File %r not in database. Not on mirrors yet? Inserting.' % self.src_rel
+        res_filearr = c.fetchone()
+        if res_filearr:
+            # file already present in the file array table. Is it also known in the hash table?
+            file_id = res_filearr[0]
+            c.execute("SELECT file_id, mtime, size FROM hash WHERE file_id = %s LIMIT 1",
+                      [file_id])
+            res_hash = c.fetchone()
+        else:
+            print 'File %r not in database. Not on mirrors yet? Will be inserted.' % self.src_rel
+            file_id = None
+            res_hash = None
+
+        if not res_hash:
+
+            if dry_run: 
+                print 'Would create hashes in db for: ', self.src_rel
+                return
+
+            if self.hb.empty:
+                self.hb.fill(verbose=verbose)
+            if self.hb.empty:
+                sys.stderr.write('skipping db hash generation\n')
+                return
+
+            c.execute("BEGIN")
             c.execute("INSERT INTO filearr (path, mirrors) VALUES (%s, '{}')",
                       [self.src_rel])
             c.execute("SELECT currval('filearr_id_seq')")
             file_id =  c.fetchone()[0]
-            c.execute("commit")
-            
-
-        c.execute("SELECT file_id, mtime, size FROM hash WHERE file_id = %s LIMIT 1",
-                  [file_id])
-        res = c.fetchone()
-
-        if not res:
-
-            if dry_run: 
-                print 'Would create hashes in db for: ', self.src_rel
-                return
-
-            if self.hb.empty:
-                self.hb.fill(verbose=verbose)
 
             c.execute("""INSERT INTO hash (file_id, mtime, size, md5, 
                                            sha1, sha256, sha1piecesize, 
_at_@ -158,10 +166,11 @@
                        self.hb.get_zparams(),
                        binascii.hexlify(''.join(self.hb.zsums))]
                       )
+            c.execute("COMMIT")
             if verbose:
                 print 'Hash was not present yet in database - inserted'
         else:
-            mtime, size = res[1], res[2]
+            mtime, size = res_hash[1], res_hash[2]
             
             if int(self.mtime) == mtime and self.size == size and not force:
                 if verbose:
_at_@ -170,6 +179,9 @@
 
             if self.hb.empty:
                 self.hb.fill(verbose=verbose)
+            if self.hb.empty:
+                sys.stderr.write('skipping db hash generation\n')
+                return
 
             c.execute("""UPDATE hash set mtime = %s, size = %s, 
                                          md5 = decode(%s, 'hex'), 
_at_@ -245,7 +257,11 @@
             s256 = sha256.sha256()
         short_read_before = False
 
-        f = open(self.src, 'rb')
+        try:
+            f = open(self.src, 'rb')
+        except IOError, e:
+            sys.stderr.write('%s\n' % e)
+            return None
 
         while 1 + 1 == 2:
             buf = f.read(PIECESIZE)




_______________________________________________
mirrorbrain-commits mailing list
Archive: http://mirrorbrain.org/archive/mirrorbrain-commits/

Note: To remove yourself from this list, send a mail with the content
 	unsubscribe
to the address mirrorbrain-commits-request_at_mirrorbrain.org