[mirrorbrain-commits] r7795 - /trunk/tools/metalink-hasher.py

Author: poeml
Date: Tue Sep  8 20:36:27 2009
New Revision: 7795

URL: http://svn.mirrorbrain.org/viewvc/mirrorbrain?rev=7795&view=rev
Log:
metalink-hasher:
- implement per-directory locking. Directories where already a job is running
  will be skipped. This allows for hassle-free parallel runs of more than one
  jobs. Note that simultaneous spawning of the script still needs to be
  controlled, because the I/O bandwidth consumed could become more than good
  for a machine.

Modified:
    trunk/tools/metalink-hasher.py

Modified: trunk/tools/metalink-hasher.py
URL: http://svn.mirrorbrain.org/viewvc/mirrorbrain/trunk/tools/metalink-hasher.py?rev=7795&r1=7794&r2=7795&view=diff
==============================================================================
--- trunk/tools/metalink-hasher.py (original)
+++ trunk/tools/metalink-hasher.py Tue Sep  8 20:36:27 2009
_at_@ -39,6 +39,7 @@
 import re
 import subprocess
 import errno
+import fcntl
 
 line_mask = re.compile('.*</*(verification|hash|pieces).*>.*')
 
_at_@ -253,9 +254,25 @@
 
             # a set offers the fastest access for "foo in ..." lookups
             src_basenames = set(os.listdir(src_dir))
-            #print 'doing', src_dir
+
+            if opts.verbose:
+                print 'looking at', src_dir
 
             dst_keep = set()
+
+            lockfile = os.path.join(dst_dir, 'LOCK')
+            try:
+                if not opts.dry_run:
+                    lock = open(lockfile, 'w')
+                    fcntl.lockf(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                dst_keep.add('LOCK')
+                if opts.verbose:
+                    print 'locked %s' % lockfile
+            except IOError, e:
+                if e.errno == errno.EWOULDBLOCK:
+                    print 'Skipping %r, which is locked' % src_dir
+                    continue
+
 
             for src_basename in sorted(src_basenames):
                 src = os.path.join(src_dir, src_basename)
_at_@ -327,11 +344,15 @@
                         try:
                             os.unlink(i_path)
                         except OSError, e:
-                            if e.errno != errno.ENOENT:
-                                sys.stderr.write('Unlink failed for %r: %s\n' \
-                                                    % (i_path, os.strerror(e.errno)))
+                            sys.stderr.write('Unlink failed for %r: %s\n' \
+                                                % (i_path, os.strerror(e.errno)))
                     unlinked_files += 1
 
+            if opts.verbose:
+                print 'unlocking', lockfile 
+            if not opts.dry_run:
+                lock.close()
+                os.unlink(lockfile)
 
         if  unlinked_files or unlinked_dirs:
             print 'Unlinked %s files, %d directories.' % (unlinked_files, unlinked_dirs)




_______________________________________________
mirrorbrain-commits mailing list
Archive: http://mirrorbrain.org/archive/mirrorbrain-commits/

Note: To remove yourself from this list, send a mail with the content
 	unsubscribe
to the address mirrorbrain-commits-request_at_mirrorbrain.org