[mirrorbrain-commits] [opensuse-svn] r6594 - trunk/tools/download-redirector-v2/scanner

Author: poeml
Date: 2009-02-25 19:18:12 -0700 (Wed, 25 Feb 2009)
New Revision: 6594

Modified:
   trunk/tools/download-redirector-v2/scanner/scanner.pl
Log:
scanner:
- implement for FTP, what was done for HTTP and rsync yesterday: 
  allow the definition of top-level directories that are scanned and ignore all
  others. Saves on database size, and makes scans a lot faster in many cases.


Modified: trunk/tools/download-redirector-v2/scanner/scanner.pl
===================================================================
--- trunk/tools/download-redirector-v2/scanner/scanner.pl	2009-02-25 22:05:38 UTC (rev 6593)
+++ trunk/tools/download-redirector-v2/scanner/scanner.pl	2009-02-26 02:18:12 UTC (rev 6594)
@@ -434,7 +434,7 @@
   -i regexp 
             Define regexp-pattern for path names to ignore. 
 	    Use '-i 0' to disable any ignore patterns. Default: @norecurse_list
-  -T /dir/  Directory to be scanned at the top level; option can be repeated.
+  -T dir    Directory to be scanned at the top level; option can be repeated.
 
 Both, names(identifier) and numbers(id) are accepted as mirror_ids.
 };
@@ -635,17 +635,32 @@
 {
   my ($identifier, $id, $url, $name, $ftp) = @_;
 
+  my $item;
+  my $included = 0;
+  foreach my $item(@top_include_list) {
+    if ($name =~ $item) {
+      $included = 1;
+    }
+  }
+  if (scalar(@top_include_list) && ("$name/" ne "/") && !$included) {
+    print "$identifier: not in top_include_list: $name\n";# if $verbose > 1;
+    return;
+  }
+
   # ignore paths matching those in @norecurse-list:
-  for my $item(@norecurse_list) {
-    return if $start_dir =~ $item;
+  for $item(@norecurse_list) {
+    if ($name =~ $item) {
+      print "$identifier: ignore match: $name matches ignored item $item, skipped.\n" if $verbose > 1;
+      return;
+    }
   }
 
+  print "$identifier: ftp dir: $name\n" if $verbose > 1;
+
   my $urlraw = $url;
   my $re = ''; $re = $1 if $url =~ s{#(.*?)$}{};
   $url =~ s{/+$}{};	# we add our own trailing slashes...
 
-  print "$identifier: ftp dir: $url/$name\n" if $verbose > 1;
-
   my $toplevel = ($ftp) ? 0 : 1;
   $ftp = ftp_connect($identifier, "$url/$name", "anonymous", $scanner_email) unless defined $ftp;
   return unless defined $ftp;

_______________________________________________
Opensuse-svn mailing list
Opensuse-svn_at_forge.novell.com
http://forge.novell.com/mailman/listinfo/opensuse-svn


_______________________________________________
mirrorbrain-commits mailing list

Note: To remove yourself from this list, send a mail with the content
 	unsubscribe
to the address mirrorbrain-commits-request_at_mirrorbrain.org