[mirrorbrain-commits] [opensuse-svn] r6430 - in trunk/tools/download-redirector-v2/mirrordoctor: . mb

From: Novell Forge SVN <noreply_at_novell.com>
Date: Wed, 11 Feb 2009 08:16:44 -0700 (MST)
Author: poeml
Date: 2009-02-11 08:16:31 -0700 (Wed, 11 Feb 2009)
New Revision: 6430

Added:
   trunk/tools/download-redirector-v2/mirrordoctor/mb/exports.py
Modified:
   trunk/tools/download-redirector-v2/mirrordoctor/mb/conn.py
   trunk/tools/download-redirector-v2/mirrordoctor/mb/testmirror.py
   trunk/tools/download-redirector-v2/mirrordoctor/mirrordoctor.py
Log:
mb edit/show: 
 - add new fields and make them editable
mb probefile: 
 - add --md5 option; when doing probe requests, it'll do actual downloads and
   show md5 sums of the responses
mb export: 
 - add --format option
 - add postgresql (insert statements) as export format


Modified: trunk/tools/download-redirector-v2/mirrordoctor/mb/conn.py
===================================================================
--- trunk/tools/download-redirector-v2/mirrordoctor/mb/conn.py	2009-02-11 14:59:29 UTC (rev 6429)
+++ trunk/tools/download-redirector-v2/mirrordoctor/mb/conn.py	2009-02-11 15:16:31 UTC (rev 6430)
@@ -7,14 +7,17 @@
 identifier     : %(identifier)s
 operatorName   : %(operatorName)s
 operatorUrl    : %(operatorUrl)s
-id             : %(id)s
 baseurl        : %(baseurl)s
 baseurlFtp     : %(baseurlFtp)s
 baseurlRsync   : %(baseurlRsync)s
 region         : %(region)s
 country        : %(country)s
+asn            : %(asn)s
+prefix         : %(prefix)s
+regionOnly     : %(regionOnly)s
 countryOnly    : %(countryOnly)s
-regionOnly     : %(regionOnly)s
+asOnly         : %(asOnly)s
+prefixOnly     : %(prefixOnly)s
 otherCountries : %(otherCountries)s
 fileMaxsize    : %(fileMaxsize)s
 publicNotes    : %(publicNotes)s
@@ -34,8 +37,12 @@
                           'baseurlRsync',
                           'region',
                           'country',
+                          'asn',
+                          'prefix',
+                          'regionOnly',
                           'countryOnly',
-                          'regionOnly',
+                          'asOnly',
+                          'prefixOnly',
                           'otherCountries',
                           'fileMaxsize',
                           'score',
@@ -56,17 +63,24 @@
                 baseurlRsync  = s.baseurlRsync,
                 region        = s.region,
                 country       = s.country,
+                asn           = s.asn,
+                prefix        = s.prefix,
+                regionOnly    = s.regionOnly,
                 countryOnly   = s.countryOnly,
-                regionOnly    = s.regionOnly,
+                asOnly        = s.asOnly,
+                prefixOnly    = s.prefixOnly,
                 otherCountries = s.otherCountries,
                 fileMaxsize   = s.fileMaxsize,
                 score         = s.score,
+                scanFpm       = s.scanFpm,
                 publicNotes   = s.publicNotes,
                 enabled       = s.enabled,
                 statusBaseurl = s.statusBaseurl,
                 comment       = s.comment,
                 admin         = s.admin,
                 adminEmail    = s.adminEmail,
+                lat           = s.lat,
+                lng           = s.lng,
                 operatorName  = s.operatorName,
                 operatorUrl   = s.operatorUrl)
 

Added: trunk/tools/download-redirector-v2/mirrordoctor/mb/exports.py
===================================================================
--- trunk/tools/download-redirector-v2/mirrordoctor/mb/exports.py	                        (rev 0)
+++ trunk/tools/download-redirector-v2/mirrordoctor/mb/exports.py	2009-02-11 15:16:31 UTC (rev 6430)
@@ -0,0 +1,60 @@
+import time
+
+postgresql_header = """
+--
+-- generated on %s
+--
+""" % time.asctime()
+
+postgresql_template = """\
+--
+-- %(identifier)s
+--
+INSERT INTO server (
+  identifier, baseurl, baseurl_ftp, baseurl_rsync, enabled, status_baseurl, 
+  region, country, asn, prefix,
+  score, comment, operator_name, operator_url, public_notes,
+  admin, admin_email, lat, lng,
+  country_only, region_only, as_only, prefix_only, 
+  other_countries, file_maxsize, scan_fpm)
+VALUES (
+  '%(identifier)s', '%(baseurl)s', '%(baseurlFtp)s', '%(baseurlRsync)s', '%(enabled)s', '%(statusBaseurl)s', 
+  '%(region)s', '%(country)s', '%(asn)s', '%(prefix)s',
+  '%(score)s', $QUOTE$%(comment)s$QUOTE$, $QUOTE$%(operatorName)s$QUOTE$, '%(operatorUrl)s', $QUOTE$%(publicNotes)s$QUOTE$,
+  $QUOTE$%(admin)s$QUOTE$, $QUOTE$%(adminEmail)s$QUOTE$, '%(lat)s', '%(lng)s',
+  '%(countryOnly)s', '%(regionOnly)s', '%(asOnly)s', '%(prefixOnly)s',
+  '%(otherCountries)s', '%(fileMaxsize)s', '%(scanFpm)s');
+"""
+
+django_header = """#!/usr/bin/env python
+import os, sys
+
+mybasepath = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, mybasepath)
+os.environ['DJANGO_SETTINGS_MODULE'] = 'mirrordjango.settings'
+
+from django.db import connection
+
+from mirrordjango.mb.models import Contact, Operator, Project, Server, Mirror
+
+"""
+
+django_template = """\
+
+# ------------------------------------------------------------
+try:
+    c = Contact.objects.get_or_create(username=%(admin)r, password='UNSET', name=%(admin)r, email=%(adminEmail)r)[0]
+except:
+    connection.connection.rollback()
+    c = None
+o = Operator.objects.get_or_create(name='%(identifier)s', logo='')[0]
+p = Project.objects.filter(name='%(project)s')
+p = p[0]
+s = Server.objects.get_or_create(identifier='%(identifier)s', operator=o, region='%(region)s', country='%(country)s', country_only='%(countryOnly)s', region_only='%(regionOnly)s', as_only='%(asOnly)s', prefix_only='%(prefixOnly)s', other_countries=%(otherCountries)r, file_maxsize='%(fileMaxsize)s', comment=%(comment)r, bandwidth=1)[0]
+m = Mirror.objects.get_or_create(http='%(baseurl)s', ftp='%(baseurlFtp)s', rsync='%(baseurlRsync)s', prio='%(score)s', project=p, server=s)[0]
+# s.mirrors.add(m)
+if c:
+    s.contacts.add(c)
+
+"""
+

Modified: trunk/tools/download-redirector-v2/mirrordoctor/mb/testmirror.py
===================================================================
--- trunk/tools/download-redirector-v2/mirrordoctor/mb/testmirror.py	2009-02-11 14:59:29 UTC (rev 6429)
+++ trunk/tools/download-redirector-v2/mirrordoctor/mb/testmirror.py	2009-02-11 15:16:31 UTC (rev 6430)
@@ -6,6 +6,7 @@
 import tempfile
 import shutil
 import socket
+import mb.util
 
 TIMEOUT = 20
 
@@ -23,9 +24,11 @@
             del os.environ[i]
 
 
-def req(baseurl, filename, http_method='GET'):
+def req(baseurl, filename, http_method='GET', do_digest=False):
 
     url = baseurl + filename
+    worked = False
+    digest = None
 
     if url.startswith('http://') or url.startswith('ftp://'):
         req = urllib2.Request(url)
@@ -35,16 +38,27 @@
 
         try:
             response = urllib2.urlopen(req)
-            return response.code
+            worked = True
         except KeyboardInterrupt:
             print >>sys.stderr, 'interrupted!'
             raise
         except:
-            return 0
+            return (0, digest)
 
+        if do_digest:
+            t = tempfile.NamedTemporaryFile()
+            while 1:
+                buf = response.read(1024*512)
+                if not buf: break
+                t.write(buf)
+            t.flush()
+            digest = mb.util.dgst(t.name)
+            t.close()
+
+        return (response.code, digest)
+
     elif url.startswith('rsync://'):
 
-        worked = False
         try:
             tmpdir = tempfile.mkdtemp(prefix='mb_probefile_')
             # note the -r; *some* option is needed because many rsync servers
@@ -52,18 +66,22 @@
             # -t (as the most harmless option) also isn't sufficient.
             cmd = 'rsync -r --timeout=%d %s %s/' % (TIMEOUT, url, tmpdir)
             (rc, out) = commands.getstatusoutput(cmd)
-            worked = os.path.exists(os.path.join(tmpdir, os.path.basename(filename)))
+            targetfile = os.path.join(tmpdir, os.path.basename(filename))
+            worked = os.path.exists(targetfile)
+            if worked and do_digest:
+                digest = mb.util.dgst(targetfile)
 
+
         finally:
             shutil.rmtree(tmpdir, ignore_errors=True)
 
         if rc != 0:
-            return 1
+            return (1, digest)
 
         if worked:
-            return 200
+            return (200, digest)
         else:
-            return 0
+            return (0, digest)
 
     else:
         raise 'unknown URL type: %r' % baseurl

Modified: trunk/tools/download-redirector-v2/mirrordoctor/mirrordoctor.py
===================================================================
--- trunk/tools/download-redirector-v2/mirrordoctor/mirrordoctor.py	2009-02-11 14:59:29 UTC (rev 6429)
+++ trunk/tools/download-redirector-v2/mirrordoctor/mirrordoctor.py	2009-02-11 15:16:31 UTC (rev 6430)
@@ -151,6 +151,8 @@
                              baseurlRsync = opts.rsync or '',
                              region       = opts.region,
                              country      = opts.country,
+                             asn          = 0,
+                             prefix       = '',
                              score        = opts.score,
                              enabled      = 0,
                              statusBaseurl = 0,
@@ -163,7 +165,9 @@
                              comment      = opts.comment or '',
                              scanFpm      = 0,
                              countryOnly  = 0,
-                             regionOnly   = 0)
+                             regionOnly   = 0,
+                             asOnly       = 0,
+                             prefixOnly   = 0)
         if self.options.debug:
             print s
 
@@ -231,6 +235,8 @@
         mb.testmirror.access_http(mirror.baseurl)
 
 
+    @cmdln.option('--md5', action='store_true',
+                        help='download and show the md5 sum')
     @cmdln.option('-m', '--mirror', 
                         help='probe only on this mirror')
     @cmdln.option('-a', '--all-mirrors', action='store_true',
@@ -257,8 +263,8 @@
             mirrors = self.conn.Server.select()
         else:
             mirrors = self.conn.Server.select(
-                         AND(self.conn.Server.q.statusBaseurl == 1, 
-                             self.conn.Server.q.enabled ==1))
+                         AND(self.conn.Server.q.statusBaseurl, 
+                             self.conn.Server.q.enabled))
 
         found_mirrors = 0
         try:
@@ -268,11 +274,16 @@
                 for baseurl in [mirror.baseurl, mirror.baseurlFtp, mirror.baseurlRsync]:
                     if baseurl == None or baseurl == '':
                         continue
-                    response = mb.testmirror.req(baseurl, filename)
+                    (response, md5) = mb.testmirror.req(baseurl, filename, do_digest=opts.md5)
                     if opts.hide_negative and response != 200:
                         continue
-                    print "%3d %-30s %s" \
-                            % (response, mirror.identifier, os.path.join(baseurl, filename))
+                    if opts.md5:
+                        print "%3d %-30s %-32s %s" \
+                                % (response, mirror.identifier, md5 or '', os.path.join(baseurl, filename))
+                    else:
+                        print "%3d %-30s %s" \
+                                % (response, mirror.identifier, os.path.join(baseurl, filename))
+
                     if response == 200: found_mirrors += 1
 
         except KeyboardInterrupt:
@@ -698,8 +709,10 @@
 
 
 
+    @cmdln.option('--format', metavar='FORMAT',
+            help='Specify the output format: [django|postgresql]')
     @cmdln.option('--project', metavar='PROJECT',
-                  help='Specify a project name.')
+                  help='Specify a project name (previously corresponding to a MirrorBrain instance).')
     def do_export(self, subcmd, opts, *args):
         """${cmd_name}: export the mirror list as text file
 
@@ -709,53 +722,50 @@
         ${cmd_option_list}
         """
 
-        if not opts.project:
-            sys.exit('specify a project name with --project')
+        import mb.exports
 
-        print """#!/usr/bin/env python
-import os, sys
+        if opts.format == 'django' and not opts.project:
+            sys.exit('For Django ORM format, specify a project name (roughly corresponding to a MirrorBrain instance) name with --project')
 
-mybasepath = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.insert(0, mybasepath)
-os.environ['DJANGO_SETTINGS_MODULE'] = 'mirrordjango.settings'
+        if not opts.format:
+            sys.exit('You need to specify an output format. See --help output.')
 
-from django.db import connection
+        if opts.format == 'django':
+            print mb.exports.django_header
 
-from mirrordjango.mb.models import Contact, Operator, Project, Server, Mirror
+            # FIXME: add new fields: operator_name, operator_url, public_notes
+            print """Project(name='%s').save()""" % opts.project
 
-"""
+        elif opts.format == 'postgresql':
+            print mb.exports.postgresql_header
 
-        # FIXME: add new fields: operator_name, operator_url, public_notes
-        print """Project(name='%s').save()""" % opts.project
+        else:
+            sys.exit('unknown format %r' % opts.format)
 
-        django_template = """\
 
-# ------------------------------------------------------------
-try:
-    c = Contact.objects.get_or_create(username=%(admin)r, password='UNSET', name=%(admin)r, email=%(adminEmail)r)[0]
-except:
-    connection.connection.rollback()
-    c = None
-o = Operator.objects.get_or_create(name='%(identifier)s', logo='')[0]
-p = Project.objects.filter(name='""" + opts.project + """')
-p = p[0]
-s = Server.objects.get_or_create(identifier='%(identifier)s', operator=o, region='%(region)s', country='%(country)s', country_only='%(countryOnly)s', region_only='%(regionOnly)s', other_countries=%(otherCountries)r, file_maxsize='%(fileMaxsize)s', comment=%(comment)r, bandwidth=1)[0]
-m = Mirror.objects.get_or_create(http='%(baseurl)s', ftp='%(baseurlFtp)s', rsync='%(baseurlRsync)s', prio='%(score)s', project=p, server=s)[0]
-# s.mirrors.add(m)
-if c:
-    s.contacts.add(c)
-
-"""
-
         mirrors = self.conn.Server.select()
         for i in mirrors:
             if i.comment == None:
                 #print 'null comment', i
                 i.comment = ''
             d = mb.conn.server2dict(i)
-            print django_template % d
+            d.update(dict(project=opts.project))
 
+            #print >>sys.stderr, d
 
+            # replace None's
+            #for i in mb.conn.server_editable_attrs:
+            for i in ['asn', 'prefix', 'asOnly', 'prefixOnly', 'lat', 'lng', 'scanFpm']:
+                if d[i] == None: d[i] = '0'
+            for i in ['prefix', 'baseurlRsync', 'admin', 'adminEmail']:
+                if d[i] == None: d[i] = ''
+
+            if opts.format == 'django':
+                print mb.exports.django_template % d
+            elif opts.format == 'postgresql':
+                print mb.exports.postgresql_template % d
+
+
 if __name__ == '__main__':
     import sys
     mirrordoctor = MirrorDoctor()

_______________________________________________
Opensuse-svn mailing list
Opensuse-svn_at_forge.novell.com
http://forge.novell.com/mailman/listinfo/opensuse-svn


_______________________________________________
mirrorbrain-commits mailing list

Note: To remove yourself from this list, send a mail with the content
 	unsubscribe
to the address mirrorbrain-commits-request_at_mirrorbrain.org
Received on 2009-02-11Z15:17:09

This archive was generated by hypermail 2.2.0 : 2009-07-10Z19:18:11 GMT