Author: poeml Date: 2009-02-11 08:16:31 -0700 (Wed, 11 Feb 2009) New Revision: 6430 Added: trunk/tools/download-redirector-v2/mirrordoctor/mb/exports.py Modified: trunk/tools/download-redirector-v2/mirrordoctor/mb/conn.py trunk/tools/download-redirector-v2/mirrordoctor/mb/testmirror.py trunk/tools/download-redirector-v2/mirrordoctor/mirrordoctor.py Log: mb edit/show: - add new fields and make them editable mb probefile: - add --md5 option; when doing probe requests, it'll do actual downloads and show md5 sums of the responses mb export: - add --format option - add postgresql (insert statements) as export format Modified: trunk/tools/download-redirector-v2/mirrordoctor/mb/conn.py =================================================================== --- trunk/tools/download-redirector-v2/mirrordoctor/mb/conn.py 2009-02-11 14:59:29 UTC (rev 6429) +++ trunk/tools/download-redirector-v2/mirrordoctor/mb/conn.py 2009-02-11 15:16:31 UTC (rev 6430) _at_@ -7,14 +7,17 @@ identifier : %(identifier)s operatorName : %(operatorName)s operatorUrl : %(operatorUrl)s -id : %(id)s baseurl : %(baseurl)s baseurlFtp : %(baseurlFtp)s baseurlRsync : %(baseurlRsync)s region : %(region)s country : %(country)s +asn : %(asn)s +prefix : %(prefix)s +regionOnly : %(regionOnly)s countryOnly : %(countryOnly)s -regionOnly : %(regionOnly)s +asOnly : %(asOnly)s +prefixOnly : %(prefixOnly)s otherCountries : %(otherCountries)s fileMaxsize : %(fileMaxsize)s publicNotes : %(publicNotes)s _at_@ -34,8 +37,12 @@ 'baseurlRsync', 'region', 'country', + 'asn', + 'prefix', + 'regionOnly', 'countryOnly', - 'regionOnly', + 'asOnly', + 'prefixOnly', 'otherCountries', 'fileMaxsize', 'score', _at_@ -56,17 +63,24 @@ baseurlRsync = s.baseurlRsync, region = s.region, country = s.country, + asn = s.asn, + prefix = s.prefix, + regionOnly = s.regionOnly, countryOnly = s.countryOnly, - regionOnly = s.regionOnly, + asOnly = s.asOnly, + prefixOnly = s.prefixOnly, otherCountries = s.otherCountries, fileMaxsize = s.fileMaxsize, score = s.score, + scanFpm = s.scanFpm, publicNotes = s.publicNotes, enabled = s.enabled, statusBaseurl = s.statusBaseurl, comment = s.comment, admin = s.admin, adminEmail = s.adminEmail, + lat = s.lat, + lng = s.lng, operatorName = s.operatorName, operatorUrl = s.operatorUrl) Added: trunk/tools/download-redirector-v2/mirrordoctor/mb/exports.py =================================================================== --- trunk/tools/download-redirector-v2/mirrordoctor/mb/exports.py (rev 0) +++ trunk/tools/download-redirector-v2/mirrordoctor/mb/exports.py 2009-02-11 15:16:31 UTC (rev 6430) _at_@ -0,0 +1,60 @@ +import time + +postgresql_header = """ +-- +-- generated on %s +-- +""" % time.asctime() + +postgresql_template = """\ +-- +-- %(identifier)s +-- +INSERT INTO server ( + identifier, baseurl, baseurl_ftp, baseurl_rsync, enabled, status_baseurl, + region, country, asn, prefix, + score, comment, operator_name, operator_url, public_notes, + admin, admin_email, lat, lng, + country_only, region_only, as_only, prefix_only, + other_countries, file_maxsize, scan_fpm) +VALUES ( + '%(identifier)s', '%(baseurl)s', '%(baseurlFtp)s', '%(baseurlRsync)s', '%(enabled)s', '%(statusBaseurl)s', + '%(region)s', '%(country)s', '%(asn)s', '%(prefix)s', + '%(score)s', $QUOTE$%(comment)s$QUOTE$, $QUOTE$%(operatorName)s$QUOTE$, '%(operatorUrl)s', $QUOTE$%(publicNotes)s$QUOTE$, + $QUOTE$%(admin)s$QUOTE$, $QUOTE$%(adminEmail)s$QUOTE$, '%(lat)s', '%(lng)s', + '%(countryOnly)s', '%(regionOnly)s', '%(asOnly)s', '%(prefixOnly)s', + '%(otherCountries)s', '%(fileMaxsize)s', '%(scanFpm)s'); +""" + +django_header = """#!/usr/bin/env python +import os, sys + +mybasepath = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, mybasepath) +os.environ['DJANGO_SETTINGS_MODULE'] = 'mirrordjango.settings' + +from django.db import connection + +from mirrordjango.mb.models import Contact, Operator, Project, Server, Mirror + +""" + +django_template = """\ + +# ------------------------------------------------------------ +try: + c = Contact.objects.get_or_create(username=%(admin)r, password='UNSET', name=%(admin)r, email=%(adminEmail)r)[0] +except: + connection.connection.rollback() + c = None +o = Operator.objects.get_or_create(name='%(identifier)s', logo='')[0] +p = Project.objects.filter(name='%(project)s') +p = p[0] +s = Server.objects.get_or_create(identifier='%(identifier)s', operator=o, region='%(region)s', country='%(country)s', country_only='%(countryOnly)s', region_only='%(regionOnly)s', as_only='%(asOnly)s', prefix_only='%(prefixOnly)s', other_countries=%(otherCountries)r, file_maxsize='%(fileMaxsize)s', comment=%(comment)r, bandwidth=1)[0] +m = Mirror.objects.get_or_create(http='%(baseurl)s', ftp='%(baseurlFtp)s', rsync='%(baseurlRsync)s', prio='%(score)s', project=p, server=s)[0] +# s.mirrors.add(m) +if c: + s.contacts.add(c) + +""" + Modified: trunk/tools/download-redirector-v2/mirrordoctor/mb/testmirror.py =================================================================== --- trunk/tools/download-redirector-v2/mirrordoctor/mb/testmirror.py 2009-02-11 14:59:29 UTC (rev 6429) +++ trunk/tools/download-redirector-v2/mirrordoctor/mb/testmirror.py 2009-02-11 15:16:31 UTC (rev 6430) _at_@ -6,6 +6,7 @@ import tempfile import shutil import socket +import mb.util TIMEOUT = 20 _at_@ -23,9 +24,11 @@ del os.environ[i] -def req(baseurl, filename, http_method='GET'): +def req(baseurl, filename, http_method='GET', do_digest=False): url = baseurl + filename + worked = False + digest = None if url.startswith('http://') or url.startswith('ftp://'): req = urllib2.Request(url) _at_@ -35,16 +38,27 @@ try: response = urllib2.urlopen(req) - return response.code + worked = True except KeyboardInterrupt: print >>sys.stderr, 'interrupted!' raise except: - return 0 + return (0, digest) + if do_digest: + t = tempfile.NamedTemporaryFile() + while 1: + buf = response.read(1024*512) + if not buf: break + t.write(buf) + t.flush() + digest = mb.util.dgst(t.name) + t.close() + + return (response.code, digest) + elif url.startswith('rsync://'): - worked = False try: tmpdir = tempfile.mkdtemp(prefix='mb_probefile_') # note the -r; *some* option is needed because many rsync servers _at_@ -52,18 +66,22 @@ # -t (as the most harmless option) also isn't sufficient. cmd = 'rsync -r --timeout=%d %s %s/' % (TIMEOUT, url, tmpdir) (rc, out) = commands.getstatusoutput(cmd) - worked = os.path.exists(os.path.join(tmpdir, os.path.basename(filename))) + targetfile = os.path.join(tmpdir, os.path.basename(filename)) + worked = os.path.exists(targetfile) + if worked and do_digest: + digest = mb.util.dgst(targetfile) + finally: shutil.rmtree(tmpdir, ignore_errors=True) if rc != 0: - return 1 + return (1, digest) if worked: - return 200 + return (200, digest) else: - return 0 + return (0, digest) else: raise 'unknown URL type: %r' % baseurl Modified: trunk/tools/download-redirector-v2/mirrordoctor/mirrordoctor.py =================================================================== --- trunk/tools/download-redirector-v2/mirrordoctor/mirrordoctor.py 2009-02-11 14:59:29 UTC (rev 6429) +++ trunk/tools/download-redirector-v2/mirrordoctor/mirrordoctor.py 2009-02-11 15:16:31 UTC (rev 6430) _at_@ -151,6 +151,8 @@ baseurlRsync = opts.rsync or '', region = opts.region, country = opts.country, + asn = 0, + prefix = '', score = opts.score, enabled = 0, statusBaseurl = 0, _at_@ -163,7 +165,9 @@ comment = opts.comment or '', scanFpm = 0, countryOnly = 0, - regionOnly = 0) + regionOnly = 0, + asOnly = 0, + prefixOnly = 0) if self.options.debug: print s _at_@ -231,6 +235,8 @@ mb.testmirror.access_http(mirror.baseurl) + _at_cmdln.option('--md5', action='store_true', + help='download and show the md5 sum') _at_cmdln.option('-m', '--mirror', help='probe only on this mirror') _at_cmdln.option('-a', '--all-mirrors', action='store_true', _at_@ -257,8 +263,8 @@ mirrors = self.conn.Server.select() else: mirrors = self.conn.Server.select( - AND(self.conn.Server.q.statusBaseurl == 1, - self.conn.Server.q.enabled ==1)) + AND(self.conn.Server.q.statusBaseurl, + self.conn.Server.q.enabled)) found_mirrors = 0 try: _at_@ -268,11 +274,16 @@ for baseurl in [mirror.baseurl, mirror.baseurlFtp, mirror.baseurlRsync]: if baseurl == None or baseurl == '': continue - response = mb.testmirror.req(baseurl, filename) + (response, md5) = mb.testmirror.req(baseurl, filename, do_digest=opts.md5) if opts.hide_negative and response != 200: continue - print "%3d %-30s %s" \ - % (response, mirror.identifier, os.path.join(baseurl, filename)) + if opts.md5: + print "%3d %-30s %-32s %s" \ + % (response, mirror.identifier, md5 or '', os.path.join(baseurl, filename)) + else: + print "%3d %-30s %s" \ + % (response, mirror.identifier, os.path.join(baseurl, filename)) + if response == 200: found_mirrors += 1 except KeyboardInterrupt: _at_@ -698,8 +709,10 @@ + _at_cmdln.option('--format', metavar='FORMAT', + help='Specify the output format: [django|postgresql]') _at_cmdln.option('--project', metavar='PROJECT', - help='Specify a project name.') + help='Specify a project name (previously corresponding to a MirrorBrain instance).') def do_export(self, subcmd, opts, *args): """${cmd_name}: export the mirror list as text file _at_@ -709,53 +722,50 @@ ${cmd_option_list} """ - if not opts.project: - sys.exit('specify a project name with --project') + import mb.exports - print """#!/usr/bin/env python -import os, sys + if opts.format == 'django' and not opts.project: + sys.exit('For Django ORM format, specify a project name (roughly corresponding to a MirrorBrain instance) name with --project') -mybasepath = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, mybasepath) -os.environ['DJANGO_SETTINGS_MODULE'] = 'mirrordjango.settings' + if not opts.format: + sys.exit('You need to specify an output format. See --help output.') -from django.db import connection + if opts.format == 'django': + print mb.exports.django_header -from mirrordjango.mb.models import Contact, Operator, Project, Server, Mirror + # FIXME: add new fields: operator_name, operator_url, public_notes + print """Project(name='%s').save()""" % opts.project -""" + elif opts.format == 'postgresql': + print mb.exports.postgresql_header - # FIXME: add new fields: operator_name, operator_url, public_notes - print """Project(name='%s').save()""" % opts.project + else: + sys.exit('unknown format %r' % opts.format) - django_template = """\ -# ------------------------------------------------------------ -try: - c = Contact.objects.get_or_create(username=%(admin)r, password='UNSET', name=%(admin)r, email=%(adminEmail)r)[0] -except: - connection.connection.rollback() - c = None -o = Operator.objects.get_or_create(name='%(identifier)s', logo='')[0] -p = Project.objects.filter(name='""" + opts.project + """') -p = p[0] -s = Server.objects.get_or_create(identifier='%(identifier)s', operator=o, region='%(region)s', country='%(country)s', country_only='%(countryOnly)s', region_only='%(regionOnly)s', other_countries=%(otherCountries)r, file_maxsize='%(fileMaxsize)s', comment=%(comment)r, bandwidth=1)[0] -m = Mirror.objects.get_or_create(http='%(baseurl)s', ftp='%(baseurlFtp)s', rsync='%(baseurlRsync)s', prio='%(score)s', project=p, server=s)[0] -# s.mirrors.add(m) -if c: - s.contacts.add(c) - -""" - mirrors = self.conn.Server.select() for i in mirrors: if i.comment == None: #print 'null comment', i i.comment = '' d = mb.conn.server2dict(i) - print django_template % d + d.update(dict(project=opts.project)) + #print >>sys.stderr, d + # replace None's + #for i in mb.conn.server_editable_attrs: + for i in ['asn', 'prefix', 'asOnly', 'prefixOnly', 'lat', 'lng', 'scanFpm']: + if d[i] == None: d[i] = '0' + for i in ['prefix', 'baseurlRsync', 'admin', 'adminEmail']: + if d[i] == None: d[i] = '' + + if opts.format == 'django': + print mb.exports.django_template % d + elif opts.format == 'postgresql': + print mb.exports.postgresql_template % d + + if __name__ == '__main__': import sys mirrordoctor = MirrorDoctor() _______________________________________________ Opensuse-svn mailing list Opensuse-svn_at_forge.novell.com http://forge.novell.com/mailman/listinfo/opensuse-svn _______________________________________________ mirrorbrain-commits mailing list Note: To remove yourself from this list, send a mail with the content unsubscribe to the address mirrorbrain-commits-request_at_mirrorbrain.orgReceived on Wed Feb 11 2009 - 15:17:09 GMT
This archive was generated by hypermail 2.3.0 : Mon Feb 20 2012 - 23:47:04 GMT