Author: poeml Date: Sat Nov 13 00:35:43 2010 New Revision: 8217 URL: http://svn.mirrorbrain.org/viewvc/mirrorbrain?rev=8217&view=rev Log: - implement Yum mirror list support. There is a new directive for it: MirrorBrainYumDir Specify query arguments mapping to a directory that must have a certain file. Syntax: arg1=<regexp> arg2=<regexp> <basedir> <mandatory_file>. Parts of basedir can be substituted with query arguments $1-$9. Patterns are forced to be anchored to start and end for security reasons. Example: MirrorBrainYumDir release=(5\.5) repo=(os|updates) arch=i586 $1/$2/i386 repodata/repomd.xml Modified: trunk/mod_mirrorbrain/mod_mirrorbrain.c Modified: trunk/mod_mirrorbrain/mod_mirrorbrain.c URL: http://svn.mirrorbrain.org/viewvc/mirrorbrain/trunk/mod_mirrorbrain/mod_mirrorbrain.c?rev=8217&r1=8216&r2=8217&view=diff ============================================================================== --- trunk/mod_mirrorbrain/mod_mirrorbrain.c (original) +++ trunk/mod_mirrorbrain/mod_mirrorbrain.c Sat Nov 13 00:35:43 2010 _at_@ -167,7 +167,7 @@ /* (meta) representations of a requested file */ enum { REDIRECT, META4, METALINK, MIRRORLIST, TORRENT, - ZSYNC, MAGNET, MD5, SHA1, SHA256, BTIH, UNKNOWN }; + ZSYNC, MAGNET, MD5, SHA1, SHA256, BTIH, YUMLIST, UNKNOWN }; static struct { int id; char *ext; _at_@ -183,6 +183,7 @@ { SHA1, "sha1" }, { SHA256, "sha256" }, { BTIH, "btih" }, + { YUMLIST, "yumlist" }, { UNKNOWN, NULL } }; _at_@ -264,6 +265,7 @@ apr_array_header_t *dhtnodes; const char *metalink_broken_test_mirrors; int metalink_magnets; + apr_array_header_t *yumdirs; const char *mirrorlist_stylesheet; const char *mirrorlist_header; const char *mirrorlist_footer; _at_@ -278,6 +280,19 @@ struct dhtnode { char *name; int port; +}; + +typedef struct yumdir yumdir_t; +struct yumdir { + char *dir; /* base dir */ + char *file; /* marker file within base dir */ + apr_array_header_t *args; /* query arguments */ +}; + +typedef struct yumarg yumarg_t; +struct yumarg { + char *key; + ap_regex_t *regexp; }; _at_@ -450,6 +465,7 @@ new->dhtnodes = apr_array_make(p, 5, sizeof (dhtnode_t)); new->metalink_broken_test_mirrors = NULL; new->metalink_magnets = UNSET; + new->yumdirs = apr_array_make(p, 10, sizeof (yumdir_t)); new->mirrorlist_stylesheet = NULL; new->mirrorlist_header = NULL; new->mirrorlist_footer = NULL; _at_@ -483,6 +499,7 @@ mrg->dhtnodes = apr_array_append(p, base->dhtnodes, add->dhtnodes); cfgMergeString(metalink_broken_test_mirrors); cfgMergeBool(metalink_magnets); + mrg->yumdirs = apr_array_append(p, base->yumdirs, add->yumdirs); cfgMergeString(mirrorlist_stylesheet); cfgMergeString(mirrorlist_header); cfgMergeString(mirrorlist_footer); _at_@ -809,6 +826,64 @@ return NULL; } + +static const char *mb_cmd_add_yumdir(cmd_parms *cmd, void *dummy, + const char *arg) +{ + server_rec *s = cmd->server; + mb_server_conf *cfg = + ap_get_module_config(s->module_config, &mirrorbrain_module); + + char *d = NULL; /* base dir */ + char *f = NULL; /* marker file within base dir */ + char *word; + apr_array_header_t *yumargs = apr_array_make(cmd->pool, 3, sizeof (yumarg_t)); + + while (*arg) { + word = ap_getword_conf(cmd->pool, &arg); + char *val = ap_strchr(word, '='); + if (!val) { + if (!d) { + d = word; + continue; + } else if (!f) { + f = word; + continue; + } else { + return "Invalid MirrorBrainYumDir parameter. Parameter must be " + "in the form 'key=value'."; + } + } + *val++ = '\0'; + + yumarg_t *a = apr_array_push(yumargs); + a->key = apr_pstrdup(cmd->pool, word); + /* we better anchor the regexp to the start and end, because when user + * data matches the regexp, we will substitute parts of the URL with it */ + a->regexp = ap_pregcomp(cmd->pool, + apr_pstrcat(cmd->pool, "^", val, "$", NULL), + AP_REG_EXTENDED); + if (!a->regexp) + return "Regular expression for ProxyRemoteMatch could not be compiled."; + }; + + if (d == NULL) + return "MirrorBrainYumDir needs a (relative) base path"; + if (f == NULL) + return "MirrorBrainYumDir needs a file name relative to the base path"; + + if (apr_is_empty_array(yumargs)) + return "MirrorBrainYumDir needs at least one query argument"; + + yumdir_t *new = apr_array_push(cfg->yumdirs); + new->dir = apr_pstrdup(cmd->pool, d); + new->file = apr_pstrdup(cmd->pool, f); + new->args = yumargs; + + return NULL; +} + + #ifdef WITH_MEMCACHE static const char *mb_cmd_memcached_on(cmd_parms *cmd, void *config, int flag) _at_@ -1255,9 +1330,13 @@ { mb_dir_conf *cfg = NULL; mb_server_conf *scfg = NULL; + char *ptr = NULL; char *uri = NULL; char *filename = NULL; + const char *basename = NULL; + const char *mirror_base = NULL; char *realfile = NULL; + yumdir_t *yum = NULL; const char *clientip = NULL; apr_sockaddr_t *clientaddr; const char *query_country = NULL; _at_@ -1278,7 +1357,7 @@ const char *prefix; /* network prefix */ int i; int mirror_cnt; - apr_size_t len; + apr_size_t len, nr; mirror_entry_t *new; mirror_entry_t *mirror; mirror_entry_t **mirrorp; _at_@ -1367,6 +1446,92 @@ if (form_lookup(r, "sha256")) { rep = SHA256; rep_ext = reps[SHA256].ext; }; if (form_lookup(r, "btih")) { rep = BTIH; rep_ext = reps[BTIH].ext; }; if (form_lookup(r, "only_hash")) { only_hash = 1; }; + + /* yum query to be parsed? */ + if (!apr_is_empty_array(scfg->yumdirs)) { + for (i = 0; i < scfg->yumdirs->nelts; i++) { + yumdir_t y; + yumarg_t a; + char *d, *f; + const char *val; + int val_len_sum = 0; + int n_matches = 0; + + y = ((yumdir_t *) scfg->yumdirs->elts)[i]; + d = y.dir; + f = y.file; + //debugLog(r, cfg, "checking against yum dir %s / %s", d, f); + for (n_matches = 0; n_matches < y.args->nelts; n_matches++) { + a = ((yumarg_t *) y.args->elts)[n_matches]; + val = form_lookup(r, a.key); + if (!val) + break; + if (ap_regexec(a.regexp, val, 0, NULL, 0)) + break; + val_len_sum += strlen(val); + //debugLog(r, cfg, "value '%s' matches regexp for '%s'", val, a.key); + } + + char *src, *dst; + char c; + + if (n_matches == y.args->nelts) { + + rep = YUMLIST; + yum = apr_pcalloc(r->pool, sizeof(yumdir_t)); + + debugLog(r, cfg, "match for yum dir %s / %s", d, f); + yum->file = apr_pstrdup(r->pool, f); + + if ((ptr = ap_strchr(d, '$'))) { + //debugLog(r, cfg, "substitution to be done"); + + src = d; + yum->dir = dst = apr_pcalloc(r->pool, strlen(d) + val_len_sum + 1); + + while ((c = *src++) != '\0') { + if (c == '$' && apr_isdigit(*src)) { + nr = *src++ - '0'; + + if (nr == 0) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "[mod_mirrorbrain] " + "cannot substitute $0 in '%s' -- use 1 or a greater digit", d); + return HTTP_INTERNAL_SERVER_ERROR; + } else if (nr > n_matches) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "[mod_mirrorbrain] " + "cannot substitute $%d in '%s' -- only %d args are defined", + nr, d, y.args->nelts); + return HTTP_INTERNAL_SERVER_ERROR; + } + + a = ((yumarg_t *) y.args->elts)[nr - 1]; + val = form_lookup(r, a.key); + + debugLog(r, cfg, "substituting $%d with '%s'", nr, val); + len = strlen(val); + memcpy(dst, val, len); + dst += len; + } else { + *dst++ = c; + } + } + *dst = '\0'; + + } else { + yum->dir = apr_pstrdup(r->pool, d); + } + debugLog(r, cfg, "yum->dir: '%s', yum->file: '%s'", yum->dir, yum->file); + + /* FIXME: maybe don't break in debug mode, to discover double matches */ + break; + } + + } + if (yum == NULL) { + debugLog(r, cfg, "yum query received, but didn't match any of the rules"); + return HTTP_NOT_FOUND; + } + } } if (!query_country _at_@ -1415,14 +1580,14 @@ debugLog(r, cfg, "FAKE File -- not looking at real files"); } else { - if (r->finfo.filetype == APR_DIR) { + if ((r->finfo.filetype == APR_DIR) && (rep != YUMLIST)) { /* if (ap_is_directory(r->pool, r->filename)) */ debugLog(r, cfg, "'%s' is a directory", r->filename); return DECLINED; } /* if the file doesn't exist, maybe a representation of it is requested */ - if (r->finfo.filetype != APR_REG) { + if ((r->finfo.filetype != APR_REG) && (rep != YUMLIST)) { debugLog(r, cfg, "File does not exist according to r->finfo"); if (r->filename[strlen(r->filename) - 1] == '.') { _at_@ -1607,6 +1772,7 @@ } + country_code = apr_table_get(r->subprocess_env, "GEOIP_COUNTRY_CODE"); country_name = apr_table_get(r->subprocess_env, "GEOIP_COUNTRY_NAME"); continent_code = apr_table_get(r->subprocess_env, "GEOIP_CONTINENT_CODE"); _at_@ -1660,17 +1826,9 @@ as, prefix, lat, lng, state_id, state_name); - /* prepare the filename to look up */ - char *ptr = realpath(r->filename, NULL); - if (ptr == NULL) { - ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, - "[mod_mirrorbrain] Error canonicalizing filename '%s'", r->filename); - return HTTP_INTERNAL_SERVER_ERROR; - } - realfile = apr_pstrdup(r->pool, ptr); - free(ptr); - - /* the basedir might contain symlinks. That needs to be taken into account. See issue #17 */ + + /* The basedir might contain symlinks. That needs to be taken into account. + * See discussion in http://mirrorbrain.org/issues/issue17 */ ptr = realpath(cfg->mirror_base, NULL); if (ptr == NULL) { /* this should never happen, because the MirrorBrainEngine directive would never _at_@ -1680,20 +1838,42 @@ "exist. Filesystem not mounted?", cfg->mirror_base); return HTTP_INTERNAL_SERVER_ERROR; } + mirror_base = apr_pstrdup(r->pool, ptr); + free(ptr); + + /* prepare the filename to look up */ + if (rep != YUMLIST) { + filename = apr_pstrdup(r->pool, r->filename); + } else { + filename = apr_pstrcat(r->pool, mirror_base, "/", yum->dir, "/", yum->file, NULL); + debugLog(r, cfg, "yum path on disk: %s", filename); + } + + ptr = realpath(filename, NULL); + if (ptr == NULL) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, + "[mod_mirrorbrain] Error canonicalizing filename '%s'", filename); + /* return HTTP_INTERNAL_SERVER_ERROR; */ + return HTTP_NOT_FOUND; + } + + realfile = apr_pstrdup(r->pool, ptr); + debugLog(r, cfg, "Canonicalized file on disk: %s", realfile); + free(ptr); + /* the leading directory needs to be stripped from the file path */ /* a directory from Apache always ends in '/'; a result from realpath() doesn't */ - filename = realfile + strlen(ptr) + 1; - free(ptr); - debugLog(r, cfg, "Canonicalized file on disk: %s", realfile); + filename = realfile + strlen(mirror_base) + 1; + + if (rep != YUMLIST) { + /* keep a filename version without leading path, because metalink clients + * will otherwise place the downloaded file into a directory hierarchy */ + if ((basename = ap_strrchr_c(filename, '/')) == NULL) + basename = filename; + else + ++basename; + } debugLog(r, cfg, "SQL file to look up: %s", filename); - - /* keep a filename version without leading path, because metalink clients - * will otherwise place the downloaded file into a directory hierarchy */ - const char *basename; - if ((basename = ap_strrchr_c(filename, '/')) == NULL) - basename = filename; - else - ++basename; _at_@ -2187,6 +2367,7 @@ case METALINK: case MIRRORLIST: case ZSYNC: + case YUMLIST: qsort(mirrors_same_prefix->elts, mirrors_same_prefix->nelts, mirrors_same_prefix->elt_size, cmp_mirror_best); qsort(mirrors_same_as->elts, mirrors_same_as->nelts, _at_@ -2296,7 +2477,7 @@ - if ((rep != REDIRECT) && (!hashbag)) { + if ((rep != REDIRECT) && (rep != YUMLIST) && (!hashbag)) { hashbag = hashbag_fill(r, dbd, filename); if (hashbag == NULL) { debugLog(r, cfg, "no hashes found in database"); _at_@ -2357,7 +2538,7 @@ } - /* return a metalink instead of doing a redirect? */ + /* So, which representation are we going to send back? */ switch (rep) { case META4: _at_@ -3268,6 +3449,19 @@ ap_set_content_type(r, "text/plain; charset=UTF-8"); ap_rprintf(r, "%s\n", magnet); setenv_give(r, "magnet"); + return OK; + + case YUMLIST: + ap_set_content_type(r, "text/plain; charset=UTF-8"); + apr_array_header_t *topten = get_n_best_mirrors(r, 10, mirrors_same_prefix, mirrors_same_as, + mirrors_same_country, mirrors_same_region, + mirrors_elsewhere); + mirrorp = (mirror_entry_t **)topten->elts; + for (i = 0; i < topten->nelts; i++) { + mirror = mirrorp[i]; + ap_rprintf(r, "%s%s/\n", mirror->baseurl, yum->dir); + } + setenv_give(r, "yumlist"); return OK; } /* end switch representation */ _at_@ -3487,6 +3681,14 @@ "Causes MirrorBrain to append a signed timestamp to redirection URLs. The " "argument is a string that defines the key to encrypt the timestamp with. " "Can be configured on directory-level."), + AP_INIT_RAW_ARGS("MirrorBrainYumDir", mb_cmd_add_yumdir, NULL, + RSRC_CONF, /* RSRC_CONF|ACCESS_CONF, */ + "Specify query arguments mapping to a directory that must have a certain file. " + "Syntax: arg1=<regexp> arg2=<regexp> <basedir> <mandatory_file>. " + "Parts of basedir can be substituted with query arguments $1-$9. " + "Patterns are forced to be anchored to start and end for security reasons. " + "Example: MirrorBrainYumDir release=(5\\.5) repo=(os|updates) arch=i586 " + "$1/$2/i386 repodata/repomd.xml"), /* to be used only in server context */ AP_INIT_TAKE1("MirrorBrainDBDQuery", mb_cmd_dbd_query, NULL, _______________________________________________ mirrorbrain-commits mailing list Archive: http://mirrorbrain.org/archive/mirrorbrain-commits/ Note: To remove yourself from this list, send a mail with the content unsubscribe to the address mirrorbrain-commits-request_at_mirrorbrain.orgReceived on Fri Nov 12 2010 - 23:35:45 GMT
This archive was generated by hypermail 2.3.0 : Mon Feb 20 2012 - 23:47:04 GMT