[mirrorbrain-commits] r7880 - /trunk/mod_mirrorbrain/mod_mirrorbrain.c

From: <poeml_at_mirrorbrain.org>
Date: Sun, 29 Nov 2009 23:33:19 -0000
Author: poeml
Date: Mon Nov 30 00:33:18 2009
New Revision: 7880

URL: http://svn.mirrorbrain.org/viewvc/mirrorbrain?rev=7880&view=rev
Log:
mod_mirrorbrain:
- Allow to configure fallback mirrors, via the Apache config, in the following form:
       MirrorBrainFallback na us ftp://linuxfreedom.com/ultimate/
       MirrorBrainFallback eu de http://www.ultimate-edition.org/~ue/
  Those are used when no reachable mirror is known in the database. Thus, these mirrors
  get requests that MirrorBrain would normally deliver itself.
  They are also used in the mirror lists (with priority 1) and metalinks, and
  country/region selection is done like for normal mirrors. They are used
  blindly, without knowing their file lists.
  This also allows to run a MB instance with a pseudo file tree (cf. recently
  added null-rsync script.) 
  A degraded mode that continues to work in case of database complete outages is easily
  achievable now, however for now the code path is less robust in that regard.
  This should be fixed later.

Modified:
    trunk/mod_mirrorbrain/mod_mirrorbrain.c

Modified: trunk/mod_mirrorbrain/mod_mirrorbrain.c
URL: http://svn.mirrorbrain.org/viewvc/mirrorbrain/trunk/mod_mirrorbrain/mod_mirrorbrain.c?rev=7880&r1=7879&r2=7880&view=diff
==============================================================================
--- trunk/mod_mirrorbrain/mod_mirrorbrain.c (original)
+++ trunk/mod_mirrorbrain/mod_mirrorbrain.c Mon Nov 30 00:33:18 2009
_at_@ -73,6 +73,14 @@
 #define UNSET (-1)
 #endif
 
+/* available since APR 1.3 */
+#ifndef APR_ARRAY_IDX
+#define APR_ARRAY_IDX(ary,i,type) (((type *)(ary)->elts)[i])
+#endif
+#ifndef APR_ARRAY_PUSH
+#define APR_ARRAY_PUSH(ary,type) (*((type *)apr_array_push(ary)))
+#endif
+
 #define MOD_MIRRORBRAIN_VER "2.10.3"
 #define VERSION_COMPONENT "mod_mirrorbrain/"MOD_MIRRORBRAIN_VER
 
_at_@ -139,6 +147,7 @@
     int min_size;
     int handle_headrequest_locally;
     const char *mirror_base;
+    apr_array_header_t *fallbacks;
     apr_array_header_t *exclude_mime;
     apr_array_header_t *exclude_agents;
     apr_array_header_t *exclude_networks;
_at_@ -270,6 +279,7 @@
     new->min_size                   = DEFAULT_MIN_MIRROR_SIZE;
     new->handle_headrequest_locally = 0;
     new->mirror_base = NULL;
+    new->fallbacks = apr_array_make(p, 10, sizeof (mirror_entry_t));
     new->exclude_mime = apr_array_make(p, 0, sizeof (char *));
     new->exclude_agents = apr_array_make(p, 0, sizeof (char *));
     new->exclude_networks = apr_array_make(p, 4, sizeof (char *));
_at_@ -294,6 +304,11 @@
     mrg->min_size = (add->min_size != DEFAULT_MIN_MIRROR_SIZE) ? add->min_size : base->min_size;
     cfgMergeInt(handle_headrequest_locally);
     cfgMergeString(mirror_base);
+    /* inheriting makes sense, but does it also make sense if the directory has its own
+     * fallback mirror directives? */
+    /* mrg->fallbacks = apr_is_empty_array(add->fallbacks) ? base->fallbacks : add->fallbacks; */
+    /* it's a merge for now */
+    mrg->fallbacks = apr_array_append(p, base->fallbacks, add->fallbacks);
     mrg->exclude_mime = apr_array_append(p, base->exclude_mime, add->exclude_mime);
     mrg->exclude_agents = apr_array_append(p, base->exclude_agents, add->exclude_agents);
     mrg->exclude_networks = apr_array_append(p, base->exclude_networks, add->exclude_networks);
_at_@ -375,6 +390,44 @@
     cfg->min_size = atoi(arg1);
     if (cfg->min_size < 0)
         return "MirrorBrainMinSize requires a non-negative integer.";
+    return NULL;
+}
+
+static const char *mb_cmd_fallback(cmd_parms *cmd, void *config,
+                                   const char *arg1, const char *arg2,
+                                   const char *arg3)
+{
+    mb_dir_conf *cfg = (mb_dir_conf *) config;
+    mirror_entry_t *new;
+    apr_uri_t uri;
+
+    if (APR_SUCCESS != apr_uri_parse(cmd->pool, arg3, &uri)) {
+        return "MirrorBrainFallback URI cannot be parsed";
+    }
+
+    new = apr_array_push(cfg->fallbacks);
+    new->id = 0;
+    new->identifier = uri.hostname;
+    new->region = apr_pstrdup(cmd->pool, arg1);
+    new->country_code = apr_pstrdup(cmd->pool, arg2);
+    new->other_countries = NULL;
+    new->as = NULL;
+    new->prefix = NULL;
+    new->region_only = 0;
+    new->country_only = 0;
+    new->as_only = 0;
+    new->prefix_only = 0;
+    new->score = 1; /* give it a minimal score (but with 0, it wouldn't be considered) */
+    new->file_maxsize = 0;
+    if (arg3[strlen(arg3) - 1] == '/') { 
+        new->baseurl = apr_pstrdup(cmd->pool, arg3);
+    } else {
+        new->baseurl = apr_pstrcat(cmd->pool, arg3, "/", NULL); 
+    }
+    ap_log_error(APLOG_MARK, APLOG_INFO, 0, NULL,
+                 "[mod_mirrorbrain] configured fallback mirror (%s:%s): %s", 
+                 new->region, new->country_code, new->baseurl);
+
     return NULL;
 }
 
_at_@ -985,7 +1038,9 @@
     if (dbd == NULL) {
         ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, 
                 "[mod_mirrorbrain] Error acquiring database connection");
-        return DECLINED; /* fail gracefully */
+        if (apr_is_empty_array(cfg->fallbacks)) {
+            return DECLINED; /* fail gracefully */
+        }
     }
     debugLog(r, cfg, "Successfully acquired database connection.");
 
_at_@ -1041,21 +1096,30 @@
                 filename, NULL) != 0) {
         ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, 
                 "[mod_mirrorbrain] Error looking up %s in database", filename);
-        return DECLINED;
+        if (apr_is_empty_array(cfg->fallbacks)) {
+            return DECLINED;
+        }
     }
 
     mirror_cnt = apr_dbd_num_tuples(dbd->driver, res);
+
     if (mirror_cnt > 0) {
         debugLog(r, cfg, "Found %d mirror%s", mirror_cnt,
                 (mirror_cnt == 1) ? "" : "s");
-    }
-    else {
-        ap_log_rerror(APLOG_MARK, APLOG_INFO, 0, r, 
-                "[mod_mirrorbrain] no mirrors found for %s", filename);
-        /* can be used for a CustomLog */
+    } else {
+        if (apr_is_empty_array(cfg->fallbacks))  {
+            ap_log_rerror(APLOG_MARK, APLOG_INFO, 0, r, 
+                    "[mod_mirrorbrain] no mirrors found for %s", filename);
+        } else {
+            ap_log_rerror(APLOG_MARK, APLOG_INFO, 0, r, 
+                    "[mod_mirrorbrain] no mirrors found for %s, "
+                    "but fallback mirrors are available", filename);
+        }
+
+        /* can be used with a CustomLog directive, conditionally logging these requests */
         apr_table_setn(r->subprocess_env, "MB_NOMIRROR", "1");
 
-        if (mirrorlist) {
+        if (mirrorlist && apr_is_empty_array(cfg->fallbacks)) {
             debugLog(r, cfg, "empty mirrorlist");
             ap_set_content_type(r, "text/html; charset=ISO-8859-1");
             ap_rputs(DOCTYPE_XHTML_1_0T
_at_@ -1078,34 +1142,24 @@
 
             ap_rputs("</body></html>\n", r);
             return OK;
-        }
-
-        /* deliver the file ourselves */
-        return DECLINED;
+        } 
+        if (!mirrorlist && apr_is_empty_array(cfg->fallbacks)) {
+            /* deliver the file ourselves */
+            debugLog(r, cfg, "have to deliver directly");
+            return DECLINED;
+        }
     }
 
 
     /* allocate space for the expected results */
     mirrors              = apr_array_make(r->pool, mirror_cnt, sizeof (mirror_entry_t));
+    /* n.b., the following arrays only hold pointers into the above array */
     mirrors_same_prefix  = apr_array_make(r->pool, 1,          sizeof (mirror_entry_t *));
     mirrors_same_as      = apr_array_make(r->pool, 1,          sizeof (mirror_entry_t *));
     mirrors_same_country = apr_array_make(r->pool, mirror_cnt, sizeof (mirror_entry_t *));
     mirrors_fallback_country = apr_array_make(r->pool, 5,      sizeof (mirror_entry_t *));
     mirrors_same_region  = apr_array_make(r->pool, mirror_cnt, sizeof (mirror_entry_t *));
     mirrors_elsewhere    = apr_array_make(r->pool, mirror_cnt, sizeof (mirror_entry_t *));
-
-
-    /* need to remind myself... how to use the pointer arrays:
-     *                                                          
-     * 1) multi line version, allowing for easier access of last added element
-     * void **new_same = (void **)apr_array_push(mirrors_same_country);
-     * *new_same = new;
-     *
-     * ap_log_rerror(APLOG_MARK, APLOG_INFO, 0, r, "[mod_mirrorbrain] new_same->identifier: %s",
-     *        ((mirror_entry_t *)*new_same)->identifier);
-     *
-     * 2) one line version
-     * *(void **)apr_array_push(mirrors_same_country) = new;                        */
 
 
     /* store the results which the database yielded, taking into account which
_at_@ -1330,19 +1384,55 @@
     mirror_entry_t *elts;
     elts = (mirror_entry_t *) mirrors->elts;
     for (i = 0; i < mirrors->nelts; i++) {
-        debugLog(r, cfg, "mirror  %3d  %-30ss", elts[i].id, elts[i].identifier);
+        debugLog(r, cfg, "mirror  %3d  %-30s", elts[i].id, elts[i].identifier);
     }
 #endif
 
 
     /* 2nd pass */
 
-    /* if we didn't found a mirror in the country: are other mirrors set to
+    /* if we didn't find a mirror in the country: are other mirrors set to
      * handle this country? */
     if (apr_is_empty_array(mirrors_same_country) 
             && !apr_is_empty_array(mirrors_fallback_country)) {
         mirrors_same_country = mirrors_fallback_country;
         debugLog(r, cfg, "no mirror in country, but found fallback_country mirrors");
+    }
+
+
+    /* 3rd pass */
+    if (apr_is_empty_array(mirrors) && ! apr_is_empty_array(cfg->fallbacks)) {
+
+        debugLog(r, cfg, "ok, need to add fallback mirrors (%d configured)", 
+                 cfg->fallbacks->nelts);
+
+        /* we copy the array, so we don't modify the one in the config */
+        mirrors = apr_array_copy(r->pool, cfg->fallbacks);
+
+        mirror_entry_t *elts;
+        elts = (mirror_entry_t *) mirrors->elts;
+        for (i = 0; i < mirrors->nelts; i++) {
+
+            elts[i].rank = (rand()>>16) * ((RAND_MAX>>16) / elts[i].score);
+            /* elts[i].identifier = apr_psprintf(r->pool, "fallback_%02d(%s)", 
+                                              i, elts[i].baseurl); */
+
+            if (strcasecmp(elts[i].country_code, country_code) == 0) {
+                *(void **)apr_array_push(mirrors_same_country) = &(elts[i]);
+                debugLog(r, cfg, "adding fallback mirror in same country: %s:%s %s", 
+                         elts[i].region, elts[i].country_code, elts[i].baseurl);
+            } 
+            else if (strcasecmp(elts[i].region, continent_code) == 0) {
+                *(void **)apr_array_push(mirrors_same_region) = &(elts[i]);
+                debugLog(r, cfg, "adding fallback mirror in same region: %s:%s %s", 
+                         elts[i].region, elts[i].country_code, elts[i].baseurl);
+            } 
+            else {
+                *(void **)apr_array_push(mirrors_elsewhere) = &(elts[i]);
+                debugLog(r, cfg, "adding fallback mirror elsewhere: %s:%s %s", 
+                         elts[i].region, elts[i].country_code, elts[i].baseurl);
+            }
+        }
     }
 
 
_at_@ -2088,6 +2178,7 @@
                   ACCESS_CONF,
                   "Regexp which determines which files will be excluded form redirecting"),
 
+    /* obsolete, and to removed later */
     AP_INIT_FLAG("MirrorBrainHandleDirectoryIndexLocally", mb_cmd_handle_dirindex_locally, NULL, 
                   OR_OPTIONS,
                   "Obsolete directive. You can remove it from your config."),
_at_@ -2100,6 +2191,12 @@
                   ACCESS_CONF,
                   "Regexp which determines for which files to look for correspondant "
                   ".torrent files, and add them into generated metalinks"),
+
+    AP_INIT_TAKE3("MirrorBrainFallback", mb_cmd_fallback, NULL, 
+                  ACCESS_CONF, 
+                  "region code, country code and base URL of a mirror that is used when no "
+                  "mirror can be found in the database. These mirrors are assumed to have "
+                  "*all* files."),
 
     /* to be used only in server context */
     AP_INIT_TAKE1("MirrorBrainDBDQuery", mb_cmd_dbdquery, NULL,




_______________________________________________
mirrorbrain-commits mailing list
Archive: http://mirrorbrain.org/archive/mirrorbrain-commits/

Note: To remove yourself from this list, send a mail with the content
 	unsubscribe
to the address mirrorbrain-commits-request_at_mirrorbrain.org
Received on Sun Nov 29 2009 - 23:33:20 GMT

This archive was generated by hypermail 2.3.0 : Mon Feb 20 2012 - 23:47:04 GMT