wvhttppool.cc

00001 /*
00002  * Worldvisions Weaver Software:
00003  *   Copyright (C) 1997-2002 Net Integration Technologies, Inc.
00004  * 
00005  * A fast, easy-to-use, parallelizing, pipelining HTTP/1.1 file retriever.
00006  * 
00007  * See wvhttppool.h.
00008  */
00009 #include <ctype.h>
00010 #include <time.h>
00011 #include "wvhttppool.h"
00012 #include "wvbufstream.h"
00013 #include "wvtcp.h"
00014 #include "strutils.h"
00015 
00016 bool WvHttpStream::global_enable_pipelining = true;
00017 int WvUrlStream::max_requests = 100;
00018 
00019 unsigned WvHash(const WvUrlStream::Target &n)
00020 {
00021     WvString key("%s%s", n.remaddr, n.username);
00022     return (WvHash(key));
00023 }
00024 
00025 
00026 WvUrlRequest::WvUrlRequest(WvStringParm _url, WvStringParm _method,
00027                 WvStringParm _headers, WvStream *content_source,
00028                 bool _create_dirs, bool _pipeline_test)
00029     : url(_url), headers(_headers)
00030 { 
00031     instream = NULL;
00032     create_dirs = _create_dirs;
00033     pipeline_test = _pipeline_test;
00034     method = _method;
00035     is_dir = false;    // for ftp primarily; set later
00036 
00037     if (pipeline_test)
00038     {
00039         outstream = NULL;
00040         putstream = NULL;
00041     }
00042     else
00043     {
00044         WvBufUrlStream *x = new WvBufUrlStream;
00045         outstream = x;
00046         x->death_notify = (WvStream **)&outstream;
00047         x->url = url;
00048 
00049         putstream = content_source;
00050     }
00051     inuse = false;
00052 }
00053 
00054 
00055 WvUrlRequest::~WvUrlRequest()
00056 {
00057     done();
00058 }
00059 
00060 
00061 void WvUrlRequest::done()
00062 {
00063     if (outstream)
00064     {
00065         outstream->death_notify = NULL;
00066         outstream->seteof();
00067         outstream = NULL; 
00068     }
00069     if (putstream)
00070         putstream = NULL;
00071     inuse = false;
00072 }
00073 
00074 
00075 void WvUrlStream::addurl(WvUrlRequest *url)
00076 {
00077     log(WvLog::Debug4, "Adding a new url: '%s'\n", url->url);
00078 
00079     assert(url->outstream);
00080 
00081     if (!url->url.isok())
00082         return;
00083 
00084     waiting_urls.append(url, false, "waiting_url");
00085     request_next();
00086 }
00087 
00088 
00089 void WvUrlStream::delurl(WvUrlRequest *url)
00090 {
00091     log(WvLog::Debug4, "Removing an url: '%s'\n", url->url);
00092 
00093     if (url == curl)
00094         doneurl();
00095     waiting_urls.unlink(url);
00096     urls.unlink(url);
00097 }
00098 
00099 
00100 WvHttpPool::WvHttpPool() 
00101     : log("HTTP Pool", WvLog::Debug), conns(10),
00102       pipeline_incompatible(50)
00103 {
00104     log("Pool initializing.\n");
00105     num_streams_created = 0;
00106 }
00107 
00108 
00109 WvHttpPool::~WvHttpPool()
00110 {
00111     log("Created %s individual session%s during this run.\n",
00112             num_streams_created, num_streams_created == 1 ? "" : "s");
00113     if (geterr())
00114         log("Error was: %s\n", errstr());
00115 
00116     // these must get zapped before the URL list, since they have pointers
00117     // to URLs.
00118     zap();
00119     conns.zap();
00120 }
00121 
00122 
00123 void WvHttpPool::pre_select(SelectInfo &si)
00124 {
00125     //    log(WvLog::Debug5, "pre_select: main:%s conns:%s urls:%s\n",
00126     //         count(), conns.count(), urls.count());
00127 
00128     WvIStreamList::pre_select(si);
00129 
00130     WvUrlStreamDict::Iter ci(conns);
00131     for (ci.rewind(); ci.next(); )
00132     {
00133         if (!ci->isok())
00134             si.msec_timeout = 0;
00135     }
00136     
00137     WvUrlRequestList::Iter i(urls);
00138     for (i.rewind(); i.next(); )
00139     {
00140         if (!i->instream)
00141         {
00142             log(WvLog::Debug4, "Checking dns for '%s'\n", i->url.gethost());
00143             if (i->url.resolve())
00144                 si.msec_timeout = 0;
00145             else
00146                 dns.pre_select(i->url.gethost(), si);    
00147         }
00148     }
00149 }
00150 
00151 
00152 bool WvHttpPool::post_select(SelectInfo &si)
00153 {
00154     bool sure = false;
00155 
00156     WvUrlStreamDict::Iter ci(conns);
00157     for (ci.rewind(); ci.next(); )
00158     {
00159         if (!ci->isok())
00160         {
00161             log(WvLog::Debug4, "Selecting true because of a dead stream.\n");
00162             unconnect(ci.ptr());
00163             ci.rewind();
00164             sure = true;
00165         }
00166     }
00167 
00168     WvUrlRequestList::Iter i(urls);
00169     for (i.rewind(); i.next(); )
00170     {
00171         if ((!i->outstream && !i->inuse) || !i->url.isok())
00172         {
00173             //log("'%s' is dead: %s/%s\n", 
00174             //  i->url, i->url.isok(), i.outstream->isok());
00175             if (!i->url.isok())
00176             {
00177                 log("URL not okay: '%s'\n", i->url);
00178                 i->done();
00179             }
00180             // nicely delete the url request
00181             WvUrlStream::Target target(i->url.getaddr(), i->url.getuser());
00182             WvUrlStream *s = conns[target];
00183             if (s)
00184                 s->delurl(i.ptr());
00185             i.xunlink();
00186             continue;
00187         }
00188 
00189         if (!i->instream)
00190         {
00191             log(WvLog::Debug4, "Checking dns for '%s'\n", i->url.gethost());
00192             if (i->url.resolve() || dns.post_select(i->url.gethost(), si))
00193             {
00194                 log(WvLog::Debug4, "Selecting true because of '%s'\n", i->url);
00195                 sure = true;
00196             }
00197         }
00198     }
00199 
00200     return WvIStreamList::post_select(si) || sure;
00201 }
00202 
00203 
00204 void WvHttpPool::execute()
00205 {
00206     WvIStreamList::execute();
00207 
00208     WvUrlRequestList::Iter i(urls);
00209     for (i.rewind(); i.next(); )
00210     {
00211         WvUrlStream *s;
00212 
00213         if (!i->outstream || !i->url.isok() || !i->url.resolve())
00214             continue; // skip it for now
00215 
00216         WvUrlStream::Target target(i->url.getaddr(), i->url.getuser());
00217 
00218         //log(WvLog::Info, "remaddr is %s; username is %s\n", target.remaddr,
00219         //    target.username);
00220         s = conns[target];
00221         //if (!s) log("conn for '%s' is not found.\n", ip);
00222 
00223         if (s && !s->isok())
00224         {
00225             unconnect(s);
00226             s = NULL;
00227         }
00228 
00229         if (!i->outstream)
00230             continue; // unconnect might have caused this URL to be marked bad
00231 
00232         if (!s)
00233         {
00234             num_streams_created++;
00235             if (!strncasecmp(i->url.getproto(), "http", 4))
00236                 s = new WvHttpStream(target.remaddr, target.username,
00237                         i->url.getproto() == "https",
00238                         pipeline_incompatible);
00239             else if (!strcasecmp(i->url.getproto(), "ftp"))
00240                 s = new WvFtpStream(target.remaddr, target.username,
00241                         i->url.getpassword());
00242             conns.add(s, true);
00243 
00244             // add it to the streamlist, so it can do things
00245             append(s, false, "http/ftp stream");
00246         }
00247 
00248         if (!i->instream)
00249         {
00250             s->addurl(i.ptr());
00251             i->instream = s;
00252         }
00253     }
00254 }
00255 
00256 
00257 WvBufUrlStream *WvHttpPool::addurl(WvStringParm _url, WvStringParm _method,
00258         WvStringParm _headers, WvStream *content_source, bool create_dirs)
00259 {
00260     log(WvLog::Debug4, "Adding a new url to pool: '%s'\n", _url);
00261     WvUrlRequest *url = new WvUrlRequest(_url, _method, _headers, content_source,
00262                                          create_dirs, false);
00263     urls.append(url, true, "addurl");
00264 
00265     return url->outstream;
00266 }
00267 
00268 
00269 void WvHttpPool::unconnect(WvUrlStream *s)
00270 {
00271     if (!s->target.username)
00272         log("Unconnecting stream to %s.\n", s->target.remaddr);
00273     else
00274         log("Unconnecting stream to %s@%s.\n", s->target.username,
00275                 s->target.remaddr);
00276 
00277     WvUrlRequestList::Iter i(urls);
00278     for (i.rewind(); i.next(); )
00279     {
00280         if (i->instream == s)
00281             i->instream = NULL;
00282     }
00283 
00284     unlink(s);
00285     conns.remove(s);
00286 }

Generated on Thu Jan 24 16:50:56 2008 for WvStreams by  doxygen 1.5.4