Main Page | Namespace List | Class Hierarchy | Alphabetical List | Class List | File List | Namespace Members | Class Members | File Members | Related Pages

wvhttppool.cc

Go to the documentation of this file.
00001 /* 00002 * Worldvisions Weaver Software: 00003 * Copyright (C) 1997-2002 Net Integration Technologies, Inc. 00004 * 00005 * A fast, easy-to-use, parallelizing, pipelining HTTP/1.1 file retriever. 00006 * 00007 * See wvhttppool.h. 00008 */ 00009 #include <ctype.h> 00010 #include <time.h> 00011 #include "wvhttppool.h" 00012 #include "wvbufstream.h" 00013 #include "wvtcp.h" 00014 #include "strutils.h" 00015 00016 bool WvHttpStream::global_enable_pipelining = true; 00017 int WvUrlStream::max_requests = 100; 00018 00019 unsigned WvHash(const WvUrlStream::Target &n) 00020 { 00021 WvString key("%s%s", n.remaddr, n.username); 00022 return (WvHash(key)); 00023 } 00024 00025 00026 WvUrlRequest::WvUrlRequest(WvStringParm _url, WvStringParm _method, 00027 WvStringParm _headers, WvStream *content_source, 00028 bool _create_dirs, bool _pipeline_test) 00029 : url(_url), headers(_headers) 00030 { 00031 instream = NULL; 00032 create_dirs = _create_dirs; 00033 pipeline_test = _pipeline_test; 00034 method = _method; 00035 is_dir = false; // for ftp primarily; set later 00036 00037 if (pipeline_test) 00038 { 00039 outstream = NULL; 00040 putstream = NULL; 00041 } 00042 else 00043 { 00044 WvBufUrlStream *x = new WvBufUrlStream; 00045 outstream = x; 00046 x->death_notify = (WvStream **)&outstream; 00047 x->url = url; 00048 00049 putstream = content_source; 00050 } 00051 inuse = false; 00052 } 00053 00054 00055 WvUrlRequest::~WvUrlRequest() 00056 { 00057 done(); 00058 } 00059 00060 00061 void WvUrlRequest::done() 00062 { 00063 if (outstream) 00064 { 00065 outstream->death_notify = NULL; 00066 outstream->seteof(); 00067 outstream = NULL; 00068 } 00069 if (putstream) 00070 putstream = NULL; 00071 inuse = false; 00072 } 00073 00074 00075 void WvUrlStream::addurl(WvUrlRequest *url) 00076 { 00077 log(WvLog::Debug4, "Adding a new url: '%s'\n", url->url); 00078 00079 assert(url->outstream); 00080 00081 if (!url->url.isok()) 00082 return; 00083 00084 waiting_urls.append(url, false, "waiting_url"); 00085 request_next(); 00086 } 00087 00088 00089 void WvUrlStream::delurl(WvUrlRequest *url) 00090 { 00091 log(WvLog::Debug4, "Removing an url: '%s'\n", url->url); 00092 00093 if (url == curl) 00094 doneurl(); 00095 waiting_urls.unlink(url); 00096 urls.unlink(url); 00097 } 00098 00099 00100 WvHttpPool::WvHttpPool() : log("HTTP Pool", WvLog::Debug), conns(10), 00101 pipeline_incompatible(50) 00102 { 00103 log("Pool initializing.\n"); 00104 num_streams_created = 0; 00105 } 00106 00107 00108 WvHttpPool::~WvHttpPool() 00109 { 00110 log("Created %s individual session%s during this run.\n", 00111 num_streams_created, num_streams_created == 1 ? "" : "s"); 00112 if (geterr()) 00113 log("Error was: %s\n", errstr()); 00114 00115 // these must get zapped before the URL list, since they have pointers 00116 // to URLs. 00117 zap(); 00118 conns.zap(); 00119 } 00120 00121 00122 bool WvHttpPool::pre_select(SelectInfo &si) 00123 { 00124 bool sure = false; 00125 00126 WvUrlStreamDict::Iter ci(conns); 00127 for (ci.rewind(); ci.next(); ) 00128 { 00129 // if (!ci->isok() || urls.isempty()) 00130 if (!ci->isok()) 00131 { 00132 unconnect(ci.ptr()); 00133 ci.rewind(); 00134 log(WvLog::Debug3, "Selecting true because of a dead stream.\n"); 00135 sure = true; 00136 } 00137 } 00138 00139 // log(WvLog::Debug5, "pre_select: main:%s conns:%s urls:%s\n", 00140 // count(), conns.count(), urls.count()); 00141 00142 WvUrlRequestList::Iter i(urls); 00143 for (i.rewind(); i.next(); ) 00144 { 00145 if ((!i->outstream && !i->inuse) || !i->url.isok()) 00146 { 00147 //log("'%s' is dead: %s/%s\n", 00148 // i->url, i->url.isok(), i.outstream->isok()); 00149 if (!i->url.isok()) 00150 { 00151 log("URL not okay: '%s'\n", i->url); 00152 i->done(); 00153 } 00154 // nicely delete the url request 00155 WvUrlStream::Target target(i->url.getaddr(), i->url.getuser()); 00156 WvUrlStream *s = conns[target]; 00157 if (s) 00158 s->delurl(i.ptr()); 00159 i.xunlink(); 00160 continue; 00161 } 00162 00163 if (!i->instream) 00164 { 00165 log(WvLog::Debug4, "Checking dns for '%s'\n", i->url.gethost()); 00166 if (i->url.resolve() || dns.pre_select(i->url.gethost(), si)) 00167 { 00168 log(WvLog::Debug4, "Selecting true because of '%s'\n", i->url); 00169 sure = true; 00170 } 00171 } 00172 } 00173 00174 if (WvStreamList::pre_select(si)) 00175 { 00176 //log("Selecting true because of list members.\n"); 00177 sure = true; 00178 } 00179 00180 return sure; 00181 } 00182 00183 00184 void WvHttpPool::execute() 00185 { 00186 WvStreamList::execute(); 00187 00188 WvUrlRequestList::Iter i(urls); 00189 for (i.rewind(); i.next(); ) 00190 { 00191 WvUrlStream *s; 00192 00193 if (!i->outstream || !i->url.isok() || !i->url.resolve()) 00194 continue; // skip it for now 00195 00196 WvUrlStream::Target target(i->url.getaddr(), i->url.getuser()); 00197 00198 //log(WvLog::Info, "remaddr is %s; username is %s\n", target.remaddr, 00199 // target.username); 00200 s = conns[target]; 00201 //if (!s) log("conn for '%s' is not found.\n", ip); 00202 00203 if (s && !s->isok()) 00204 { 00205 unconnect(s); 00206 s = NULL; 00207 } 00208 00209 if (!i->outstream) 00210 continue; // unconnect might have caused this URL to be marked bad 00211 00212 if (!s) 00213 { 00214 num_streams_created++; 00215 if (!strncasecmp(i->url.getproto(), "http", 4)) 00216 s = new WvHttpStream(target.remaddr, target.username, 00217 i->url.getproto() == "https", 00218 pipeline_incompatible); 00219 else if (!strcasecmp(i->url.getproto(), "ftp")) 00220 s = new WvFtpStream(target.remaddr, target.username, 00221 i->url.getpassword()); 00222 conns.add(s, true); 00223 00224 // add it to the streamlist, so it can do things 00225 append(s, false, "http/ftp stream"); 00226 } 00227 00228 if (!i->instream) 00229 { 00230 s->addurl(i.ptr()); 00231 i->instream = s; 00232 } 00233 } 00234 } 00235 00236 00237 WvBufUrlStream *WvHttpPool::addurl(WvStringParm _url, WvStringParm _method, 00238 WvStringParm _headers, WvStream *content_source, bool create_dirs) 00239 { 00240 log(WvLog::Debug4, "Adding a new url to pool: '%s'\n", _url); 00241 WvUrlRequest *url = new WvUrlRequest(_url, _method, _headers, content_source, 00242 create_dirs, false); 00243 urls.append(url, true, "addurl"); 00244 00245 return url->outstream; 00246 } 00247 00248 00249 void WvHttpPool::unconnect(WvUrlStream *s) 00250 { 00251 if (!s->target.username) 00252 log("Unconnecting stream to %s.\n", s->target.remaddr); 00253 else 00254 log("Unconnecting stream to %s@%s.\n", s->target.username, 00255 s->target.remaddr); 00256 00257 WvUrlRequestList::Iter i(urls); 00258 for (i.rewind(); i.next(); ) 00259 { 00260 if (i->instream == s) 00261 i->instream = NULL; 00262 } 00263 00264 unlink(s); 00265 conns.remove(s); 00266 }

Generated on Tue Oct 5 01:09:20 2004 for WvStreams by doxygen 1.3.7