00001
00002
00003
00004
00005
00006
00007
00008
00009
#include <ctype.h>
00010
#include <time.h>
00011
#include "wvhttppool.h"
00012
#include "wvbufstream.h"
00013
#include "wvtcp.h"
00014
#include "strutils.h"
00015
00016 bool WvHttpStream::global_enable_pipelining =
true;
00017 int WvUrlStream::max_requests = 100;
00018
00019 unsigned WvHash(
const WvUrlStream::Target &n)
00020 {
00021
WvString key(
"%s%s", n.
remaddr, n.
username);
00022
return (
WvHash(key));
00023 }
00024
00025
00026 WvUrlRequest::WvUrlRequest(
WvStringParm _url,
WvStringParm _method,
00027
WvStringParm _headers,
WvStream *content_source,
00028
bool _create_dirs,
bool _pipeline_test)
00029 : url(_url), headers(_headers)
00030 {
00031
instream = NULL;
00032
create_dirs = _create_dirs;
00033
pipeline_test = _pipeline_test;
00034
method = _method;
00035
is_dir =
false;
00036
00037
if (
pipeline_test)
00038 {
00039
outstream = NULL;
00040
putstream = NULL;
00041 }
00042
else
00043 {
00044
WvBufUrlStream *x =
new WvBufUrlStream;
00045
outstream = x;
00046 x->
death_notify = (
WvStream **)&
outstream;
00047 x->
url =
url;
00048
00049
putstream = content_source;
00050 }
00051
inuse =
false;
00052 }
00053
00054
00055 WvUrlRequest::~WvUrlRequest()
00056 {
00057
done();
00058 }
00059
00060
00061 void WvUrlRequest::done()
00062 {
00063
if (
outstream)
00064 {
00065
outstream->
death_notify = NULL;
00066
outstream->
seteof();
00067
outstream = NULL;
00068 }
00069
if (
putstream)
00070
putstream = NULL;
00071
inuse =
false;
00072 }
00073
00074
00075 void WvUrlStream::addurl(
WvUrlRequest *url)
00076 {
00077
log(WvLog::Debug4,
"Adding a new url: '%s'\n", url->
url);
00078
00079 assert(url->
outstream);
00080
00081
if (!url->
url.
isok())
00082
return;
00083
00084
waiting_urls.append(url,
false,
"waiting_url");
00085
request_next();
00086 }
00087
00088
00089 void WvUrlStream::delurl(
WvUrlRequest *url)
00090 {
00091
log(WvLog::Debug4,
"Removing an url: '%s'\n", url->
url);
00092
00093
if (url ==
curl)
00094
doneurl();
00095
waiting_urls.unlink(url);
00096
urls.unlink(url);
00097 }
00098
00099
00100 WvHttpPool::WvHttpPool() :
log("HTTP Pool",
WvLog::Debug), conns(10),
00101 pipeline_incompatible(50)
00102 {
00103
log(
"Pool initializing.\n");
00104 num_streams_created = 0;
00105 }
00106
00107
00108 WvHttpPool::~WvHttpPool()
00109 {
00110
log(
"Created %s individual session%s during this run.\n",
00111 num_streams_created, num_streams_created == 1 ?
"" :
"s");
00112
if (
geterr())
00113
log(
"Error was: %s\n",
errstr());
00114
00115
00116
00117 zap();
00118 conns.zap();
00119 }
00120
00121
00122 bool WvHttpPool::pre_select(SelectInfo &si)
00123 {
00124
bool sure =
false;
00125
00126 WvUrlStreamDict::Iter ci(conns);
00127
for (ci.rewind(); ci.next(); )
00128 {
00129
00130
if (!ci->isok())
00131 {
00132 unconnect(ci.ptr());
00133 ci.rewind();
00134
log(WvLog::Debug3,
"Selecting true because of a dead stream.\n");
00135 sure =
true;
00136 }
00137 }
00138
00139
00140
00141
00142 WvUrlRequestList::Iter i(urls);
00143
for (i.rewind(); i.next(); )
00144 {
00145
if ((!i->outstream && !i->inuse) || !i->url.isok())
00146 {
00147
00148
00149
if (!i->url.isok())
00150 {
00151
log(
"URL not okay: '%s'\n", i->url);
00152 i->done();
00153 }
00154
00155
WvUrlStream::Target target(i->url.getaddr(), i->url.getuser());
00156
WvUrlStream *s = conns[target];
00157
if (s)
00158 s->
delurl(i.ptr());
00159 i.xunlink();
00160
continue;
00161 }
00162
00163
if (!i->instream)
00164 {
00165
log(WvLog::Debug4,
"Checking dns for '%s'\n", i->url.gethost());
00166
if (i->url.resolve() || dns.
pre_select(i->url.gethost(), si))
00167 {
00168
log(WvLog::Debug4,
"Selecting true because of '%s'\n", i->url);
00169 sure =
true;
00170 }
00171 }
00172 }
00173
00174
if (WvStreamList::pre_select(si))
00175 {
00176
00177 sure =
true;
00178 }
00179
00180
return sure;
00181 }
00182
00183
00184 void WvHttpPool::execute()
00185 {
00186
WvStreamList::execute();
00187
00188 WvUrlRequestList::Iter i(urls);
00189
for (i.rewind(); i.next(); )
00190 {
00191
WvUrlStream *s;
00192
00193
if (!i->outstream || !i->url.isok() || !i->url.resolve())
00194
continue;
00195
00196
WvUrlStream::Target target(i->url.getaddr(), i->url.getuser());
00197
00198
00199
00200 s = conns[target];
00201
00202
00203
if (s && !s->
isok())
00204 {
00205 unconnect(s);
00206 s = NULL;
00207 }
00208
00209
if (!i->outstream)
00210
continue;
00211
00212
if (!s)
00213 {
00214 num_streams_created++;
00215
if (!strncasecmp(i->url.getproto(),
"http", 4))
00216 s =
new WvHttpStream(target.
remaddr, target.
username,
00217 i->url.getproto() ==
"https",
00218 pipeline_incompatible);
00219
else if (!strcasecmp(i->url.getproto(),
"ftp"))
00220 s =
new WvFtpStream(target.
remaddr, target.
username,
00221 i->url.getpassword());
00222 conns.add(s,
true);
00223
00224
00225 append(s,
false,
"http/ftp stream");
00226 }
00227
00228
if (!i->instream)
00229 {
00230 s->
addurl(i.ptr());
00231 i->instream = s;
00232 }
00233 }
00234 }
00235
00236
00237 WvBufUrlStream *
WvHttpPool::addurl(
WvStringParm _url,
WvStringParm _method,
00238
WvStringParm _headers,
WvStream *content_source,
bool create_dirs)
00239 {
00240
log(WvLog::Debug4,
"Adding a new url to pool: '%s'\n", _url);
00241
WvUrlRequest *url =
new WvUrlRequest(_url, _method, _headers, content_source,
00242 create_dirs,
false);
00243 urls.append(url,
true,
"addurl");
00244
00245
return url->
outstream;
00246 }
00247
00248
00249
void WvHttpPool::unconnect(
WvUrlStream *s)
00250 {
00251
if (!s->
target.username)
00252
log(
"Unconnecting stream to %s.\n", s->
target.remaddr);
00253
else
00254
log(
"Unconnecting stream to %s@%s.\n", s->
target.username,
00255 s->
target.remaddr);
00256
00257 WvUrlRequestList::Iter i(urls);
00258
for (i.rewind(); i.next(); )
00259 {
00260
if (i->instream == s)
00261 i->instream = NULL;
00262 }
00263
00264
unlink(s);
00265 conns.remove(s);
00266 }