Main Page | Namespace List | Class Hierarchy | Alphabetical List | Class List | File List | Namespace Members | Class Members | File Members | Related Pages

wvhttppool.h

Go to the documentation of this file.
00001 /* -*- Mode: C++ -*- 00002 * Worldvisions Weaver Software: 00003 * Copyright (C) 1997-2002 Net Integration Technologies, Inc. 00004 * 00005 * A fast, easy-to-use, parallelizing, pipelining HTTP/1.1 file retriever. 00006 * 00007 * Just create a WvHttpPool object, add it to your list, and use pool.addurl() 00008 * to get a WvStream* that gives you the file you requested. 00009 */ 00010 #ifndef __WVHTTPPOOL_H 00011 #define __WVHTTPPOOL_H 00012 00013 #include "ftpparse.h" 00014 #include "wvurl.h" 00015 #include "wvstreamlist.h" 00016 #include "wvstreamclone.h" 00017 #include "wvlog.h" 00018 #include "wvhashtable.h" 00019 #include "wvhttp.h" 00020 #include "wvbufstream.h" 00021 #include "wvbuf.h" 00022 00023 class WvBufUrlStream; 00024 class WvUrlStream; 00025 class WvHttpStream; 00026 00027 static const WvString DEFAULT_ANON_PW("weasels@"); 00028 00029 class WvUrlRequest 00030 { 00031 public: 00032 WvUrl url; 00033 WvString headers; 00034 WvUrlStream *instream; 00035 WvBufUrlStream *outstream; 00036 WvStream *putstream; 00037 00038 bool pipeline_test; 00039 bool inuse; 00040 bool is_dir; 00041 bool create_dirs; 00042 WvString method; 00043 00044 WvUrlRequest(WvStringParm _url, WvStringParm _method, WvStringParm _headers, 00045 WvStream *content_source, bool _create_dirs, bool _pipeline_test); 00046 ~WvUrlRequest(); 00047 00048 void done(); 00049 }; 00050 00051 DeclareWvList(WvUrlRequest); 00052 00053 00054 struct WvUrlLink 00055 { 00056 WvString linkname; 00057 WvUrl url; 00058 00059 WvUrlLink::WvUrlLink(WvStringParm _linkname, WvStringParm _url) 00060 : linkname(_linkname), url(_url) 00061 {} 00062 }; 00063 DeclareWvList(WvUrlLink); 00064 00065 00066 class WvBufUrlStream : public WvBufStream 00067 { 00068 public: 00069 WvString url; 00070 WvString proto; 00071 WvUrlLinkList links; // HTML links or FTP directory listing 00072 00073 // HTTP stuff... 00074 WvString version; 00075 int status; 00076 WvHTTPHeaderDict headers; 00077 00078 WvBufUrlStream() : status(0), headers(10) 00079 {} 00080 virtual ~WvBufUrlStream() 00081 {} 00082 }; 00083 00084 DeclareWvTable(WvIPPortAddr); 00085 00086 00087 class WvUrlStream : public WvStreamClone 00088 { 00089 public: 00090 class Target 00091 { 00092 public: 00093 WvIPPortAddr remaddr; 00094 WvString username; 00095 00096 Target(const WvIPPortAddr &_remaddr, WvStringParm _username) 00097 : remaddr(_remaddr), username(_username) {} 00098 00099 ~Target() {} 00100 00101 bool operator== (const Target &n2) const 00102 { return (username == n2.username && remaddr == n2.remaddr); } 00103 }; 00104 Target target; 00105 static int max_requests; 00106 00107 protected: 00108 WvLog log; 00109 WvUrlRequestList urls, waiting_urls; 00110 int request_count; 00111 WvUrlRequest *curl; // current url 00112 virtual void doneurl() = 0; 00113 virtual void request_next() = 0; 00114 00115 public: 00116 WvUrlStream(const WvIPPortAddr &_remaddr, WvStringParm _username, 00117 WvStringParm logname) 00118 : WvStreamClone(new WvTCPConn(_remaddr)), target(_remaddr, _username), 00119 log(logname, WvLog::Debug) 00120 { 00121 request_count = 0; 00122 curl = NULL; 00123 } 00124 00125 virtual ~WvUrlStream() {}; 00126 00127 virtual void close() = 0; 00128 void addurl(WvUrlRequest *url); 00129 void delurl(WvUrlRequest *url); 00130 // only implemented in WvHttpStream 00131 virtual size_t remaining() 00132 { return 0; } 00133 00134 virtual void execute() = 0; 00135 }; 00136 00137 unsigned WvHash(const WvUrlStream::Target &n); 00138 00139 DeclareWvDict(WvUrlStream, WvUrlStream::Target, target); 00140 00141 00142 class WvHttpStream : public WvUrlStream 00143 { 00144 public: 00145 static bool global_enable_pipelining; 00146 bool enable_pipelining; 00147 00148 private: 00149 int pipeline_test_count; 00150 bool ssl; 00151 bool sent_url_request; // Have we sent a request to the server yet? 00152 WvIPPortAddrTable &pipeline_incompatible; 00153 WvString http_response, pipeline_test_response; 00154 WvDynBuf putstream_data; 00155 00156 enum { Unknown, Chunked, ContentLength, Infinity } encoding; 00157 size_t bytes_remaining; 00158 bool in_chunk_trailer, last_was_pipeline_test; 00159 00160 virtual void doneurl(); 00161 virtual void request_next(); 00162 void start_pipeline_test(WvUrl *url); 00163 WvString request_str(WvUrlRequest *url, bool keep_alive); 00164 void send_request(WvUrlRequest *url); 00165 void pipelining_is_broken(int why); 00166 00167 public: 00168 WvHttpStream(const WvIPPortAddr &_remaddr, WvStringParm _username, 00169 bool ssl, WvIPPortAddrTable &_pipeline_incompatible); 00170 virtual ~WvHttpStream(); 00171 00172 virtual void close(); 00173 virtual bool pre_select(SelectInfo &si); 00174 virtual bool post_select(SelectInfo &si); 00175 virtual void execute(); 00176 virtual size_t remaining() 00177 { return bytes_remaining; } 00178 }; 00179 00180 00181 class WvFtpStream : public WvUrlStream 00182 { 00183 bool logged_in, pasv_acked; 00184 WvString password; 00185 WvTCPConn *data; 00186 time_t last_request_time; 00187 00188 virtual void doneurl(); 00189 virtual void request_next(); 00190 00191 // Disregard all lines that are of the form "xxx-", meaning that another 00192 // line follows. Only the last line is important for us. 00193 char *get_important_line(int timeout); 00194 00195 // Parse response to "PASV" command and returns a pointer to the address 00196 // of the data port (or NULL if it can't parse the response).. 00197 // This mucks about with line. 00198 WvIPPortAddr *parse_pasv_response(char *line); 00199 00200 WvString parse_for_links(char *line); 00201 00202 public: 00203 WvFtpStream(const WvIPPortAddr &_remaddr, WvStringParm _username, 00204 WvStringParm _password); 00205 virtual ~WvFtpStream(); 00206 00207 virtual bool pre_select(SelectInfo &si); 00208 virtual bool post_select(SelectInfo &si); 00209 virtual void close(); 00210 virtual void execute(); 00211 }; 00212 00213 00214 // FIXME: Rename this to WvUrlPool someday. 00215 class WvHttpPool : public WvStreamList 00216 { 00217 WvLog log; 00218 WvResolver dns; 00219 WvUrlStreamDict conns; 00220 WvUrlRequestList urls; 00221 int num_streams_created; 00222 00223 WvIPPortAddrTable pipeline_incompatible; 00224 00225 public: 00226 WvHttpPool(); 00227 virtual ~WvHttpPool(); 00228 00229 virtual bool pre_select(SelectInfo &si); 00230 virtual void execute(); 00231 00232 WvBufUrlStream *addurl(WvStringParm _url, WvStringParm _method = "GET", 00233 WvStringParm _headers = "", 00234 WvStream *content_source = NULL, 00235 bool create_dirs = false); 00236 00237 // For URL uploads. create_dirs should be true if you want all 00238 // non-existent directories in _url to be created. 00239 // WvBufUrlStream *addputurl(WvStringParm _url, WvStringParm _headers, 00240 // WvStream *s, bool create_dirs = false); 00241 private: 00242 void unconnect(WvUrlStream *s); 00243 00244 public: 00245 bool idle() const 00246 { return !urls.count(); } 00247 }; 00248 00249 00250 #endif // __WVHTTPPOOL_H

Generated on Tue Oct 5 01:09:20 2004 for WvStreams by doxygen 1.3.7