Main Page | Namespace List | Class Hierarchy | Alphabetical List | Class List | Directories | File List | Namespace Members | Class Members | File Members | Related Pages

wvhttppool.h

Go to the documentation of this file.
00001 /* -*- Mode: C++ -*-
00002  * Worldvisions Weaver Software:
00003  *   Copyright (C) 1997-2002 Net Integration Technologies, Inc.
00004  *
00005  * A fast, easy-to-use, parallelizing, pipelining HTTP/1.1 file retriever.
00006  * 
00007  * Just create a WvHttpPool object, add it to your list, and use pool.addurl()
00008  * to get a WvStream* that gives you the file you requested.
00009  */ 
00010 #ifndef __WVHTTPPOOL_H
00011 #define __WVHTTPPOOL_H
00012 
00013 #include "ftpparse.h"
00014 #include "wvurl.h"
00015 #include "wvstreamlist.h"
00016 #include "wvstreamclone.h"
00017 #include "wvlog.h"
00018 #include "wvhashtable.h"
00019 #include "wvhttp.h"
00020 #include "wvbufstream.h"
00021 #include "wvbuf.h"
00022 
00023 class WvBufUrlStream;
00024 class WvUrlStream;
00025 class WvHttpStream;
00026 
00027 static const WvString DEFAULT_ANON_PW("weasels@");
00028 
00029 class WvUrlRequest
00030 {
00031 public:
00032     WvUrl url;
00033     WvString headers;
00034     WvUrlStream *instream;
00035     WvBufUrlStream *outstream;
00036     WvStream *putstream;
00037 
00038     bool pipeline_test;
00039     bool inuse;
00040     bool is_dir;
00041     bool create_dirs;
00042     WvString method;
00043     
00044     WvUrlRequest(WvStringParm _url, WvStringParm _method, WvStringParm _headers,
00045                  WvStream *content_source, bool _create_dirs, bool _pipeline_test);
00046     ~WvUrlRequest();
00047     
00048     void done();
00049 };
00050 
00051 DeclareWvList(WvUrlRequest);
00052 
00053 
00054 struct WvUrlLink
00055 {
00056     WvString linkname;
00057     WvUrl url;
00058 
00059     WvUrlLink::WvUrlLink(WvStringParm _linkname, WvStringParm _url)
00060         : linkname(_linkname), url(_url)
00061     {}
00062 };
00063 DeclareWvList(WvUrlLink);
00064 
00065 
00066 class WvBufUrlStream : public WvBufStream
00067 {
00068 public:
00069     WvString url;
00070     WvString proto;
00071     WvUrlLinkList links;  // HTML links or FTP directory listing
00072 
00073     // HTTP stuff...
00074     WvString version;
00075     int status;
00076     WvHTTPHeaderDict headers; 
00077 
00078     WvBufUrlStream() : status(0), headers(10)
00079         {}
00080     virtual ~WvBufUrlStream()
00081         {}
00082 };
00083 
00084 DeclareWvTable(WvIPPortAddr);
00085 
00086 
00087 class WvUrlStream : public WvStreamClone
00088 {
00089 public:
00090     class Target
00091     {
00092     public:
00093         WvIPPortAddr remaddr;
00094         WvString username;
00095 
00096         Target(const WvIPPortAddr &_remaddr, WvStringParm _username)
00097             : remaddr(_remaddr), username(_username) {}
00098 
00099         ~Target() {}
00100 
00101         bool operator== (const Target &n2) const
00102         { return (username == n2.username && remaddr == n2.remaddr); }
00103     };
00104     Target target;
00105     static int max_requests;
00106 
00107 protected:
00108     WvLog log;
00109     WvUrlRequestList urls, waiting_urls;
00110     int request_count;
00111     WvUrlRequest *curl; // current url
00112     virtual void doneurl() = 0;
00113     virtual void request_next() = 0;
00114 
00115 public:
00116     WvUrlStream(const WvIPPortAddr &_remaddr, WvStringParm _username, 
00117         WvStringParm logname)
00118         : WvStreamClone(new WvTCPConn(_remaddr)), target(_remaddr, _username),
00119           log(logname, WvLog::Debug)
00120     {
00121         request_count = 0;
00122         curl = NULL;
00123     }
00124 
00125     virtual ~WvUrlStream() {};
00126 
00127     virtual void close() = 0;
00128     void addurl(WvUrlRequest *url);
00129     void delurl(WvUrlRequest *url);
00130     // only implemented in WvHttpStream
00131     virtual size_t remaining()
00132     { return 0; }
00133     
00134     virtual void execute() = 0;
00135 };
00136 
00137 unsigned WvHash(const WvUrlStream::Target &n);
00138 
00139 DeclareWvDict(WvUrlStream, WvUrlStream::Target, target);
00140 
00141 
00142 class WvHttpStream : public WvUrlStream
00143 {
00144 public:
00145     static bool global_enable_pipelining;
00146     bool enable_pipelining;
00147     
00148 private:
00149     int pipeline_test_count;
00150     bool ssl;
00151     bool sent_url_request;      // Have we sent a request to the server yet?
00152     WvIPPortAddrTable &pipeline_incompatible;
00153     WvString http_response, pipeline_test_response;
00154     WvDynBuf putstream_data;
00155     
00156     enum { Unknown, Chunked, ContentLength, Infinity } encoding;
00157     size_t bytes_remaining;
00158     bool in_chunk_trailer, last_was_pipeline_test;
00159 
00160     virtual void doneurl();
00161     virtual void request_next();
00162     void start_pipeline_test(WvUrl *url);
00163     WvString request_str(WvUrlRequest *url, bool keep_alive);
00164     void send_request(WvUrlRequest *url);
00165     void pipelining_is_broken(int why);
00166     
00167 public:
00168     WvHttpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
00169          bool ssl, WvIPPortAddrTable &_pipeline_incompatible);
00170     virtual ~WvHttpStream();
00171 
00172     virtual void close();
00173     virtual bool pre_select(SelectInfo &si);
00174     virtual bool post_select(SelectInfo &si);
00175     virtual void execute();
00176     virtual size_t remaining()
00177     { return bytes_remaining; }
00178 };
00179 
00180 
00181 class WvFtpStream : public WvUrlStream
00182 {
00183     bool logged_in, pasv_acked;
00184     WvString password;
00185     WvTCPConn *data;
00186     time_t last_request_time;
00187 
00188     virtual void doneurl();
00189     virtual void request_next();
00190 
00191     // Disregard all lines that are of the form "xxx-", meaning that another
00192     // line follows.  Only the last line is important for us.
00193     char *get_important_line(int timeout);
00194 
00195     // Parse response to "PASV" command and returns a pointer to the address
00196     // of the data port (or NULL if it can't parse the response)..
00197     // This mucks about with line.
00198     WvIPPortAddr *parse_pasv_response(char *line);
00199 
00200     WvString parse_for_links(char *line);
00201 
00202 public:
00203     WvFtpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
00204                 WvStringParm _password);
00205     virtual ~WvFtpStream();
00206 
00207     virtual bool pre_select(SelectInfo &si);
00208     virtual bool post_select(SelectInfo &si);
00209     virtual void close();
00210     virtual void execute();
00211 };
00212 
00213 
00214 // FIXME: Rename this to WvUrlPool someday.
00215 class WvHttpPool : public WvStreamList
00216 {
00217     WvLog log;
00218     WvResolver dns;
00219     WvUrlStreamDict conns;
00220     WvUrlRequestList urls;
00221     int num_streams_created;
00222     
00223     WvIPPortAddrTable pipeline_incompatible;
00224     
00225 public:
00226     WvHttpPool();
00227     virtual ~WvHttpPool();
00228     
00229     virtual bool pre_select(SelectInfo &si);
00230     virtual void execute();
00231     
00232     WvBufUrlStream *addurl(WvStringParm _url, WvStringParm _method = "GET",
00233                             WvStringParm _headers = "",
00234                             WvStream *content_source = NULL,
00235                             bool create_dirs = false);
00236 
00237     // For URL uploads.  create_dirs should be true if you want all
00238     // non-existent directories in _url to be created.
00239 //    WvBufUrlStream *addputurl(WvStringParm _url, WvStringParm _headers,
00240 //                            WvStream *s, bool create_dirs = false);
00241 private:
00242     void unconnect(WvUrlStream *s);
00243     
00244 public:
00245     bool idle() const 
00246         { return !urls.count(); }
00247 };
00248 
00249 
00250 #endif // __WVHTTPPOOL_H

Generated on Wed Dec 15 15:08:11 2004 for WvStreams by  doxygen 1.3.9.1