Main Page | Namespace List | Class Hierarchy | Alphabetical List | Class List | Directories | File List | Namespace Members | Class Members | File Members | Related Pages

wvftpstream.cc

Go to the documentation of this file.
00001 /*
00002  * Worldvisions Weaver Software:
00003  *   Copyright (C) 1997-2002 Net Integration Technologies, Inc.
00004  * 
00005  * A fast, easy-to-use, parallelizing, pipelining HTTP/1.1 file retriever.
00006  * 
00007  * See wvhttppool.h.
00008  */
00009 #include <ctype.h>
00010 #include <time.h>
00011 #include "wvhttppool.h"
00012 #include "wvbufstream.h"
00013 #include "wvtcp.h"
00014 #include "wvsslstream.h"
00015 #include "strutils.h"
00016 
00017 WvFtpStream::WvFtpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
00018                 WvStringParm _password)
00019     : WvUrlStream(_remaddr, _username, WvString("FTP %s", _remaddr))
00020 {
00021     data = NULL;
00022     logged_in = false;
00023     password = _password;
00024     uses_continue_select = true;
00025     last_request_time = time(0);
00026     alarm(60000); // timeout if no connection, or something goes wrong
00027 }
00028 
00029 
00030 WvFtpStream::~WvFtpStream()
00031 {
00032     close();
00033 }
00034 
00035 
00036 void WvFtpStream::doneurl()
00037 {
00038     log("Done URL: %s\n", curl->url);
00039 
00040     curl->done();
00041     curl = NULL;
00042     if (data)
00043     {
00044         delete data;
00045         data = NULL;
00046     }
00047     urls.unlink_first();
00048     last_request_time = time(0);
00049     alarm(60000);
00050     request_next();
00051 }
00052 
00053 
00054 void WvFtpStream::request_next()
00055 {
00056     // don't do a request if we've done too many already or we have none
00057     // waiting.
00058     if (request_count >= max_requests || waiting_urls.isempty())
00059         return;
00060 
00061     if (!urls.isempty())
00062         return;
00063 
00064     // okay then, we really do want to send a new request.
00065     WvUrlRequest *url = waiting_urls.first();
00066 
00067     waiting_urls.unlink_first();
00068 
00069     request_count++;
00070     log("Request #%s: %s\n", request_count, url->url);
00071     urls.append(url, false, "request_url");
00072     alarm(0);
00073 }
00074 
00075 
00076 void WvFtpStream::close()
00077 {
00078     if (isok())
00079         log("Closing.\n");
00080     WvStreamClone::close();
00081 
00082     if (geterr())
00083     {
00084         // if there was an error, count the first URL as done.  This prevents
00085         // retrying indefinitely.
00086         if (!curl && !urls.isempty())
00087             curl = urls.first();
00088         if (!curl && !waiting_urls.isempty())
00089             curl = waiting_urls.first();
00090         if (curl)
00091             log("URL '%s' is FAILED\n", curl->url);
00092         if (curl) 
00093             curl->done();
00094     }
00095 
00096     if (curl)
00097         curl->done();
00098 }
00099 
00100 
00101 char *WvFtpStream::get_important_line(int timeout)
00102 {
00103     char *line;
00104     do
00105     {
00106         line = getline(timeout);
00107         if (!line)
00108             return NULL;
00109     }
00110     while (line[3] == '-');
00111     return line;
00112 }
00113 
00114 
00115 bool WvFtpStream::pre_select(SelectInfo &si)
00116 {
00117     SelectRequest oldwant = si.wants;
00118 
00119     if (WvUrlStream::pre_select(si))
00120         return true;
00121 
00122     if (data && data->pre_select(si))
00123         return true;
00124 
00125     if (curl && curl->putstream && curl->putstream->pre_select(si))
00126         return true;
00127 
00128     si.wants = oldwant;
00129 
00130     return false;
00131 }
00132 
00133 
00134 bool WvFtpStream::post_select(SelectInfo &si)
00135 {
00136     SelectRequest oldwant = si.wants;
00137 
00138     if (WvUrlStream::post_select(si))
00139         return true;
00140 
00141     if (data && data->post_select(si))
00142         return true;
00143 
00144     if (curl && curl->putstream && curl->putstream->post_select(si))
00145         return true;
00146 
00147     si.wants = oldwant;
00148 
00149     return false;
00150 }
00151 
00152 
00153 void WvFtpStream::execute()
00154 {
00155     char buf[1024], *line;
00156     WvStreamClone::execute();
00157 
00158     if (alarm_was_ticking && ((last_request_time + 60) <= time(0)))
00159     {
00160         log(WvLog::Debug4, "urls count: %s\n", urls.count());
00161         if (urls.isempty())
00162             close(); // timed out, but not really an error
00163 
00164         return;
00165     }
00166 
00167     if (!logged_in)
00168     {
00169         line = get_important_line(60000);
00170         if (!line)
00171             return;
00172         if (strncmp(line, "220", 3))
00173         {
00174             log("Server rejected connection: %s\n", line);
00175             seterr("server rejected connection");
00176             return;
00177         }
00178 
00179         log(WvLog::Debug5, "Got greeting: %s\n", line);
00180         write(WvString("USER %s\r\n",
00181                     !target.username ? "anonymous" :
00182                     target.username.cstr()));
00183         line = get_important_line(60000);
00184         if (!line)
00185             return;
00186 
00187         if (!strncmp(line, "230", 3))
00188         {
00189             log(WvLog::Debug5, "Server doesn't need password.\n");
00190             logged_in = true;        // No password needed;
00191         }
00192         else if (!strncmp(line, "33", 2))
00193         {
00194             write(WvString("PASS %s\r\n", !password ? DEFAULT_ANON_PW :
00195                         password));
00196             line = get_important_line(60000);
00197             if (!line)
00198                 return;
00199 
00200             if (line[0] == '2')
00201             {
00202                 log(WvLog::Debug5, "Authenticated.\n");
00203                 logged_in = true;
00204             }
00205             else
00206             {
00207                 log("Strange response to PASS command: %s\n", line);
00208                 seterr("strange response to PASS command");
00209                 return;
00210             }
00211         }
00212         else
00213         {
00214             log("Strange response to USER command: %s\n", line);
00215             seterr("strange response to USER command");
00216             return;
00217         }
00218 
00219         write("TYPE I\r\n");
00220         line = get_important_line(60000);
00221         if (!line)
00222             return;
00223 
00224         if (strncmp(line, "200", 3))
00225         {
00226             log("Strange response to TYPE I command: %s\n", line);
00227             seterr("strange response to TYPE I command");
00228             return;
00229         }
00230     }
00231 
00232     if (!curl && !urls.isempty())
00233     {
00234         curl = urls.first();
00235 
00236         write(WvString("CWD %s\r\n", curl->url.getfile()));
00237         line = get_important_line(60000);
00238         if (!line)
00239             return;
00240 
00241         if (!strncmp(line, "250", 3))
00242         {
00243             log(WvLog::Debug5, "This is a directory.\n");
00244             curl->is_dir = true;
00245         }
00246 
00247         write("PASV\r\n");
00248         line = get_important_line(60000);
00249         if (!line)
00250             return;
00251 
00252         WvIPPortAddr *dataip = parse_pasv_response(line);
00253 
00254         if (!dataip)
00255             return;
00256 
00257         log(WvLog::Debug4, "Data port is %s.\n", *dataip);
00258         // Open data connection.
00259         data = new WvTCPConn(*dataip);
00260         if (!data)
00261         {
00262             log("Can't open data connection.\n");
00263             seterr("can't open data connection");
00264             return;
00265         }
00266 
00267         if (curl->is_dir)
00268         {
00269             if (!curl->putstream)
00270             {
00271                 write(WvString("LIST %s\r\n", curl->url.getfile()));
00272                 if (curl->outstream)
00273                 {
00274                     WvString url_no_pw("ftp://%s%s%s%s", curl->url.getuser(),
00275                             !!curl->url.getuser() ? "@" : "", 
00276                             curl->url.gethost(),
00277                             curl->url.getfile());
00278                     curl->outstream->write(
00279                             WvString(
00280                                 "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML "
00281                                 "4.01//EN\">\n"
00282                                 "<html>\n<head>\n<title>%s</title>\n"
00283                                 "<meta http-equiv=\"Content-Type\" "
00284                                 "content=\"text/html; "
00285                                 "charset=ISO-8859-1\">\n"
00286                                 "<base href=\"%s\"/>\n</head>\n"
00287                                 "<style type=\"text/css\">\n"
00288                                 "img { border: 0; padding: 0 2px; vertical-align: "
00289                                 "text-bottom; }\n"
00290                                 "td  { font-family: monospace; padding: 2px 3px; "
00291                                 "text-align: right; vertical-align: bottom; }\n"
00292                                 "td:first-child { text-align: left; padding: "
00293                                 "2px 10px 2px 3px; }\n"
00294                                 "table { border: 0; }\n"
00295                                 "a.symlink { font-style: italic; }\n"
00296                                 "</style>\n<body>\n"
00297                                 "<h1>Index of %s</h1>\n"
00298                                 "<hr/><table>\n", url_no_pw, curl->url, url_no_pw 
00299                                 ));
00300                 }
00301             }
00302             else
00303             {
00304                 log("Target is a directory.\n");
00305                 seterr("target is a directory");
00306                 doneurl();
00307                 return;
00308             }
00309         }
00310         else if (!curl->putstream)
00311             write(WvString("RETR %s\r\n", curl->url.getfile()));
00312         else
00313         {
00314             if (curl->create_dirs)
00315             {
00316                 write(WvString("CWD %s\r\n", getdirname(curl->url.getfile())));
00317                 line = get_important_line(60000);
00318                 if (!line)
00319                     return;
00320 
00321                 if (strncmp(line, "250", 3))
00322                 {
00323                     log("Path doesn't exist; creating directories...\n");
00324                     // create missing directories.
00325                     WvString current_dir("");
00326                     WvStringList dirs;
00327                     dirs.split(getdirname(curl->url.getfile()), "/");
00328                     WvStringList::Iter i(dirs);
00329                     for (i.rewind(); i.next(); )
00330                     {
00331                         current_dir.append(WvString("/%s", i()));
00332                         write(WvString("MKD %s\r\n", current_dir));
00333                         line = get_important_line(60000);
00334                         if (!line)
00335                             return;
00336                     }
00337                 }
00338             }
00339             write(WvString("STOR %s\r\n", curl->url.getfile()));
00340         }
00341 
00342         log(WvLog::Debug5, "Waiting for response to STOR/RETR/LIST\n");
00343         line = get_important_line(60000);
00344         log("Response: %s\n", line);
00345         if (!line)
00346             doneurl();
00347         else if (strncmp(line, "150", 3))
00348         {
00349             log("Strange response to %s command: %s\n", 
00350                     curl->putstream ? "STOR" : "RETR", line);
00351             seterr(WvString("strange response to %s command",
00352                         curl->putstream ? "STOR" : "RETR"));
00353             doneurl();
00354         }
00355 
00356     }
00357 
00358     if (curl)
00359     {
00360         if (curl->is_dir)
00361         {
00362             line = data->getline(0);
00363             if (line && curl->outstream)
00364             {
00365                 WvString output_line(parse_for_links(line));
00366                 if (!!output_line)
00367                     curl->outstream->write(output_line);
00368                 else
00369                     curl->outstream->write("Unknown format of LIST "
00370                             "response\n");
00371             }
00372         }
00373         else
00374         {
00375             if (curl->putstream)
00376             {
00377                 int len = curl->putstream->read(buf, sizeof(buf));
00378                 log(WvLog::Debug5, "Read %s bytes.\n", len);
00379 
00380                 if (len)
00381                     data->write(buf, len);
00382             }
00383             else
00384             {
00385                 int len = data->read(buf, sizeof(buf));
00386                 log(WvLog::Debug5, "Read %s bytes.\n", len);
00387 
00388                 if (len && curl->outstream)
00389                     curl->outstream->write(buf, len);
00390             }
00391         }
00392 
00393         if (!data->isok() || (curl->putstream && !curl->putstream->isok()))
00394         {
00395             if (curl->putstream && data->isok())
00396                 data->close();
00397             line = get_important_line(60000);
00398             if (!line)
00399             {
00400                 doneurl();
00401                 return;
00402             }
00403 
00404             if (strncmp(line, "226", 3))
00405                 log("Unexpected message: %s\n", line);
00406 
00407             if (curl->is_dir)
00408             {
00409                 if (curl->outstream)
00410                     curl->outstream->write("</table><hr/></body>\n"
00411                             "</html>\n");
00412                 write("CWD /\r\n");
00413                 log(WvLog::Debug5, "Waiting for response to CWD /\n");
00414                 line = get_important_line(60000);
00415                 if (!line)
00416                     return;
00417 
00418                 if (strncmp(line, "250", 3))
00419                     log("Strange resonse to \"CWD /\": %s\n", line);
00420                 // Don't bother failing here.
00421             }
00422             doneurl();
00423         }
00424     }
00425 }
00426 
00427 
00428 WvString WvFtpStream::parse_for_links(char *line)
00429 {
00430     WvString output_line("");
00431     trim_string(line);
00432 
00433     if (curl->is_dir && curl->outstream)
00434     {
00435         struct ftpparse fp;
00436         int res = ftpparse(&fp, line, strlen(line));
00437         if (res)
00438         {
00439             char *linkname = (char *) alloca(fp.namelen+1);
00440             int i;
00441             for (i = 0; i < fp.namelen; i++)
00442             {
00443                 if (fp.name[i] >= 32)
00444                     linkname[i] = fp.name[i];
00445                 else
00446                 {
00447                     linkname[i] = '?';
00448                 }
00449             }
00450             linkname[i] = 0;
00451 
00452             WvString linkurl(curl->url);
00453             if (linkurl.cstr()[linkurl.len()-1] != '/')
00454                 linkurl.append("/");
00455             linkurl.append(linkname);
00456             WvUrlLink *link = new WvUrlLink(linkname, linkurl);
00457             curl->outstream->links.append(link, true);
00458 
00459             output_line.append("<tr>\n");
00460 
00461             output_line.append(WvString(" <td>%s%s</td>\n", linkname,
00462                         fp.flagtrycwd ? "/" : ""));
00463 
00464             if (fp.flagtryretr)
00465             {
00466                 if (!fp.sizetype)
00467                     output_line.append(" <td>? bytes</td>\n");
00468                 else
00469                     output_line.append(WvString(" <td>%s bytes</td>\n",
00470                                 fp.size));
00471                 if (fp.mtimetype > 0)
00472                     output_line.append(WvString(" <td>%s</td>\n", (fp.mtime)));
00473                 else
00474                     output_line.append(" <td>?</td>\n");
00475             }
00476             else
00477                 output_line.append(" <td></td>\n");
00478 
00479             output_line.append("</tr>\n");
00480         }
00481     }
00482     return output_line;
00483 }
00484 
00485 
00486 WvIPPortAddr *WvFtpStream::parse_pasv_response(char *line)
00487 {
00488     if (strncmp(line, "227 ", 4))
00489     {
00490         log("Strange response to PASV command: %s\n", line);
00491         seterr("strange response to PASV command");
00492         return NULL;
00493     }
00494 
00495     char *p = &line[3];
00496     while (!isdigit(*p))
00497     {
00498         if (*p == '\0' || *p == '\r' || *p == '\n')
00499         {
00500             log("Couldn't parse PASV response: %s\n", line);
00501             seterr("couldn't parse response to PASV command");
00502             return NULL;
00503         }
00504         p++;
00505     }
00506     char *ipstart = p;
00507 
00508     for (int i = 0; i < 4; i++)
00509     {
00510         p = strchr(p, ',');
00511         if (!p)
00512         {
00513             log("Couldn't parse PASV IP: %s\n", line);
00514             seterr("couldn't parse PASV IP");
00515             return NULL;
00516         }
00517         *p = '.';
00518     }
00519     *p = '\0';
00520     WvString pasvip(ipstart);
00521     p++;
00522     int pasvport;
00523     pasvport = atoi(p)*256;
00524     p = strchr(p, ',');
00525     if (!p)
00526     {
00527         log("Couldn't parse PASV IP port: %s\n", line);
00528         seterr("couldn't parse PASV IP port");
00529         return NULL;
00530     }
00531     pasvport += atoi(++p);
00532 
00533     WvIPPortAddr *res = new WvIPPortAddr(pasvip.cstr(), pasvport);
00534 
00535     return res;
00536 }

Generated on Wed Dec 15 15:08:11 2004 for WvStreams by  doxygen 1.3.9.1