wvftpstream.cc

00001 /*
00002  * Worldvisions Weaver Software:
00003  *   Copyright (C) 1997-2002 Net Integration Technologies, Inc.
00004  * 
00005  * A fast, easy-to-use, parallelizing, pipelining HTTP/1.1 file retriever.
00006  * 
00007  * See wvhttppool.h.
00008  */
00009 
00010 #ifdef __GNUC__
00011 # define alloca __builtin_alloca
00012 #else
00013 # ifdef _MSC_VER
00014 #  include <malloc.h>
00015 #  define alloca _alloca
00016 # else
00017 #  if HAVE_ALLOCA_H
00018 #   include <alloca.h>
00019 #  else
00020 #   ifdef _AIX
00021 #pragma alloca
00022 #   else
00023 #    ifndef alloca /* predefined by HP cc +Olibcalls */
00024 char *alloca ();
00025 #    endif
00026 #   endif
00027 #  endif
00028 # endif
00029 #endif
00030 
00031 #include <ctype.h>
00032 #include <time.h>
00033 #include "wvhttppool.h"
00034 #include "wvbufstream.h"
00035 #include "wvtcp.h"
00036 #include "wvsslstream.h"
00037 #include "strutils.h"
00038 #include <malloc.h> // for alloca()... FIXME: which we shouldn't be using!
00039 
00040 WvFtpStream::WvFtpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
00041                 WvStringParm _password)
00042     : WvUrlStream(_remaddr, _username, WvString("FTP %s", _remaddr)),
00043       cont(WvContCallback(this, &WvFtpStream::real_execute))
00044 {
00045     data = NULL;
00046     logged_in = false;
00047     password = _password;
00048     last_request_time = time(0);
00049     alarm(60000); // timeout if no connection, or something goes wrong
00050 }
00051 
00052 
00053 void WvFtpStream::doneurl()
00054 {
00055     log("Done URL: %s\n", curl->url);
00056 
00057     curl->done();
00058     curl = NULL;
00059     WVRELEASE(data);
00060     urls.unlink_first();
00061     last_request_time = time(0);
00062     alarm(60000);
00063     request_next();
00064     // We just processed the last url in the queue,
00065     // so go away.
00066     if (urls.isempty() && waiting_urls.isempty())
00067         close();
00068 }
00069 
00070 
00071 void WvFtpStream::request_next()
00072 {
00073     // don't do a request if we've done too many already or we have none
00074     // waiting.
00075     if (request_count >= max_requests || waiting_urls.isempty())
00076         return;
00077 
00078     if (!urls.isempty())
00079         return;
00080 
00081     // okay then, we really do want to send a new request.
00082     WvUrlRequest *url = waiting_urls.first();
00083 
00084     waiting_urls.unlink_first();
00085 
00086     request_count++;
00087     log("Request #%s: %s\n", request_count, url->url);
00088     urls.append(url, false, "request_url");
00089     alarm(0);
00090 }
00091 
00092 
00093 void WvFtpStream::close()
00094 {
00095     if (isok())
00096         log("Closing.\n");
00097     WvStreamClone::close();
00098 
00099     if (geterr())
00100     {
00101         // if there was an error, count the first URL as done.  This prevents
00102         // retrying indefinitely.
00103         if (!curl && !urls.isempty())
00104             curl = urls.first();
00105         if (!curl && !waiting_urls.isempty())
00106             curl = waiting_urls.first();
00107         if (curl)
00108             log("URL '%s' is FAILED\n", curl->url);
00109         if (curl) 
00110             curl->done();
00111     }
00112 
00113     if (curl)
00114         curl->done();
00115 }
00116 
00117 
00118 char *WvFtpStream::get_important_line()
00119 {
00120     char *line;
00121     do
00122     {
00123         line = blocking_getline(-1);
00124         if (!line)
00125             return NULL;
00126     }
00127     while (line[3] == '-');
00128     log(WvLog::Debug5, ">> %s\n", line);
00129     return line;
00130 }
00131 
00132 
00133 bool WvFtpStream::pre_select(SelectInfo &si)
00134 {
00135     SelectRequest oldwant = si.wants;
00136 
00137     if (WvUrlStream::pre_select(si))
00138         return true;
00139 
00140     if (data && data->pre_select(si))
00141         return true;
00142 
00143     if (curl && curl->putstream && curl->putstream->pre_select(si))
00144         return true;
00145 
00146     si.wants = oldwant;
00147 
00148     return false;
00149 }
00150 
00151 
00152 bool WvFtpStream::post_select(SelectInfo &si)
00153 {
00154     SelectRequest oldwant = si.wants;
00155 
00156     if (WvUrlStream::post_select(si))
00157         return true;
00158 
00159     if (data && data->post_select(si))
00160         return true;
00161 
00162     if (curl && curl->putstream && curl->putstream->post_select(si))
00163         return true;
00164 
00165     si.wants = oldwant;
00166 
00167     return false;
00168 }
00169 
00170 
00171 void *WvFtpStream::real_execute(void*)
00172 {
00173     WvString line;
00174     WvStreamClone::execute();
00175 
00176     if (alarm_was_ticking && ((last_request_time + 60) <= time(0)))
00177     {
00178         log(WvLog::Debug4, "urls count: %s\n", urls.count());
00179         if (urls.isempty())
00180             close(); // timed out, but not really an error
00181 
00182         return 0;
00183     }
00184 
00185     if (!logged_in)
00186     {
00187         line = get_important_line();
00188         if (!line)
00189         {
00190             seterr("Server not reachable: %s\n",strerror(errno));
00191             return 0;
00192         }
00193             
00194         if (strncmp(line, "220", 3))
00195         {
00196             log("Server rejected connection: %s\n", line);
00197             seterr("server rejected connection");
00198             return 0;
00199         }
00200         print("USER %s\r\n", !target.username ? WvString("anonymous") :
00201                     target.username);
00202         line = get_important_line();
00203         if (!line)
00204             return 0;
00205 
00206         if (!strncmp(line, "230", 3))
00207         {
00208             log(WvLog::Info, "Server doesn't need password.\n");
00209             logged_in = true;        // No password needed;
00210         }
00211         else if (!strncmp(line, "33", 2))
00212         {
00213             print("PASS %s\r\n", !password ? DEFAULT_ANON_PW : password);
00214             
00215             line = get_important_line();
00216             if (!line)
00217                 return 0;
00218 
00219             if (line[0] == '2')
00220             {
00221                 log(WvLog::Info, "Authenticated.\n");
00222                 logged_in = true;
00223             }
00224             else
00225             {
00226                 log("Strange response to PASS command: %s\n", line);
00227                 seterr("strange response to PASS command");
00228                 return 0;
00229             }
00230         }
00231         else
00232         {
00233             log("Strange response to USER command: %s\n", line);
00234             seterr("strange response to USER command");
00235             return 0;
00236         }
00237 
00238         print("TYPE I\r\n");
00239         log(WvLog::Debug5, "<< TYPE I\n");
00240         line = get_important_line();
00241         if (!line)
00242             return 0;
00243         
00244         if (strncmp(line, "200", 3))
00245         {
00246             log("Strange response to TYPE I command: %s\n", line);
00247             seterr("strange response to TYPE I command");
00248             return 0;
00249         }
00250     }
00251 
00252     if (!curl && !urls.isempty())
00253     {
00254         curl = urls.first();
00255 
00256         print("CWD %s\r\n", curl->url.getfile());
00257         line = get_important_line();
00258         if (!line)
00259             return 0;
00260 
00261         if (!strncmp(line, "250", 3))
00262         {
00263             log(WvLog::Debug5, "This is a directory.\n");
00264             curl->is_dir = true;
00265         }
00266 
00267         print("PASV\r\n");
00268         line = get_important_line();
00269         if (!line)
00270             return 0;
00271         WvIPPortAddr *dataip = parse_pasv_response(line.edit());
00272 
00273         if (!dataip)
00274             return 0;
00275 
00276         log(WvLog::Debug4, "Data port is %s.\n", *dataip);
00277         // Open data connection.
00278         data = new WvTCPConn(*dataip);
00279         if (!data)
00280         {
00281             log("Can't open data connection.\n");
00282             seterr("can't open data connection");
00283             return 0;
00284         }
00285 
00286         if (curl->is_dir)
00287         {
00288             if (!curl->putstream)
00289             {
00290                 print("LIST %s\r\n", curl->url.getfile());
00291                 if (curl->outstream)
00292                 {
00293                     WvString url_no_pw("ftp://%s%s%s%s", curl->url.getuser(),
00294                             !!curl->url.getuser() ? "@" : "", 
00295                             curl->url.gethost(),
00296                             curl->url.getfile());
00297                     curl->outstream->print("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML "
00298                                            "4.01//EN\">\n"
00299                                            "<html>\n<head>\n<title>%s</title>\n"
00300                                            "<meta http-equiv=\"Content-Type\" "
00301                                            "content=\"text/html; "
00302                                            "charset=ISO-8859-1\">\n"
00303                                            "<base href=\"%s\"/>\n</head>\n"
00304                                            "<style type=\"text/css\">\n"
00305                                            "img { border: 0; padding: 0 2px; vertical-align: "
00306                                            "text-bottom; }\n"
00307                                            "td  { font-family: monospace; padding: 2px 3px; "
00308                                            "text-align: right; vertical-align: bottom; }\n"
00309                                            "td:first-child { text-align: left; padding: "
00310                                            "2px 10px 2px 3px; }\n"
00311                                            "table { border: 0; }\n"
00312                                            "a.symlink { font-style: italic; }\n"
00313                                            "</style>\n<body>\n"
00314                                            "<h1>Index of %s</h1>\n"
00315                                            "<hr/><table>\n", url_no_pw, curl->url, url_no_pw 
00316                                            );
00317                 }
00318             }
00319             else
00320             {
00321                 log("Target is a directory.\n");
00322                 seterr("target is a directory");
00323                 doneurl();
00324                 return 0;
00325             }
00326         }
00327         else if (!curl->putstream)
00328             print("RETR %s\r\n", curl->url.getfile());
00329         else
00330         {
00331             if (curl->create_dirs)
00332             {
00333                 print("CWD %s\r\n", getdirname(curl->url.getfile()));
00334                 line = get_important_line();
00335                 if (!line)
00336                     return 0;
00337                 if (strncmp(line, "250", 3))
00338                 {
00339                     log("Path doesn't exist; creating directories...\n");
00340                     // create missing directories.
00341                     WvString current_dir("");
00342                     WvStringList dirs;
00343                     dirs.split(getdirname(curl->url.getfile()), "/");
00344                     WvStringList::Iter i(dirs);
00345                     for (i.rewind(); i.next(); )
00346                     {
00347                         current_dir.append(WvString("/%s", i()));
00348                         print("MKD %s\r\n", current_dir);
00349                         line = get_important_line();
00350                         if (!line)
00351                             return 0;
00352                     }
00353                 }
00354             }
00355             print("STOR %s\r\n", curl->url.getfile());
00356         }
00357 
00358         log(WvLog::Debug5, "Waiting for response to %s\n", curl->putstream ? "STOR" : 
00359             curl->is_dir ? "LIST" : "RETR");
00360         line = get_important_line();
00361 
00362         if (!line)
00363             doneurl();
00364         else if (strncmp(line, "150", 3))
00365         {
00366             log("Strange response to %s command: %s\n", 
00367                     curl->putstream ? "STOR" : "RETR", line);
00368             seterr(WvString("strange response to %s command",
00369                         curl->putstream ? "STOR" : "RETR"));
00370             doneurl();
00371         }
00372 
00373     }
00374 
00375     if (curl)
00376     {
00377         if (curl->is_dir)
00378         {
00379             line = data->blocking_getline(-1);
00380             if (line && curl->outstream)
00381             {
00382                 WvString output_line(parse_for_links(line.edit()));
00383                 if (!!output_line)
00384                     curl->outstream->write(output_line);
00385                 else
00386                     curl->outstream->write("Unknown format of LIST "
00387                             "response\n");
00388             }
00389         }
00390         else
00391         {
00392             char buf[1024];
00393             
00394             if (curl->putstream)
00395             {
00396                 while (curl->putstream->isreadable())
00397                 {
00398                     int len = curl->putstream->read(buf, sizeof(buf));
00399                     log(WvLog::Debug5, "Read %s bytes.\n%s\n", len, hexdump_buffer(buf, len));
00400 
00401                     if (len)
00402                     {
00403                         int wrote = data->write(buf, len);
00404                         log(WvLog::Debug5,"Wrote %s bytes\n", wrote);
00405                         data->flush(0);
00406                     }
00407                 }
00408                 curl->putstream->close();
00409             }
00410             else
00411             {
00412                 while (data->isreadable() && curl->outstream->isok())
00413                 {
00414                     int len = data->read(buf, sizeof(buf));
00415                     log(WvLog::Debug5, "Read %s bytes from remote.\n", len);
00416                     
00417                     if (len && curl->outstream)
00418                     {
00419                         int wrote = curl->outstream->write(buf, len);
00420                         log(WvLog::Debug5, "Wrote %s bytes to local.\n", wrote);
00421                     }
00422                 }
00423             }
00424         }
00425 
00426         if (!data->isok() || (curl->putstream && !curl->putstream->isok()))
00427         {
00428             log("OK, we should have finished writing!\n");
00429             if (curl->putstream && data->isok())
00430                 data->close();
00431             line = get_important_line();
00432             if (!line)
00433             {
00434                 doneurl();
00435                 return 0;
00436             }
00437 
00438             if (strncmp(line, "226", 3))
00439                 log("Unexpected message: %s\n", line);
00440 
00441             if (curl->is_dir)
00442             {
00443                 if (curl->outstream)
00444                     curl->outstream->write("</table><hr/></body>\n"
00445                             "</html>\n");
00446                 write("CWD /\r\n");
00447                 log(WvLog::Debug5, "Waiting for response to CWD /\n");
00448                 line = get_important_line();
00449                 if (!line)
00450                     return 0;
00451 
00452                 if (strncmp(line, "250", 3))
00453                     log("Strange resonse to \"CWD /\": %s\n", line);
00454                 // Don't bother failing here.
00455             }
00456             doneurl();
00457         }
00458         else
00459         {
00460             log("Why are we here??\n");
00461         }
00462     }
00463 
00464     return 0;
00465 }
00466 
00467 
00468 void WvFtpStream::execute()
00469 {
00470     real_execute(0);
00471 }
00472 
00473 
00474 WvString WvFtpStream::parse_for_links(char *line)
00475 {
00476     WvString output_line("");
00477     trim_string(line);
00478 
00479     if (curl->is_dir && curl->outstream)
00480     {
00481         struct ftpparse fp;
00482         int res = ftpparse(&fp, line, strlen(line));
00483         if (res)
00484         {
00485             char *linkname = (char *)alloca(fp.namelen+1);
00486             int i;
00487             for (i = 0; i < fp.namelen; i++)
00488             {
00489                 if (fp.name[i] >= 32)
00490                     linkname[i] = fp.name[i];
00491                 else
00492                 {
00493                     linkname[i] = '?';
00494                 }
00495             }
00496             linkname[i] = 0;
00497 
00498             WvString linkurl(curl->url);
00499             if (linkurl.cstr()[linkurl.len()-1] != '/')
00500                 linkurl.append("/");
00501             linkurl.append(linkname);
00502             WvUrlLink *link = new WvUrlLink(linkname, linkurl);
00503             curl->outstream->links.append(link, true);
00504 
00505             output_line.append("<tr>\n");
00506 
00507             output_line.append(WvString(" <td>%s%s</td>\n", linkname,
00508                         fp.flagtrycwd ? "/" : ""));
00509 
00510             if (fp.flagtryretr)
00511             {
00512                 if (!fp.sizetype)
00513                     output_line.append(" <td>? bytes</td>\n");
00514                 else
00515                     output_line.append(WvString(" <td>%s bytes</td>\n",
00516                                 fp.size));
00517                 if (fp.mtimetype > 0)
00518                     output_line.append(WvString(" <td>%s</td>\n", (fp.mtime)));
00519                 else
00520                     output_line.append(" <td>?</td>\n");
00521             }
00522             else
00523                 output_line.append(" <td></td>\n");
00524 
00525             output_line.append("</tr>\n");
00526         }
00527     }
00528     return output_line;
00529 }
00530 
00531 
00532 WvIPPortAddr *WvFtpStream::parse_pasv_response(char *line)
00533 {
00534     if (strncmp(line, "227 ", 4))
00535     {
00536         log("Strange response to PASV command: %s\n", line);
00537         seterr("strange response to PASV command");
00538         return NULL;
00539     }
00540 
00541     char *p = &line[3];
00542     while (!isdigit(*p))
00543     {
00544         if (*p == '\0' || *p == '\r' || *p == '\n')
00545         {
00546             log("Couldn't parse PASV response: %s\n", line);
00547             seterr("couldn't parse response to PASV command");
00548             return NULL;
00549         }
00550         p++;
00551     }
00552     char *ipstart = p;
00553 
00554     for (int i = 0; i < 4; i++)
00555     {
00556         p = strchr(p, ',');
00557         if (!p)
00558         {
00559             log("Couldn't parse PASV IP: %s\n", line);
00560             seterr("couldn't parse PASV IP");
00561             return NULL;
00562         }
00563         *p = '.';
00564     }
00565     *p = '\0';
00566     WvString pasvip(ipstart);
00567     p++;
00568     int pasvport;
00569     pasvport = atoi(p)*256;
00570     p = strchr(p, ',');
00571     if (!p)
00572     {
00573         log("Couldn't parse PASV IP port: %s\n", line);
00574         seterr("couldn't parse PASV IP port");
00575         return NULL;
00576     }
00577     pasvport += atoi(++p);
00578 
00579     WvIPPortAddr *res = new WvIPPortAddr(pasvip.cstr(), pasvport);
00580 
00581     return res;
00582 }

Generated on Mon Feb 5 10:54:29 2007 for WvStreams by  doxygen 1.5.1