Main Page | Modules | Data Structures | Directories | File List | Data Fields | Globals

tools/klone/main.c

00001 /*
00002  * Copyright (c) 2005, 2006 by KoanLogic s.r.l. <http://www.koanlogic.com>
00003  * All rights reserved.
00004  *
00005  * This file is part of KLone, and as such it is subject to the license stated
00006  * in the LICENSE file which you have received as part of this distribution.
00007  *
00008  * $Id: main.c,v 1.40 2007/09/15 16:36:12 tat Exp $
00009  */
00010 
00011 #include "klone_conf.h"
00012 #include <sys/stat.h>
00013 #ifdef OS_UNIX
00014 #include <sys/dir.h>
00015 #endif
00016 #include <sys/types.h>
00017 #include <dirent.h>
00018 #include <stdlib.h>
00019 #include <ctype.h>
00020 #include <stdio.h>
00021 #include <fcntl.h>
00022 #include <unistd.h>
00023 #include <u/libu.h>
00024 #include <klone/os.h>
00025 #include <klone/request.h>
00026 #include <klone/response.h>
00027 #include <klone/translat.h>
00028 #include <klone/utils.h>
00029 #include <klone/run.h>
00030 #include <klone/mime_map.h>
00031 #include <klone/version.h>
00032 #include "pm.h"
00033 
00034 int facility = LOG_LOCAL0;
00035 
00036 /* command list enums */
00037 enum command_e { CMD_UNKNOWN, CMD_TRANS, CMD_IMPORT };
00038 
00039 /* runtime flags */
00040 enum flags_e { FLAG_NONE, FLAG_VERBOSE };
00041 
00042 typedef struct 
00043 {
00044     char *file_in, *file_out;   /* [trans] input, output file       */
00045     char *uri;                  /* [trans] translated file uri      */
00046     int verbose;                /* >0 when verbose mode is on       */
00047     char **arg;                 /* argv                             */
00048     size_t narg;                /* argc                             */
00049     int cmd;                    /* command to run                   */
00050     char *base_uri;             /* site base uri                    */
00051     int encrypt;                /* >0 when encryption is enabled    */
00052     int compress;               /* >0 when compress is enabled      */
00053     pm_t *comp_patt;            /* compress file pattern            */
00054     pm_t *enc_patt;             /* encrypt file pattern             */
00055     pm_t *excl_patt;            /* exclude file pattern             */
00056     char *key_file;             /* encryption key file name         */
00057     io_t *iom, *iod, *ior;      /* io makefile, io deps             */
00058     size_t ndir, nfile;         /* dir and file count               */
00059     size_t nexcl;               /* # of excluded files (-x)         */
00060 } context_t;
00061 
00062 context_t *ctx;
00063 
00064 #define KL1_FILE_FMT "pg_%s.%s"
00065 
00066 static void usage(void)
00067 {
00068     static const char * us = 
00069 "Usage: klone [-hvV] -c COMMAND OPTIONS ARGUMENTS\n"
00070 "Version: %s - Copyright (c) 2005, 2006, 2007 KoanLogic s.r.l.\n"
00071 "All rights reserved.\n"
00072 "\n"
00073 "       -h            display this help\n"
00074 "       -v            verbose mode\n"
00075 "       -V            print KLone version and exit\n"
00076 "       -c command    command to execute (see COMMAND LIST)\n"
00077 "\n"
00078 "\n"
00079 "    COMMAND LIST:\n"
00080 "       import        import a directory tree in the embedded filesystem\n"
00081 "       translate     convert a file or a dynamic web page to a C file\n"
00082 "\n"
00083 "\n"
00084 "    COMMANDS SYNTAX:\n"
00085 "\n"
00086 "       import OPTIONS dir\n"
00087 "         -b URI      base URI\n"
00088 "         -x pattern  exclude all files whose URI match the given pattern (*)\n"
00089 #ifdef HAVE_LIBOPENSSL
00090 "         -e pattern  encrypt all files whose URI match the given pattern (*)\n"
00091 "         -k key_file encryption key filename\n"
00092 #endif
00093 #ifdef HAVE_LIBZ
00094 "         -z          compress all compressable content (based on MIME types)\n"
00095 "         -Z pattern  compress all files whose URI match the given pattern (*)\n"
00096 #endif
00097 "         dir         directory tree path\n"
00098 "\n"
00099 "         (*) may be used more then once\n"
00100 "\n"
00101 "       translate OPTIONS\n"
00102 #ifdef HAVE_LIBOPENSSL
00103 "         -E          encrypt file content\n"
00104 #endif
00105 "         -i file     input file\n"
00106 #ifdef HAVE_LIBOPENSSL
00107 "         -k key_file encryption key filename\n"
00108 #endif
00109 "         -o file     output file\n"
00110 "         -u URI      URI of translated page\n"
00111 "                     (KLONE_CIPHER_KEY environ var is used if not provided)\n"
00112 #ifdef HAVE_LIBZ
00113 "         -z          compress file content\n"
00114 #endif
00115 "\n";
00116 
00117     fprintf(stderr, us, klone_version());
00118 
00119     exit(EXIT_FAILURE);
00120 }
00121 
00122 static void remove_trailing_slash(char *s)
00123 {
00124     size_t len;
00125     
00126     dbg_ifb (s == NULL) return;
00127     
00128     len = strlen(s);
00129     if(len && s[len - 1] == '/')
00130         s[len - 1] = 0;
00131 }
00132 
00133 static int parse_opt(int argc, char **argv)
00134 {
00135     int ret;
00136     char opts[512];
00137 
00138     if(argc == 1)
00139         usage();
00140 
00141     /* common switches */
00142     strcpy(opts, "hvVx:b:i:o:u:c:");
00143 
00144     /* encryption switches */
00145 #ifdef HAVE_LIBOPENSSL
00146     strcat(opts, "k:e:E");
00147 #endif
00148 
00149     /* compression switches */
00150 #ifdef HAVE_LIBZ
00151     strcat(opts, "zZ:");
00152 #endif
00153 
00154     while((ret = getopt(argc, argv, opts)) != -1)
00155     {
00156         switch(ret)
00157         {
00158         case 'v': /* verbose on */
00159             ctx->verbose++;
00160             break;
00161         case 'V': /* print name/version info and exit */
00162             u_print_version_and_exit();
00163             break;
00164 
00165 #ifdef HAVE_LIBOPENSSL
00166         case 'E': /* encryption on */
00167             ctx->encrypt = 1;
00168             break;
00169         case 'e': /* encrypt file pattern */
00170             dbg_err_if(pm_add(ctx->enc_patt, u_strdup(optarg)));
00171             break;
00172         case 'k': /* encryption key filename */
00173             ctx->key_file = u_strdup(optarg);
00174             warn_err_if(ctx->key_file == NULL);
00175             break;
00176 #endif
00177 
00178 #ifdef HAVE_LIBZ
00179         case 'Z': /* compress file pattern */
00180             ctx->compress = 1;
00181             dbg_err_if(pm_add(ctx->comp_patt, u_strdup(optarg)));
00182             break;
00183         case 'z': /* compress */
00184             ctx->compress = 1;
00185             break;
00186 #endif
00187         case 'c': /* command */
00188             if(!strcasecmp(optarg, "import"))
00189                 ctx->cmd = CMD_IMPORT;
00190             else if(!strcasecmp(optarg, "translate"))
00191                 ctx->cmd = CMD_TRANS;
00192             else
00193                 con_err("unknown command: %s", optarg);
00194             break;
00195         case 'i': /* input file */
00196             ctx->file_in = u_strdup(optarg);
00197             warn_err_if(ctx->file_in == NULL);
00198             break;
00199         case 'x': /* exclude pattern */
00200             dbg_err_if(pm_add(ctx->excl_patt, u_strdup(optarg)));
00201             break;
00202         case 'b': /* base_uri */
00203             ctx->base_uri = u_strdup(optarg);
00204             warn_err_if(ctx->base_uri == NULL);
00205 
00206             if(ctx->base_uri[0] != '/')
00207                 klone_die("base URI must be absolute "
00208                           "(i.e. must start with a '/')");
00209 
00210             remove_trailing_slash(ctx->base_uri);
00211 
00212             break;
00213         case 'o': /* output file */
00214             ctx->file_out = u_strdup(optarg);
00215             warn_err_if(ctx->file_out == NULL);
00216             break;
00217         case 'u': /* translated page uri */
00218             /* skip the first char to avoid MSYS path translation bug
00219              * (see klone-site.c) */
00220             ctx->uri = u_strdup(1+optarg);
00221             warn_err_if(ctx->uri == NULL);
00222 
00223             if(ctx->uri[0] != '/')
00224                 klone_die("URI must be absolute (i.e. must start with a '/')");
00225 
00226             remove_trailing_slash(ctx->uri);
00227 
00228             break;
00229         default:
00230         case 'h': 
00231             usage();
00232         }
00233     }
00234 
00235     klone_die_if(ctx->cmd == 0, "missing command argument (-c)");
00236     ctx->narg = argc - optind;  /* # of args left */
00237     ctx->arg = argv + optind;   
00238 
00239     return 0;
00240 err:
00241     return ~0;
00242 }
00243 
00244 static int set_key_from_file(trans_info_t *pti, const char *key_file)
00245 {
00246     io_t *io = NULL;
00247 
00248     dbg_err_if (pti == NULL);
00249     dbg_err_if (key_file == NULL);
00250     
00251     dbg_err_if(u_file_open(key_file, O_RDONLY, &io));
00252 
00253     dbg_err_if(io_read(io, pti->key, CODEC_CIPHER_KEY_SIZE) <= 0);
00254     
00255     io_free(io);
00256 
00257     return 0;
00258 err:
00259     return ~0;
00260 }
00261 
00262 static int command_trans(void)
00263 {
00264     trans_info_t ti;
00265     const mime_map_t *mm;
00266     struct stat st;
00267     char *key_env;
00268     int key_found = 0;
00269 
00270     if(ctx->narg != 0)
00271         usage();    /* no argument allowed */
00272 
00273     memset(&ti, 0, sizeof(trans_info_t));
00274 
00275     klone_die_if(!ctx->file_in, "input file name required (-i file)");
00276     klone_die_if(!ctx->file_out, "output file name required (-o file)");
00277     klone_die_if(!ctx->uri, "translated page URI required (-u uri)");
00278 
00279     if(ctx->verbose)
00280         con("translating %s to %s (uri: %s)", ctx->file_in, ctx->file_out, 
00281             ctx->uri);
00282 
00283     /* input file */
00284     strncpy(ti.file_in, ctx->file_in, U_FILENAME_MAX);
00285 
00286     /* output file */
00287     strncpy(ti.file_out, ctx->file_out, U_FILENAME_MAX);
00288 
00289     /* uri */
00290     strncpy(ti.uri, ctx->uri, URI_BUFSZ);
00291 
00292     /* zero out the key (some byte could not be overwritten with small keys) */
00293     memset(ti.key, 0, CODEC_CIPHER_KEY_SIZE);
00294 
00295     /* sanity checks */
00296     con_err_ifm(ctx->key_file && !ctx->encrypt, "-k used but -E is missing");
00297 
00298     /* encryption key */
00299     key_env = getenv("KLONE_CIPHER_KEY");
00300     if(key_env && strlen(key_env))
00301     {
00302         key_found = 1;
00303         strncpy(ti.key, key_env, CODEC_CIPHER_KEY_SIZE);
00304     }
00305 
00306     /* if -k has been used the overwrite KLONE_CIPHER_KEY env var (if present)*/
00307     if(ctx->key_file)
00308     {
00309         key_found = 1;
00310         con_err_ifm(set_key_from_file(&ti, ctx->key_file), 
00311             "error reading key file [%s]", ctx->key_file);
00312     }
00313 
00314     if(ctx->encrypt)
00315     {
00316         if(!key_found)
00317             con_err("encryption key required (use -k or KLONE_CIPHER_KEY "
00318                     "environ variable)");
00319         ti.encrypt = 1;
00320     }
00321 
00322     /* set MIME type */
00323     if((mm = u_get_mime_map(ctx->file_in)) != NULL)
00324         strncpy(ti.mime_type, mm->mime_type, MIME_BUFSZ);
00325     else
00326         strncpy(ti.mime_type, "application/octet-stream", MIME_BUFSZ);
00327 
00328     /* compress if requested and the file is compressable (by MIME type) */
00329     if(ctx->compress)
00330         ti.comp = 1;
00331 
00332     /* be sure that the input file exists */
00333     klone_die_if(stat(ctx->file_in, &st), "input file not found");
00334 
00335     ti.file_size = st.st_size;
00336     ti.mtime = st.st_mtime; 
00337 
00338     /* translate it */
00339     dbg_err_if(translate(&ti));
00340 
00341     return 0;
00342 err:
00343     /* delete output file on error */
00344     unlink(ti.file_out);
00345     con(" ");
00346     return ~0;
00347 }
00348 
00349 static int is_cpp(const char *file_in)
00350 {
00351     size_t l;
00352 
00353     dbg_err_if (file_in == NULL);
00354 
00355     l = strlen(file_in);
00356     if(l < 4)
00357         return 0;
00358 
00359     /* if the file name ends with "[Cc][Cc]" consider it a c++ file */
00360     if(tolower(file_in[--l]) == 'c' && tolower(file_in[--l]) == 'c')
00361         return 1; /* c++ */
00362 
00363 err:
00364     return 0;
00365 }
00366 
00367 static int cb_file(struct dirent *de, const char *path , void *arg)
00368 {
00369     static const char *prefix = "$(srcdir)";
00370     const mime_map_t *mm;
00371     char uri_md5[MD5_DIGEST_BUFSZ];
00372     char file_in[U_FILENAME_MAX], uri[URI_BUFSZ], *base_uri = (char*)arg;
00373     const char *ext;
00374     int enc = 0, zip = 0;
00375 
00376     dbg_err_if (de == NULL);
00377     dbg_err_if (path == NULL);
00378     dbg_err_if (arg == NULL);
00379 
00380     /* input file */
00381     if(path[0] == U_PATH_SEPARATOR)
00382     {   /* absolute path */
00383         dbg_err_if(u_snprintf(file_in, U_FILENAME_MAX, "%s/%s", path, 
00384             de->d_name));
00385     } else if(isupper(path[0]) && path[1] == ':') {
00386         /* absolute path Windows (X:/....) */
00387         dbg_err_if(u_snprintf(file_in, U_FILENAME_MAX, "%s/%s", path, 
00388             de->d_name));
00389     } else {
00390         /* relative path, use $(srcdir) */
00391         dbg_err_if(u_snprintf(file_in, U_FILENAME_MAX, "%s/%s/%s", prefix, 
00392             path, de->d_name));
00393     }
00394 
00395     /* base uri */
00396     dbg_err_if(u_snprintf(uri, URI_BUFSZ, "%s/%s", base_uri, de->d_name));
00397     dbg_err_if(u_md5(uri, strlen(uri), uri_md5));
00398 
00399     /* if the URI match the given exclude pattern then skip it */
00400     if(!pm_is_empty(ctx->excl_patt) && pm_match(ctx->excl_patt, uri))
00401     {
00402         if(ctx->verbose)
00403             con("%s skipped", uri);
00404 
00405         ctx->nexcl++;
00406 
00407         return 0; /* skip it */
00408     } 
00409 
00410     ctx->nfile++;
00411 
00412     /* if the URI match the given encrypt pattern then encrypt it */
00413     if(!pm_is_empty(ctx->enc_patt) && pm_match(ctx->enc_patt, uri))
00414         enc = 1;
00415 
00416     if(ctx->compress) /* -z or -Z have been used */
00417     {
00418         /* if the URI match the given compress pattern then compress it */
00419         if(!pm_is_empty(ctx->comp_patt))
00420         {   /* compression enabled basing on file URI pattern */
00421             if(pm_match(ctx->comp_patt, uri))
00422                 zip = 1;
00423         } else {
00424             /* compression enabled basing on URI MIME types */
00425             if((mm = u_get_mime_map(uri)) != NULL)
00426                 zip = mm->comp;
00427         }
00428     }
00429 
00430     /* print out some info */
00431     if(ctx->verbose == 1)
00432         con("%s (encrypted: %s, compressed: %s)", 
00433             uri, enc ? "yes" : "no", zip ? "yes" : "no");
00434     else if(ctx->verbose > 1)
00435         con("%s -> %s (encrypted: %s, compressed: %s)", 
00436             file_in + strlen(prefix), uri, 
00437             enc ? "yes" : "no", zip ? "yes" : "no");
00438 
00439     ext = u_match_ext(file_in, "klx") ? "cc" : "c";
00440 
00441     dbg_err_if(io_printf(ctx->iom, " \\\n" KL1_FILE_FMT, uri_md5, ext) < 0);
00442 
00443     dbg_err_if(io_printf(ctx->ior, "KLONE_REGISTER(action,%s);\n", uri_md5) <0);
00444 
00445     /* we're adding a '/' before the uri (that will be removed by klone.exe) 
00446      * to avoid MSYS (win32) automatic path translation oddity */
00447     dbg_err_if(io_printf(ctx->iod, 
00448             "\n" KL1_FILE_FMT 
00449             ": %s\n\t$(KLONE) -c translate -i $< -o $@ -u /%s %s %s %s %s\n", 
00450             uri_md5, ext, file_in, uri, 
00451             zip ? "-z" : "",
00452             enc ? "-E" : "", 
00453             enc && ctx->key_file ? "-k" : "", 
00454             enc && ctx->key_file ? ctx->key_file  : ""
00455             ) < 0);
00456 
00457     return 0;
00458 err:
00459     return ~0;
00460 }
00461 
00462 static int cb_dir(struct dirent *de, const char *path , void *arg)
00463 {
00464     char dir[U_FILENAME_MAX], base_uri[URI_BUFSZ], *cur_uri = (char*)arg;
00465 
00466     dbg_err_if (de == NULL);
00467     dbg_err_if (path == NULL);
00468     dbg_err_if (arg == NULL);
00469     
00470     ctx->ndir++;
00471 
00472     dbg_err_if(u_snprintf(dir, U_FILENAME_MAX, "%s/%s", path, de->d_name));
00473 
00474     dbg_err_if(u_snprintf(base_uri, URI_BUFSZ, "%s/%s", cur_uri, de->d_name));
00475 
00476     u_foreach_dir_item(dir, S_IFREG, cb_file, (void*)base_uri);
00477 
00478     u_foreach_dir_item(dir, S_IFDIR, cb_dir, (void*)base_uri);
00479 
00480     return 0;
00481 err:
00482     return ~0;
00483 }
00484 
00485 static int print_register_header(io_t *out)
00486 {
00487     dbg_err_if (out == NULL);
00488  
00489     dbg_err_if(io_printf(out, "#include <klone_conf.h>\n") < 0);
00490     dbg_err_if(io_printf(out, "#include <klone/hook.h>\n") < 0);
00491     dbg_err_if(io_printf(out, "static void do_register(int);\n") < 0);
00492     dbg_err_if(io_printf(out, "void unregister_pages(void);\n") < 0);
00493     dbg_err_if(io_printf(out, "void register_pages(void);\n") < 0);
00494 
00495     dbg_err_if(io_printf(out, 
00496         "void unregister_pages(void) { \n"
00497         "do_register(0); }\n"
00498         ) < 0);
00499     dbg_err_if(io_printf(out, 
00500         "void register_pages(void) { \n"
00501         "do_register(1);\n"
00502         "#ifdef ENABLE_HOOKS\n"
00503         "    hooks_setup(); \n"
00504         "#endif \n"
00505         "}\n") < 0);
00506     dbg_err_if(io_printf(out, 
00507         "static void do_register(int action) {\n") < 0);
00508     dbg_err_if(io_printf(out,
00509         "#define KLONE_REGISTER(a, md5)     \\\n"
00510         "    do {                           \\\n"
00511         "    void module_init_##md5(void);  \\\n"
00512         "    void module_term_##md5(void);  \\\n"
00513         "    if(a) module_init_##md5();     \\\n"
00514         "    else module_term_##md5();      \\\n"
00515         "    } while(0)                     \n") < 0);
00516 
00517     return 0;
00518 err:
00519     return ~0;
00520 }
00521 
00522 static int print_register_footer(io_t *out)
00523 {
00524     dbg_err_if (out == NULL);
00525 
00526     dbg_err_if(io_printf(out, "#undef KLONE_REGISTER\n") < 0);
00527     dbg_err_if(io_printf(out, "}\n") < 0);
00528 
00529     return 0;
00530 err:
00531     return ~0;
00532 }
00533 
00534 static int trans_site(char *root_dir, char *base_uri)
00535 {
00536     dbg_err_if (root_dir == NULL);
00537     dbg_err_if (base_uri == NULL);
00538     
00539     /* makefile */
00540     dbg_err_if(u_file_open("autogen.mk", O_CREAT | O_TRUNC | O_WRONLY, 
00541                 &ctx->iom));
00542     dbg_err_if(u_file_open("autogen.dps", O_CREAT | O_TRUNC | O_WRONLY, 
00543                 &ctx->iod));
00544 
00545     dbg_err_if(u_file_open("register.c", O_CREAT | O_TRUNC | O_WRONLY, 
00546                 &ctx->ior));
00547 
00548     dbg_err_if(print_register_header(ctx->ior));
00549 
00550     dbg_err_if(io_printf(ctx->iom, "embfs_rootdir=%s\n", root_dir) < 0);
00551     dbg_err_if(io_printf(ctx->iom, "autogen_src= ") < 0);
00552 
00553     /* for each file call cb_file */
00554     u_foreach_dir_item(root_dir, S_IFREG, cb_file, base_uri);
00555     /* for each directory call cb_dir */
00556     u_foreach_dir_item(root_dir, S_IFDIR, cb_dir, base_uri);
00557 
00558     dbg_err_if(print_register_footer(ctx->ior));
00559 
00560     io_free(ctx->ior);
00561     io_free(ctx->iod);
00562     io_free(ctx->iom);
00563 
00564     return 0;
00565 err:
00566     if(ctx->ior)
00567         io_free(ctx->ior);
00568     if(ctx->iod)
00569         io_free(ctx->iod);
00570     if(ctx->iom)
00571         io_free(ctx->iom);
00572     return ~0;
00573 }
00574 
00575 static int command_import(void)
00576 {
00577     char *root_dir, *base_uri;
00578 
00579     if(ctx->narg != 1)
00580         usage();    /* just on directory expected */
00581 
00582     root_dir = ctx->arg[0];
00583     dbg_err_if(root_dir == NULL);
00584 
00585     if((base_uri = ctx->base_uri) == NULL)
00586     {
00587         base_uri = u_strdup("");
00588         dbg_err_if (base_uri == NULL);
00589     }
00590 
00591     dbg_err_if(trans_site(root_dir, base_uri));
00592 
00593     con("%lu dirs and %lu files imported, %lu files skipped", 
00594             ctx->ndir, ctx->nfile, ctx->nexcl);
00595 
00596     return 0;
00597 err:
00598     con("import error");
00599     return ~0;
00600 }
00601 
00602 static int dispatch_command(void)
00603 {
00604     switch(ctx->cmd)
00605     {
00606     case CMD_TRANS:
00607         dbg_err_if(command_trans());
00608         break;
00609     case CMD_IMPORT:
00610         dbg_err_if(command_import());
00611         break;
00612     default:
00613         con_err("unknown command");
00614     }
00615 
00616     return 0;
00617 err:
00618     return ~0;
00619 }
00620 
00621 int main(int argc, char **argv)
00622 {
00623     context_t context;
00624 
00625     ctx = &context;
00626 
00627     /* zero-out the context */
00628     memset(ctx, 0, sizeof(context_t));
00629 
00630     /* init pattern matching objects */
00631     dbg_err_if(pm_create(&ctx->comp_patt));
00632     dbg_err_if(pm_create(&ctx->enc_patt));
00633     dbg_err_if(pm_create(&ctx->excl_patt));
00634 
00635     /* parse command line switches and set ctx->cmd and params */
00636     dbg_err_if(parse_opt(argc, argv));
00637 
00638     /* run the command */
00639     dbg_err_if(dispatch_command());
00640 
00641     return EXIT_SUCCESS;
00642 err:
00643     return EXIT_FAILURE;
00644 }