Libav
|
00001 /* 00002 * VMware Screen Codec (VMnc) decoder 00003 * Copyright (c) 2006 Konstantin Shishkov 00004 * 00005 * This file is part of FFmpeg. 00006 * 00007 * FFmpeg is free software; you can redistribute it and/or 00008 * modify it under the terms of the GNU Lesser General Public 00009 * License as published by the Free Software Foundation; either 00010 * version 2.1 of the License, or (at your option) any later version. 00011 * 00012 * FFmpeg is distributed in the hope that it will be useful, 00013 * but WITHOUT ANY WARRANTY; without even the implied warranty of 00014 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 00015 * Lesser General Public License for more details. 00016 * 00017 * You should have received a copy of the GNU Lesser General Public 00018 * License along with FFmpeg; if not, write to the Free Software 00019 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 00020 */ 00021 00028 #include <stdio.h> 00029 #include <stdlib.h> 00030 00031 #include "libavutil/intreadwrite.h" 00032 #include "avcodec.h" 00033 00034 enum EncTypes { 00035 MAGIC_WMVd = 0x574D5664, 00036 MAGIC_WMVe, 00037 MAGIC_WMVf, 00038 MAGIC_WMVg, 00039 MAGIC_WMVh, 00040 MAGIC_WMVi, 00041 MAGIC_WMVj 00042 }; 00043 00044 enum HexTile_Flags { 00045 HT_RAW = 1, // tile is raw 00046 HT_BKG = 2, // background color is present 00047 HT_FG = 4, // foreground color is present 00048 HT_SUB = 8, // subrects are present 00049 HT_CLR = 16 // each subrect has own color 00050 }; 00051 00052 /* 00053 * Decoder context 00054 */ 00055 typedef struct VmncContext { 00056 AVCodecContext *avctx; 00057 AVFrame pic; 00058 00059 int bpp; 00060 int bpp2; 00061 int bigendian; 00062 uint8_t pal[768]; 00063 int width, height; 00064 00065 /* cursor data */ 00066 int cur_w, cur_h; 00067 int cur_x, cur_y; 00068 int cur_hx, cur_hy; 00069 uint8_t* curbits, *curmask; 00070 uint8_t* screendta; 00071 } VmncContext; 00072 00073 /* read pixel value from stream */ 00074 static av_always_inline int vmnc_get_pixel(const uint8_t* buf, int bpp, int be) { 00075 switch(bpp * 2 + be) { 00076 case 2: 00077 case 3: return *buf; 00078 case 4: return AV_RL16(buf); 00079 case 5: return AV_RB16(buf); 00080 case 8: return AV_RL32(buf); 00081 case 9: return AV_RB32(buf); 00082 default: return 0; 00083 } 00084 } 00085 00086 static void load_cursor(VmncContext *c, const uint8_t *src) 00087 { 00088 int i, j, p; 00089 const int bpp = c->bpp2; 00090 uint8_t *dst8 = c->curbits; 00091 uint16_t *dst16 = (uint16_t*)c->curbits; 00092 uint32_t *dst32 = (uint32_t*)c->curbits; 00093 00094 for(j = 0; j < c->cur_h; j++) { 00095 for(i = 0; i < c->cur_w; i++) { 00096 p = vmnc_get_pixel(src, bpp, c->bigendian); 00097 src += bpp; 00098 if(bpp == 1) *dst8++ = p; 00099 if(bpp == 2) *dst16++ = p; 00100 if(bpp == 4) *dst32++ = p; 00101 } 00102 } 00103 dst8 = c->curmask; 00104 dst16 = (uint16_t*)c->curmask; 00105 dst32 = (uint32_t*)c->curmask; 00106 for(j = 0; j < c->cur_h; j++) { 00107 for(i = 0; i < c->cur_w; i++) { 00108 p = vmnc_get_pixel(src, bpp, c->bigendian); 00109 src += bpp; 00110 if(bpp == 1) *dst8++ = p; 00111 if(bpp == 2) *dst16++ = p; 00112 if(bpp == 4) *dst32++ = p; 00113 } 00114 } 00115 } 00116 00117 static void put_cursor(uint8_t *dst, int stride, VmncContext *c, int dx, int dy) 00118 { 00119 int i, j; 00120 int w, h, x, y; 00121 w = c->cur_w; 00122 if(c->width < c->cur_x + c->cur_w) w = c->width - c->cur_x; 00123 h = c->cur_h; 00124 if(c->height < c->cur_y + c->cur_h) h = c->height - c->cur_y; 00125 x = c->cur_x; 00126 y = c->cur_y; 00127 if(x < 0) { 00128 w += x; 00129 x = 0; 00130 } 00131 if(y < 0) { 00132 h += y; 00133 y = 0; 00134 } 00135 00136 if((w < 1) || (h < 1)) return; 00137 dst += x * c->bpp2 + y * stride; 00138 00139 if(c->bpp2 == 1) { 00140 uint8_t* cd = c->curbits, *msk = c->curmask; 00141 for(j = 0; j < h; j++) { 00142 for(i = 0; i < w; i++) 00143 dst[i] = (dst[i] & cd[i]) ^ msk[i]; 00144 msk += c->cur_w; 00145 cd += c->cur_w; 00146 dst += stride; 00147 } 00148 } else if(c->bpp2 == 2) { 00149 uint16_t* cd = (uint16_t*)c->curbits, *msk = (uint16_t*)c->curmask; 00150 uint16_t* dst2; 00151 for(j = 0; j < h; j++) { 00152 dst2 = (uint16_t*)dst; 00153 for(i = 0; i < w; i++) 00154 dst2[i] = (dst2[i] & cd[i]) ^ msk[i]; 00155 msk += c->cur_w; 00156 cd += c->cur_w; 00157 dst += stride; 00158 } 00159 } else if(c->bpp2 == 4) { 00160 uint32_t* cd = (uint32_t*)c->curbits, *msk = (uint32_t*)c->curmask; 00161 uint32_t* dst2; 00162 for(j = 0; j < h; j++) { 00163 dst2 = (uint32_t*)dst; 00164 for(i = 0; i < w; i++) 00165 dst2[i] = (dst2[i] & cd[i]) ^ msk[i]; 00166 msk += c->cur_w; 00167 cd += c->cur_w; 00168 dst += stride; 00169 } 00170 } 00171 } 00172 00173 /* fill rectangle with given color */ 00174 static av_always_inline void paint_rect(uint8_t *dst, int dx, int dy, int w, int h, int color, int bpp, int stride) 00175 { 00176 int i, j; 00177 dst += dx * bpp + dy * stride; 00178 if(bpp == 1){ 00179 for(j = 0; j < h; j++) { 00180 memset(dst, color, w); 00181 dst += stride; 00182 } 00183 }else if(bpp == 2){ 00184 uint16_t* dst2; 00185 for(j = 0; j < h; j++) { 00186 dst2 = (uint16_t*)dst; 00187 for(i = 0; i < w; i++) { 00188 *dst2++ = color; 00189 } 00190 dst += stride; 00191 } 00192 }else if(bpp == 4){ 00193 uint32_t* dst2; 00194 for(j = 0; j < h; j++) { 00195 dst2 = (uint32_t*)dst; 00196 for(i = 0; i < w; i++) { 00197 dst2[i] = color; 00198 } 00199 dst += stride; 00200 } 00201 } 00202 } 00203 00204 static av_always_inline void paint_raw(uint8_t *dst, int w, int h, const uint8_t* src, int bpp, int be, int stride) 00205 { 00206 int i, j, p; 00207 for(j = 0; j < h; j++) { 00208 for(i = 0; i < w; i++) { 00209 p = vmnc_get_pixel(src, bpp, be); 00210 src += bpp; 00211 switch(bpp){ 00212 case 1: 00213 dst[i] = p; 00214 break; 00215 case 2: 00216 ((uint16_t*)dst)[i] = p; 00217 break; 00218 case 4: 00219 ((uint32_t*)dst)[i] = p; 00220 break; 00221 } 00222 } 00223 dst += stride; 00224 } 00225 } 00226 00227 static int decode_hextile(VmncContext *c, uint8_t* dst, const uint8_t* src, int ssize, int w, int h, int stride) 00228 { 00229 int i, j, k; 00230 int bg = 0, fg = 0, rects, color, flags, xy, wh; 00231 const int bpp = c->bpp2; 00232 uint8_t *dst2; 00233 int bw = 16, bh = 16; 00234 const uint8_t *ssrc=src; 00235 00236 for(j = 0; j < h; j += 16) { 00237 dst2 = dst; 00238 bw = 16; 00239 if(j + 16 > h) bh = h - j; 00240 for(i = 0; i < w; i += 16, dst2 += 16 * bpp) { 00241 if(src - ssrc >= ssize) { 00242 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); 00243 return -1; 00244 } 00245 if(i + 16 > w) bw = w - i; 00246 flags = *src++; 00247 if(flags & HT_RAW) { 00248 if(src - ssrc > ssize - bw * bh * bpp) { 00249 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); 00250 return -1; 00251 } 00252 paint_raw(dst2, bw, bh, src, bpp, c->bigendian, stride); 00253 src += bw * bh * bpp; 00254 } else { 00255 if(flags & HT_BKG) { 00256 bg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp; 00257 } 00258 if(flags & HT_FG) { 00259 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp; 00260 } 00261 rects = 0; 00262 if(flags & HT_SUB) 00263 rects = *src++; 00264 color = !!(flags & HT_CLR); 00265 00266 paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride); 00267 00268 if(src - ssrc > ssize - rects * (color * bpp + 2)) { 00269 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); 00270 return -1; 00271 } 00272 for(k = 0; k < rects; k++) { 00273 if(color) { 00274 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp; 00275 } 00276 xy = *src++; 00277 wh = *src++; 00278 paint_rect(dst2, xy >> 4, xy & 0xF, (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride); 00279 } 00280 } 00281 } 00282 dst += stride * 16; 00283 } 00284 return src - ssrc; 00285 } 00286 00287 static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPacket *avpkt) 00288 { 00289 const uint8_t *buf = avpkt->data; 00290 int buf_size = avpkt->size; 00291 VmncContext * const c = avctx->priv_data; 00292 uint8_t *outptr; 00293 const uint8_t *src = buf; 00294 int dx, dy, w, h, depth, enc, chunks, res, size_left; 00295 00296 c->pic.reference = 1; 00297 c->pic.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE; 00298 if(avctx->reget_buffer(avctx, &c->pic) < 0){ 00299 av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n"); 00300 return -1; 00301 } 00302 00303 c->pic.key_frame = 0; 00304 c->pic.pict_type = FF_P_TYPE; 00305 00306 //restore screen after cursor 00307 if(c->screendta) { 00308 int i; 00309 w = c->cur_w; 00310 if(c->width < c->cur_x + w) w = c->width - c->cur_x; 00311 h = c->cur_h; 00312 if(c->height < c->cur_y + h) h = c->height - c->cur_y; 00313 dx = c->cur_x; 00314 if(dx < 0) { 00315 w += dx; 00316 dx = 0; 00317 } 00318 dy = c->cur_y; 00319 if(dy < 0) { 00320 h += dy; 00321 dy = 0; 00322 } 00323 if((w > 0) && (h > 0)) { 00324 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0]; 00325 for(i = 0; i < h; i++) { 00326 memcpy(outptr, c->screendta + i * c->cur_w * c->bpp2, w * c->bpp2); 00327 outptr += c->pic.linesize[0]; 00328 } 00329 } 00330 } 00331 src += 2; 00332 chunks = AV_RB16(src); src += 2; 00333 while(chunks--) { 00334 dx = AV_RB16(src); src += 2; 00335 dy = AV_RB16(src); src += 2; 00336 w = AV_RB16(src); src += 2; 00337 h = AV_RB16(src); src += 2; 00338 enc = AV_RB32(src); src += 4; 00339 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0]; 00340 size_left = buf_size - (src - buf); 00341 switch(enc) { 00342 case MAGIC_WMVd: // cursor 00343 if(size_left < 2 + w * h * c->bpp2 * 2) { 00344 av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", 2 + w * h * c->bpp2 * 2, size_left); 00345 return -1; 00346 } 00347 src += 2; 00348 c->cur_w = w; 00349 c->cur_h = h; 00350 c->cur_hx = dx; 00351 c->cur_hy = dy; 00352 if((c->cur_hx > c->cur_w) || (c->cur_hy > c->cur_h)) { 00353 av_log(avctx, AV_LOG_ERROR, "Cursor hot spot is not in image: %ix%i of %ix%i cursor size\n", c->cur_hx, c->cur_hy, c->cur_w, c->cur_h); 00354 c->cur_hx = c->cur_hy = 0; 00355 } 00356 c->curbits = av_realloc(c->curbits, c->cur_w * c->cur_h * c->bpp2); 00357 c->curmask = av_realloc(c->curmask, c->cur_w * c->cur_h * c->bpp2); 00358 c->screendta = av_realloc(c->screendta, c->cur_w * c->cur_h * c->bpp2); 00359 load_cursor(c, src); 00360 src += w * h * c->bpp2 * 2; 00361 break; 00362 case MAGIC_WMVe: // unknown 00363 src += 2; 00364 break; 00365 case MAGIC_WMVf: // update cursor position 00366 c->cur_x = dx - c->cur_hx; 00367 c->cur_y = dy - c->cur_hy; 00368 break; 00369 case MAGIC_WMVg: // unknown 00370 src += 10; 00371 break; 00372 case MAGIC_WMVh: // unknown 00373 src += 4; 00374 break; 00375 case MAGIC_WMVi: // ServerInitialization struct 00376 c->pic.key_frame = 1; 00377 c->pic.pict_type = FF_I_TYPE; 00378 depth = *src++; 00379 if(depth != c->bpp) { 00380 av_log(avctx, AV_LOG_INFO, "Depth mismatch. Container %i bpp, Frame data: %i bpp\n", c->bpp, depth); 00381 } 00382 src++; 00383 c->bigendian = *src++; 00384 if(c->bigendian & (~1)) { 00385 av_log(avctx, AV_LOG_INFO, "Invalid header: bigendian flag = %i\n", c->bigendian); 00386 return -1; 00387 } 00388 //skip the rest of pixel format data 00389 src += 13; 00390 break; 00391 case MAGIC_WMVj: // unknown 00392 src += 2; 00393 break; 00394 case 0x00000000: // raw rectangle data 00395 if((dx + w > c->width) || (dy + h > c->height)) { 00396 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height); 00397 return -1; 00398 } 00399 if(size_left < w * h * c->bpp2) { 00400 av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", w * h * c->bpp2, size_left); 00401 return -1; 00402 } 00403 paint_raw(outptr, w, h, src, c->bpp2, c->bigendian, c->pic.linesize[0]); 00404 src += w * h * c->bpp2; 00405 break; 00406 case 0x00000005: // HexTile encoded rectangle 00407 if((dx + w > c->width) || (dy + h > c->height)) { 00408 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height); 00409 return -1; 00410 } 00411 res = decode_hextile(c, outptr, src, size_left, w, h, c->pic.linesize[0]); 00412 if(res < 0) 00413 return -1; 00414 src += res; 00415 break; 00416 default: 00417 av_log(avctx, AV_LOG_ERROR, "Unsupported block type 0x%08X\n", enc); 00418 chunks = 0; // leave chunks decoding loop 00419 } 00420 } 00421 if(c->screendta){ 00422 int i; 00423 //save screen data before painting cursor 00424 w = c->cur_w; 00425 if(c->width < c->cur_x + w) w = c->width - c->cur_x; 00426 h = c->cur_h; 00427 if(c->height < c->cur_y + h) h = c->height - c->cur_y; 00428 dx = c->cur_x; 00429 if(dx < 0) { 00430 w += dx; 00431 dx = 0; 00432 } 00433 dy = c->cur_y; 00434 if(dy < 0) { 00435 h += dy; 00436 dy = 0; 00437 } 00438 if((w > 0) && (h > 0)) { 00439 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0]; 00440 for(i = 0; i < h; i++) { 00441 memcpy(c->screendta + i * c->cur_w * c->bpp2, outptr, w * c->bpp2); 00442 outptr += c->pic.linesize[0]; 00443 } 00444 outptr = c->pic.data[0]; 00445 put_cursor(outptr, c->pic.linesize[0], c, c->cur_x, c->cur_y); 00446 } 00447 } 00448 *data_size = sizeof(AVFrame); 00449 *(AVFrame*)data = c->pic; 00450 00451 /* always report that the buffer was completely consumed */ 00452 return buf_size; 00453 } 00454 00455 00456 00457 /* 00458 * 00459 * Init VMnc decoder 00460 * 00461 */ 00462 static av_cold int decode_init(AVCodecContext *avctx) 00463 { 00464 VmncContext * const c = avctx->priv_data; 00465 00466 c->avctx = avctx; 00467 00468 c->width = avctx->width; 00469 c->height = avctx->height; 00470 00471 c->bpp = avctx->bits_per_coded_sample; 00472 c->bpp2 = c->bpp/8; 00473 00474 switch(c->bpp){ 00475 case 8: 00476 avctx->pix_fmt = PIX_FMT_PAL8; 00477 break; 00478 case 16: 00479 avctx->pix_fmt = PIX_FMT_RGB555; 00480 break; 00481 case 32: 00482 avctx->pix_fmt = PIX_FMT_RGB32; 00483 break; 00484 default: 00485 av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp); 00486 } 00487 00488 return 0; 00489 } 00490 00491 00492 00493 /* 00494 * 00495 * Uninit VMnc decoder 00496 * 00497 */ 00498 static av_cold int decode_end(AVCodecContext *avctx) 00499 { 00500 VmncContext * const c = avctx->priv_data; 00501 00502 if (c->pic.data[0]) 00503 avctx->release_buffer(avctx, &c->pic); 00504 00505 av_free(c->curbits); 00506 av_free(c->curmask); 00507 av_free(c->screendta); 00508 return 0; 00509 } 00510 00511 AVCodec vmnc_decoder = { 00512 "vmnc", 00513 AVMEDIA_TYPE_VIDEO, 00514 CODEC_ID_VMNC, 00515 sizeof(VmncContext), 00516 decode_init, 00517 NULL, 00518 decode_end, 00519 decode_frame, 00520 CODEC_CAP_DR1, 00521 .long_name = NULL_IF_CONFIG_SMALL("VMware Screen Codec / VMware Video"), 00522 }; 00523