• Main Page
  • Related Pages
  • Modules
  • Data Structures
  • Files
  • File List
  • Globals

libavcodec/vmnc.c

Go to the documentation of this file.
00001 /*
00002  * VMware Screen Codec (VMnc) decoder
00003  * Copyright (c) 2006 Konstantin Shishkov
00004  *
00005  * This file is part of FFmpeg.
00006  *
00007  * FFmpeg is free software; you can redistribute it and/or
00008  * modify it under the terms of the GNU Lesser General Public
00009  * License as published by the Free Software Foundation; either
00010  * version 2.1 of the License, or (at your option) any later version.
00011  *
00012  * FFmpeg is distributed in the hope that it will be useful,
00013  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00014  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00015  * Lesser General Public License for more details.
00016  *
00017  * You should have received a copy of the GNU Lesser General Public
00018  * License along with FFmpeg; if not, write to the Free Software
00019  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
00020  */
00021 
00028 #include <stdio.h>
00029 #include <stdlib.h>
00030 
00031 #include "libavutil/intreadwrite.h"
00032 #include "avcodec.h"
00033 
00034 enum EncTypes {
00035     MAGIC_WMVd = 0x574D5664,
00036     MAGIC_WMVe,
00037     MAGIC_WMVf,
00038     MAGIC_WMVg,
00039     MAGIC_WMVh,
00040     MAGIC_WMVi,
00041     MAGIC_WMVj
00042 };
00043 
00044 enum HexTile_Flags {
00045     HT_RAW =  1, // tile is raw
00046     HT_BKG =  2, // background color is present
00047     HT_FG  =  4, // foreground color is present
00048     HT_SUB =  8, // subrects are present
00049     HT_CLR = 16  // each subrect has own color
00050 };
00051 
00052 /*
00053  * Decoder context
00054  */
00055 typedef struct VmncContext {
00056     AVCodecContext *avctx;
00057     AVFrame pic;
00058 
00059     int bpp;
00060     int bpp2;
00061     int bigendian;
00062     uint8_t pal[768];
00063     int width, height;
00064 
00065     /* cursor data */
00066     int cur_w, cur_h;
00067     int cur_x, cur_y;
00068     int cur_hx, cur_hy;
00069     uint8_t* curbits, *curmask;
00070     uint8_t* screendta;
00071 } VmncContext;
00072 
00073 /* read pixel value from stream */
00074 static av_always_inline int vmnc_get_pixel(const uint8_t* buf, int bpp, int be) {
00075     switch(bpp * 2 + be) {
00076     case 2:
00077     case 3: return *buf;
00078     case 4: return AV_RL16(buf);
00079     case 5: return AV_RB16(buf);
00080     case 8: return AV_RL32(buf);
00081     case 9: return AV_RB32(buf);
00082     default: return 0;
00083     }
00084 }
00085 
00086 static void load_cursor(VmncContext *c, const uint8_t *src)
00087 {
00088     int i, j, p;
00089     const int bpp = c->bpp2;
00090     uint8_t  *dst8  = c->curbits;
00091     uint16_t *dst16 = (uint16_t*)c->curbits;
00092     uint32_t *dst32 = (uint32_t*)c->curbits;
00093 
00094     for(j = 0; j < c->cur_h; j++) {
00095         for(i = 0; i < c->cur_w; i++) {
00096             p = vmnc_get_pixel(src, bpp, c->bigendian);
00097             src += bpp;
00098             if(bpp == 1) *dst8++ = p;
00099             if(bpp == 2) *dst16++ = p;
00100             if(bpp == 4) *dst32++ = p;
00101         }
00102     }
00103     dst8 = c->curmask;
00104     dst16 = (uint16_t*)c->curmask;
00105     dst32 = (uint32_t*)c->curmask;
00106     for(j = 0; j < c->cur_h; j++) {
00107         for(i = 0; i < c->cur_w; i++) {
00108             p = vmnc_get_pixel(src, bpp, c->bigendian);
00109             src += bpp;
00110             if(bpp == 1) *dst8++ = p;
00111             if(bpp == 2) *dst16++ = p;
00112             if(bpp == 4) *dst32++ = p;
00113         }
00114     }
00115 }
00116 
00117 static void put_cursor(uint8_t *dst, int stride, VmncContext *c, int dx, int dy)
00118 {
00119     int i, j;
00120     int w, h, x, y;
00121     w = c->cur_w;
00122     if(c->width < c->cur_x + c->cur_w) w = c->width - c->cur_x;
00123     h = c->cur_h;
00124     if(c->height < c->cur_y + c->cur_h) h = c->height - c->cur_y;
00125     x = c->cur_x;
00126     y = c->cur_y;
00127     if(x < 0) {
00128         w += x;
00129         x = 0;
00130     }
00131     if(y < 0) {
00132         h += y;
00133         y = 0;
00134     }
00135 
00136     if((w < 1) || (h < 1)) return;
00137     dst += x * c->bpp2 + y * stride;
00138 
00139     if(c->bpp2 == 1) {
00140         uint8_t* cd = c->curbits, *msk = c->curmask;
00141         for(j = 0; j < h; j++) {
00142             for(i = 0; i < w; i++)
00143                 dst[i] = (dst[i] & cd[i]) ^ msk[i];
00144             msk += c->cur_w;
00145             cd += c->cur_w;
00146             dst += stride;
00147         }
00148     } else if(c->bpp2 == 2) {
00149         uint16_t* cd = (uint16_t*)c->curbits, *msk = (uint16_t*)c->curmask;
00150         uint16_t* dst2;
00151         for(j = 0; j < h; j++) {
00152             dst2 = (uint16_t*)dst;
00153             for(i = 0; i < w; i++)
00154                 dst2[i] = (dst2[i] & cd[i]) ^ msk[i];
00155             msk += c->cur_w;
00156             cd += c->cur_w;
00157             dst += stride;
00158         }
00159     } else if(c->bpp2 == 4) {
00160         uint32_t* cd = (uint32_t*)c->curbits, *msk = (uint32_t*)c->curmask;
00161         uint32_t* dst2;
00162         for(j = 0; j < h; j++) {
00163             dst2 = (uint32_t*)dst;
00164             for(i = 0; i < w; i++)
00165                 dst2[i] = (dst2[i] & cd[i]) ^ msk[i];
00166             msk += c->cur_w;
00167             cd += c->cur_w;
00168             dst += stride;
00169         }
00170     }
00171 }
00172 
00173 /* fill rectangle with given color */
00174 static av_always_inline void paint_rect(uint8_t *dst, int dx, int dy, int w, int h, int color, int bpp, int stride)
00175 {
00176     int i, j;
00177     dst += dx * bpp + dy * stride;
00178     if(bpp == 1){
00179         for(j = 0; j < h; j++) {
00180             memset(dst, color, w);
00181             dst += stride;
00182         }
00183     }else if(bpp == 2){
00184         uint16_t* dst2;
00185         for(j = 0; j < h; j++) {
00186             dst2 = (uint16_t*)dst;
00187             for(i = 0; i < w; i++) {
00188                 *dst2++ = color;
00189             }
00190             dst += stride;
00191         }
00192     }else if(bpp == 4){
00193         uint32_t* dst2;
00194         for(j = 0; j < h; j++) {
00195             dst2 = (uint32_t*)dst;
00196             for(i = 0; i < w; i++) {
00197                 dst2[i] = color;
00198             }
00199             dst += stride;
00200         }
00201     }
00202 }
00203 
00204 static av_always_inline void paint_raw(uint8_t *dst, int w, int h, const uint8_t* src, int bpp, int be, int stride)
00205 {
00206     int i, j, p;
00207     for(j = 0; j < h; j++) {
00208         for(i = 0; i < w; i++) {
00209             p = vmnc_get_pixel(src, bpp, be);
00210             src += bpp;
00211             switch(bpp){
00212             case 1:
00213                 dst[i] = p;
00214                 break;
00215             case 2:
00216                 ((uint16_t*)dst)[i] = p;
00217                 break;
00218             case 4:
00219                 ((uint32_t*)dst)[i] = p;
00220                 break;
00221             }
00222         }
00223         dst += stride;
00224     }
00225 }
00226 
00227 static int decode_hextile(VmncContext *c, uint8_t* dst, const uint8_t* src, int ssize, int w, int h, int stride)
00228 {
00229     int i, j, k;
00230     int bg = 0, fg = 0, rects, color, flags, xy, wh;
00231     const int bpp = c->bpp2;
00232     uint8_t *dst2;
00233     int bw = 16, bh = 16;
00234     const uint8_t *ssrc=src;
00235 
00236     for(j = 0; j < h; j += 16) {
00237         dst2 = dst;
00238         bw = 16;
00239         if(j + 16 > h) bh = h - j;
00240         for(i = 0; i < w; i += 16, dst2 += 16 * bpp) {
00241             if(src - ssrc >= ssize) {
00242                 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
00243                 return -1;
00244             }
00245             if(i + 16 > w) bw = w - i;
00246             flags = *src++;
00247             if(flags & HT_RAW) {
00248                 if(src - ssrc > ssize - bw * bh * bpp) {
00249                     av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
00250                     return -1;
00251                 }
00252                 paint_raw(dst2, bw, bh, src, bpp, c->bigendian, stride);
00253                 src += bw * bh * bpp;
00254             } else {
00255                 if(flags & HT_BKG) {
00256                     bg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
00257                 }
00258                 if(flags & HT_FG) {
00259                     fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
00260                 }
00261                 rects = 0;
00262                 if(flags & HT_SUB)
00263                     rects = *src++;
00264                 color = !!(flags & HT_CLR);
00265 
00266                 paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride);
00267 
00268                 if(src - ssrc > ssize - rects * (color * bpp + 2)) {
00269                     av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
00270                     return -1;
00271                 }
00272                 for(k = 0; k < rects; k++) {
00273                     if(color) {
00274                         fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
00275                     }
00276                     xy = *src++;
00277                     wh = *src++;
00278                     paint_rect(dst2, xy >> 4, xy & 0xF, (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride);
00279                 }
00280             }
00281         }
00282         dst += stride * 16;
00283     }
00284     return src - ssrc;
00285 }
00286 
00287 static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPacket *avpkt)
00288 {
00289     const uint8_t *buf = avpkt->data;
00290     int buf_size = avpkt->size;
00291     VmncContext * const c = avctx->priv_data;
00292     uint8_t *outptr;
00293     const uint8_t *src = buf;
00294     int dx, dy, w, h, depth, enc, chunks, res, size_left;
00295 
00296     c->pic.reference = 1;
00297     c->pic.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE;
00298     if(avctx->reget_buffer(avctx, &c->pic) < 0){
00299         av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
00300         return -1;
00301     }
00302 
00303     c->pic.key_frame = 0;
00304     c->pic.pict_type = FF_P_TYPE;
00305 
00306     //restore screen after cursor
00307     if(c->screendta) {
00308         int i;
00309         w = c->cur_w;
00310         if(c->width < c->cur_x + w) w = c->width - c->cur_x;
00311         h = c->cur_h;
00312         if(c->height < c->cur_y + h) h = c->height - c->cur_y;
00313         dx = c->cur_x;
00314         if(dx < 0) {
00315             w += dx;
00316             dx = 0;
00317         }
00318         dy = c->cur_y;
00319         if(dy < 0) {
00320             h += dy;
00321             dy = 0;
00322         }
00323         if((w > 0) && (h > 0)) {
00324             outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
00325             for(i = 0; i < h; i++) {
00326                 memcpy(outptr, c->screendta + i * c->cur_w * c->bpp2, w * c->bpp2);
00327                 outptr += c->pic.linesize[0];
00328             }
00329         }
00330     }
00331     src += 2;
00332     chunks = AV_RB16(src); src += 2;
00333     while(chunks--) {
00334         dx = AV_RB16(src); src += 2;
00335         dy = AV_RB16(src); src += 2;
00336         w  = AV_RB16(src); src += 2;
00337         h  = AV_RB16(src); src += 2;
00338         enc = AV_RB32(src); src += 4;
00339         outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
00340         size_left = buf_size - (src - buf);
00341         switch(enc) {
00342         case MAGIC_WMVd: // cursor
00343             if(size_left < 2 + w * h * c->bpp2 * 2) {
00344                 av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", 2 + w * h * c->bpp2 * 2, size_left);
00345                 return -1;
00346             }
00347             src += 2;
00348             c->cur_w = w;
00349             c->cur_h = h;
00350             c->cur_hx = dx;
00351             c->cur_hy = dy;
00352             if((c->cur_hx > c->cur_w) || (c->cur_hy > c->cur_h)) {
00353                 av_log(avctx, AV_LOG_ERROR, "Cursor hot spot is not in image: %ix%i of %ix%i cursor size\n", c->cur_hx, c->cur_hy, c->cur_w, c->cur_h);
00354                 c->cur_hx = c->cur_hy = 0;
00355             }
00356             c->curbits = av_realloc(c->curbits, c->cur_w * c->cur_h * c->bpp2);
00357             c->curmask = av_realloc(c->curmask, c->cur_w * c->cur_h * c->bpp2);
00358             c->screendta = av_realloc(c->screendta, c->cur_w * c->cur_h * c->bpp2);
00359             load_cursor(c, src);
00360             src += w * h * c->bpp2 * 2;
00361             break;
00362         case MAGIC_WMVe: // unknown
00363             src += 2;
00364             break;
00365         case MAGIC_WMVf: // update cursor position
00366             c->cur_x = dx - c->cur_hx;
00367             c->cur_y = dy - c->cur_hy;
00368             break;
00369         case MAGIC_WMVg: // unknown
00370             src += 10;
00371             break;
00372         case MAGIC_WMVh: // unknown
00373             src += 4;
00374             break;
00375         case MAGIC_WMVi: // ServerInitialization struct
00376             c->pic.key_frame = 1;
00377             c->pic.pict_type = FF_I_TYPE;
00378             depth = *src++;
00379             if(depth != c->bpp) {
00380                 av_log(avctx, AV_LOG_INFO, "Depth mismatch. Container %i bpp, Frame data: %i bpp\n", c->bpp, depth);
00381             }
00382             src++;
00383             c->bigendian = *src++;
00384             if(c->bigendian & (~1)) {
00385                 av_log(avctx, AV_LOG_INFO, "Invalid header: bigendian flag = %i\n", c->bigendian);
00386                 return -1;
00387             }
00388             //skip the rest of pixel format data
00389             src += 13;
00390             break;
00391         case MAGIC_WMVj: // unknown
00392             src += 2;
00393             break;
00394         case 0x00000000: // raw rectangle data
00395             if((dx + w > c->width) || (dy + h > c->height)) {
00396                 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height);
00397                 return -1;
00398             }
00399             if(size_left < w * h * c->bpp2) {
00400                 av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", w * h * c->bpp2, size_left);
00401                 return -1;
00402             }
00403             paint_raw(outptr, w, h, src, c->bpp2, c->bigendian, c->pic.linesize[0]);
00404             src += w * h * c->bpp2;
00405             break;
00406         case 0x00000005: // HexTile encoded rectangle
00407             if((dx + w > c->width) || (dy + h > c->height)) {
00408                 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height);
00409                 return -1;
00410             }
00411             res = decode_hextile(c, outptr, src, size_left, w, h, c->pic.linesize[0]);
00412             if(res < 0)
00413                 return -1;
00414             src += res;
00415             break;
00416         default:
00417             av_log(avctx, AV_LOG_ERROR, "Unsupported block type 0x%08X\n", enc);
00418             chunks = 0; // leave chunks decoding loop
00419         }
00420     }
00421     if(c->screendta){
00422         int i;
00423         //save screen data before painting cursor
00424         w = c->cur_w;
00425         if(c->width < c->cur_x + w) w = c->width - c->cur_x;
00426         h = c->cur_h;
00427         if(c->height < c->cur_y + h) h = c->height - c->cur_y;
00428         dx = c->cur_x;
00429         if(dx < 0) {
00430             w += dx;
00431             dx = 0;
00432         }
00433         dy = c->cur_y;
00434         if(dy < 0) {
00435             h += dy;
00436             dy = 0;
00437         }
00438         if((w > 0) && (h > 0)) {
00439             outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
00440             for(i = 0; i < h; i++) {
00441                 memcpy(c->screendta + i * c->cur_w * c->bpp2, outptr, w * c->bpp2);
00442                 outptr += c->pic.linesize[0];
00443             }
00444             outptr = c->pic.data[0];
00445             put_cursor(outptr, c->pic.linesize[0], c, c->cur_x, c->cur_y);
00446         }
00447     }
00448     *data_size = sizeof(AVFrame);
00449     *(AVFrame*)data = c->pic;
00450 
00451     /* always report that the buffer was completely consumed */
00452     return buf_size;
00453 }
00454 
00455 
00456 
00457 /*
00458  *
00459  * Init VMnc decoder
00460  *
00461  */
00462 static av_cold int decode_init(AVCodecContext *avctx)
00463 {
00464     VmncContext * const c = avctx->priv_data;
00465 
00466     c->avctx = avctx;
00467 
00468     c->width = avctx->width;
00469     c->height = avctx->height;
00470 
00471     c->bpp = avctx->bits_per_coded_sample;
00472     c->bpp2 = c->bpp/8;
00473 
00474     switch(c->bpp){
00475     case 8:
00476         avctx->pix_fmt = PIX_FMT_PAL8;
00477         break;
00478     case 16:
00479         avctx->pix_fmt = PIX_FMT_RGB555;
00480         break;
00481     case 32:
00482         avctx->pix_fmt = PIX_FMT_RGB32;
00483         break;
00484     default:
00485         av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp);
00486     }
00487 
00488     return 0;
00489 }
00490 
00491 
00492 
00493 /*
00494  *
00495  * Uninit VMnc decoder
00496  *
00497  */
00498 static av_cold int decode_end(AVCodecContext *avctx)
00499 {
00500     VmncContext * const c = avctx->priv_data;
00501 
00502     if (c->pic.data[0])
00503         avctx->release_buffer(avctx, &c->pic);
00504 
00505     av_free(c->curbits);
00506     av_free(c->curmask);
00507     av_free(c->screendta);
00508     return 0;
00509 }
00510 
00511 AVCodec vmnc_decoder = {
00512     "vmnc",
00513     AVMEDIA_TYPE_VIDEO,
00514     CODEC_ID_VMNC,
00515     sizeof(VmncContext),
00516     decode_init,
00517     NULL,
00518     decode_end,
00519     decode_frame,
00520     CODEC_CAP_DR1,
00521     .long_name = NULL_IF_CONFIG_SMALL("VMware Screen Codec / VMware Video"),
00522 };
00523 

Generated on Fri Sep 16 2011 17:17:45 for FFmpeg by  doxygen 1.7.1