OpenVDB  0.104.0
Compression.h
Go to the documentation of this file.
1 
2 //
3 // Copyright (c) 2012 DreamWorks Animation LLC
4 //
5 // All rights reserved. This software is distributed under the
6 // Mozilla Public License 2.0 ( http://www.mozilla.org/MPL/2.0/ )
7 //
8 // Redistributions of source code must retain the above copyright
9 // and license notice and the following restrictions and disclaimer.
10 //
11 // * Neither the name of DreamWorks Animation nor the names of
12 // its contributors may be used to endorse or promote products derived
13 // from this software without specific prior written permission.
14 //
15 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY INDIRECT, INCIDENTAL,
20 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 // IN NO EVENT SHALL THE COPYRIGHT HOLDERS' AND CONTRIBUTORS' AGGREGATE
27 // LIABILITY FOR ALL CLAIMS REGARDLESS OF THEIR BASIS EXCEED US$250.00.
28 //
30 
31 #ifndef OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
32 #define OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
33 
34 #include <openvdb/Types.h>
35 #include <openvdb/math/Math.h> // for negative()
36 #include <boost/scoped_array.hpp>
37 #include <algorithm>
38 #include <iostream>
39 #include <string>
40 #include <vector>
41 
42 
43 namespace openvdb {
45 namespace OPENVDB_VERSION_NAME {
46 namespace io {
47 
68 enum {
70  COMPRESS_ZIP = 0x1,
72 };
73 
75 OPENVDB_API std::string compressionToString(uint32_t flags);
76 
77 
79 
80 
83 enum {
84  /*0*/ NO_MASK_OR_INACTIVE_VALS, // no inactive vals, or all inactive vals are +background
85  /*1*/ NO_MASK_AND_MINUS_BG, // all inactive vals are -background
86  /*2*/ NO_MASK_AND_ONE_INACTIVE_VAL, // all inactive vals have the same non-background val
87  /*3*/ MASK_AND_NO_INACTIVE_VALS, // mask selects between -background and +background
88  /*4*/ MASK_AND_ONE_INACTIVE_VAL, // mask selects between backgd and one other inactive val
89  /*5*/ MASK_AND_TWO_INACTIVE_VALS // mask selects between two non-background inactive vals
90 };
91 
92 
94 
95 
98 template<typename T>
99 struct RealToHalf {
100  enum { isReal = false }; // unless otherwise specified, type T is not a floating-point type
101  typedef T HalfT; // type T's half float analogue is T itself
102 };
103 template<> struct RealToHalf<float> { enum { isReal = true }; typedef half HalfT; };
104 template<> struct RealToHalf<double> { enum { isReal = true }; typedef half HalfT; };
105 template<> struct RealToHalf<Vec2s> { enum { isReal = true }; typedef Vec2H HalfT; };
106 template<> struct RealToHalf<Vec2d> { enum { isReal = true }; typedef Vec2H HalfT; };
107 template<> struct RealToHalf<Vec3s> { enum { isReal = true }; typedef Vec3H HalfT; };
108 template<> struct RealToHalf<Vec3d> { enum { isReal = true }; typedef Vec3H HalfT; };
109 
110 
111 OPENVDB_API void zipToStream(std::ostream&, const char* data, size_t numBytes);
112 OPENVDB_API void unzipFromStream(std::istream&, char* data, size_t numBytes);
113 
121 template<typename T>
122 inline void
123 readData(std::istream& is, T* data, Index count, bool compressed)
124 {
125  if (compressed) {
126  unzipFromStream(is, reinterpret_cast<char*>(data), sizeof(T) * count);
127  } else {
128  is.read(reinterpret_cast<char*>(data), sizeof(T) * count);
129  }
130 }
131 
133 template<>
134 inline void
135 readData<std::string>(std::istream& is, std::string* data, Index count, bool /*compressed*/)
136 {
137  for (Index i = 0; i < count; ++i) {
138  size_t len = 0;
139  is >> len;
140  //data[i].resize(len);
141  //is.read(&(data[i][0]), len);
142 
143  std::string buffer(len+1, ' ');
144  is.read(&buffer[0], len+1 );
145  data[i].assign(buffer, 0, len);
146  }
147 }
148 
153 template<bool IsReal, typename T> struct HalfReader;
155 template<typename T>
156 struct HalfReader</*IsReal=*/false, T> {
157  static inline void read(std::istream& is, T* data, Index count, bool compressed) {
158  readData(is, data, count, compressed);
159  }
160 };
162 template<typename T>
163 struct HalfReader</*IsReal=*/true, T> {
164  typedef typename RealToHalf<T>::HalfT HalfT;
165  static inline void read(std::istream& is, T* data, Index count, bool compressed) {
166  if (count < 1) return;
167  std::vector<HalfT> halfData(count); // temp buffer into which to read half float values
168  readData<HalfT>(is, reinterpret_cast<HalfT*>(&halfData[0]), count, compressed);
169  // Copy half float values from the temporary buffer to the full float output array.
170  std::copy(halfData.begin(), halfData.end(), data);
171  }
172 };
173 
174 
182 template<typename T>
183 inline void
184 writeData(std::ostream &os, const T *data, Index count, bool compress)
185 {
186  if (compress) {
187  zipToStream(os, reinterpret_cast<const char*>(data), sizeof(T) * count);
188  } else {
189  os.write(reinterpret_cast<const char*>(data), sizeof(T) * count);
190  }
191 }
192 
195 template<>
196 inline void
197 writeData<std::string>(std::ostream& os, const std::string* data, Index count, bool /*compress*/)
198 {
199  for (Index i = 0; i < count; ++i) {
200  const size_t len = data[i].size();
201  os << len;
202  os.write(data[i].c_str(), len+1);
203  //os.write(&(data[i][0]), len );
204  }
205 }
206 
211 template<bool IsReal, typename T> struct HalfWriter;
213 template<typename T>
214 struct HalfWriter</*IsReal=*/false, T> {
215  static inline void write(std::ostream& os, const T* data, Index count, bool compress) {
216  writeData(os, data, count, compress);
217  }
218 };
220 template<typename T>
221 struct HalfWriter</*IsReal=*/true, T> {
222  typedef typename RealToHalf<T>::HalfT HalfT;
223  static inline void write(std::ostream& os, const T* data, Index count, bool compress) {
224  if (count < 1) return;
225  // Convert full float values to half float, then output the half float array.
226  std::vector<HalfT> halfData(count);
227  std::copy(data, data + count, halfData.begin());
228  writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compress);
229  }
230 };
231 #ifdef _MSC_VER
232 
233 template<>
234 struct HalfWriter</*IsReal=*/true, double> {
235  typedef RealToHalf<double>::HalfT HalfT;
236  static inline void write(std::ostream& os, const double* data, Index count, bool compress) {
237  if (count < 1) return;
238  // Convert full float values to half float, then output the half float array.
239  std::vector<HalfT> halfData(count);
240  for (Index i = 0; i < count; ++i) halfData[i] = float(data[i]);
241  writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compress);
242  }
243 };
244 #endif // _MSC_VER
245 
246 
248 
249 
262 template<typename ValueT, typename MaskT>
263 inline void
264 readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount,
265  const MaskT& valueMask, bool fromHalf)
266 {
267  // Get the stream's compression settings.
268  const uint32_t compression = getDataCompression(is);
269  const bool
270  zipped = compression & COMPRESS_ZIP,
271  maskCompressed = compression & COMPRESS_ACTIVE_MASK;
272 
273  int8_t metadata = NO_MASK_OR_INACTIVE_VALS;
275  // Read the flag that specifies what, if any, additional metadata
276  // (selection mask and/or inactive value(s)) is saved.
277  is.read(reinterpret_cast<char*>(&metadata), /*bytes=*/1);
278  }
279 
280  ValueT background = zeroVal<ValueT>();
281  if (const void* bgPtr = getGridBackgroundValuePtr(is)) {
282  background = *static_cast<const ValueT*>(bgPtr);
283  }
284  ValueT inactiveVal1 = background;
285  ValueT inactiveVal0 =
286  ((metadata == NO_MASK_OR_INACTIVE_VALS) ? background : negative(background));
287 
288  if (metadata != NO_MASK_OR_INACTIVE_VALS &&
289  metadata != NO_MASK_AND_MINUS_BG &&
290  metadata != MASK_AND_NO_INACTIVE_VALS)
291  {
292  // Read one of at most two distinct inactive values.
293  is.read(reinterpret_cast<char*>(&inactiveVal0), sizeof(ValueT));
294  if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
295  // Read the second of two distinct inactive values.
296  is.read(reinterpret_cast<char*>(&inactiveVal1), sizeof(ValueT));
297  }
298  }
299 
300  MaskT selectionMask;
301  if (metadata != NO_MASK_OR_INACTIVE_VALS &&
302  metadata != NO_MASK_AND_MINUS_BG &&
303  metadata != NO_MASK_AND_ONE_INACTIVE_VAL)
304  {
305  // For use in mask compression (only), read the bitmask that selects
306  // between two distinct inactive values.
307  selectionMask.load(is);
308  }
309 
310  ValueT* tempBuf = destBuf;
311  boost::scoped_array<ValueT> scopedTempBuf;
312 
313  Index tempCount = destCount;
314  if (maskCompressed && getFormatVersion(is) >= OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION) {
315  tempCount = valueMask.countOn();
316  if (tempCount != destCount) {
317  // If this node has inactive voxels, allocate a temporary buffer
318  // into which to read just the active values.
319  scopedTempBuf.reset(new ValueT[tempCount]);
320  tempBuf = scopedTempBuf.get();
321  }
322  }
323 
324  // Read in the buffer.
325  if (fromHalf) {
326  HalfReader<RealToHalf<ValueT>::isReal, ValueT>::read(is, tempBuf, tempCount, zipped);
327  } else {
328  readData<ValueT>(is, tempBuf, tempCount, zipped);
329  }
330 
331  // If mask compression is enabled and the number of active values read into
332  // the temp buffer is smaller than the size of the destination buffer,
333  // then there are missing (inactive) values.
334  if (maskCompressed && tempCount != destCount) {
335  // Restore inactive values, using the background value and, if available,
336  // the inside/outside mask. (For fog volumes, the destination buffer is assumed
337  // to be initialized to background value zero, so inactive values can be ignored.)
338  for (Index destIdx = 0, tempIdx = 0; destIdx < MaskT::BIT_SIZE; ++destIdx) {
339  if (valueMask.isOn(destIdx)) {
340  // Copy a saved active value into this node's buffer.
341  destBuf[destIdx] = tempBuf[tempIdx];
342  ++tempIdx;
343  } else {
344  // Reconstruct an unsaved inactive value and copy it into this node's buffer.
345  destBuf[destIdx] = (selectionMask.isOn(destIdx) ? inactiveVal1 : inactiveVal0);
346  }
347  }
348  }
349 }
350 
351 
364 template<typename ValueT, typename MaskT>
365 inline void
366 writeCompressedValues(std::ostream& os, ValueT* srcBuf, Index srcCount,
367  const MaskT& valueMask, const MaskT& childMask, bool toHalf)
368 {
369  struct Local {
370  // Comparison function for values
371  static inline bool eq(const ValueT& a, const ValueT& b) {
372  return math::isExactlyEqual(a, b);
373  }
374  };
375 
376  // Get the stream's compression settings.
377  const uint32_t compress = getDataCompression(os);
378  const bool
379  zip = compress & COMPRESS_ZIP,
380  maskCompress = compress & COMPRESS_ACTIVE_MASK;
381 
382  Index tempCount = srcCount;
383  ValueT* tempBuf = srcBuf;
384  boost::scoped_array<ValueT> scopedTempBuf;
385 
386  int8_t metadata = NO_MASK_OR_INACTIVE_VALS;
387 
388  if (!maskCompress) {
389  os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
390  } else {
391  // A valid level set's inactive values are either +background (outside)
392  // or -background (inside), and a fog volume's inactive values are all zero.
393  // Rather than write out all of these values, we can store just the active values
394  // (given that the value mask specifies their positions) and, if necessary,
395  // an inside/outside bitmask.
396 
397  const ValueT zero = zeroVal<ValueT>();
398  ValueT background = zero;
399  if (const void* bgPtr = getGridBackgroundValuePtr(os)) {
400  background = *static_cast<const ValueT*>(bgPtr);
401  }
402 
404  ValueT inactiveVal[2] = { background, background };
405  int numUniqueInactiveVals = 0;
406  for (typename MaskT::OffIterator it = valueMask.beginOff();
407  numUniqueInactiveVals < 3 && it; ++it)
408  {
409  const Index32 idx = it.pos();
410 
411  // Skip inactive values that are actually child node pointers.
412  if (childMask.isOn(idx)) continue;
413 
414  const ValueT& val = srcBuf[idx];
415  const bool unique = !(
416  (numUniqueInactiveVals > 0 && Local::eq(val, inactiveVal[0])) ||
417  (numUniqueInactiveVals > 1 && Local::eq(val, inactiveVal[1]))
418  );
419  if (unique) {
420  if (numUniqueInactiveVals < 2) inactiveVal[numUniqueInactiveVals] = val;
421  ++numUniqueInactiveVals;
422  }
423  }
424 
425  metadata = NO_MASK_OR_INACTIVE_VALS;
426 
427  if (numUniqueInactiveVals == 1) {
428  if (!Local::eq(inactiveVal[0], background)) {
429  if (Local::eq(inactiveVal[0], negative(background))) {
430  metadata = NO_MASK_AND_MINUS_BG;
431  } else {
432  metadata = NO_MASK_AND_ONE_INACTIVE_VAL;
433  }
434  }
435  } else if (numUniqueInactiveVals == 2) {
436  metadata = NO_MASK_OR_INACTIVE_VALS;
437  if (!Local::eq(inactiveVal[0], background) && !Local::eq(inactiveVal[1], background)) {
438  // If neither inactive value is equal to the background, both values
439  // need to be saved, along with a mask that selects between them.
440  metadata = MASK_AND_TWO_INACTIVE_VALS;
441 
442  } else if (Local::eq(inactiveVal[1], background)) {
443  if (Local::eq(inactiveVal[0], negative(background))) {
444  // If the second inactive value is equal to the background and
445  // the first is equal to -background, neither value needs to be saved,
446  // but save a mask that selects between -background and +background.
447  metadata = MASK_AND_NO_INACTIVE_VALS;
448  } else {
449  // If the second inactive value is equal to the background, only
450  // the first value needs to be saved, along with a mask that selects
451  // between it and the background.
452  metadata = MASK_AND_ONE_INACTIVE_VAL;
453  }
454  } else if (Local::eq(inactiveVal[0], background)) {
455  if (Local::eq(inactiveVal[1], negative(background))) {
456  // If the first inactive value is equal to the background and
457  // the second is equal to -background, neither value needs to be saved,
458  // but save a mask that selects between -background and +background.
459  metadata = MASK_AND_NO_INACTIVE_VALS;
460  std::swap(inactiveVal[0], inactiveVal[1]);
461  } else {
462  // If the first inactive value is equal to the background, swap it
463  // with the second value and save only that value, along with a mask
464  // that selects between it and the background.
465  std::swap(inactiveVal[0], inactiveVal[1]);
466  metadata = MASK_AND_ONE_INACTIVE_VAL;
467  }
468  }
469  }
470 
471  os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
472 
473  if (metadata != NO_MASK_OR_INACTIVE_VALS &&
474  metadata != NO_MASK_AND_MINUS_BG &&
475  metadata != MASK_AND_NO_INACTIVE_VALS)
476  {
477  // Write one of at most two distinct inactive values.
478  os.write(reinterpret_cast<const char*>(&inactiveVal[0]), sizeof(ValueT));
479  if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
480  // Write the second of two distinct inactive values.
481  os.write(reinterpret_cast<const char*>(&inactiveVal[1]), sizeof(ValueT));
482  }
483  }
484 
485  if (metadata == NO_MASK_OR_INACTIVE_VALS && numUniqueInactiveVals > 2) {
486  // If there are more than two unique inactive values, the entire input buffer
487  // needs to be saved (both active and inactive values).
490  } else {
491  // Create a new array to hold just the active values.
492  scopedTempBuf.reset(new ValueT[srcCount]);
493  tempBuf = scopedTempBuf.get();
494 
495  if (metadata == NO_MASK_OR_INACTIVE_VALS ||
496  metadata == NO_MASK_AND_MINUS_BG ||
497  metadata == NO_MASK_AND_ONE_INACTIVE_VAL)
498  {
499  // Copy active values to the contiguous array.
500  tempCount = 0;
501  for (typename MaskT::OnIterator it = valueMask.beginOn(); it; ++it, ++tempCount) {
502  tempBuf[tempCount] = srcBuf[it.pos()];
503  }
504  } else {
505  // Copy active values to a new, contiguous array and populate a bitmask
506  // that selects between two distinct inactive values.
507  MaskT selectionMask;
508  tempCount = 0;
509  for (Index srcIdx = 0; srcIdx < srcCount; ++srcIdx) {
510  if (valueMask.isOn(srcIdx)) { // active value
511  tempBuf[tempCount] = srcBuf[srcIdx];
512  ++tempCount;
513  } else { // inactive value
514  if (Local::eq(srcBuf[srcIdx], inactiveVal[1])) {
515  selectionMask.setOn(srcIdx); // inactive value 1
516  } // else inactive value 0
517  }
518  }
519  assert(tempCount == valueMask.countOn());
520 
521  // Write out the mask that selects between two inactive values.
522  selectionMask.save(os);
523  }
524  }
525  }
526 
527  // Write out the buffer.
528  if (toHalf) {
529  HalfWriter<RealToHalf<ValueT>::isReal, ValueT>::write(os, tempBuf, tempCount, zip);
530  } else {
531  writeData(os, tempBuf, tempCount, zip);
532  }
533 }
534 
535 } // namespace io
536 } // namespace OPENVDB_VERSION_NAME
537 } // namespace openvdb
538 
539 #endif // OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
540 
541 // Copyright (c) 2012 DreamWorks Animation LLC
542 // All rights reserved. This software is distributed under the
543 // Mozilla Public License 2.0 ( http://www.mozilla.org/MPL/2.0/ )