Main MRPT website > C++ reference
MRPT logo

CFeature.h

Go to the documentation of this file.
00001 /* +---------------------------------------------------------------------------+
00002    |          The Mobile Robot Programming Toolkit (MRPT) C++ library          |
00003    |                                                                           |
00004    |                   http://mrpt.sourceforge.net/                            |
00005    |                                                                           |
00006    |   Copyright (C) 2005-2011  University of Malaga                           |
00007    |                                                                           |
00008    |    This software was written by the Machine Perception and Intelligent    |
00009    |      Robotics Lab, University of Malaga (Spain).                          |
00010    |    Contact: Jose-Luis Blanco  <jlblanco@ctima.uma.es>                     |
00011    |                                                                           |
00012    |  This file is part of the MRPT project.                                   |
00013    |                                                                           |
00014    |     MRPT is free software: you can redistribute it and/or modify          |
00015    |     it under the terms of the GNU General Public License as published by  |
00016    |     the Free Software Foundation, either version 3 of the License, or     |
00017    |     (at your option) any later version.                                   |
00018    |                                                                           |
00019    |   MRPT is distributed in the hope that it will be useful,                 |
00020    |     but WITHOUT ANY WARRANTY; without even the implied warranty of        |
00021    |     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         |
00022    |     GNU General Public License for more details.                          |
00023    |                                                                           |
00024    |     You should have received a copy of the GNU General Public License     |
00025    |     along with MRPT.  If not, see <http://www.gnu.org/licenses/>.         |
00026    |                                                                           |
00027    +---------------------------------------------------------------------------+ */
00028 #ifndef CFeature_H
00029 #define CFeature_H
00030 
00031 #include <mrpt/utils/CImage.h>
00032 #include <mrpt/utils/stl_extensions.h>
00033 #include <mrpt/math/CMatrix.h>
00034 #include <mrpt/math/ops_matrices.h>
00035 #include <mrpt/math/KDTreeCapable.h>
00036 
00037 #include <mrpt/vision/link_pragmas.h>
00038 
00039 namespace mrpt
00040 {
00041         namespace vision
00042         {
00043                 using namespace mrpt::utils;
00044                 using namespace mrpt::math;
00045                 using namespace std;
00046 
00047                 class CFeatureList;
00048                 class CMatchedFeatureList;
00049 
00050                 /** Definition of a feature ID
00051                 */
00052                 typedef uint64_t TFeatureID;
00053 
00054                 /** Types of features - This means that the point has been detected with this algorithm, which is independent of additional descriptors a feature may also have
00055                 */
00056                 enum TFeatureType
00057                 {
00058                         featNotDefined = -1,    //!< Non-defined feature (also used for Occupancy features)
00059                         featKLT = 0,                    //!< Kanade-Lucas-Tomasi feature [SHI'94]
00060                         featHarris,                             //!< Harris border and corner detector [HARRIS]
00061                         featBCD,                                //!< Binary corder detector
00062                         featSIFT,                               //!< Scale Invariant Feature Transform [LOWE'04]
00063                         featSURF,                               //!< Speeded Up Robust Feature [BAY'06]
00064                         featBeacon,                             //!< A especial case: this is not an image feature, but a 2D/3D beacon (used for range-only SLAM from mrpt::slam::CLandmark)
00065                         featFAST                                //!< FAST feature detector ("Faster and better: A machine learning approach to corner detection", E. Rosten, R. Porter and T. Drummond, PAMI, 2009).
00066                 };
00067 
00068                 /** The bitwise OR combination of values of TDescriptorType are used in CFeatureExtraction::computeDescriptors to indicate which descriptors are to be computed for features.
00069                   */
00070                 enum TDescriptorType
00071                 {
00072                         descAny                         = 0,  //!< Used in some methods to mean "any of the present descriptors"
00073                         descSIFT            = 1,  //!< SIFT descriptors
00074                         descSURF                        = 2,  //!< SURF descriptors
00075                         descSpinImages      = 4,  //!< Intensity-domain spin image descriptors
00076                         descPolarImages     = 8,  //!< Polar image descriptor
00077                         descLogPolarImages      = 16  //!< Log-Polar image descriptor
00078                 };
00079 
00080                 enum TFeatureTrackStatus
00081                 {
00082                         // Init value
00083                         status_IDLE     = 0,    //!< Inactive (right after detection, and before being tried to track)
00084 
00085                         // Ok:
00086                         status_TRACKED  = 5,    //!< Feature correctly tracked
00087 
00088                         // Bad:
00089                         status_OOB              = 1,    //!< Feature felt Out Of Bounds
00090                         status_LOST     = 10,   //!< Unable to track this feature
00091 
00092                         // KLT specific:
00093                         statusKLT_IDLE  = 0,    //!< Inactive
00094                         statusKLT_OOB   = 1,    //!< Out Of Bounds      (Value identical to status_OOB)
00095                         statusKLT_SMALL_DET     = 2,    //!< Determinant of the matrix too small
00096                         statusKLT_LARGE_RESIDUE = 3,    //!< Error too big
00097                         statusKLT_MAX_RESIDUE   = 4,
00098                         statusKLT_TRACKED       = 5,    //!< Feature correctly tracked (Value identical to status_TRACKED)
00099                         statusKLT_MAX_ITERATIONS        = 6     //!< Iteration maximum reached
00100                 };
00101 
00102                 enum TListIdx
00103                 {
00104                     firstList = 0,
00105                     secondList,
00106                     bothLists
00107                 };
00108 
00109                 typedef TFeatureTrackStatus TKLTFeatureStatus; //!< For backward compatibility
00110 
00111                 /****************************************************
00112                                                 Class CFEATURE
00113                 *****************************************************/
00114                 DEFINE_SERIALIZABLE_PRE_CUSTOM_BASE_LINKAGE( CFeature, mrpt::utils::CSerializable, VISION_IMPEXP )
00115 
00116                 /** A generic 2D feature from an image, extracted with \a CFeatureExtraction
00117                   * Each feature may have one or more descriptors (see \a descriptors), in addition to an image patch.
00118                   * The (Euclidean) distance between descriptors in a pair of features can be computed with  descriptorDistanceTo,
00119                   *  while the similarity of the patches is given by patchCorrelationTo.
00120                   */
00121                 class VISION_IMPEXP CFeature : public mrpt::utils::CSerializable
00122                 {
00123                         friend class CFeatureList;
00124                         friend class CMatchedFeatureList;
00125 
00126                         DEFINE_SERIALIZABLE( CFeature )
00127 
00128                 public:
00129                         float                           x,y;                    //!< Coordinates in the image
00130                         TFeatureID                      ID;                             //!< ID of the feature
00131                         CImage                          patch;                  //!< A patch of the image surrounding the feature
00132                         uint16_t                        patchSize;              //!< Size of the patch (patchSize x patchSize) (it must be an odd number)
00133                         TFeatureType            type;                   //!< Type of the feature: featNotDefined, featSIFT, featKLT,    featHarris, featSURF, featBeacon
00134                         TFeatureTrackStatus     track_status;   //!< Status of the feature tracking process (old name: KLT_status)
00135                         float                           response;               //!< A measure of the "goodness" of the feature (old name: KLT_val)
00136                         float                           orientation;    //!< Main orientation of the feature
00137                         float                           scale;                  //!< Feature scale into the scale space
00138                         uint8_t                         IDSourceImage;  //!< ID of the image from which the feature was extracted (JL says: ?????)
00139                         uint16_t            nTimesSeen;     //!< Number of frames it has been seen in a sequence of images.
00140                         uint16_t            nTimesNotSeen;  //!< Number of frames it has not been seen in a sequence of images.
00141                         uint16_t            nTimesLastSeen; //!< Number of frames since it was seen for the last time.
00142 
00143             double                          depth;              //!< The estimated depth in 3D of this feature wrt the camera in the current frame
00144             double                          initialDepth;       //!< The estimated depth in 3D of this feature wrt the camera that took its image
00145             TPoint3D                        p3D;                //!< The estimated 3D point of this feature wrt its camera
00146             deque<double>                   multiScales;        //!< A set of scales where the multi-resolution descriptor has been computed
00147             deque<vector<double> >          multiOrientations;  //!< A vector of main orientations (there is a vector of orientations for each scale)
00148             deque<vector<vector<int32_t> > >    multiHashCoeffs;    //!< A set of vectors containing the coefficients for a HASH table of descriptors
00149                         bool isPointFeature() const;                            //!< Return false only for Blob detectors (SIFT, SURF)
00150 
00151                         /** All the possible descriptors this feature may have */
00152                         struct VISION_IMPEXP TDescriptors
00153                         {
00154                                 TDescriptors();  // Initialization
00155 
00156                                 std::vector<uint8_t>            SIFT;                           //!< Feature descriptor
00157                                 std::vector<float>                          SURF;                               //!< Feature descriptor
00158                                 std::vector<float>                          SpinImg;                    //!< The 2D histogram as a single row
00159                                 uint16_t                                            SpinImg_range_rows;     //!< The number of rows (corresponding to range bins in the 2D histogram) of the original matrix from which SpinImg was extracted as a vector.
00160                                 mrpt::math::CMatrix                         PolarImg;                   //!< A polar image centered at the interest point
00161                                 mrpt::math::CMatrix                         LogPolarImg;                //!< A log-polar image centered at the interest point
00162                                 bool                                                polarImgsNoRotation;    //!< If set to true (manually, default=false) the call to "descriptorDistanceTo" will not consider all the rotations between polar image descriptors (PolarImg, LogPolarImg)
00163                                 deque<vector<vector<int32_t> > >    multiSIFTDescriptors;   //!< A set of SIFT-like descriptors for each orientation and scale of the multiResolution feature (there is a vector of descriptors for each scale)
00164 
00165                                 bool hasDescriptorSIFT() const { return !SIFT.empty(); };                       //!< Whether this feature has this kind of descriptor
00166                                 bool hasDescriptorSURF() const { return !SURF.empty(); }                        //!< Whether this feature has this kind of descriptor
00167                                 bool hasDescriptorSpinImg() const { return !SpinImg.empty(); };                 //!< Whether this feature has this kind of descriptor
00168                                 bool hasDescriptorPolarImg() const { return PolarImg.rows()!=0; } ;             //!< Whether this feature has this kind of descriptor
00169                                 bool hasDescriptorLogPolarImg() const { return LogPolarImg.rows()!=0; } ;       //!< Whether this feature has this kind of descriptor
00170                                 bool hasDescriptorMultiSIFT() const {
00171                     return (multiSIFTDescriptors.size() > 0 && multiSIFTDescriptors[0].size() > 0); //!< Whether this feature has this kind of descriptor
00172                 }
00173                         }
00174                         descriptors;
00175 
00176                         /** Return the first found descriptor, as a matrix.
00177                           * \return false on error, i.e. there is no valid descriptor.
00178                           */
00179                         bool getFirstDescriptorAsMatrix(mrpt::math::CMatrixFloat &desc) const;
00180 
00181                         /** Computes the normalized cross-correlation between the patches of this and another feature (normalized in the range [0,1], such as 0=best, 1=worst).
00182                           *  \note If this or the other features does not have patches or they are of different sizes, an exception will be raised.
00183                           * \sa descriptorDistanceTo
00184                           */
00185                         float patchCorrelationTo( const CFeature &oFeature) const;
00186 
00187                         /** Computes the Euclidean Distance between this feature's and other feature's descriptors, using the given descriptor or the first present one.
00188                           *  \note If descriptorToUse is not descAny and that descriptor is not present in one of the features, an exception will be raised.
00189                           * \sa patchCorrelationTo
00190                           */
00191                         float descriptorDistanceTo( const CFeature &oFeature,  TDescriptorType descriptorToUse = descAny, bool normalize_distances = true ) const;
00192 
00193                         /** Computes the Euclidean Distance between "this" and the "other" descriptors */
00194                         float descriptorSIFTDistanceTo( const CFeature &oFeature, bool normalize_distances = true ) const;
00195 
00196                         /** Computes the Euclidean Distance between "this" and the "other" descriptors */
00197                         float descriptorSURFDistanceTo( const CFeature &oFeature, bool normalize_distances = true  ) const;
00198 
00199                         /** Computes the Euclidean Distance between "this" and the "other" descriptors */
00200                         float descriptorSpinImgDistanceTo( const CFeature &oFeature, bool normalize_distances = true ) const;
00201 
00202                         /** Returns the minimum Euclidean Distance between "this" and the "other" polar image descriptor, for the best shift in orientation.
00203                           * \param oFeature The other feature to compare with.
00204                           * \param minDistAngle The placeholder for the angle at which the smallest distance is found.
00205                           * \return The distance for the best orientation (minimum distance).
00206                           */
00207                         float descriptorPolarImgDistanceTo(
00208                                 const CFeature &oFeature,
00209                                 float &minDistAngle,
00210                                 bool normalize_distances = true ) const;
00211 
00212                         /** Returns the minimum Euclidean Distance between "this" and the "other" log-polar image descriptor, for the best shift in orientation.
00213                           * \param oFeature The other feature to compare with.
00214                           * \param minDistAngle The placeholder for the angle at which the smallest distance is found.
00215                           * \return The distance for the best orientation (minimum distance).
00216                           */
00217                         float descriptorLogPolarImgDistanceTo(
00218                                 const CFeature &oFeature,
00219                                 float &minDistAngle,
00220                                 bool normalize_distances = true ) const;
00221 
00222                         /** Save the feature to a text file in this format:
00223               *    "%% Dump of mrpt::vision::CFeatureList. Each line format is:\n"
00224               *    "%% ID TYPE X Y ORIENTATION SCALE TRACK_STATUS RESPONSE HAS_SIFT [SIFT] HAS_SURF [SURF] HAS_MULTI [MULTI]_i"
00225               *    "%% |---------------------- feature ------------------| |---------------------- descriptors ------------------------|"
00226               *    "%% with:\n"
00227               *    "%%  TYPE  : The used detector: 0:KLT, 1: Harris, 2: BCD, 3: SIFT, 4: SURF, 5: Beacon, 6: FAST\n"
00228               *    "%%  HAS_* : 1 if a descriptor of that type is associated to the feature."
00229               *    "%%  SIFT  : Present if HAS_SIFT=1: N DESC_0 ... DESC_N-1"
00230               *    "%%  SURF  : Present if HAS_SURF=1: N DESC_0 ... DESC_N-1"
00231               *    "%%  MULTI : Present if HAS_MULTI=1: SCALE ORI N DESC_0 ... DESC_N-1"
00232               *    "%%-------------------------------------------------------------------------------------------\n");
00233                         */
00234             void saveToTextFile( const std::string &filename, bool APPEND = false );
00235 
00236                         /** Get the type of the feature
00237                         */
00238                         TFeatureType get_type() const { return type; }
00239 
00240                         /** Dump feature information into a text stream */
00241                         void dumpToTextStream( mrpt::utils::CStream &out) const;
00242 
00243                         void dumpToConsole() const;
00244 
00245                         /** Constructor
00246                         */
00247                         CFeature();
00248 
00249                         /** Virtual destructor */
00250                         virtual ~CFeature() {}
00251 
00252 
00253                 protected:
00254 
00255                         /** Internal function used by "descriptorLogPolarImgDistanceTo" and "descriptorPolarImgDistanceTo"
00256                           */
00257                         static float internal_distanceBetweenPolarImages(
00258                                 const mrpt::math::CMatrix &desc1,
00259                                 const mrpt::math::CMatrix &desc2,
00260                                 float &minDistAngle,
00261                                 bool normalize_distances,
00262                                 bool dont_shift_angle );
00263 
00264                 }; // end of class
00265 
00266 
00267                 /****************************************************
00268                                                 Class CFEATURELIST
00269                 *****************************************************/
00270                 /** A list of visual features, to be used as output by detectors, as input/output by trackers, etc.
00271                   */
00272                 class VISION_IMPEXP CFeatureList : public mrpt::math::KDTreeCapable  //public std::deque<CFeaturePtr>
00273                 {
00274                 protected:
00275                         typedef std::deque<CFeaturePtr> TInternalFeatList;
00276 
00277                         TInternalFeatList  m_feats; //!< The actual container with the list of features
00278 
00279                 public:
00280                         /** The type of the first feature in the list */
00281                         inline TFeatureType get_type() const { return empty() ? featNotDefined : (*begin())->get_type(); }
00282 
00283                         /** Save feature list to a text file */
00284                         void saveToTextFile( const std::string &fileName, bool APPEND = false );
00285 
00286                         /** Save feature list to a text file */
00287                         void loadFromTextFile( const std::string &fileName );
00288 
00289                         /** Copies the content of another CFeatureList inside this one. The inner features are also copied. */
00290                         void copyListFrom( const CFeatureList &otherList );
00291 
00292                         /** Get the maximum ID into the list */
00293                         TFeatureID getMaxID() const;
00294 
00295                         /** Get a reference to a Feature from its ID */
00296                         CFeaturePtr getByID( TFeatureID ID ) const;
00297 
00298             /** Get a vector of references to a subset of features from their IDs */
00299             void getByMultiIDs( const vector<TFeatureID> &IDs, vector<CFeaturePtr> &out, vector<int> &outIndex ) const;
00300 
00301                         /** Get a reference to the nearest feature to the a given 2D point (version returning distance to closest feature in "max_dist")
00302                         *   \param x [IN] The query point x-coordinate
00303                         *   \param y [IN] The query point y-coordinate
00304                         *   \param max_dist [IN/OUT] At input: The maximum distance to search for. At output: The actual distance to the feature.
00305                         *  \return A reference to the found feature, or a NULL smart pointer if none found.
00306                         *  \note See also all the available KD-tree search methods, listed in mrpt::math::KDTreeCapable
00307                         */
00308                         CFeaturePtr nearest( const float x, const float y, double &max_dist ) const;
00309 
00310                         /** Constructor */
00311                         CFeatureList();
00312 
00313                         /** Virtual destructor */
00314                         virtual ~CFeatureList();
00315 
00316                         /** Call this when the list of features has been modified so the KD-tree is marked as outdated. */
00317                         inline void mark_kdtree_as_outdated() const { kdtree_mark_as_outdated(); }
00318 
00319                         /** @name Method and datatypes to emulate a STL container
00320                             @{ */
00321                         typedef TInternalFeatList::iterator iterator;
00322                         typedef TInternalFeatList::const_iterator const_iterator;
00323 
00324                         typedef TInternalFeatList::reverse_iterator reverse_iterator;
00325                         typedef TInternalFeatList::const_reverse_iterator const_reverse_iterator;
00326 
00327                         inline iterator begin() { return m_feats.begin(); }
00328                         inline iterator end() { return m_feats.end(); }
00329                         inline const_iterator begin() const { return m_feats.begin(); }
00330                         inline const_iterator end() const { return m_feats.end(); }
00331 
00332                         inline iterator erase(const iterator it)  { mark_kdtree_as_outdated(); return m_feats.erase(it); }
00333 
00334                         inline bool empty() const  { return m_feats.empty(); }
00335                         inline size_t size() const { return m_feats.size(); }
00336 
00337                         inline void clear() { m_feats.clear(); mark_kdtree_as_outdated(); }
00338                         inline void resize(size_t N) { m_feats.resize(N); }
00339 
00340                         inline void push_front(const CFeaturePtr &f) { mark_kdtree_as_outdated();  m_feats.push_front(f); }
00341                         inline void push_back(const CFeaturePtr &f) { mark_kdtree_as_outdated();  m_feats.push_back(f); }
00342 
00343                         inline CFeaturePtr & operator [](const unsigned int index) { return m_feats[index]; }
00344                         inline const CFeaturePtr & operator [](const unsigned int index) const  { return m_feats[index]; }
00345 
00346                         /** @} */
00347 
00348                         /** @name Virtual methods that MUST be implemented by children classes of KDTreeCapable
00349                             @{ */
00350 
00351                         /** Must return the number of data points */
00352                         virtual size_t kdtree_get_point_count() const { return size(); }
00353 
00354                         /** Must fill out the data points in "data", such as the i'th point will be stored in (data[i][0],...,data[i][nDims-1]). */
00355                         virtual void kdtree_fill_point_data(ANNpointArray &data, const int nDims) const;
00356                         /** @} */
00357 
00358 
00359                 }; // end of class
00360 
00361                 /****************************************************
00362                                         Class CMATCHEDFEATURELIST
00363                 *****************************************************/
00364                 /** A list of features
00365                 */
00366                 class VISION_IMPEXP CMatchedFeatureList : public std::deque< std::pair<CFeaturePtr,CFeaturePtr> >
00367                 {
00368                 public:
00369                         /** The type of the first feature in the list */
00370                         inline TFeatureType get_type() const { return empty() ? featNotDefined : (begin()->first)->get_type(); }
00371 
00372                         /** Save list of matched features to a text file */
00373                         void saveToTextFile(const std::string &fileName);
00374 
00375             /** Returns the matching features as two separate CFeatureLists */
00376                         void getBothFeatureLists( CFeatureList &list1, CFeatureList &list2 );
00377 
00378                         /** Returns a smart pointer to the feature with the provided ID or a empty one if not found */
00379                         CFeaturePtr getByID( const TFeatureID & ID, const TListIdx &idx );
00380 
00381                         /** Returns the maximum ID of the features in the list. If the max ID has been already set up, this method just returns it.
00382                             Otherwise, this method finds, stores and returns it.*/
00383                         void getMaxID( const TListIdx &idx, TFeatureID & firstListID, TFeatureID & secondListID );
00384 
00385                         /** Updates the value of the maximum ID of the features in the matched list, i.e. it explicitly searches for the max ID and updates the member variables. */
00386                         void updateMaxID( const TListIdx &idx );
00387 
00388             /** Explicitly set the max IDs values to certain values */
00389             inline void setLeftMaxID( const TFeatureID &leftID ){ m_leftMaxID = leftID; }
00390             inline void setRightMaxID( const TFeatureID &rightID ){ m_rightMaxID = rightID; }
00391                         inline void setMaxIDs( const TFeatureID &leftID, const TFeatureID &rightID )
00392             {
00393                 setLeftMaxID(leftID);
00394                 setRightMaxID(rightID);
00395             };
00396 
00397                         /** Constructor */
00398                         CMatchedFeatureList();
00399 
00400                         /** Virtual destructor */
00401                         virtual ~CMatchedFeatureList();
00402         protected:
00403             TFeatureID m_leftMaxID, m_rightMaxID;
00404                 }; // end of class
00405 
00406         } // end of namespace
00407 
00408         namespace utils
00409         {
00410                 using namespace ::mrpt::vision;
00411                 // Specialization must occur in the same namespace
00412                 MRPT_DECLARE_TTYPENAME_PTR(CFeature)
00413         }
00414 
00415 
00416 } // end of namespace
00417 
00418 #endif
00419 



Page generated by Doxygen 1.7.3 for MRPT 0.9.4 SVN:exported at Tue Jan 25 21:56:31 UTC 2011