_concurrent_queue_internal.h

00001 /*
00002     Copyright 2005-2011 Intel Corporation.  All Rights Reserved.
00003 
00004     The source code contained or described herein and all documents related
00005     to the source code ("Material") are owned by Intel Corporation or its
00006     suppliers or licensors.  Title to the Material remains with Intel
00007     Corporation or its suppliers and licensors.  The Material is protected
00008     by worldwide copyright laws and treaty provisions.  No part of the
00009     Material may be used, copied, reproduced, modified, published, uploaded,
00010     posted, transmitted, distributed, or disclosed in any way without
00011     Intel's prior express written permission.
00012 
00013     No license under any patent, copyright, trade secret or other
00014     intellectual property right is granted to or conferred upon you by
00015     disclosure or delivery of the Materials, either expressly, by
00016     implication, inducement, estoppel or otherwise.  Any license under such
00017     intellectual property rights must be express and approved by Intel in
00018     writing.
00019 */
00020 
00021 #ifndef __TBB_concurrent_queue_internal_H
00022 #define __TBB_concurrent_queue_internal_H
00023 
00024 #include "tbb_stddef.h"
00025 #include "tbb_machine.h"
00026 #include "atomic.h"
00027 #include "spin_mutex.h"
00028 #include "cache_aligned_allocator.h"
00029 #include "tbb_exception.h"
00030 #include "tbb_profiling.h"
00031 #include <new>
00032 
00033 #if !TBB_USE_EXCEPTIONS && _MSC_VER
00034     // Suppress "C++ exception handler used, but unwind semantics are not enabled" warning in STL headers
00035     #pragma warning (push)
00036     #pragma warning (disable: 4530)
00037 #endif
00038 
00039 #include <iterator>
00040 
00041 #if !TBB_USE_EXCEPTIONS && _MSC_VER
00042     #pragma warning (pop)
00043 #endif
00044 
00045 namespace tbb {
00046 
00047 #if !__TBB_TEMPLATE_FRIENDS_BROKEN
00048 
00049 // forward declaration
00050 namespace strict_ppl {
00051 template<typename T, typename A> class concurrent_queue;
00052 }
00053 
00054 template<typename T, typename A> class concurrent_bounded_queue;
00055 
00056 namespace deprecated {
00057 template<typename T, typename A> class concurrent_queue;
00058 }
00059 #endif
00060 
00062 namespace strict_ppl {
00063 
00065 namespace internal {
00066 
00067 using namespace tbb::internal;
00068 
00069 typedef size_t ticket;
00070 
00071 template<typename T> class micro_queue ;
00072 template<typename T> class micro_queue_pop_finalizer ;
00073 template<typename T> class concurrent_queue_base_v3;
00074 
00076 
00079 struct concurrent_queue_rep_base : no_copy {
00080     template<typename T> friend class micro_queue;
00081     template<typename T> friend class concurrent_queue_base_v3;
00082 
00083 protected:
00085     static const size_t phi = 3;
00086 
00087 public:
00088     // must be power of 2
00089     static const size_t n_queue = 8;
00090 
00092     struct page {
00093         page* next;
00094         uintptr_t mask; 
00095     };
00096 
00097     atomic<ticket> head_counter;
00098     char pad1[NFS_MaxLineSize-sizeof(atomic<ticket>)];
00099     atomic<ticket> tail_counter;
00100     char pad2[NFS_MaxLineSize-sizeof(atomic<ticket>)];
00101 
00103     size_t items_per_page;
00104 
00106     size_t item_size;
00107 
00109     atomic<size_t> n_invalid_entries;
00110 
00111     char pad3[NFS_MaxLineSize-sizeof(size_t)-sizeof(size_t)-sizeof(atomic<size_t>)];
00112 } ;
00113 
00114 inline bool is_valid_page(const concurrent_queue_rep_base::page* p) {
00115     return uintptr_t(p)>1;
00116 }
00117 
00119 
00122 class concurrent_queue_page_allocator
00123 {
00124     template<typename T> friend class micro_queue ;
00125     template<typename T> friend class micro_queue_pop_finalizer ;
00126 protected:
00127     virtual ~concurrent_queue_page_allocator() {}
00128 private:
00129     virtual concurrent_queue_rep_base::page* allocate_page() = 0;
00130     virtual void deallocate_page( concurrent_queue_rep_base::page* p ) = 0;
00131 } ;
00132 
00133 #if _MSC_VER && !defined(__INTEL_COMPILER)
00134 // unary minus operator applied to unsigned type, result still unsigned
00135 #pragma warning( push )
00136 #pragma warning( disable: 4146 )
00137 #endif
00138 
00140 
00142 template<typename T>
00143 class micro_queue : no_copy {
00144     typedef concurrent_queue_rep_base::page page;
00145 
00147     class destroyer: no_copy {
00148         T& my_value;
00149     public:
00150         destroyer( T& value ) : my_value(value) {}
00151         ~destroyer() {my_value.~T();}          
00152     };
00153 
00154     void copy_item( page& dst, size_t index, const void* src ) {
00155         new( &get_ref(dst,index) ) T(*static_cast<const T*>(src)); 
00156     }
00157 
00158     void copy_item( page& dst, size_t dindex, const page& src, size_t sindex ) {
00159         new( &get_ref(dst,dindex) ) T( get_ref(const_cast<page&>(src),sindex) );
00160     }
00161 
00162     void assign_and_destroy_item( void* dst, page& src, size_t index ) {
00163         T& from = get_ref(src,index);
00164         destroyer d(from);
00165         *static_cast<T*>(dst) = from;
00166     }
00167 
00168     void spin_wait_until_my_turn( atomic<ticket>& counter, ticket k, concurrent_queue_rep_base& rb ) const ;
00169 
00170 public:
00171     friend class micro_queue_pop_finalizer<T>;
00172 
00173     struct padded_page: page {
00175         padded_page(); 
00177         void operator=( const padded_page& );
00179         T last;
00180     };
00181 
00182     static T& get_ref( page& p, size_t index ) {
00183         return (&static_cast<padded_page*>(static_cast<void*>(&p))->last)[index];
00184     }
00185 
00186     atomic<page*> head_page;
00187     atomic<ticket> head_counter;
00188 
00189     atomic<page*> tail_page;
00190     atomic<ticket> tail_counter;
00191 
00192     spin_mutex page_mutex;
00193     
00194     void push( const void* item, ticket k, concurrent_queue_base_v3<T>& base ) ;
00195 
00196     bool pop( void* dst, ticket k, concurrent_queue_base_v3<T>& base ) ;
00197 
00198     micro_queue& assign( const micro_queue& src, concurrent_queue_base_v3<T>& base ) ;
00199 
00200     page* make_copy( concurrent_queue_base_v3<T>& base, const page* src_page, size_t begin_in_page, size_t end_in_page, ticket& g_index ) ;
00201 
00202     void invalidate_page_and_rethrow( ticket k ) ;
00203 };
00204 
00205 template<typename T>
00206 void micro_queue<T>::spin_wait_until_my_turn( atomic<ticket>& counter, ticket k, concurrent_queue_rep_base& rb ) const {
00207     atomic_backoff backoff;
00208     do {
00209         backoff.pause();
00210         if( counter&1 ) {
00211             ++rb.n_invalid_entries;
00212             throw_exception( eid_bad_last_alloc );
00213         }
00214     } while( counter!=k ) ;
00215 }
00216 
00217 template<typename T>
00218 void micro_queue<T>::push( const void* item, ticket k, concurrent_queue_base_v3<T>& base ) {
00219     k &= -concurrent_queue_rep_base::n_queue;
00220     page* p = NULL;
00221     size_t index = k/concurrent_queue_rep_base::n_queue & (base.my_rep->items_per_page-1);
00222     if( !index ) {
00223         __TBB_TRY {
00224             concurrent_queue_page_allocator& pa = base;
00225             p = pa.allocate_page();
00226         } __TBB_CATCH (...) {
00227             ++base.my_rep->n_invalid_entries;
00228             invalidate_page_and_rethrow( k );
00229         }
00230         p->mask = 0;
00231         p->next = NULL;
00232     }
00233 
00234     if( tail_counter!=k ) spin_wait_until_my_turn( tail_counter, k, *base.my_rep );
00235     call_itt_notify(acquired, &tail_counter);
00236         
00237     if( p ) {
00238         spin_mutex::scoped_lock lock( page_mutex );
00239         page* q = tail_page;
00240         if( is_valid_page(q) )
00241             q->next = p;
00242         else
00243             head_page = p; 
00244         tail_page = p;
00245     } else {
00246         p = tail_page;
00247     }
00248     __TBB_TRY {
00249         copy_item( *p, index, item );
00250         // If no exception was thrown, mark item as present.
00251         itt_hide_store_word(p->mask,  p->mask | uintptr_t(1)<<index);
00252         call_itt_notify(releasing, &tail_counter);
00253         tail_counter += concurrent_queue_rep_base::n_queue; 
00254     } __TBB_CATCH (...) {
00255         ++base.my_rep->n_invalid_entries;
00256         call_itt_notify(releasing, &tail_counter);
00257         tail_counter += concurrent_queue_rep_base::n_queue; 
00258         __TBB_RETHROW();
00259     }
00260 }
00261 
00262 template<typename T>
00263 bool micro_queue<T>::pop( void* dst, ticket k, concurrent_queue_base_v3<T>& base ) {
00264     k &= -concurrent_queue_rep_base::n_queue;
00265     if( head_counter!=k ) spin_wait_until_eq( head_counter, k );
00266     call_itt_notify(acquired, &head_counter);
00267     if( tail_counter==k ) spin_wait_while_eq( tail_counter, k );
00268     call_itt_notify(acquired, &tail_counter);
00269     page& p = *head_page;
00270     __TBB_ASSERT( &p, NULL );
00271     size_t index = k/concurrent_queue_rep_base::n_queue & (base.my_rep->items_per_page-1);
00272     bool success = false; 
00273     {
00274         micro_queue_pop_finalizer<T> finalizer( *this, base, k+concurrent_queue_rep_base::n_queue, index==base.my_rep->items_per_page-1 ? &p : NULL ); 
00275         if( p.mask & uintptr_t(1)<<index ) {
00276             success = true;
00277             assign_and_destroy_item( dst, p, index );
00278         } else {
00279             --base.my_rep->n_invalid_entries;
00280         }
00281     }
00282     return success;
00283 }
00284 
00285 template<typename T>
00286 micro_queue<T>& micro_queue<T>::assign( const micro_queue<T>& src, concurrent_queue_base_v3<T>& base ) {
00287     head_counter = src.head_counter;
00288     tail_counter = src.tail_counter;
00289     page_mutex   = src.page_mutex;
00290 
00291     const page* srcp = src.head_page;
00292     if( is_valid_page(srcp) ) {
00293         ticket g_index = head_counter;
00294         __TBB_TRY {
00295             size_t n_items  = (tail_counter-head_counter)/concurrent_queue_rep_base::n_queue;
00296             size_t index = head_counter/concurrent_queue_rep_base::n_queue & (base.my_rep->items_per_page-1);
00297             size_t end_in_first_page = (index+n_items<base.my_rep->items_per_page)?(index+n_items):base.my_rep->items_per_page;
00298 
00299             head_page = make_copy( base, srcp, index, end_in_first_page, g_index );
00300             page* cur_page = head_page;
00301 
00302             if( srcp != src.tail_page ) {
00303                 for( srcp = srcp->next; srcp!=src.tail_page; srcp=srcp->next ) {
00304                     cur_page->next = make_copy( base, srcp, 0, base.my_rep->items_per_page, g_index );
00305                     cur_page = cur_page->next;
00306                 }
00307 
00308                 __TBB_ASSERT( srcp==src.tail_page, NULL );
00309                 size_t last_index = tail_counter/concurrent_queue_rep_base::n_queue & (base.my_rep->items_per_page-1);
00310                 if( last_index==0 ) last_index = base.my_rep->items_per_page;
00311 
00312                 cur_page->next = make_copy( base, srcp, 0, last_index, g_index );
00313                 cur_page = cur_page->next;
00314             }
00315             tail_page = cur_page;
00316         } __TBB_CATCH (...) {
00317             invalidate_page_and_rethrow( g_index );
00318         }
00319     } else {
00320         head_page = tail_page = NULL;
00321     }
00322     return *this;
00323 }
00324 
00325 template<typename T>
00326 void micro_queue<T>::invalidate_page_and_rethrow( ticket k ) {
00327     // Append an invalid page at address 1 so that no more pushes are allowed.
00328     page* invalid_page = (page*)uintptr_t(1);
00329     {
00330         spin_mutex::scoped_lock lock( page_mutex );
00331         itt_store_word_with_release(tail_counter, k+concurrent_queue_rep_base::n_queue+1);
00332         page* q = tail_page;
00333         if( is_valid_page(q) )
00334             q->next = invalid_page;
00335         else
00336             head_page = invalid_page;
00337         tail_page = invalid_page;
00338     }
00339     __TBB_RETHROW();
00340 }
00341 
00342 template<typename T>
00343 concurrent_queue_rep_base::page* micro_queue<T>::make_copy( concurrent_queue_base_v3<T>& base, const concurrent_queue_rep_base::page* src_page, size_t begin_in_page, size_t end_in_page, ticket& g_index ) {
00344     concurrent_queue_page_allocator& pa = base;
00345     page* new_page = pa.allocate_page();
00346     new_page->next = NULL;
00347     new_page->mask = src_page->mask;
00348     for( ; begin_in_page!=end_in_page; ++begin_in_page, ++g_index )
00349         if( new_page->mask & uintptr_t(1)<<begin_in_page )
00350             copy_item( *new_page, begin_in_page, *src_page, begin_in_page );
00351     return new_page;
00352 }
00353 
00354 template<typename T>
00355 class micro_queue_pop_finalizer: no_copy {
00356     typedef concurrent_queue_rep_base::page page;
00357     ticket my_ticket;
00358     micro_queue<T>& my_queue;
00359     page* my_page; 
00360     concurrent_queue_page_allocator& allocator;
00361 public:
00362     micro_queue_pop_finalizer( micro_queue<T>& queue, concurrent_queue_base_v3<T>& b, ticket k, page* p ) :
00363         my_ticket(k), my_queue(queue), my_page(p), allocator(b)
00364     {}
00365     ~micro_queue_pop_finalizer() ;
00366 };
00367 
00368 template<typename T>
00369 micro_queue_pop_finalizer<T>::~micro_queue_pop_finalizer() {
00370     page* p = my_page;
00371     if( is_valid_page(p) ) {
00372         spin_mutex::scoped_lock lock( my_queue.page_mutex );
00373         page* q = p->next;
00374         my_queue.head_page = q;
00375         if( !is_valid_page(q) ) {
00376             my_queue.tail_page = NULL;
00377         }
00378     }
00379     itt_store_word_with_release(my_queue.head_counter, my_ticket);
00380     if( is_valid_page(p) ) {
00381         allocator.deallocate_page( p );
00382     }
00383 }
00384 
00385 #if _MSC_VER && !defined(__INTEL_COMPILER)
00386 #pragma warning( pop )
00387 #endif // warning 4146 is back
00388 
00389 template<typename T> class concurrent_queue_iterator_rep ;
00390 template<typename T> class concurrent_queue_iterator_base_v3;
00391 
00393 
00396 template<typename T>
00397 struct concurrent_queue_rep : public concurrent_queue_rep_base {
00398     micro_queue<T> array[n_queue];
00399 
00401     static size_t index( ticket k ) {
00402         return k*phi%n_queue;
00403     }
00404 
00405     micro_queue<T>& choose( ticket k ) {
00406         // The formula here approximates LRU in a cache-oblivious way.
00407         return array[index(k)];
00408     }
00409 };
00410 
00412 
00416 template<typename T>
00417 class concurrent_queue_base_v3: public concurrent_queue_page_allocator {
00419     concurrent_queue_rep<T>* my_rep;
00420 
00421     friend struct concurrent_queue_rep<T>;
00422     friend class micro_queue<T>;
00423     friend class concurrent_queue_iterator_rep<T>;
00424     friend class concurrent_queue_iterator_base_v3<T>;
00425 
00426 protected:
00427     typedef typename concurrent_queue_rep<T>::page page;
00428 
00429 private:
00430     typedef typename micro_queue<T>::padded_page padded_page;
00431 
00432     /* override */ virtual page *allocate_page() {
00433         concurrent_queue_rep<T>& r = *my_rep;
00434         size_t n = sizeof(padded_page) + (r.items_per_page-1)*sizeof(T);
00435         return reinterpret_cast<page*>(allocate_block ( n ));
00436     }
00437 
00438     /* override */ virtual void deallocate_page( concurrent_queue_rep_base::page *p ) {
00439         concurrent_queue_rep<T>& r = *my_rep;
00440         size_t n = sizeof(padded_page) + (r.items_per_page-1)*sizeof(T);
00441         deallocate_block( reinterpret_cast<void*>(p), n );
00442     }
00443 
00445     virtual void *allocate_block( size_t n ) = 0;
00446 
00448     virtual void deallocate_block( void *p, size_t n ) = 0;
00449 
00450 protected:
00451     concurrent_queue_base_v3();
00452 
00453     /* override */ virtual ~concurrent_queue_base_v3() {
00454 #if TBB_USE_ASSERT
00455         size_t nq = my_rep->n_queue;
00456         for( size_t i=0; i<nq; i++ )
00457             __TBB_ASSERT( my_rep->array[i].tail_page==NULL, "pages were not freed properly" );
00458 #endif /* TBB_USE_ASSERT */
00459         cache_aligned_allocator<concurrent_queue_rep<T> >().deallocate(my_rep,1);
00460     }
00461 
00463     void internal_push( const void* src ) {
00464         concurrent_queue_rep<T>& r = *my_rep;
00465         ticket k = r.tail_counter++;
00466         r.choose(k).push( src, k, *this );
00467     }
00468 
00470 
00471     bool internal_try_pop( void* dst ) ;
00472 
00474     size_t internal_size() const ;
00475 
00477     bool internal_empty() const ;
00478 
00480     /* note that the name may be misleading, but it remains so due to a historical accident. */
00481     void internal_finish_clear() ;
00482 
00484     void internal_throw_exception() const {
00485         throw_exception( eid_bad_alloc );
00486     }
00487 
00489     void assign( const concurrent_queue_base_v3& src ) ;
00490 };
00491 
00492 template<typename T>
00493 concurrent_queue_base_v3<T>::concurrent_queue_base_v3() {
00494     const size_t item_size = sizeof(T);
00495     my_rep = cache_aligned_allocator<concurrent_queue_rep<T> >().allocate(1);
00496     __TBB_ASSERT( (size_t)my_rep % NFS_GetLineSize()==0, "alignment error" );
00497     __TBB_ASSERT( (size_t)&my_rep->head_counter % NFS_GetLineSize()==0, "alignment error" );
00498     __TBB_ASSERT( (size_t)&my_rep->tail_counter % NFS_GetLineSize()==0, "alignment error" );
00499     __TBB_ASSERT( (size_t)&my_rep->array % NFS_GetLineSize()==0, "alignment error" );
00500     memset(my_rep,0,sizeof(concurrent_queue_rep<T>));
00501     my_rep->item_size = item_size;
00502     my_rep->items_per_page = item_size<=8 ? 32 :
00503                              item_size<=16 ? 16 : 
00504                              item_size<=32 ? 8 :
00505                              item_size<=64 ? 4 :
00506                              item_size<=128 ? 2 :
00507                              1;
00508 }
00509 
00510 template<typename T>
00511 bool concurrent_queue_base_v3<T>::internal_try_pop( void* dst ) {
00512     concurrent_queue_rep<T>& r = *my_rep;
00513     ticket k;
00514     do {
00515         k = r.head_counter;
00516         for(;;) {
00517             if( r.tail_counter<=k ) {
00518                 // Queue is empty 
00519                 return false;
00520             }
00521             // Queue had item with ticket k when we looked.  Attempt to get that item.
00522             ticket tk=k;
00523 #if defined(_MSC_VER) && defined(_Wp64)
00524     #pragma warning (push)
00525     #pragma warning (disable: 4267)
00526 #endif
00527             k = r.head_counter.compare_and_swap( tk+1, tk );
00528 #if defined(_MSC_VER) && defined(_Wp64)
00529     #pragma warning (pop)
00530 #endif
00531             if( k==tk )
00532                 break;
00533             // Another thread snatched the item, retry.
00534         }
00535     } while( !r.choose( k ).pop( dst, k, *this ) );
00536     return true;
00537 }
00538 
00539 template<typename T>
00540 size_t concurrent_queue_base_v3<T>::internal_size() const {
00541     concurrent_queue_rep<T>& r = *my_rep;
00542     __TBB_ASSERT( sizeof(ptrdiff_t)<=sizeof(size_t), NULL );
00543     ticket hc = r.head_counter;
00544     size_t nie = r.n_invalid_entries;
00545     ticket tc = r.tail_counter;
00546     __TBB_ASSERT( hc!=tc || !nie, NULL );
00547     ptrdiff_t sz = tc-hc-nie;
00548     return sz<0 ? 0 :  size_t(sz);
00549 }
00550 
00551 template<typename T>
00552 bool concurrent_queue_base_v3<T>::internal_empty() const {
00553     concurrent_queue_rep<T>& r = *my_rep;
00554     ticket tc = r.tail_counter;
00555     ticket hc = r.head_counter;
00556     // if tc!=r.tail_counter, the queue was not empty at some point between the two reads.
00557     return tc==r.tail_counter && tc==hc+r.n_invalid_entries ;
00558 }
00559 
00560 template<typename T>
00561 void concurrent_queue_base_v3<T>::internal_finish_clear() {
00562     concurrent_queue_rep<T>& r = *my_rep;
00563     size_t nq = r.n_queue;
00564     for( size_t i=0; i<nq; ++i ) {
00565         page* tp = r.array[i].tail_page;
00566         if( is_valid_page(tp) ) {
00567             __TBB_ASSERT( r.array[i].head_page==tp, "at most one page should remain" );
00568             deallocate_page( tp );
00569             r.array[i].tail_page = NULL;
00570         } else 
00571             __TBB_ASSERT( !is_valid_page(r.array[i].head_page), "head page pointer corrupt?" );
00572     }
00573 }
00574 
00575 template<typename T>
00576 void concurrent_queue_base_v3<T>::assign( const concurrent_queue_base_v3& src ) {
00577     concurrent_queue_rep<T>& r = *my_rep;
00578     r.items_per_page = src.my_rep->items_per_page;
00579 
00580     // copy concurrent_queue_rep.
00581     r.head_counter = src.my_rep->head_counter;
00582     r.tail_counter = src.my_rep->tail_counter;
00583     r.n_invalid_entries = src.my_rep->n_invalid_entries;
00584 
00585     // copy micro_queues
00586     for( size_t i = 0; i<r.n_queue; ++i )
00587         r.array[i].assign( src.my_rep->array[i], *this);
00588 
00589     __TBB_ASSERT( r.head_counter==src.my_rep->head_counter && r.tail_counter==src.my_rep->tail_counter, 
00590             "the source concurrent queue should not be concurrently modified." );
00591 }
00592 
00593 template<typename Container, typename Value> class concurrent_queue_iterator;
00594 
00595 template<typename T>
00596 class concurrent_queue_iterator_rep: no_assign {
00597     typedef typename micro_queue<T>::padded_page padded_page;
00598 public:
00599     ticket head_counter;
00600     const concurrent_queue_base_v3<T>& my_queue;
00601     typename concurrent_queue_base_v3<T>::page* array[concurrent_queue_rep<T>::n_queue];
00602     concurrent_queue_iterator_rep( const concurrent_queue_base_v3<T>& queue ) :
00603         head_counter(queue.my_rep->head_counter),
00604         my_queue(queue)
00605     {
00606         for( size_t k=0; k<concurrent_queue_rep<T>::n_queue; ++k )
00607             array[k] = queue.my_rep->array[k].head_page;
00608     }
00609 
00611     bool get_item( T*& item, size_t k ) ;
00612 };
00613 
00614 template<typename T>
00615 bool concurrent_queue_iterator_rep<T>::get_item( T*& item, size_t k ) {
00616     if( k==my_queue.my_rep->tail_counter ) {
00617         item = NULL;
00618         return true;
00619     } else {
00620         typename concurrent_queue_base_v3<T>::page* p = array[concurrent_queue_rep<T>::index(k)];
00621         __TBB_ASSERT(p,NULL);
00622         size_t i = k/concurrent_queue_rep<T>::n_queue & (my_queue.my_rep->items_per_page-1);
00623         item = &micro_queue<T>::get_ref(*p,i);
00624         return (p->mask & uintptr_t(1)<<i)!=0;
00625     }
00626 }
00627 
00629 
00630 template<typename Value>
00631 class concurrent_queue_iterator_base_v3 : no_assign {
00633 
00634     concurrent_queue_iterator_rep<Value>* my_rep;
00635 
00636     template<typename C, typename T, typename U>
00637     friend bool operator==( const concurrent_queue_iterator<C,T>& i, const concurrent_queue_iterator<C,U>& j );
00638 
00639     template<typename C, typename T, typename U>
00640     friend bool operator!=( const concurrent_queue_iterator<C,T>& i, const concurrent_queue_iterator<C,U>& j );
00641 protected:
00643     Value* my_item;
00644 
00646     concurrent_queue_iterator_base_v3() : my_rep(NULL), my_item(NULL) {
00647 #if __TBB_GCC_OPTIMIZER_ORDERING_BROKEN
00648         __asm__ __volatile__("": : :"memory");
00649 #endif
00650     }
00651 
00653     concurrent_queue_iterator_base_v3( const concurrent_queue_iterator_base_v3& i )
00654     : no_assign(), my_rep(NULL), my_item(NULL) {
00655         assign(i);
00656     }
00657 
00659     concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3<Value>& queue ) ;
00660 
00662     void assign( const concurrent_queue_iterator_base_v3<Value>& other ) ;
00663 
00665     void advance() ;
00666 
00668     ~concurrent_queue_iterator_base_v3() {
00669         cache_aligned_allocator<concurrent_queue_iterator_rep<Value> >().deallocate(my_rep, 1);
00670         my_rep = NULL;
00671     }
00672 };
00673 
00674 template<typename Value>
00675 concurrent_queue_iterator_base_v3<Value>::concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3<Value>& queue ) {
00676     my_rep = cache_aligned_allocator<concurrent_queue_iterator_rep<Value> >().allocate(1);
00677     new( my_rep ) concurrent_queue_iterator_rep<Value>(queue);
00678     size_t k = my_rep->head_counter;
00679     if( !my_rep->get_item(my_item, k) ) advance();
00680 }
00681 
00682 template<typename Value>
00683 void concurrent_queue_iterator_base_v3<Value>::assign( const concurrent_queue_iterator_base_v3<Value>& other ) {
00684     if( my_rep!=other.my_rep ) {
00685         if( my_rep ) {
00686             cache_aligned_allocator<concurrent_queue_iterator_rep<Value> >().deallocate(my_rep, 1);
00687             my_rep = NULL;
00688         }
00689         if( other.my_rep ) {
00690             my_rep = cache_aligned_allocator<concurrent_queue_iterator_rep<Value> >().allocate(1);
00691             new( my_rep ) concurrent_queue_iterator_rep<Value>( *other.my_rep );
00692         }
00693     }
00694     my_item = other.my_item;
00695 }
00696 
00697 template<typename Value>
00698 void concurrent_queue_iterator_base_v3<Value>::advance() {
00699     __TBB_ASSERT( my_item, "attempt to increment iterator past end of queue" );  
00700     size_t k = my_rep->head_counter;
00701     const concurrent_queue_base_v3<Value>& queue = my_rep->my_queue;
00702 #if TBB_USE_ASSERT
00703     Value* tmp;
00704     my_rep->get_item(tmp,k);
00705     __TBB_ASSERT( my_item==tmp, NULL );
00706 #endif /* TBB_USE_ASSERT */
00707     size_t i = k/concurrent_queue_rep<Value>::n_queue & (queue.my_rep->items_per_page-1);
00708     if( i==queue.my_rep->items_per_page-1 ) {
00709         typename concurrent_queue_base_v3<Value>::page*& root = my_rep->array[concurrent_queue_rep<Value>::index(k)];
00710         root = root->next;
00711     }
00712     // advance k
00713     my_rep->head_counter = ++k;
00714     if( !my_rep->get_item(my_item, k) ) advance();
00715 }
00716 
00718 
00719 template<typename T> struct tbb_remove_cv {typedef T type;};
00720 template<typename T> struct tbb_remove_cv<const T> {typedef T type;};
00721 template<typename T> struct tbb_remove_cv<volatile T> {typedef T type;};
00722 template<typename T> struct tbb_remove_cv<const volatile T> {typedef T type;};
00723 
00725 
00727 template<typename Container, typename Value>
00728 class concurrent_queue_iterator: public concurrent_queue_iterator_base_v3<typename tbb_remove_cv<Value>::type>,
00729         public std::iterator<std::forward_iterator_tag,Value> {
00730 #if !__TBB_TEMPLATE_FRIENDS_BROKEN
00731     template<typename T, class A>
00732     friend class ::tbb::strict_ppl::concurrent_queue;
00733 #else
00734 public: // workaround for MSVC
00735 #endif 
00737     concurrent_queue_iterator( const concurrent_queue_base_v3<Value>& queue ) :
00738         concurrent_queue_iterator_base_v3<typename tbb_remove_cv<Value>::type>(queue)
00739     {
00740     }
00741 
00742 public:
00743     concurrent_queue_iterator() {}
00744 
00745     concurrent_queue_iterator( const concurrent_queue_iterator<Container,typename Container::value_type>& other ) :
00746         concurrent_queue_iterator_base_v3<typename tbb_remove_cv<Value>::type>(other)
00747     {}
00748 
00750     concurrent_queue_iterator& operator=( const concurrent_queue_iterator& other ) {
00751         this->assign(other);
00752         return *this;
00753     }
00754 
00756     Value& operator*() const {
00757         return *static_cast<Value*>(this->my_item);
00758     }
00759 
00760     Value* operator->() const {return &operator*();}
00761 
00763     concurrent_queue_iterator& operator++() {
00764         this->advance();
00765         return *this;
00766     }
00767 
00769     Value* operator++(int) {
00770         Value* result = &operator*();
00771         operator++();
00772         return result;
00773     }
00774 }; // concurrent_queue_iterator
00775 
00776 
00777 template<typename C, typename T, typename U>
00778 bool operator==( const concurrent_queue_iterator<C,T>& i, const concurrent_queue_iterator<C,U>& j ) {
00779     return i.my_item==j.my_item;
00780 }
00781 
00782 template<typename C, typename T, typename U>
00783 bool operator!=( const concurrent_queue_iterator<C,T>& i, const concurrent_queue_iterator<C,U>& j ) {
00784     return i.my_item!=j.my_item;
00785 }
00786 
00787 } // namespace internal
00788 
00790 
00791 } // namespace strict_ppl
00792 
00794 namespace internal {
00795 
00796 class concurrent_queue_rep;
00797 class concurrent_queue_iterator_rep;
00798 class concurrent_queue_iterator_base_v3;
00799 template<typename Container, typename Value> class concurrent_queue_iterator;
00800 
00802 
00804 class concurrent_queue_base_v3: no_copy {
00806     concurrent_queue_rep* my_rep;
00807 
00808     friend class concurrent_queue_rep;
00809     friend struct micro_queue;
00810     friend class micro_queue_pop_finalizer;
00811     friend class concurrent_queue_iterator_rep;
00812     friend class concurrent_queue_iterator_base_v3;
00813 protected:
00815     struct page {
00816         page* next;
00817         uintptr_t mask; 
00818     };
00819 
00821     ptrdiff_t my_capacity;
00822    
00824     size_t items_per_page;
00825 
00827     size_t item_size;
00828 
00829 #if __TBB_GCC_3_3_PROTECTED_BROKEN
00830 public:
00831 #endif
00832     template<typename T>
00833     struct padded_page: page {
00835         padded_page(); 
00837         void operator=( const padded_page& );
00839         T last;
00840     };
00841 
00842 private:
00843     virtual void copy_item( page& dst, size_t index, const void* src ) = 0;
00844     virtual void assign_and_destroy_item( void* dst, page& src, size_t index ) = 0;
00845 protected:
00846     __TBB_EXPORTED_METHOD concurrent_queue_base_v3( size_t item_size );
00847     virtual __TBB_EXPORTED_METHOD ~concurrent_queue_base_v3();
00848 
00850     void __TBB_EXPORTED_METHOD internal_push( const void* src );
00851 
00853     void __TBB_EXPORTED_METHOD internal_pop( void* dst );
00854 
00856     bool __TBB_EXPORTED_METHOD internal_push_if_not_full( const void* src );
00857 
00859 
00860     bool __TBB_EXPORTED_METHOD internal_pop_if_present( void* dst );
00861 
00863     ptrdiff_t __TBB_EXPORTED_METHOD internal_size() const;
00864 
00866     bool __TBB_EXPORTED_METHOD internal_empty() const;
00867 
00869     void __TBB_EXPORTED_METHOD internal_set_capacity( ptrdiff_t capacity, size_t element_size );
00870 
00872     virtual page *allocate_page() = 0;
00873 
00875     virtual void deallocate_page( page *p ) = 0;
00876 
00878     /* note that the name may be misleading, but it remains so due to a historical accident. */
00879     void __TBB_EXPORTED_METHOD internal_finish_clear() ;
00880 
00882     void __TBB_EXPORTED_METHOD internal_throw_exception() const;
00883 
00885     void __TBB_EXPORTED_METHOD assign( const concurrent_queue_base_v3& src ) ;
00886 
00887 private:
00888     virtual void copy_page_item( page& dst, size_t dindex, const page& src, size_t sindex ) = 0;
00889 };
00890 
00892 
00893 class concurrent_queue_iterator_base_v3 {
00895 
00896     concurrent_queue_iterator_rep* my_rep;
00897 
00898     template<typename C, typename T, typename U>
00899     friend bool operator==( const concurrent_queue_iterator<C,T>& i, const concurrent_queue_iterator<C,U>& j );
00900 
00901     template<typename C, typename T, typename U>
00902     friend bool operator!=( const concurrent_queue_iterator<C,T>& i, const concurrent_queue_iterator<C,U>& j );
00903 
00904     void initialize( const concurrent_queue_base_v3& queue, size_t offset_of_data );
00905 protected:
00907     void* my_item;
00908 
00910     concurrent_queue_iterator_base_v3() : my_rep(NULL), my_item(NULL) {}
00911 
00913     concurrent_queue_iterator_base_v3( const concurrent_queue_iterator_base_v3& i ) : my_rep(NULL), my_item(NULL) {
00914         assign(i);
00915     }
00916 
00918 
00919     __TBB_EXPORTED_METHOD concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3& queue );
00920 
00922     __TBB_EXPORTED_METHOD concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3& queue, size_t offset_of_data );
00923 
00925     void __TBB_EXPORTED_METHOD assign( const concurrent_queue_iterator_base_v3& i );
00926 
00928     void __TBB_EXPORTED_METHOD advance();
00929 
00931     __TBB_EXPORTED_METHOD ~concurrent_queue_iterator_base_v3();
00932 };
00933 
00934 typedef concurrent_queue_iterator_base_v3 concurrent_queue_iterator_base;
00935 
00937 
00939 template<typename Container, typename Value>
00940 class concurrent_queue_iterator: public concurrent_queue_iterator_base,
00941         public std::iterator<std::forward_iterator_tag,Value> {
00942 
00943 #if !defined(_MSC_VER) || defined(__INTEL_COMPILER)
00944     template<typename T, class A>
00945     friend class ::tbb::concurrent_bounded_queue;
00946 
00947     template<typename T, class A>
00948     friend class ::tbb::deprecated::concurrent_queue;
00949 #else
00950 public: // workaround for MSVC
00951 #endif 
00953     concurrent_queue_iterator( const concurrent_queue_base_v3& queue ) :
00954         concurrent_queue_iterator_base_v3(queue,__TBB_offsetof(concurrent_queue_base_v3::padded_page<Value>,last))
00955     {
00956     }
00957 
00958 public:
00959     concurrent_queue_iterator() {}
00960 
00963     concurrent_queue_iterator( const concurrent_queue_iterator<Container,typename Container::value_type>& other ) :
00964         concurrent_queue_iterator_base_v3(other)
00965     {}
00966 
00968     concurrent_queue_iterator& operator=( const concurrent_queue_iterator& other ) {
00969         assign(other);
00970         return *this;
00971     }
00972 
00974     Value& operator*() const {
00975         return *static_cast<Value*>(my_item);
00976     }
00977 
00978     Value* operator->() const {return &operator*();}
00979 
00981     concurrent_queue_iterator& operator++() {
00982         advance();
00983         return *this;
00984     }
00985 
00987     Value* operator++(int) {
00988         Value* result = &operator*();
00989         operator++();
00990         return result;
00991     }
00992 }; // concurrent_queue_iterator
00993 
00994 
00995 template<typename C, typename T, typename U>
00996 bool operator==( const concurrent_queue_iterator<C,T>& i, const concurrent_queue_iterator<C,U>& j ) {
00997     return i.my_item==j.my_item;
00998 }
00999 
01000 template<typename C, typename T, typename U>
01001 bool operator!=( const concurrent_queue_iterator<C,T>& i, const concurrent_queue_iterator<C,U>& j ) {
01002     return i.my_item!=j.my_item;
01003 }
01004 
01005 } // namespace internal;
01006 
01008 
01009 } // namespace tbb
01010 
01011 #endif /* __TBB_concurrent_queue_internal_H */

Copyright © 2005-2011 Intel Corporation. All Rights Reserved.

Intel, Pentium, Intel Xeon, Itanium, Intel XScale and VTune are registered trademarks or trademarks of Intel Corporation or its subsidiaries in the United States and other countries.

* Other names and brands may be claimed as the property of others.