Commit 8dc5a6c6 authored by unknown's avatar unknown
Browse files

bug#16771 - ndb dd - fix leak of extents 


storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp:
  Add debug printouts
storage/ndb/src/kernel/blocks/dbtup/DbtupDiskAlloc.cpp:
  Add debug printouts
  Fix leak of extents
storage/ndb/src/kernel/blocks/pgman.cpp:
  Add debug printouts
parent d5e109ea

storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp

+2 −1
Original line number Diff line number Diff line
@@ -565,7 +565,6 @@ typedef Ptr<Fragoperrec> FragoperrecPtr; 
     * 

     */

    STATIC_CONST( SZ = EXTENT_SEARCH_MATRIX_SIZE );

    Uint32 m_extent_search_matrix[SZ]; // 4x4

    DLList<Extent_info>::Head m_free_extents[SZ];

    Uint32 m_total_extent_free_space_thresholds[EXTENT_SEARCH_MATRIX_ROWS];

    Uint32 m_page_free_bits_map[EXTENT_SEARCH_MATRIX_COLS];
@@ -593,6 +592,8 @@ typedef Ptr<Fragoperrec> FragoperrecPtr; 
    SLList<Extent_info, Extent_list_t>::Head m_extent_list;

  };

  

  void dump_disk_alloc(Disk_alloc_info&);



struct Fragrecord {

  Uint32 nextStartRange;

  Uint32 currentPageRange;

storage/ndb/src/kernel/blocks/dbtup/DbtupDiskAlloc.cpp

+155 −50
Original line number Diff line number Diff line
@@ -17,6 +17,112 @@ 
#define DBTUP_C

#include "Dbtup.hpp"



static

NdbOut&

operator<<(NdbOut& out, const Ptr<Dbtup::Page> & ptr)

{

  out << "[ Page: ptr.i: " << ptr.i 

      << " [ m_file_no: " << ptr.p->m_file_no

      << " m_page_no: " << ptr.p->m_page_no << "]"

      << " list_index: " << ptr.p->list_index 

      << " free_space: " << ptr.p->free_space

      << " uncommitted_used_space: " << ptr.p->uncommitted_used_space

      << " ]";

  return out;

}



static

NdbOut&

operator<<(NdbOut& out, const Ptr<Dbtup::Page_request> & ptr)

{

  out << "[ Page_request: ptr.i: " << ptr.i

      << " " << ptr.p->m_key

      << " m_estimated_free_space: " << ptr.p->m_estimated_free_space

      << " m_list_index: " << ptr.p->m_list_index

      << " m_frag_ptr_i: " << ptr.p->m_frag_ptr_i

      << " m_extent_info_ptr: " << ptr.p->m_extent_info_ptr

      << " m_ref_count: " << ptr.p->m_ref_count

      << " m_uncommitted_used_space: " << ptr.p->m_uncommitted_used_space

      << " ]";

  

  return out;

}



static

NdbOut&

operator<<(NdbOut& out, const Ptr<Dbtup::Extent_info> & ptr)

{

  out << "[ Extent_info: ptr.i " << ptr.i

      << " " << ptr.p->m_key

      << " m_first_page_no: " << ptr.p->m_first_page_no

      << " m_free_space: " << ptr.p->m_free_space

      << " m_free_matrix_pos: " << ptr.p->m_free_matrix_pos

      << " m_free_page_count: [";



  for(Uint32 i = 0; i<Dbtup::EXTENT_SEARCH_MATRIX_COLS; i++)

    out << " " << ptr.p->m_free_page_count[i];

  out << " ] ]";



  return out;

}



void 

Dbtup::dump_disk_alloc(Dbtup::Disk_alloc_info & alloc)

{

  ndbout_c("dirty pages");

  for(Uint32 i = 0; i<MAX_FREE_LIST; i++)

  {

    printf("  %d : ", i);

    Ptr<Page> ptr;

    ArrayPool<Page> *pool= (ArrayPool<Page>*)&m_global_page_pool;

    LocalDLList<Page> list(*pool, alloc.m_dirty_pages[i]);

    for(list.first(ptr); !ptr.isNull(); list.next(ptr))

    {

      ndbout << ptr << " ";

    }

    ndbout_c("");

  }

  ndbout_c("page requests");

  for(Uint32 i = 0; i<MAX_FREE_LIST; i++)

  {

    printf("  %d : ", i);

    Ptr<Page_request> ptr;

    LocalDLList<Page_request> list(c_page_request_pool, 

				   alloc.m_page_requests[i]);

    for(list.first(ptr); !ptr.isNull(); list.next(ptr))

    {

      ndbout << ptr << " ";

    }

    ndbout_c("");

  }



  ndbout_c("Extent matrix");

  for(Uint32 i = 0; i<alloc.SZ; i++)

  {

    printf("  %d : ", i);

    Ptr<Extent_info> ptr;

    LocalDLList<Extent_info> list(c_extent_pool, alloc.m_free_extents[i]);

    for(list.first(ptr); !ptr.isNull(); list.next(ptr))

    {

      ndbout << ptr << " ";

    }

    ndbout_c("");

  }



  if (alloc.m_curr_extent_info_ptr_i != RNIL)

  {

    Ptr<Extent_info> ptr;

    c_extent_pool.getPtr(ptr, alloc.m_curr_extent_info_ptr_i);

    ndbout << "current extent: " << ptr << endl;

  }

}



#if defined VM_TRACE || true

#define ddassert(x) do { if(unlikely(!(x))) { dump_disk_alloc(alloc); ndbrequire(false); } } while(0)

#else

#define ddassert(x)

#endif



Dbtup::Disk_alloc_info::Disk_alloc_info(const Tablerec* tabPtrP, 

					Uint32 extent_size)

{
@@ -60,19 +166,19 @@ Dbtup::Disk_alloc_info::find_extent(Uint32 sz) const 
   * Find the biggest available (with most free space)

   * Return position in matrix

   */

  Uint32 col = calc_page_free_bits(sz);

  Uint32 mask= EXTENT_SEARCH_MATRIX_COLS - 1;

  for(Uint32 i= 0; i<EXTENT_SEARCH_MATRIX_SIZE; i++)

  {

    // Check that it can cater for request

    if (m_extent_search_matrix[i] < sz)

    if (!m_free_extents[i].isEmpty())

    {

      i = (i + mask) & ~mask;

      continue;

      return i;

    }

    

    if (!m_free_extents[i].isEmpty())

    if ((i & mask) >= col)

    {

      return i;

      i = (i & ~mask) + mask;

    }

  }
@@ -93,12 +199,6 @@ Dbtup::Disk_alloc_info::calc_extent_pos(const Extent_info* extP) const 
   *     absolutly last

   */

  {    

    

    printf("free space %d free_page_thresholds ", free);

    for(Uint32 i = 0; i<EXTENT_SEARCH_MATRIX_ROWS; i++)

      printf("%d ", m_total_extent_free_space_thresholds[i]);

    ndbout_c("");

    

    const Uint32 *arr= m_total_extent_free_space_thresholds;

    for(; free < * arr++; row++)

      assert(row < EXTENT_SEARCH_MATRIX_ROWS);
@@ -123,11 +223,6 @@ Dbtup::Disk_alloc_info::calc_extent_pos(const Extent_info* extP) const 
   */

  Uint32 pos= (row * (mask + 1)) + (col & mask);

  

  printf("free space %d free_page_count ", free);

  for(Uint32 i = 0; i<EXTENT_SEARCH_MATRIX_COLS; i++)

    printf("%d ", extP->m_free_page_count[i]);

  ndbout_c(" -> row: %d col: %d -> pos= %d", row, col, pos);



  assert(pos < EXTENT_SEARCH_MATRIX_SIZE);

  return pos;

}
@@ -237,7 +332,9 @@ Dbtup::disk_page_prealloc(Signal* signal, 
       *   and since it couldn't accomadate the request

       *   we put it on the free list

       */

      alloc.m_curr_extent_info_ptr_i = RNIL;

      Uint32 pos= alloc.calc_extent_pos(ext.p);

      ext.p->m_free_matrix_pos = pos;

      LocalDLList<Extent_info> list(c_extent_pool, alloc.m_free_extents[pos]);

      list.add(ext);

    }
@@ -290,7 +387,14 @@ Dbtup::disk_page_prealloc(Signal* signal, 
    alloc.m_curr_extent_info_ptr_i= ext.i;

    ext.p->m_free_matrix_pos= RNIL;

    pageBits= tsman.alloc_page_from_extent(&ext.p->m_key, bits);

    ndbassert(pageBits >= 0);

#ifdef VM_TRACE

    ddassert(pageBits >= 0);

#else

    if (unlikely(pageBits < 0))

    {

      return -AllocExtentReq::NoExtentAvailable;

    }

#endif

  }

  

  /**
@@ -305,18 +409,18 @@ Dbtup::disk_page_prealloc(Signal* signal, 
   */

  Uint32 size= alloc.calc_page_free_space((Uint32)pageBits);

  

  ndbassert(size >= sz);

  ddassert(size >= sz);

  Uint32 new_size = size - sz;   // Subtract alloc rec

  req.p->m_estimated_free_space= new_size; // Store on page request



  Uint32 newPageBits= alloc.calc_page_free_bits(new_size);

  if (newPageBits != (Uint32)pageBits)

  {

    ndbassert(ext.p->m_free_page_count[pageBits] > 0);

    ddassert(ext.p->m_free_page_count[pageBits] > 0);

    ext.p->m_free_page_count[pageBits]--;

    ext.p->m_free_page_count[newPageBits]++;

  }

  ndbassert(ext.p->m_free_space >= sz);

  ddassert(ext.p->m_free_space >= sz);

  ext.p->m_free_space -= sz;

  

  // And put page request in correct free list
@@ -365,13 +469,13 @@ Dbtup::disk_page_prealloc_dirty_page(Disk_alloc_info & alloc, 
				     Ptr<Page> pagePtr, 

				     Uint32 old_idx, Uint32 sz)

{

  ndbassert(pagePtr.p->list_index == old_idx);

  ddassert(pagePtr.p->list_index == old_idx);



  Uint32 free= pagePtr.p->free_space;

  Uint32 used= pagePtr.p->uncommitted_used_space + sz;

  Uint32 ext= pagePtr.p->m_extent_info_ptr;

  

  ndbassert(free >= used);

  ddassert(free >= used);

  Ptr<Extent_info> extentPtr;

  c_extent_pool.getPtr(extentPtr, ext);
@@ -385,14 +489,14 @@ Dbtup::disk_page_prealloc_dirty_page(Disk_alloc_info & alloc, 
    old_list.remove(pagePtr);

    new_list.add(pagePtr);



    ndbassert(extentPtr.p->m_free_page_count[old_idx]);

    ddassert(extentPtr.p->m_free_page_count[old_idx]);

    extentPtr.p->m_free_page_count[old_idx]--;

    extentPtr.p->m_free_page_count[new_idx]++;

    pagePtr.p->list_index= new_idx;  

  }



  pagePtr.p->uncommitted_used_space = used;

  ndbassert(extentPtr.p->m_free_space >= sz);

  ddassert(extentPtr.p->m_free_space >= sz);

  extentPtr.p->m_free_space -= sz;

  Uint32 old_pos= extentPtr.p->m_free_matrix_pos;

  if (old_pos != RNIL) // Current extent
@@ -417,7 +521,7 @@ Dbtup::disk_page_prealloc_transit_page(Disk_alloc_info& alloc, 
				       Ptr<Page_request> req, 

				       Uint32 old_idx, Uint32 sz)

{

  ndbassert(req.p->m_list_index == old_idx);

  ddassert(req.p->m_list_index == old_idx);



  Uint32 free= req.p->m_estimated_free_space;

  Uint32 used= req.p->m_uncommitted_used_space + sz;
@@ -426,7 +530,7 @@ Dbtup::disk_page_prealloc_transit_page(Disk_alloc_info& alloc, 
  Ptr<Extent_info> extentPtr;

  c_extent_pool.getPtr(extentPtr, ext);



  ndbassert(free >= sz);

  ddassert(free >= sz);

  Uint32 new_idx= alloc.calc_page_free_bits(free - sz);

  

  if (old_idx != new_idx)
@@ -437,7 +541,7 @@ Dbtup::disk_page_prealloc_transit_page(Disk_alloc_info& alloc, 
    old_list.remove(req);

    new_list.add(req);



    ndbassert(extentPtr.p->m_free_page_count[old_idx]);

    ddassert(extentPtr.p->m_free_page_count[old_idx]);

    extentPtr.p->m_free_page_count[old_idx]--;

    extentPtr.p->m_free_page_count[new_idx]++;

    req.p->m_list_index= new_idx;
@@ -445,7 +549,7 @@ Dbtup::disk_page_prealloc_transit_page(Disk_alloc_info& alloc, 


  req.p->m_uncommitted_used_space = used;

  req.p->m_estimated_free_space = free - sz;

  ndbassert(extentPtr.p->m_free_space >= sz);

  ddassert(extentPtr.p->m_free_space >= sz);

  extentPtr.p->m_free_space -= sz;

  Uint32 old_pos= extentPtr.p->m_free_matrix_pos;

  if (old_pos != RNIL) // Current extent
@@ -551,11 +655,11 @@ Dbtup::disk_page_prealloc_callback_common(Signal* signal, 
   * 3) register callback in pgman (unmap callback)

   * 4) inform pgman about current users

   */

  ndbassert((page->list_index & 0x8000) == 0x8000);

  ndbassert(page->m_extent_info_ptr == req.p->m_extent_info_ptr);

  ndbassert(page->m_page_no == req.p->m_key.m_page_no);

  ndbassert(page->m_file_no == req.p->m_key.m_file_no);

  Disk_alloc_info& alloc= fragPtr.p->m_disk_alloc_info;

  ddassert((page->list_index & 0x8000) == 0x8000);

  ddassert(page->m_extent_info_ptr == req.p->m_extent_info_ptr);

  ddassert(page->m_page_no == req.p->m_key.m_page_no);

  ddassert(page->m_file_no == req.p->m_key.m_file_no);

  

  Uint32 old_idx = req.p->m_list_index;

  Uint32 free= req.p->m_estimated_free_space;
@@ -564,9 +668,9 @@ Dbtup::disk_page_prealloc_callback_common(Signal* signal, 
  Uint32 real_free = page->free_space;

  Uint32 real_used = used + page->uncommitted_used_space;

 

  ndbassert(real_free >= free);

  ndbassert(real_free >= real_used);

  ndbassert(alloc.calc_page_free_bits(free) == old_idx);

  ddassert(real_free >= free);

  ddassert(real_free >= real_used);

  ddassert(alloc.calc_page_free_bits(free) == old_idx);

  Uint32 new_idx= alloc.calc_page_free_bits(real_free - real_used);

  

  /**
@@ -587,7 +691,7 @@ Dbtup::disk_page_prealloc_callback_common(Signal* signal, 
    

    if (old_idx != new_idx)

    {

      ndbassert(extentPtr.p->m_free_page_count[old_idx]);

      ddassert(extentPtr.p->m_free_page_count[old_idx]);

      extentPtr.p->m_free_page_count[old_idx]--;

      extentPtr.p->m_free_page_count[new_idx]++;

    }
@@ -721,13 +825,14 @@ Dbtup::disk_page_alloc(Signal* signal, 
		       Local_key* key, PagePtr pagePtr, Uint32 gci)

{

  Uint32 logfile_group_id= fragPtrP->m_logfile_group_id;

  Disk_alloc_info& alloc= fragPtrP->m_disk_alloc_info;



  Uint64 lsn;

  Uint32 old_free = pagePtr.p->free_space;

  Uint32 old_bits= fragPtrP->m_disk_alloc_info.calc_page_free_bits(old_free);

  Uint32 old_bits= alloc.calc_page_free_bits(old_free);

  if (tabPtrP->m_attributes[DD].m_no_of_varsize == 0)

  {

    ndbassert(pagePtr.p->uncommitted_used_space > 0);

    ddassert(pagePtr.p->uncommitted_used_space > 0);

    pagePtr.p->uncommitted_used_space--;

    key->m_page_idx= ((Fix_page*)pagePtr.p)->alloc_record();

    lsn= disk_page_undo_alloc(pagePtr.p, key, 1, gci, logfile_group_id);
@@ -735,7 +840,7 @@ Dbtup::disk_page_alloc(Signal* signal, 
  else

  {

    Uint32 sz= key->m_page_idx;

    ndbassert(pagePtr.p->uncommitted_used_space >= sz);

    ddassert(pagePtr.p->uncommitted_used_space >= sz);

    pagePtr.p->uncommitted_used_space -= sz;

    key->m_page_idx= ((Var_page*)pagePtr.p)->

      alloc_record(sz, (Var_page*)ctemp_page, 0);
@@ -744,7 +849,7 @@ Dbtup::disk_page_alloc(Signal* signal, 
  }



  Uint32 new_free = pagePtr.p->free_space;

  Uint32 new_bits= fragPtrP->m_disk_alloc_info.calc_page_free_bits(new_free);

  Uint32 new_bits= alloc.calc_page_free_bits(new_free);

  

  if (old_bits != new_bits)

  {
@@ -806,20 +911,20 @@ Dbtup::disk_page_free(Signal *signal, 


  Uint32 ext = pagePtr.p->m_extent_info_ptr;

  Uint32 used = pagePtr.p->uncommitted_used_space;

  ndbassert(old_free >= used);

  ndbassert(new_free >= used);

  ndbassert(new_free >= old_free);

  ddassert(old_free >= used);

  ddassert(new_free >= used);

  ddassert(new_free >= old_free);

  page_idx = pagePtr.p->list_index;

  Uint32 old_idx = page_idx & 0x7FFF;

  Uint32 new_idx = alloc.calc_page_free_bits(new_free - used);

  ndbassert(alloc.calc_page_free_bits(old_free - used) == old_idx);

  ddassert(alloc.calc_page_free_bits(old_free - used) == old_idx);



  Ptr<Extent_info> extentPtr;

  c_extent_pool.getPtr(extentPtr, ext);



  if (old_idx != new_idx)

  {

    ndbassert(extentPtr.p->m_free_page_count[old_idx]);

    ddassert(extentPtr.p->m_free_page_count[old_idx]);

    extentPtr.p->m_free_page_count[old_idx]--;

    extentPtr.p->m_free_page_count[new_idx]++;
@@ -915,16 +1020,16 @@ Dbtup::disk_page_abort_prealloc_callback_1(Signal* signal, 
  Uint32 ext = pagePtr.p->m_extent_info_ptr;



  Uint32 old_idx = page_idx & 0x7FFF;

  ndbassert(free >= used);

  ndbassert(used >= sz);

  ndbassert(alloc.calc_page_free_bits(free - used) == old_idx);

  ddassert(free >= used);

  ddassert(used >= sz);

  ddassert(alloc.calc_page_free_bits(free - used) == old_idx);

  Uint32 new_idx = alloc.calc_page_free_bits(free - used + sz);



  Ptr<Extent_info> extentPtr;

  c_extent_pool.getPtr(extentPtr, ext);

  if (old_idx != new_idx)

  {

    ndbassert(extentPtr.p->m_free_page_count[old_idx]);

    ddassert(extentPtr.p->m_free_page_count[old_idx]);

    extentPtr.p->m_free_page_count[old_idx]--;

    extentPtr.p->m_free_page_count[new_idx]++;

storage/ndb/src/kernel/blocks/pgman.cpp

+20 −0
Original line number Diff line number Diff line
@@ -40,6 +40,12 @@ 
#define dbg(x)

#endif



#if 1

#define DBG_LCP(x)

#else

#define DBG_LCP(x) ndbout << x

#endif



Pgman::Pgman(const Configuration & conf) :

  SimulatedBlock(PGMAN, conf),

  m_file_map(m_data_buffer_pool),
@@ -1083,6 +1089,7 @@ Pgman::execLCP_FRAG_ORD(Signal* signal) 
  LcpFragOrd* ord = (LcpFragOrd*)signal->getDataPtr();

  ndbrequire(ord->lcpId >= m_last_lcp_complete + 1 || m_last_lcp_complete == 0);

  m_last_lcp = ord->lcpId;

  DBG_LCP("execLCP_FRAG_ORD" << endl);



  ndbrequire(!m_lcp_outstanding);

  ndbrequire(m_lcp_copy_page_free);
@@ -1104,6 +1111,8 @@ Pgman::execEND_LCP_REQ(Signal* signal) 
  EndLcpReq* req = (EndLcpReq*)signal->getDataPtr();

  m_end_lcp_req = *req;



  DBG_LCP("execEND_LCP_REQ" << endl);



#ifdef VM_TRACE

  debugOut

    << "PGMAN: execEND_LCP_REQ"
@@ -1117,6 +1126,7 @@ Pgman::execEND_LCP_REQ(Signal* signal) 
    ndbrequire(! m_lcp_loop_on);

    signal->theData[0] = m_end_lcp_req.senderData;

    sendSignal(m_end_lcp_req.senderRef, GSN_END_LCP_CONF, signal, 1, JBB);

    DBG_LCP("GSN_END_LCP_CONF" << endl);

  }



  m_last_lcp_complete = m_last_lcp;
@@ -1149,6 +1159,8 @@ Pgman::process_lcp(Signal* signal) 
      Ptr<Page_entry>& ptr = iter.curr;

      Uint16 state = ptr.p->m_state;



      DBG_LCP("PROCESS LCP: " << ptr);

      

      if (ptr.p->m_last_lcp < m_last_lcp &&

          (state & Page_entry::DIRTY))

      {
@@ -1159,6 +1171,7 @@ Pgman::process_lcp(Signal* signal) 
        }

        if (state & Page_entry::BUSY)

        {

	  DBG_LCP(" BUSY" << endl);

          break;  // wait for it

        }

        if (state & Page_entry::LOCKED)
@@ -1169,6 +1182,7 @@ Pgman::process_lcp(Signal* signal) 
           */

          if (!m_lcp_copy_page_free)

          {

	    DBG_LCP(" !m_lcp_copy_page_free" << endl);

            break;

          }

          m_lcp_copy_page_free = false;
@@ -1183,10 +1197,12 @@ Pgman::process_lcp(Signal* signal) 
        }

        else if (state & Page_entry::PAGEOUT)

        {

	  DBG_LCP(" PAGEOUT -> state |= LCP" << endl);

          set_page_state(ptr, state | Page_entry::LCP);

        }

        else

        {

	  DBG_LCP(" pageout()" << endl);

          ptr.p->m_state |= Page_entry::LCP;

          pageout(signal, ptr);

        }
@@ -1205,11 +1221,15 @@ Pgman::process_lcp(Signal* signal) 
    {

      signal->theData[0] = m_end_lcp_req.senderData;

      sendSignal(m_end_lcp_req.senderRef, GSN_END_LCP_CONF, signal, 1, JBB);

      DBG_LCP("GSN_END_LCP_CONF" << endl);

    }

    DBG_LCP(" -- RETURN FALSE" << endl);

    m_last_lcp_complete = m_last_lcp;

    m_lcp_curr_bucket = ~(Uint32)0;

    return false;

  }



  DBG_LCP(" -- RETURN TRUE" << endl);

  return true;

}