Commit f4b56000 authored by unknown's avatar unknown
Browse files

WL#2347 - Load independent heartbeats 

Reset missed heartbeat count on receipt of signal from node.

This fixes a bug where that under high network load, the heartbeat packets could be
delayed, causing the appearance of node failure (due to lost heartbeats).


ndb/include/kernel/NodeInfo.hpp:
  Add m_heartbeat_cnt to track missed heartbeats
ndb/include/transporter/TransporterCallback.hpp:
  add prototype for transporter_recv_from()
  
  Called on receipt from a node.
ndb/src/common/transporter/TransporterRegistry.cpp:
  Add calls to transporter_receive_from when data is received (before unpack)
ndb/src/kernel/blocks/qmgr/Qmgr.hpp:
  remove NodeRec::alarmCount. missed heartbeat count now kept in NodeInfo
ndb/src/kernel/blocks/qmgr/QmgrMain.cpp:
  Use NodeInfo::m_heartbeat_cnt for missed heartbeat count
ndb/src/kernel/vm/TransporterCallback.cpp:
  add transporter_recv_from(), which is called on receipt of signals.
  It resets missed heartbeat count for that node.
ndb/src/ndbapi/ClusterMgr.cpp:
  Use NodeInfo::m_heartbeat_cnt for missed heartbeat count
ndb/src/ndbapi/ClusterMgr.hpp:
  Use NodeInfo::m_heartbeat_cnt instead of ClusterMgr::Node::hbSent for missed
  heartbeat count.
  
  We now use the same storage for API and Kernel heartbeats.
  
  Add ClusterMgr::hb_received(nodeId) to reset hbSent (as if we received a heartbeat,
  but callable from elsewhere - e.g. when signal received)
ndb/src/ndbapi/TransporterFacade.cpp:
  Implement transporter_recv_from for ndbapi - which resets hbSent
ndb/src/ndbapi/TransporterFacade.hpp:
  Add hb_received(nodeId)
parent ef45f4b6

ndb/include/kernel/NodeInfo.hpp

+2 −0
Original line number Diff line number Diff line
@@ -41,6 +41,7 @@ public: 
  Uint32 m_type;          ///< Node type

  Uint32 m_connectCount;  ///< No of times connected

  bool   m_connected;     ///< Node is connected

  Uint32 m_heartbeat_cnt; ///< Missed heartbeats

  

  friend NdbOut & operator<<(NdbOut&, const NodeInfo&); 

};
@@ -52,6 +53,7 @@ NodeInfo::NodeInfo(){ 
  m_signalVersion = 0;

  m_type = INVALID;

  m_connectCount = 0;

  m_heartbeat_cnt= 0;

}



inline

ndb/include/transporter/TransporterCallback.hpp

+3 −0
Original line number Diff line number Diff line
@@ -342,4 +342,7 @@ enum TransporterError { 
void 

reportError(void * callbackObj, NodeId nodeId, TransporterError errorCode); 



void

transporter_recv_from(void* callbackObj, NodeId node);

 

#endif

ndb/src/common/transporter/TransporterRegistry.cpp

+4 −0
Original line number Diff line number Diff line
@@ -918,6 +918,7 @@ TransporterRegistry::performReceive() 
      NodeId remoteNodeId;

      Uint32 * readPtr;

      Uint32 sz = theOSEReceiver->getReceiveData(&remoteNodeId, &readPtr);

      transporter_recv_from(callbackObj, remoteNodeId);

      Uint32 szUsed = unpack(readPtr,

			     sz,

			     remoteNodeId,
@@ -953,6 +954,7 @@ TransporterRegistry::performReceive() 
	  {

	    Uint32 * ptr;

	    Uint32 sz = t->getReceiveData(&ptr);

	    transporter_recv_from(callbackObj, nodeId);

	    Uint32 szUsed = unpack(ptr, sz, nodeId, ioStates[nodeId]);

	    t->updateReceiveDataPtr(szUsed);

          }
@@ -976,6 +978,7 @@ TransporterRegistry::performReceive() 
      {

	Uint32 * readPtr, * eodPtr;

	t->getReceivePtr(&readPtr, &eodPtr);

	transporter_recv_from(callbackObj, nodeId);

	Uint32 *newPtr = unpack(readPtr, eodPtr, nodeId, ioStates[nodeId]);

	t->updateReceivePtr(newPtr);

      }
@@ -993,6 +996,7 @@ TransporterRegistry::performReceive() 
      {

	Uint32 * readPtr, * eodPtr;

	t->getReceivePtr(&readPtr, &eodPtr);

	transporter_recv_from(callbackObj, nodeId);

	Uint32 *newPtr = unpack(readPtr, eodPtr, nodeId, ioStates[nodeId]);

	t->updateReceivePtr(newPtr);

      }

ndb/src/kernel/blocks/qmgr/Qmgr.hpp

+1 −2
Original line number Diff line number Diff line
@@ -118,7 +118,6 @@ public: 
  struct NodeRec {

    UintR ndynamicId;

    Phase phase;

    UintR alarmCount;



    QmgrState sendPrepFailReqStatus;

    QmgrState sendCommitFailReqStatus;

ndb/src/kernel/blocks/qmgr/QmgrMain.cpp

+27 −26
Original line number Diff line number Diff line
@@ -66,7 +66,7 @@ void Qmgr::execCM_HEARTBEAT(Signal* signal) 
  jamEntry();

  hbNodePtr.i = signal->theData[0];

  ptrCheckGuard(hbNodePtr, MAX_NDB_NODES, nodeRec);

  hbNodePtr.p->alarmCount = 0;

  setNodeInfo(hbNodePtr.i).m_heartbeat_cnt= 0;

  return;

}//Qmgr::execCM_HEARTBEAT()
@@ -1040,7 +1040,7 @@ void Qmgr::execCM_ADD(Signal* signal) 
    jam();

    ndbrequire(addNodePtr.p->phase == ZSTARTING);

    addNodePtr.p->phase = ZRUNNING;

    addNodePtr.p->alarmCount = 0;

    setNodeInfo(addNodePtr.i).m_heartbeat_cnt= 0;

    c_clusterNodes.set(addNodePtr.i);

    findNeighbours(signal);
@@ -1078,7 +1078,7 @@ Qmgr::joinedCluster(Signal* signal, NodeRecPtr nodePtr){ 
   * NODES IN THE CLUSTER.

   */

  nodePtr.p->phase = ZRUNNING;

  nodePtr.p->alarmCount = 0;

  setNodeInfo(nodePtr.i).m_heartbeat_cnt= 0;

  findNeighbours(signal);

  c_clusterNodes.set(nodePtr.i);

  c_start.reset();
@@ -1299,7 +1299,7 @@ void Qmgr::findNeighbours(Signal* signal) 
       *---------------------------------------------------------------------*/

      fnNodePtr.i = cneighbourl;

      ptrCheckGuard(fnNodePtr, MAX_NDB_NODES, nodeRec);

      fnNodePtr.p->alarmCount = 0;

      setNodeInfo(fnNodePtr.i).m_heartbeat_cnt= 0;

    }//if

  }//if
@@ -1348,7 +1348,7 @@ void Qmgr::initData(Signal* signal) 
      nodePtr.p->phase = ZAPI_INACTIVE;

    }



    nodePtr.p->alarmCount = 0;

    setNodeInfo(nodePtr.i).m_heartbeat_cnt= 0;

    nodePtr.p->sendPrepFailReqStatus = Q_NOT_ACTIVE;

    nodePtr.p->sendCommitFailReqStatus = Q_NOT_ACTIVE;

    nodePtr.p->sendPresToStatus = Q_NOT_ACTIVE;
@@ -1550,18 +1550,18 @@ void Qmgr::checkHeartbeat(Signal* signal) 
  }//if

  ptrCheckGuard(nodePtr, MAX_NDB_NODES, nodeRec);

  

  nodePtr.p->alarmCount ++;

  setNodeInfo(nodePtr.i).m_heartbeat_cnt++;

  ndbrequire(nodePtr.p->phase == ZRUNNING);

  ndbrequire(getNodeInfo(nodePtr.i).m_type == NodeInfo::DB);



  if(nodePtr.p->alarmCount > 2){

  if(getNodeInfo(nodePtr.i).m_heartbeat_cnt > 2){

    signal->theData[0] = NDB_LE_MissedHeartbeat;

    signal->theData[1] = nodePtr.i;

    signal->theData[2] = nodePtr.p->alarmCount - 1;

    signal->theData[2] = getNodeInfo(nodePtr.i).m_heartbeat_cnt - 1;

    sendSignal(CMVMI_REF, GSN_EVENT_REP, signal, 3, JBB);

  }



  if (nodePtr.p->alarmCount > 4) {

  if (getNodeInfo(nodePtr.i).m_heartbeat_cnt > 4) {

    jam();

    /**----------------------------------------------------------------------

     * OUR LEFT NEIGHBOUR HAVE KEPT QUIET FOR THREE CONSECUTIVE HEARTBEAT
@@ -1593,16 +1593,16 @@ void Qmgr::apiHbHandlingLab(Signal* signal) 


    if (TnodePtr.p->phase == ZAPI_ACTIVE){

      jam();

      TnodePtr.p->alarmCount ++;

      setNodeInfo(TnodePtr.i).m_heartbeat_cnt++;

      

      if(TnodePtr.p->alarmCount > 2){

      if(getNodeInfo(TnodePtr.i).m_heartbeat_cnt > 2){

	signal->theData[0] = NDB_LE_MissedHeartbeat;

	signal->theData[1] = nodeId;

	signal->theData[2] = TnodePtr.p->alarmCount - 1;

	signal->theData[2] = getNodeInfo(TnodePtr.i).m_heartbeat_cnt - 1;

	sendSignal(CMVMI_REF, GSN_EVENT_REP, signal, 3, JBB);

      }

      

      if (TnodePtr.p->alarmCount > 4) {

      if (getNodeInfo(TnodePtr.i).m_heartbeat_cnt > 4) {

        jam();

	/*------------------------------------------------------------------*/

	/* THE API NODE HAS NOT SENT ANY HEARTBEAT FOR THREE SECONDS.
@@ -1634,16 +1634,17 @@ void Qmgr::checkStartInterface(Signal* signal) 
    ptrAss(nodePtr, nodeRec);

    if (nodePtr.p->phase == ZFAIL_CLOSING) {

      jam();

      nodePtr.p->alarmCount = nodePtr.p->alarmCount + 1;

      setNodeInfo(nodePtr.i).m_heartbeat_cnt++;

      if (c_connectedNodes.get(nodePtr.i)){

        jam();

	/*-------------------------------------------------------------------*/

	// We need to ensure that the connection is not restored until it has 

	// been disconnected for at least three seconds.

	/*-------------------------------------------------------------------*/

        nodePtr.p->alarmCount = 0;

        setNodeInfo(nodePtr.i).m_heartbeat_cnt= 0;

      }//if

      if ((nodePtr.p->alarmCount > 3) && (nodePtr.p->failState == NORMAL)) {

      if ((getNodeInfo(nodePtr.i).m_heartbeat_cnt > 3)

	  && (nodePtr.p->failState == NORMAL)) {

	/**------------------------------------------------------------------

	 * WE HAVE DISCONNECTED THREE SECONDS AGO. WE ARE NOW READY TO 

	 * CONNECT AGAIN AND ACCEPT NEW REGISTRATIONS FROM THIS NODE.
@@ -1659,18 +1660,18 @@ void Qmgr::checkStartInterface(Signal* signal) 
          nodePtr.p->phase = ZINIT;

        }//if



        nodePtr.p->alarmCount = 0;

        setNodeInfo(nodePtr.i).m_heartbeat_cnt= 0;

        signal->theData[0] = 0;

        signal->theData[1] = nodePtr.i;

        sendSignal(CMVMI_REF, GSN_OPEN_COMREQ, signal, 2, JBA);

      } else {

	if(((nodePtr.p->alarmCount + 1) % 60) == 0){

	if(((getNodeInfo(nodePtr.i).m_heartbeat_cnt + 1) % 60) == 0){

	  char buf[100];

	  BaseString::snprintf(buf, sizeof(buf), 

		   "Failure handling of node %d has not completed in %d min."

		   " - state = %d",

		   nodePtr.i, 

		   (nodePtr.p->alarmCount + 1)/60,

		   (getNodeInfo(nodePtr.i).m_heartbeat_cnt + 1)/60,

		   nodePtr.p->failState);

	  warningEvent(buf);

	}
@@ -1718,7 +1719,7 @@ void Qmgr::sendApiFailReq(Signal* signal, Uint16 failedNodeNo) 
   * WE ONLY NEED TO SET PARAMETERS TO ENABLE A NEW CONNECTION IN A FEW 

   * SECONDS. 

   *-------------------------------------------------------------------------*/

  failedNodePtr.p->alarmCount = 0;

  setNodeInfo(failedNodePtr.i).m_heartbeat_cnt= 0;



  CloseComReqConf * const closeCom = (CloseComReqConf *)&signal->theData[0];
@@ -1871,7 +1872,7 @@ void Qmgr::node_failed(Signal* signal, Uint16 aFailedNode) 
      /*---------------------------------------------------------------------*/

      failedNodePtr.p->failState = NORMAL;

      failedNodePtr.p->phase = ZFAIL_CLOSING;

      failedNodePtr.p->alarmCount = 0;

      setNodeInfo(failedNodePtr.i).m_heartbeat_cnt= 0;



      CloseComReqConf * const closeCom = 

	(CloseComReqConf *)&signal->theData[0];
@@ -1966,7 +1967,7 @@ void Qmgr::execAPI_REGREQ(Signal* signal) 


  setNodeInfo(apiNodePtr.i).m_version = version;



  apiNodePtr.p->alarmCount = 0;

  setNodeInfo(apiNodePtr.i).m_heartbeat_cnt= 0;



  ApiRegConf * const apiRegConf = (ApiRegConf *)&signal->theData[0];

  apiRegConf->qmgrRef = reference();
@@ -2484,7 +2485,7 @@ void Qmgr::execCOMMIT_FAILREQ(Signal* signal) 
      ptrCheckGuard(nodePtr, MAX_NDB_NODES, nodeRec);

      nodePtr.p->phase = ZFAIL_CLOSING;

      nodePtr.p->failState = WAITING_FOR_NDB_FAILCONF;

      nodePtr.p->alarmCount = 0;

      setNodeInfo(nodePtr.i).m_heartbeat_cnt= 0;

      c_clusterNodes.clear(nodePtr.i);

    }//for

    /*----------------------------------------------------------------------*/
@@ -2742,7 +2743,7 @@ void Qmgr::failReport(Signal* signal, 
    failedNodePtr.p->sendPrepFailReqStatus = Q_NOT_ACTIVE;

    failedNodePtr.p->sendCommitFailReqStatus = Q_NOT_ACTIVE;

    failedNodePtr.p->sendPresToStatus = Q_NOT_ACTIVE;

    failedNodePtr.p->alarmCount = 0;

    setNodeInfo(failedNodePtr.i).m_heartbeat_cnt= 0;

    if (aSendFailRep == ZTRUE) {

      jam();

      if (failedNodePtr.i != getOwnNodeId()) {