bug#28445 - Heartbeat does not start until first API_REGREQ is recevied (31f6b1d0) · Commits · Software / OSDI20 Artifacts / mariadb

storage/ndb/src/kernel/blocks/cmvmi/Cmvmi.cpp

+29 −28

Original line number	Diff line number	Diff line
		@@ -421,9 +421,10 @@ void Cmvmi::execCLOSE_COMREQ(Signal* signal)
		// Uint32 noOfNodes = closeCom->noOfNodes;

		jamEntry();
		for (unsigned i = 0; i < MAX_NODES; i++){
		if(NodeBitmask::get(closeCom->theNodes, i)){

		for (unsigned i = 0; i < MAX_NODES; i++)
		{
		if(NodeBitmask::get(closeCom->theNodes, i))
		{
		jam();

		//-----------------------------------------------------
		@@ -437,7 +438,9 @@ void Cmvmi::execCLOSE_COMREQ(Signal* signal)
		globalTransporterRegistry.do_disconnect(i);
		}
		}
		if (failNo != 0) {

		if (failNo != 0)
		{
		jam();
		signal->theData[0] = userRef;
		signal->theData[1] = failNo;
		@@ -456,13 +459,21 @@ void Cmvmi::execOPEN_COMREQ(Signal* signal)
		jamEntry();

		const Uint32 len = signal->getLength();
		if(len == 2){

		if(len == 2)
		{
		#ifdef ERROR_INSERT
		if (! ((ERROR_INSERTED(9000) \|\| ERROR_INSERTED(9002))
		&& c_error_9000_nodes_mask.get(tStartingNode)))
		#endif
		{
		if (globalData.theStartLevel != NodeState::SL_STARTED &&
		(getNodeInfo(tStartingNode).m_type != NodeInfo::DB &&
		getNodeInfo(tStartingNode).m_type != NodeInfo::MGM))
		{
		jam();
		goto done;
		}

		globalTransporterRegistry.do_connect(tStartingNode);
		globalTransporterRegistry.setIOState(tStartingNode, HaltIO);

		@@ -475,9 +486,11 @@ void Cmvmi::execOPEN_COMREQ(Signal* signal)
		//-----------------------------------------------------
		}
		} else {
		for(unsigned int i = 1; i < MAX_NODES; i++ ) {
		for(unsigned int i = 1; i < MAX_NODES; i++ )
		{
		jam();
		if (i != getOwnNodeId() && getNodeInfo(i).m_type == tData2){
		if (i != getOwnNodeId() && getNodeInfo(i).m_type == tData2)
		{
		jam();

		#ifdef ERROR_INSERT
		@@ -496,6 +509,7 @@ void Cmvmi::execOPEN_COMREQ(Signal* signal)
		}
		}

		done:
		if (userRef != 0) {
		jam();
		signal->theData[0] = tStartingNode;
		@@ -537,22 +551,8 @@ void Cmvmi::execDISCONNECT_REP(Signal *signal)
		const NodeInfo::NodeType type = getNodeInfo(hostId).getType();
		ndbrequire(type != NodeInfo::INVALID);

		if(type == NodeInfo::DB \|\| globalData.theStartLevel == NodeState::SL_STARTED){
		jam();
		DisconnectRep * const rep = (DisconnectRep *)&signal->theData[0];
		rep->nodeId = hostId;
		rep->err = errNo;
		sendSignal(QMGR_REF, GSN_DISCONNECT_REP, signal,
		DisconnectRep::SignalLength, JBA);
		} else if((globalData.theStartLevel == NodeState::SL_CMVMI \|\|
		globalData.theStartLevel == NodeState::SL_STARTING)
		&& type == NodeInfo::MGM) {
		/**
		* Someone disconnected during cmvmi period
		*/
		jam();
		globalTransporterRegistry.do_connect(hostId);
		}

		cancelSubscription(hostId);

		@@ -587,6 +587,8 @@ void Cmvmi::execCONNECT_REP(Signal *signal){
		*/
		if(type == NodeInfo::MGM){
		jam();
		signal->theData[0] = hostId;
		sendSignal(QMGR_REF, GSN_CONNECT_REP, signal, 1, JBA);
		} else {
		/**
		* Dont allow api nodes to connect
		@@ -802,6 +804,8 @@ Cmvmi::execSTART_ORD(Signal* signal) {
		}
		}
		}

		EXECUTE_DIRECT(QMGR, GSN_START_ORD, signal, 1);
		return ;
		}

		@@ -830,9 +834,6 @@ Cmvmi::execSTART_ORD(Signal* signal) {
		* Do Restart
		*/

		globalScheduler.clear();
		globalTimeQueue.clear();

		// Disconnect all nodes as part of the system restart.
		// We need to ensure that we are starting up
		// without any connected nodes.

storage/ndb/src/kernel/blocks/qmgr/Qmgr.hpp

+3 −0

Original line number	Diff line number	Diff line
		@@ -266,6 +266,8 @@ private:
		void execALLOC_NODEID_REF(Signal *);
		void completeAllocNodeIdReq(Signal *);

		void execSTART_ORD(Signal*);

		// Arbitration signals
		void execARBIT_CFG(Signal* signal);
		void execARBIT_PREPREQ(Signal* signal);
		@@ -281,6 +283,7 @@ private:
		void check_readnodes_reply(Signal* signal, Uint32 nodeId, Uint32 gsn);
		Uint32 check_startup(Signal* signal);

		void api_failed(Signal* signal, Uint32 aFailedNode);
		void node_failed(Signal* signal, Uint16 aFailedNode);
		void checkStartInterface(Signal* signal);
		void failReport(Signal* signal,

storage/ndb/src/kernel/blocks/qmgr/QmgrInit.cpp

+22 −4

Original line number	Diff line number	Diff line
		@@ -31,10 +31,6 @@ void Qmgr::initData()
		cnoCommitFailedNodes = 0;
		c_maxDynamicId = 0;
		c_clusterNodes.clear();

		Uint32 hbDBAPI = 500;
		setHbApiDelay(hbDBAPI);
		c_connectedNodes.set(getOwnNodeId());
		c_stopReq.senderRef = 0;

		/**
		@@ -43,6 +39,27 @@ void Qmgr::initData()
		ndbrequire((Uint32)NodeInfo::DB == 0);
		ndbrequire((Uint32)NodeInfo::API == 1);
		ndbrequire((Uint32)NodeInfo::MGM == 2);

		NodeRecPtr nodePtr;
		nodePtr.i = getOwnNodeId();
		ptrAss(nodePtr, nodeRec);
		nodePtr.p->blockRef = reference();

		c_connectedNodes.set(getOwnNodeId());
		setNodeInfo(getOwnNodeId()).m_version = NDB_VERSION;


		/**
		* Timeouts
		*/
		const ndb_mgm_configuration_iterator * p =
		m_ctx.m_config.getOwnConfigIterator();
		ndbrequire(p != 0);

		Uint32 hbDBAPI = 1500;
		ndb_mgm_get_int_parameter(p, CFG_DB_API_HEARTBEAT_INTERVAL, &hbDBAPI);

		setHbApiDelay(hbDBAPI);
		}//Qmgr::initData()

		void Qmgr::initRecords()
		@@ -113,6 +130,7 @@ Qmgr::Qmgr(Block_context& ctx)
		addRecSignal(GSN_DIH_RESTARTREF, &Qmgr::execDIH_RESTARTREF);
		addRecSignal(GSN_DIH_RESTARTCONF, &Qmgr::execDIH_RESTARTCONF);
		addRecSignal(GSN_NODE_VERSION_REP, &Qmgr::execNODE_VERSION_REP);
		addRecSignal(GSN_START_ORD, &Qmgr::execSTART_ORD);

		initData();
		}//Qmgr::Qmgr()

storage/ndb/src/kernel/blocks/qmgr/QmgrMain.cpp

+189 −173

Original line number	Diff line number	Diff line
		@@ -238,6 +238,38 @@ Qmgr::execREAD_CONFIG_REQ(Signal* signal)
		ReadConfigConf::SignalLength, JBB);
		}

		void
		Qmgr::execSTART_ORD(Signal* signal)
		{
		/**
		* Start timer handling
		*/
		signal->theData[0] = ZTIMER_HANDLING;
		sendSignal(QMGR_REF, GSN_CONTINUEB, signal, 1, JBB);

		NodeRecPtr nodePtr;
		for (nodePtr.i = 1; nodePtr.i < MAX_NODES; nodePtr.i++)
		{
		ptrAss(nodePtr, nodeRec);
		nodePtr.p->ndynamicId = 0;
		if(getNodeInfo(nodePtr.i).m_type == NodeInfo::DB)
		{
		nodePtr.p->phase = ZINIT;
		c_definedNodes.set(nodePtr.i);
		} else {
		nodePtr.p->phase = ZAPI_INACTIVE;
		}

		setNodeInfo(nodePtr.i).m_heartbeat_cnt= 0;
		nodePtr.p->sendPrepFailReqStatus = Q_NOT_ACTIVE;
		nodePtr.p->sendCommitFailReqStatus = Q_NOT_ACTIVE;
		nodePtr.p->sendPresToStatus = Q_NOT_ACTIVE;
		nodePtr.p->failState = NORMAL;
		nodePtr.p->rcv[0] = 0;
		nodePtr.p->rcv[1] = 0;
		}//for
		}

		/*
		4.2 ADD NODE MODULE*/
		/##########################################################################/
		@@ -298,8 +330,6 @@ void Qmgr::startphase1(Signal* signal)
		nodePtr.i = getOwnNodeId();
		ptrAss(nodePtr, nodeRec);
		nodePtr.p->phase = ZSTARTING;
		nodePtr.p->blockRef = reference();
		c_connectedNodes.set(nodePtr.i);

		signal->theData[0] = reference();
		sendSignal(DBDIH_REF, GSN_DIH_RESTARTREQ, signal, 1, JBB);
		@@ -371,11 +401,14 @@ void Qmgr::execCONNECT_REP(Signal* signal)
		case ZFAIL_CLOSING:
		jam();
		return;
		case ZINIT:
		ndbrequire(false);
		case ZAPI_ACTIVE:
		case ZAPI_INACTIVE:
		return;
		case ZINIT:
		ndbrequire(getNodeInfo(nodeId).m_type == NodeInfo::MGM);
		break;
		default:
		ndbrequire(false);
		}

		if (getNodeInfo(nodeId).getType() != NodeInfo::DB)
		@@ -1212,12 +1245,6 @@ void Qmgr::execCM_REGREF(Signal* signal)
		{
		jam();
		electionWon(signal);

		/**
		* Start timer handling
		*/
		signal->theData[0] = ZTIMER_HANDLING;
		sendSignal(QMGR_REF, GSN_CONTINUEB, signal, 10, JBB);
		}

		return;
		@@ -1855,12 +1882,6 @@ Qmgr::joinedCluster(Signal* signal, NodeRecPtr nodePtr){

		sendSttorryLab(signal);

		/**
		* Start timer handling
		*/
		signal->theData[0] = ZTIMER_HANDLING;
		sendSignal(QMGR_REF, GSN_CONTINUEB, signal, 10, JBB);

		sendCmAckAdd(signal, getOwnNodeId(), CmAdd::CommitNew);
		}

		@@ -2094,25 +2115,6 @@ void Qmgr::findNeighbours(Signal* signal)
		/---------------------------------------------------------------------------/
		void Qmgr::initData(Signal* signal)
		{
		NodeRecPtr nodePtr;
		for (nodePtr.i = 1; nodePtr.i < MAX_NODES; nodePtr.i++) {
		ptrAss(nodePtr, nodeRec);
		nodePtr.p->ndynamicId = 0;
		if(getNodeInfo(nodePtr.i).m_type == NodeInfo::DB){
		nodePtr.p->phase = ZINIT;
		c_definedNodes.set(nodePtr.i);
		} else {
		nodePtr.p->phase = ZAPI_INACTIVE;
		}

		setNodeInfo(nodePtr.i).m_heartbeat_cnt= 0;
		nodePtr.p->sendPrepFailReqStatus = Q_NOT_ACTIVE;
		nodePtr.p->sendCommitFailReqStatus = Q_NOT_ACTIVE;
		nodePtr.p->sendPresToStatus = Q_NOT_ACTIVE;
		nodePtr.p->failState = NORMAL;
		nodePtr.p->rcv[0] = 0;
		nodePtr.p->rcv[1] = 0;
		}//for
		cfailureNr = 1;
		ccommitFailureNr = 1;
		cprepareFailureNr = 1;
		@@ -2146,13 +2148,11 @@ void Qmgr::initData(Signal* signal)
		ndbrequire(p != 0);

		Uint32 hbDBDB = 1500;
		Uint32 hbDBAPI = 1500;
		Uint32 arbitTimeout = 1000;
		c_restartPartialTimeout = 30000;
		c_restartPartionedTimeout = 60000;
		c_restartFailureTimeout = ~0;
		ndb_mgm_get_int_parameter(p, CFG_DB_HEARTBEAT_INTERVAL, &hbDBDB);
		ndb_mgm_get_int_parameter(p, CFG_DB_API_HEARTBEAT_INTERVAL, &hbDBAPI);
		ndb_mgm_get_int_parameter(p, CFG_DB_ARBIT_TIMEOUT, &arbitTimeout);
		ndb_mgm_get_int_parameter(p, CFG_DB_START_PARTIAL_TIMEOUT,
		&c_restartPartialTimeout);
		@@ -2177,7 +2177,6 @@ void Qmgr::initData(Signal* signal)
		}

		setHbDelay(hbDBDB);
		setHbApiDelay(hbDBAPI);
		setArbitTimeout(arbitTimeout);

		arbitRec.state = ARBIT_NULL; // start state for all nodes
		@@ -2204,7 +2203,6 @@ void Qmgr::initData(Signal* signal)

		execARBIT_CFG(signal);
		}
		setNodeInfo(getOwnNodeId()).m_version = NDB_VERSION;
		}//Qmgr::initData()


		@@ -2244,13 +2242,15 @@ void Qmgr::timerHandlingLab(Signal* signal)
		checkStartInterface(signal);
		}

		if (cactivateApiCheck != 0) {
		jam();
		if (hb_api_timer.check(TcurrentTime)) {
		if (hb_api_timer.check(TcurrentTime))
		{
		jam();
		hb_api_timer.reset();
		apiHbHandlingLab(signal);
		}//if
		}

		if (cactivateApiCheck != 0) {
		jam();
		if (clatestTransactionCheck == 0) {
		//-------------------------------------------------------------
		// Initialise the Transaction check timer.
		@@ -2367,18 +2367,21 @@ void Qmgr::apiHbHandlingLab(Signal* signal)
		if(type == NodeInfo::INVALID)
		continue;

		if (TnodePtr.p->phase == ZAPI_ACTIVE){
		if (c_connectedNodes.get(nodeId))
		{
		jam();
		setNodeInfo(TnodePtr.i).m_heartbeat_cnt++;

		if(getNodeInfo(TnodePtr.i).m_heartbeat_cnt > 2){
		if(getNodeInfo(TnodePtr.i).m_heartbeat_cnt > 2)
		{
		signal->theData[0] = NDB_LE_MissedHeartbeat;
		signal->theData[1] = nodeId;
		signal->theData[2] = getNodeInfo(TnodePtr.i).m_heartbeat_cnt - 1;
		sendSignal(CMVMI_REF, GSN_EVENT_REP, signal, 3, JBB);
		}

		if (getNodeInfo(TnodePtr.i).m_heartbeat_cnt > 4) {
		if (getNodeInfo(TnodePtr.i).m_heartbeat_cnt > 4)
		{
		jam();
		/------------------------------------------------------------------/
		/* THE API NODE HAS NOT SENT ANY HEARTBEAT FOR THREE SECONDS.
		@@ -2391,7 +2394,7 @@ void Qmgr::apiHbHandlingLab(Signal* signal)
		signal->theData[1] = nodeId;
		sendSignal(CMVMI_REF, GSN_EVENT_REP, signal, 2, JBB);

		node_failed(signal, nodeId);
		api_failed(signal, nodeId);
		}//if
		}//if
		}//for
		@@ -2480,26 +2483,6 @@ void Qmgr::sendApiFailReq(Signal* signal, Uint16 failedNodeNo)
		sendSignal(DBTC_REF, GSN_API_FAILREQ, signal, 2, JBA);
		sendSignal(DBDICT_REF, GSN_API_FAILREQ, signal, 2, JBA);
		sendSignal(SUMA_REF, GSN_API_FAILREQ, signal, 2, JBA);

		/**-------------------------------------------------------------------------
		* THE OTHER NODE WAS AN API NODE. THE COMMUNICATION LINK IS ALREADY
		* BROKEN AND THUS NO ACTION IS NEEDED TO BREAK THE CONNECTION.
		* WE ONLY NEED TO SET PARAMETERS TO ENABLE A NEW CONNECTION IN A FEW
		* SECONDS.
		-------------------------------------------------------------------------/
		setNodeInfo(failedNodePtr.i).m_heartbeat_cnt= 0;
		setNodeInfo(failedNodePtr.i).m_version = 0;
		recompute_version_info(getNodeInfo(failedNodePtr.i).m_type);

		CloseComReqConf * const closeCom = (CloseComReqConf *)&signal->theData[0];

		closeCom->xxxBlockRef = reference();
		closeCom->failNo = 0;
		closeCom->noOfNodes = 1;
		NodeBitmask::clear(closeCom->theNodes);
		NodeBitmask::set(closeCom->theNodes, failedNodePtr.i);
		sendSignal(CMVMI_REF, GSN_CLOSE_COMREQ, signal,
		CloseComReqConf::SignalLength, JBA);
		}//Qmgr::sendApiFailReq()

		void Qmgr::execAPI_FAILREQ(Signal* signal)
		@@ -2512,20 +2495,7 @@ void Qmgr::execAPI_FAILREQ(Signal* signal)

		ndbrequire(getNodeInfo(failedNodePtr.i).getType() != NodeInfo::DB);

		// ignore if api not active
		if (failedNodePtr.p->phase != ZAPI_ACTIVE)
		{
		jam();
		// But send to SUMA anyway...
		sendSignal(SUMA_REF, GSN_API_FAILREQ, signal, 2, JBA);
		return;
		}

		signal->theData[0] = NDB_LE_Disconnected;
		signal->theData[1] = failedNodePtr.i;
		sendSignal(CMVMI_REF, GSN_EVENT_REP, signal, 2, JBB);

		node_failed(signal, failedNodePtr.i);
		api_failed(signal, signal->theData[0]);
		}

		void Qmgr::execAPI_FAILCONF(Signal* signal)
		@@ -2649,6 +2619,13 @@ void Qmgr::execDISCONNECT_REP(Signal* signal)
		ndbrequire(false);
		}

		if (getNodeInfo(nodeId).getType() != NodeInfo::DB)
		{
		jam();
		api_failed(signal, nodeId);
		return;
		}

		switch(nodePtr.p->phase){
		case ZRUNNING:
		jam();
		@@ -2685,8 +2662,8 @@ void Qmgr::node_failed(Signal* signal, Uint16 aFailedNode)
		failedNodePtr.i = aFailedNode;
		ptrCheckGuard(failedNodePtr, MAX_NODES, nodeRec);

		if (getNodeInfo(failedNodePtr.i).getType() == NodeInfo::DB){
		jam();
		ndbrequire(getNodeInfo(failedNodePtr.i).getType() == NodeInfo::DB);

		/**---------------------------------------------------------------------
		* THE OTHER NODE IS AN NDB NODE, WE HANDLE IT AS IF A HEARTBEAT
		* FAILURE WAS DISCOVERED.
		@@ -2726,25 +2703,68 @@ void Qmgr::node_failed(Signal* signal, Uint16 aFailedNode)
		return;
		}

		void
		Qmgr::api_failed(Signal* signal, Uint32 nodeId)
		{
		NodeRecPtr failedNodePtr;
		/**------------------------------------------------------------------------
		* A COMMUNICATION LINK HAS BEEN DISCONNECTED. WE MUST TAKE SOME ACTION
		* DUE TO THIS.
		-----------------------------------------------------------------------/
		failedNodePtr.i = nodeId;
		ptrCheckGuard(failedNodePtr, MAX_NODES, nodeRec);

		if (failedNodePtr.p->phase == ZFAIL_CLOSING)
		{
		/**
		* API code
		* Failure handling already in progress
		*/
		jam();
		if (failedNodePtr.p->phase != ZFAIL_CLOSING){
		return;
		}

		if (failedNodePtr.p->phase == ZAPI_ACTIVE)
		{
		jam();
		sendApiFailReq(signal, nodeId);
		arbitRec.code = ArbitCode::ApiFail;
		handleArbitApiFail(signal, nodeId);
		}
		else
		{
		/**
		* Always inform SUMA
		*/
		jam();
		//-------------------------------------------------------------------------
		// The API was active and has now failed. We need to initiate API failure
		// handling. If the API had already failed then we can ignore this
		// discovery.
		//-------------------------------------------------------------------------
		signal->theData[0] = nodeId;
		signal->theData[1] = QMGR_REF;
		sendSignal(SUMA_REF, GSN_API_FAILREQ, signal, 2, JBA);
		failedNodePtr.p->failState = NORMAL;
		}

		failedNodePtr.p->phase = ZFAIL_CLOSING;
		setNodeInfo(failedNodePtr.i).m_heartbeat_cnt= 0;
		setNodeInfo(failedNodePtr.i).m_version = 0;
		recompute_version_info(getNodeInfo(failedNodePtr.i).m_type);

		sendApiFailReq(signal, aFailedNode);
		arbitRec.code = ArbitCode::ApiFail;
		handleArbitApiFail(signal, aFailedNode);
		}//if
		return;
		}//Qmgr::node_failed()
		CloseComReqConf * const closeCom = (CloseComReqConf *)&signal->theData[0];
		closeCom->xxxBlockRef = reference();
		closeCom->failNo = 0;
		closeCom->noOfNodes = 1;
		NodeBitmask::clear(closeCom->theNodes);
		NodeBitmask::set(closeCom->theNodes, failedNodePtr.i);
		sendSignal(CMVMI_REF, GSN_CLOSE_COMREQ, signal,
		CloseComReqConf::SignalLength, JBA);

		if (getNodeInfo(failedNodePtr.i).getType() == NodeInfo::MGM)
		{
		/**
		* Allow MGM do reconnect "directly"
		*/
		jam();
		setNodeInfo(failedNodePtr.i).m_heartbeat_cnt = 3;
		}
		}

		/**--------------------------------------------------------------------------
		* AN API NODE IS REGISTERING. IF FOR THE FIRST TIME WE WILL ENABLE
		@@ -4963,7 +4983,6 @@ Qmgr::execDUMP_STATE_ORD(Signal* signal)
		c_start.m_president_candidate_gci);
		infoEvent("ctoStatus = %d\n", ctoStatus);
		for(Uint32 i = 1; i<MAX_NDB_NODES; i++){
		if(getNodeInfo(i).getType() == NodeInfo::DB){
		NodeRecPtr nodePtr;
		nodePtr.i = i;
		ptrCheckGuard(nodePtr, MAX_NDB_NODES, nodeRec);
		@@ -4997,9 +5016,6 @@ Qmgr::execDUMP_STATE_ORD(Signal* signal)
		infoEvent(buf);
		}
		}
		default:
		;
		}//switch

		#ifdef ERROR_INSERT
		if (signal->theData[0] == 935 && signal->getLength() == 2)