Sunday, September 16, 2018

MONGODB REPLICATION FAILOVER TESTING


MONGODB FAILOVER TESTING 


Primary Server      : 1     (Port number: 6666)
Secondary Servers : 2  ( Port number:7777 , 8888)
Arbiter Server        : 1  (Port number: 9999)



Verify the Replication Configuration from Primary Server


MongoDB Enterprise OurReplica:PRIMARY> rs.conf()
{
        "_id" : "OurReplica",
        "version" : 22,
        "protocolVersion" : NumberLong(1),
        "writeConcernMajorityJournalDefault" : true,
        "members" : [
                {
                        "_id" : 0,
                        "host" : "127.0.0.1:6666",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 1,
                        "tags" : {

                        },
                        "slaveDelay" : NumberLong(0),
                        "votes" : 1
                },
                {
                        "_id" : 1,
                        "host" : "127.0.0.1:7777",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 1,
                        "tags" : {

                        },
                        "slaveDelay" : NumberLong(0),
                        "votes" : 1
                },
                {
                        "_id" : 2,
                        "host" : "127.0.0.1:8888",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 1,
                        "tags" : {

                        },
                        "slaveDelay" : NumberLong(0),
                        "votes" : 1
                },
                {
                        "_id" : 3,
                        "host" : "127.0.0.1:9999",
                        "arbiterOnly" : true,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 0,
                        "tags" : {

                        },
                        "slaveDelay" : NumberLong(0),
                        "votes" : 1
                }
        ],
        "settings" : {
                "chainingAllowed" : true,
                "heartbeatIntervalMillis" : 2000,
                "heartbeatTimeoutSecs" : 10,
                "electionTimeoutMillis" : 10000,
                "catchUpTimeoutMillis" : -1,
                "catchUpTakeoverDelayMillis" : 30000,
                "getLastErrorModes" : {

                },
                "getLastErrorDefaults" : {
                        "w" : 1,
                        "wtimeout" : 0
                },
                "replicaSetId" : ObjectId("5b9b61a67b1454a13fc7610a")
        }
}
MongoDB Enterprise OurReplica:PRIMARY>


Verify the Replication status from Primary Server


MongoDB Enterprise OurReplica:PRIMARY> rs.status()
{
        "set" : "OurReplica",
        "date" : ISODate("2018-09-14T19:48:59.705Z"),
        "myState" : 1,
        "term" : NumberLong(8),
        "syncingTo" : "",
        "syncSourceHost" : "",
        "syncSourceId" : -1,
        "heartbeatIntervalMillis" : NumberLong(2000),
        "optimes" : {
                "lastCommittedOpTime" : {
                        "ts" : Timestamp(1536954538, 1),
                        "t" : NumberLong(8)
                },
                "readConcernMajorityOpTime" : {
                        "ts" : Timestamp(1536954538, 1),
                        "t" : NumberLong(8)
                },
                "appliedOpTime" : {
                        "ts" : Timestamp(1536954538, 1),
                        "t" : NumberLong(8)
                },
                "durableOpTime" : {
                        "ts" : Timestamp(1536954538, 1),
                        "t" : NumberLong(8)
                }
        },
        "lastStableCheckpointTimestamp" : Timestamp(1536954518, 1),
        "members" : [
                {
                        "_id" : 0,
                        "name" : "127.0.0.1:6666",
                        "health" : 1,
                        "state" : 1,
                        "stateStr" : "PRIMARY",
                        "uptime" : 675,
                        "optime" : {
                                "ts" : Timestamp(1536954538, 1),
                                "t" : NumberLong(8)
                        },
                        "optimeDate" : ISODate("2018-09-14T19:48:58Z"),
                        "syncingTo" : "",
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "",
                        "electionTime" : Timestamp(1536953897, 1),
                        "electionDate" : ISODate("2018-09-14T19:38:17Z"),
                        "configVersion" : 22,
                        "self" : true,
                        "lastHeartbeatMessage" : ""
                },
                {
                        "_id" : 1,
                        "name" : "127.0.0.1:7777",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 647,
                        "optime" : {
                                "ts" : Timestamp(1536954538, 1),
                                "t" : NumberLong(8)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1536954538, 1),
                                "t" : NumberLong(8)
                        },
                        "optimeDate" : ISODate("2018-09-14T19:48:58Z"),
                        "optimeDurableDate" : ISODate("2018-09-14T19:48:58Z"),
                        "lastHeartbeat" : ISODate("2018-09-14T19:48:59.294Z"),
                        "lastHeartbeatRecv" : ISODate("2018-09-14T19:48:59.291Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncingTo" : "127.0.0.1:6666",
                        "syncSourceHost" : "127.0.0.1:6666",
                        "syncSourceId" : 0,
                        "infoMessage" : "",
                        "configVersion" : 22
                },
                {
                        "_id" : 2,
                        "name" : "127.0.0.1:8888",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 454,
                        "optime" : {
                                "ts" : Timestamp(1536954538, 1),
                                "t" : NumberLong(8)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1536954538, 1),
                                "t" : NumberLong(8)
                        },
                        "optimeDate" : ISODate("2018-09-14T19:48:58Z"),
                        "optimeDurableDate" : ISODate("2018-09-14T19:48:58Z"),
                        "lastHeartbeat" : ISODate("2018-09-14T19:48:59.294Z"),
                        "lastHeartbeatRecv" : ISODate("2018-09-14T19:48:59.293Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncingTo" : "127.0.0.1:6666",
                        "syncSourceHost" : "127.0.0.1:6666",
                        "syncSourceId" : 0,
                        "infoMessage" : "",
                        "configVersion" : 22
                },
                {
                        "_id" : 3,
                        "name" : "127.0.0.1:9999",
                        "health" : 1,
                        "state" : 7,
                        "stateStr" : "ARBITER",
                        "uptime" : 314,
                        "lastHeartbeat" : ISODate("2018-09-14T19:48:59.294Z"),
                        "lastHeartbeatRecv" : ISODate("2018-09-14T19:48:59.323Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncingTo" : "",
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "",
                        "configVersion" : 22
                }
        ],
        "ok" : 1,
        "operationTime" : Timestamp(1536954538, 1),
        "$clusterTime" : {
                "clusterTime" : Timestamp(1536954538, 1),
                "signature" : {
                        "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                        "keyId" : NumberLong(0)
                }
        }
}
MongoDB Enterprise OurReplica:PRIMARY>

MongoDB Enterprise OurReplica:PRIMARY> use FOver_Test
switched to db FOver_Test
MongoDB Enterprise OurReplica:PRIMARY> db.createCollection("Failover_test_doc");
{
        "ok" : 1,
        "operationTime" : Timestamp(1536954642, 1),
        "$clusterTime" : {
                "clusterTime" : Timestamp(1536954642, 1),
                "signature" : {
                        "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                        "keyId" : NumberLong(0)
                }
        }
}
MongoDB Enterprise OurReplica:PRIMARY>
MongoDB Enterprise OurReplica:PRIMARY>

MongoDB Enterprise OurReplica:SECONDARY> rs.slaveOk()
MongoDB Enterprise OurReplica:SECONDARY>
MongoDB Enterprise OurReplica:SECONDARY> show dbs
AshokDB     0.000GB
FOver_Test  0.000GB
RepDB       0.000GB
admin       0.000GB
config      0.000GB
local       0.000GB
MongoDB Enterprise OurReplica:SECONDARY> use FOver_Test
switched to db FOver_Test
MongoDB Enterprise OurReplica:SECONDARY> show collections
Failover_test_doc
MongoDB Enterprise OurReplica:SECONDARY>


Terminate Primary Server :


Killed Primary Server with process id : 7320

[root@nosqlDB mongodb]# ps -ef | grep mongodb.conf
root      7320     1  0 01:07 ?        00:00:06 mongod -f mongodb.conf
root      7427     1  0 01:08 ?        00:00:05 mongod -f mongodb.conf
root      7534     1  0 01:11 ?        00:00:04 mongod -f mongodb.conf
root      7659     1  0 01:12 ?        00:00:02 mongod -f mongodb.conf
 [root@nosqlDB mongodb]# kill -9 7320
[root@nosqlDB mongodb]#

MongoDB Enterprise OurReplica:PRIMARY>
2018-09-15T01:25:02.028+0530 I NETWORK  [js] trying reconnect to 127.0.0.1:6666 failed
2018-09-15T01:25:02.029+0530 I NETWORK  [js] reconnect 127.0.0.1:6666 failed failed
MongoDB Enterprise >


Now the Primary MongoDB server (Port:6666) is terminated and the Secondary Server (Port: 8888) is failover to PRIMARY automatically.


/usr/server4/mongodb
[root@nosqlDB mongodb]# mongo --port 8888

MongoDB shell version v4.0.0

connecting to: mongodb://127.0.0.1:8888/

MongoDB server version: 4.0.0
Server has startup warnings:
2018-09-15T01:11:07.994+0530 I STORAGE  [initandlisten]
2018-09-15T01:11:07.994+0530 I STORAGE  [initandlisten] ** WARNING: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine
2018-09-15T01:11:07.994+0530 I STORAGE  [initandlisten] **          See http://dochub.mongodb.org/core/prodnotes-filesystem
2018-09-15T01:11:08.792+0530 I CONTROL  [initandlisten]
2018-09-15T01:11:08.792+0530 I CONTROL  [initandlisten] ** WARNING: Access control is not enabled for the database.
2018-09-15T01:11:08.792+0530 I CONTROL  [initandlisten] **          Read and write access to data and configuration is unrestricted.
2018-09-15T01:11:08.792+0530 I CONTROL  [initandlisten] ** WARNING: You are running this process as the root user, which is not recommended.
2018-09-15T01:11:08.792+0530 I CONTROL  [initandlisten]
2018-09-15T01:11:08.792+0530 I CONTROL  [initandlisten]
2018-09-15T01:11:08.792+0530 I CONTROL  [initandlisten] ** WARNING: /sys/kernel/mm/transparent_hugepage/enabled is 'always'.
2018-09-15T01:11:08.792+0530 I CONTROL  [initandlisten] **        We suggest setting it to 'never'
2018-09-15T01:11:08.792+0530 I CONTROL  [initandlisten]
2018-09-15T01:11:08.792+0530 I CONTROL  [initandlisten] ** WARNING: /sys/kernel/mm/transparent_hugepage/defrag is 'always'.
2018-09-15T01:11:08.792+0530 I CONTROL  [initandlisten] **        We suggest setting it to 'never'
2018-09-15T01:11:08.792+0530 I CONTROL  [initandlisten]
MongoDB Enterprise OurReplica:PRIMARY>


Verify the Replication status from Primary Server



MongoDB Enterprise OurReplica:PRIMARY> rs.status()
{
        "set" : "OurReplica",
        "date" : ISODate("2018-09-14T19:57:30.901Z"),
        "myState" : 1,
        "term" : NumberLong(9),
        "syncingTo" : "",
        "syncSourceHost" : "",
        "syncSourceId" : -1,
        "heartbeatIntervalMillis" : NumberLong(2000),
        "optimes" : {
                "lastCommittedOpTime" : {
                        "ts" : Timestamp(1536954888, 1),
                        "t" : NumberLong(8)
                },
                "readConcernMajorityOpTime" : {
                        "ts" : Timestamp(1536954888, 1),
                        "t" : NumberLong(8)
                },
                "appliedOpTime" : {
                        "ts" : Timestamp(1536955049, 1),
                        "t" : NumberLong(9)
                },
                "durableOpTime" : {
                        "ts" : Timestamp(1536955049, 1),
                        "t" : NumberLong(9)
                }
        },
        "lastStableCheckpointTimestamp" : Timestamp(1536954888, 1),
        "members" : [
                {
                        "_id" : 0,
                        "name" : "127.0.0.1:6666",
                        "health" : 0,
                        "state" : 8,
                        "stateStr" : "(not reachable/healthy)",
                        "uptime" : 0,
                        "optime" : {
                                "ts" : Timestamp(0, 0),
                                "t" : NumberLong(-1)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(0, 0),
                                "t" : NumberLong(-1)
                        },
                        "optimeDate" : ISODate("1970-01-01T00:00:00Z"),
                        "optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"),
                        "lastHeartbeat" : ISODate("2018-09-14T19:57:30.105Z"),
                        "lastHeartbeatRecv" : ISODate("2018-09-14T19:54:57.461Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "Error connecting to 127.0.0.1:6666 :: caused by :: Connection refused",
                        "syncingTo" : "",
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "",
                        "configVersion" : -1
                },
                {
                        "_id" : 1,
                        "name" : "127.0.0.1:7777",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 966,
                        "optime" : {
                                "ts" : Timestamp(1536955049, 1),
                                "t" : NumberLong(9)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1536955049, 1),
                                "t" : NumberLong(9)
                        },
                        "optimeDate" : ISODate("2018-09-14T19:57:29Z"),
                        "optimeDurableDate" : ISODate("2018-09-14T19:57:29Z"),
                        "lastHeartbeat" : ISODate("2018-09-14T19:57:30.094Z"),
                        "lastHeartbeatRecv" : ISODate("2018-09-14T19:57:29.375Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncingTo" : "127.0.0.1:8888",
                        "syncSourceHost" : "127.0.0.1:8888",
                        "syncSourceId" : 2,
                        "infoMessage" : "",
                        "configVersion" : 22
                },
                {
                        "_id" : 2,
                        "name" : "127.0.0.1:8888",
                        "health" : 1,
                        "state" : 1,
                        "stateStr" : "PRIMARY",
                        "uptime" : 983,
                        "optime" : {
                                "ts" : Timestamp(1536955049, 1),
                                "t" : NumberLong(9)
                        },
                        "optimeDate" : ISODate("2018-09-14T19:57:29Z"),
                        "syncingTo" : "",
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "",
                        "electionTime" : Timestamp(1536954908, 1),
                        "electionDate" : ISODate("2018-09-14T19:55:08Z"),
                        "configVersion" : 22,
                        "self" : true,
                        "lastHeartbeatMessage" : ""
                },
                {
                        "_id" : 3,
                        "name" : "127.0.0.1:9999",
                        "health" : 1,
                        "state" : 7,
                        "stateStr" : "ARBITER",
                        "uptime" : 825,
                        "lastHeartbeat" : ISODate("2018-09-14T19:57:30.094Z"),
                        "lastHeartbeatRecv" : ISODate("2018-09-14T19:57:29.507Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncingTo" : "",
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "",
                        "configVersion" : 22
                }
        ],
        "ok" : 1,
        "operationTime" : Timestamp(1536955049, 1),
        "$clusterTime" : {
                "clusterTime" : Timestamp(1536955049, 1),
                "signature" : {
                        "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                        "keyId" : NumberLong(0)
                }
        }
}
MongoDB Enterprise OurReplica:PRIMARY>


Verify the Replication Configuration from New Primary Server (Port:8888)


MongoDB Enterprise OurReplica:PRIMARY> rs.conf()
{
        "_id" : "OurReplica",
        "version" : 22,
        "protocolVersion" : NumberLong(1),
        "writeConcernMajorityJournalDefault" : true,
        "members" : [
                {
                        "_id" : 0,
                        "host" : "127.0.0.1:6666",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 1,
                        "tags" : {

                        },
                        "slaveDelay" : NumberLong(0),
                        "votes" : 1
                },
                {
                        "_id" : 1,
                        "host" : "127.0.0.1:7777",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 1,
                        "tags" : {

                        },
                        "slaveDelay" : NumberLong(0),
                        "votes" : 1
                },
                {
                        "_id" : 2,
                        "host" : "127.0.0.1:8888",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 1,
                        "tags" : {

                        },
                        "slaveDelay" : NumberLong(0),
                        "votes" : 1
                },
                {
                        "_id" : 3,
                        "host" : "127.0.0.1:9999",
                        "arbiterOnly" : true,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 0,
                        "tags" : {

                        },
                        "slaveDelay" : NumberLong(0),
                        "votes" : 1
                }
        ],
        "settings" : {
                "chainingAllowed" : true,
                "heartbeatIntervalMillis" : 2000,
                "heartbeatTimeoutSecs" : 10,
                "electionTimeoutMillis" : 10000,
                "catchUpTimeoutMillis" : -1,
                "catchUpTakeoverDelayMillis" : 30000,
                "getLastErrorModes" : {

                },
                "getLastErrorDefaults" : {
                        "w" : 1,
                        "wtimeout" : 0
                },
                "replicaSetId" : ObjectId("5b9b61a67b1454a13fc7610a")
        }
}
MongoDB Enterprise OurReplica:PRIMARY>


Now Failover back to earlier PRIMARY state (Port : 6666) :


The mongod service is started back with port 6666


[root@nosqlDB mongodb]# mongod -f mongodb.conf
2018-09-15T01:38:12.742+0530 I CONTROL  [main] Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none'
about to fork child process, waiting until server is ready for connections.
forked process: 8019
child process started successfully, parent exiting
[root@nosqlDB mongodb]# mongo --port 6666
MongoDB shell version v4.0.0
connecting to: mongodb://127.0.0.1:6666/
MongoDB server version: 4.0.0
Server has startup warnings:
2018-09-15T01:38:12.762+0530 I STORAGE  [initandlisten]
2018-09-15T01:38:12.762+0530 I STORAGE  [initandlisten] ** WARNING: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine
2018-09-15T01:38:12.762+0530 I STORAGE  [initandlisten] **          See http://dochub.mongodb.org/core/prodnotes-filesystem
2018-09-15T01:38:13.868+0530 I CONTROL  [initandlisten]
2018-09-15T01:38:13.868+0530 I CONTROL  [initandlisten] ** WARNING: Access control is not enabled for the database.
2018-09-15T01:38:13.868+0530 I CONTROL  [initandlisten] **          Read and write access to data and configuration is unrestricted.
2018-09-15T01:38:13.868+0530 I CONTROL  [initandlisten] ** WARNING: You are running this process as the root user, which is not recommended.
2018-09-15T01:38:13.868+0530 I CONTROL  [initandlisten]
2018-09-15T01:38:13.869+0530 I CONTROL  [initandlisten]
2018-09-15T01:38:13.869+0530 I CONTROL  [initandlisten] ** WARNING: /sys/kernel/mm/transparent_hugepage/enabled is 'always'.
2018-09-15T01:38:13.869+0530 I CONTROL  [initandlisten] **        We suggest setting it to 'never'
2018-09-15T01:38:13.869+0530 I CONTROL  [initandlisten]
2018-09-15T01:38:13.869+0530 I CONTROL  [initandlisten] ** WARNING: /sys/kernel/mm/transparent_hugepage/defrag is 'always'.
2018-09-15T01:38:13.869+0530 I CONTROL  [initandlisten] **        We suggest setting it to 'never'
2018-09-15T01:38:13.869+0530 I CONTROL  [initandlisten]
MongoDB Enterprise OurReplica:SECONDARY>


The new primary server (8888) is terminated. Now the server is failover to original PRIMARY state (port: 6666).


[root@nosqlDB mongodb]# kill -9 7534
[root@nosqlDB mongodb]#


MongoDB Enterprise OurReplica:PRIMARY>
2018-09-15T01:40:37.239+0530 I NETWORK  [js] trying reconnect to 127.0.0.1:8888 failed
2018-09-15T01:40:37.239+0530 I NETWORK  [js] reconnect 127.0.0.1:8888 failed failed
MongoDB Enterprise >

I just logged out from Secondary Server and reconnected it again with port number 6666 from the original primary server and it is converted into the PRIMARY server state



MongoDB Enterprise OurReplica:SECONDARY> exit
bye

[root@nosqlDB mongodb]# mongo --port 6666
MongoDB shell version v4.0.0
connecting to: mongodb://127.0.0.1:6666/
MongoDB server version: 4.0.0
Server has startup warnings:
2018-09-15T01:38:12.762+0530 I STORAGE  [initandlisten]
2018-09-15T01:38:12.762+0530 I STORAGE  [initandlisten] ** WARNING: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine
2018-09-15T01:38:12.762+0530 I STORAGE  [initandlisten] **          See http://dochub.mongodb.org/core/prodnotes-filesystem
2018-09-15T01:38:13.868+0530 I CONTROL  [initandlisten]
2018-09-15T01:38:13.868+0530 I CONTROL  [initandlisten] ** WARNING: Access control is not enabled for the database.
2018-09-15T01:38:13.868+0530 I CONTROL  [initandlisten] **          Read and write access to data and configuration is unrestricted.
2018-09-15T01:38:13.868+0530 I CONTROL  [initandlisten] ** WARNING: You are running this process as the root user, which is not recommended.
2018-09-15T01:38:13.868+0530 I CONTROL  [initandlisten]
2018-09-15T01:38:13.869+0530 I CONTROL  [initandlisten]
2018-09-15T01:38:13.869+0530 I CONTROL  [initandlisten] ** WARNING: /sys/kernel/mm/transparent_hugepage/enabled is 'always'.
2018-09-15T01:38:13.869+0530 I CONTROL  [initandlisten] **        We suggest setting it to 'never'
2018-09-15T01:38:13.869+0530 I CONTROL  [initandlisten]
2018-09-15T01:38:13.869+0530 I CONTROL  [initandlisten] ** WARNING: /sys/kernel/mm/transparent_hugepage/defrag is 'always'.
2018-09-15T01:38:13.869+0530 I CONTROL  [initandlisten] **        We suggest setting it to 'never'
2018-09-15T01:38:13.869+0530 I CONTROL  [initandlisten]
MongoDB Enterprise OurReplica:PRIMARY

[root@nosqlDB mongodb]# mongo --port 8888
MongoDB shell version v4.0.0
connecting to: mongodb://127.0.0.1:8888/
MongoDB server version: 4.0.0
Server has startup warnings:
2018-09-15T01:43:50.478+0530 I STORAGE  [initandlisten]
2018-09-15T01:43:50.478+0530 I STORAGE  [initandlisten] ** WARNING: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine
2018-09-15T01:43:50.478+0530 I STORAGE  [initandlisten] **          See http://dochub.mongodb.org/core/prodnotes-filesystem
2018-09-15T01:43:51.744+0530 I CONTROL  [initandlisten]
2018-09-15T01:43:51.744+0530 I CONTROL  [initandlisten] ** WARNING: Access control is not enabled for the database.
2018-09-15T01:43:51.744+0530 I CONTROL  [initandlisten] **          Read and write access to data and configuration is unrestricted.
2018-09-15T01:43:51.744+0530 I CONTROL  [initandlisten] ** WARNING: You are running this process as the root user, which is not recommended.
2018-09-15T01:43:51.744+0530 I CONTROL  [initandlisten]
2018-09-15T01:43:51.744+0530 I CONTROL  [initandlisten]
2018-09-15T01:43:51.744+0530 I CONTROL  [initandlisten] ** WARNING: /sys/kernel/mm/transparent_hugepage/enabled is 'always'.
2018-09-15T01:43:51.744+0530 I CONTROL  [initandlisten] **        We suggest setting it to 'never'
2018-09-15T01:43:51.744+0530 I CONTROL  [initandlisten]
2018-09-15T01:43:51.744+0530 I CONTROL  [initandlisten] ** WARNING: /sys/kernel/mm/transparent_hugepage/defrag is 'always'.
2018-09-15T01:43:51.744+0530 I CONTROL  [initandlisten] **        We suggest setting it to 'never'
2018-09-15T01:43:51.744+0530 I CONTROL  [initandlisten]
MongoDB Enterprise OurReplica:SECONDARY>






Now Server is back to previous state: ( Primary : 6666, Secondary Server: 7777/8888, Arbiter: 9999)

MongoDB Enterprise OurReplica:PRIMARY> rs.status()
{
        "set" : "OurReplica",
        "date" : ISODate("2018-09-14T20:16:36.462Z"),
        "myState" : 1,
        "term" : NumberLong(10),
        "syncingTo" : "",
        "syncSourceHost" : "",
        "syncSourceId" : -1,
        "heartbeatIntervalMillis" : NumberLong(2000),
        "optimes" : {
                "lastCommittedOpTime" : {
                        "ts" : Timestamp(1536956194, 1),
                        "t" : NumberLong(10)
                },
                "readConcernMajorityOpTime" : {
                        "ts" : Timestamp(1536956194, 1),
                        "t" : NumberLong(10)
                },
                "appliedOpTime" : {
                        "ts" : Timestamp(1536956194, 1),
                        "t" : NumberLong(10)
                },
                "durableOpTime" : {
                        "ts" : Timestamp(1536956194, 1),
                        "t" : NumberLong(10)
                }
        },
        "lastStableCheckpointTimestamp" : Timestamp(1536956174, 1),
        "members" : [
                {
                        "_id" : 0,
                        "name" : "127.0.0.1:6666",
                        "health" : 1,
                        "state" : 1,
                        "stateStr" : "PRIMARY",
                        "uptime" : 504,
                        "optime" : {
                                "ts" : Timestamp(1536956194, 1),
                                "t" : NumberLong(10)
                        },
                        "optimeDate" : ISODate("2018-09-14T20:16:34Z"),
                        "syncingTo" : "",
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "",
                        "electionTime" : Timestamp(1536955842, 1),
                        "electionDate" : ISODate("2018-09-14T20:10:42Z"),
                        "configVersion" : 22,
                        "self" : true,
                        "lastHeartbeatMessage" : ""
                },
                {
                        "_id" : 1,
                        "name" : "127.0.0.1:7777",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 502,
                        "optime" : {
                                "ts" : Timestamp(1536956194, 1),
                                "t" : NumberLong(10)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1536956194, 1),
                                "t" : NumberLong(10)
                        },
                        "optimeDate" : ISODate("2018-09-14T20:16:34Z"),
                        "optimeDurableDate" : ISODate("2018-09-14T20:16:34Z"),
                        "lastHeartbeat" : ISODate("2018-09-14T20:16:36.329Z"),
                        "lastHeartbeatRecv" : ISODate("2018-09-14T20:16:34.808Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncingTo" : "127.0.0.1:6666",
                        "syncSourceHost" : "127.0.0.1:6666",
                        "syncSourceId" : 0,
                        "infoMessage" : "",
                        "configVersion" : 22
                },
                {
                        "_id" : 2,
                        "name" : "127.0.0.1:8888",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 164,
                        "optime" : {
                                "ts" : Timestamp(1536956194, 1),
                                "t" : NumberLong(10)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1536956194, 1),
                                "t" : NumberLong(10)
                        },
                        "optimeDate" : ISODate("2018-09-14T20:16:34Z"),
                        "optimeDurableDate" : ISODate("2018-09-14T20:16:34Z"),
                        "lastHeartbeat" : ISODate("2018-09-14T20:16:36.329Z"),
                        "lastHeartbeatRecv" : ISODate("2018-09-14T20:16:34.884Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncingTo" : "127.0.0.1:7777",
                        "syncSourceHost" : "127.0.0.1:7777",
                        "syncSourceId" : 1,
                        "infoMessage" : "",
                        "configVersion" : 22
                },
                {
                        "_id" : 3,
                        "name" : "127.0.0.1:9999",
                        "health" : 1,
                        "state" : 7,
                        "stateStr" : "ARBITER",
                        "uptime" : 502,
                        "lastHeartbeat" : ISODate("2018-09-14T20:16:36.329Z"),
                        "lastHeartbeatRecv" : ISODate("2018-09-14T20:16:35.911Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncingTo" : "",
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "",
                        "configVersion" : 22
                }
        ],
        "ok" : 1,
        "operationTime" : Timestamp(1536956194, 1),
        "$clusterTime" : {
                "clusterTime" : Timestamp(1536956194, 1),
                "signature" : {
                        "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                        "keyId" : NumberLong(0)
                }
        }
}
MongoDB Enterprise OurReplica:PRIMARY>

Step by Step YugabyteDB 2.11 (Open Source) Distributed DB - Multi-node Cluster Setup on RHEL

Scope - ·        Infrastructure planning and requirement for installation of the multi-node cluster database ·        Prerequisites Software...