DRBD + keepalived實現文件實時同步和雙機熱備

安裝DRBD

系統初始化設置

注意： 需要有數據盤或者多的分區

yum update -y

關閉防火墻

	systemctl stop firewalld
	systemctl disable firewalld

修改host文件

	vim /etc/hosts
	192.168.1.240 Primary kylin-01
	192.168.1.241 Secondary kylin-02

關閉SELINUX

	vim /etc/sysconfig/selinux
	SELINUX=disabled

安裝依賴

yum install gcc libxslt-devel libxslt perl keyutils-libs-devel net-tools -y

下載源碼編譯安裝

drbd需要兩個安裝報：drbd drbd-utils

drbd 安裝

	wget https://pkg.linbit.com//downloads/drbd/9/drbd-9.2.8.tar.gz
	tar -zxvf drbd-9.2.8.tar.gz
	cd drbd-9.2.8
	make && make install

drbd-utils 安裝

	wget https://pkg.linbit.com//downloads/drbd/utils/drbd-utils-9.27.0.tar.gz
	tar -zxvf drbd-utils-9.27.0.tar.gz
	cd drbd-utils-9.27.0
	./configure --prefix=/usr/local/drbd --without-83support --with-udev --with-initscripttype=systemd --without-manual
	make && make install

安裝配置路徑：/usr/local/drbd/etc/drbd.d 安裝路徑： /usr/sbin/drbdsetup /usr/sbin/drbdmeta /usr/sbin/drbdadm

配置drbd

磁盤分區 此處不要格式化磁盤 fdisk /dev/sdb

全局配置

global_common.conf內容如下：

	# DRBD is the result of over a decade of development by LINBIT.
	# In case you need professional services for DRBD or have
	# feature requests visit http://www.linbit.com
	global {
	usage-count yes;
	# Decide what kind of udev symlinks you want for "implicit" volumes
	# (those without explicit volume {} block, implied vnr=0):
	# /dev/drbd/by-resource// (explicit volumes)
	# /dev/drbd/by-resource/ (default for implict)
	udev-always-use-vnr; # treat implicit the same as explicit volumes
	# minor-count dialog-refresh disable-ip-verification
	# cmd-timeout-short 5; cmd-timeout-medium 121; cmd-timeout-long 600;
	}
	common {
	handlers {
	# These are EXAMPLE handlers only.
	# They may have severe implications,
	# like hard resetting the node under certain circumstances.
	# Be careful when choosing your poison.
	# IMPORTANT: most of the following scripts symlink to "notify.sh" which tries to send mail via "mail".
	# If you intend to use this notify.sh script make sure that "mail" is installed.
	#
	pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
	pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
	local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";
	# fence-peer "/usr/lib/drbd/crm-fence-peer.sh";
	# split-brain "/usr/lib/drbd/notify-split-brain.sh root";
	# out-of-sync "/usr/lib/drbd/notify-out-of-sync.sh root";
	# before-resync-target "/usr/lib/drbd/snapshot-resync-target-lvm.sh -p 15 -- -c 16k";
	# after-resync-target /usr/lib/drbd/unsnapshot-resync-target-lvm.sh;
	# quorum-lost "/usr/lib/drbd/notify-quorum-lost.sh root";
	# disconnected /bin/true;
	}
	startup {
	# wfc-timeout degr-wfc-timeout outdated-wfc-timeout wait-after-sb
	}
	options {
	# cpu-mask on-no-data-accessible
	# RECOMMENDED for three or more storage nodes with DRBD 9:
	# quorum majority;
	# on-no-quorum suspend-io | io-error;
	}
	disk {
	on-io-error detach; #配置I/O錯誤處理策略為分離
	# size on-io-error fencing disk-barrier disk-flushes
	# disk-drain md-flushes resync-rate resync-after al-extents
	# c-plan-ahead c-delay-target c-fill-target c-max-rate
	# c-min-rate disk-timeout
	}
	net {
	# protocol timeout max-epoch-size max-buffers
	# connect-int ping-int sndbuf-size rcvbuf-size ko-count
	# allow-two-primaries cram-hmac-alg shared-secret after-sb-0pri
	# after-sb-1pri after-sb-2pri always-asbp rr-conflict
	# ping-timeout data-integrity-alg tcp-cork on-congestion
	# congestion-fill congestion-extents csums-alg verify-alg
	# use-rle
	}
	}

資源配置

在node1、node2 上分別建立drbd.res： drbd.d目錄下新建資源文件（drbd.res） vim /usr/local/drbd/etc/drbd.d/drbd.res

	resource r1 { #這個r1是定義資源的名字
	protocol C;
	on kylin-01 { #on開頭，后面是主機名稱
	device /dev/drbd0; #drbd設備名稱
	disk /dev/sdb1; #drbd0使用的磁盤分區為sdb1
	address 192.168.1.240:7789; #設置drbd監聽地址與端口
	meta-disk internal;
	}
	on kylin-02 { #on開頭，后面是主機名稱
	device /dev/drbd0; #drbd設備名稱
	disk /dev/sdb1; #drbd0使用的磁盤分區為sdb1
	address 192.168.1.241:7789; #設置drbd監聽地址與端口
	meta-disk internal;
	}
	}

在node1、node2 上初始化資源、啟動drbd：

	[root@kylin-01 drbd.d]# drbdadm create-md r1
	initializing activity log
	initializing bitmap (640 KB) to all zero
	Writing meta data...
	New drbd meta data block successfully created.
	#啟動 兩個節點需要同時啟動才生效
	[root@kylin-01 drbd.d]# systemctl start drbd
	# 開機啟動
	[root@kylin-01 drbd.d]# systemctl enable drbd
	# 查看狀態
	[root@kylin-01 drbd.d]# systemctl status drbd
	# 查看狀態
	netstat -anput|grep 7789
	tcp 0 0 192.168.1.240:33015 192.168.1.241:7789 ESTABLISHED -
	tcp 0 0 192.168.1.240:40897 192.168.1.241:7789 ESTABLISHED -

檢查資源狀態：

	#查看節點角色狀態：
	drbdadm role r1
	注：第一次啟動drbd時，兩個drbd節點默認都處于Secondary狀態

主節點執行設置主節點：

	#初始化資源
	drbdadm primary --force r1
	#查看資源狀態
	drbdadm status r1
	r1 role:Primary
	disk:UpToDate
	kylin-02 role:Secondary
	replication:SyncSource peer-disk:Inconsistent done:6.16
	#查看同步狀態
	cat /proc/drbd
	version: 9.2.8 (api:2/proto:86-122)
	GIT-hash: e163b05a76254c0f51f999970e861d72bb16409a build by root@kylin-01, 2024-04-08 1556
	Transports (api tcp (9.2.8)
	會把主機上的數據傳到備機，開始會顯示同步進度，過一會顯示狀態都是”實時”,表示數據同步完成了。接下來就可以使用DRBD了。

DRBD使用（測試）

你現在可以把主機上的DRBD設備掛載到一個目錄上進行使用，備機的DRBD設備無法被掛載,因為它是用來接收主機數據的,由DRBD負責操作.

格式化文件系統（文件格式根據自己的系統環境選擇）

mkfs.ext4 /dev/drbd0

掛載此文件系統

	mkdir /data
	mount /dev/drbd0 /data/

在掛載data目錄中創建一個測試文件，然后卸載掛載目錄，然后切換主備節點，在備用節點上查看剛剛建立的測試文件還是否存在

node1:

	[root@node1 ~]# mkdir /data/test
	將node1變為備用節點
	[root@node1 ~]# umount /data/
	[root@node1 ~]# drbdadm secondary r1
	[root@node1 ~]# drbdadm role r1
	Secondary

node2:

	將node2變為主節點
	[root@node2 ~]# drbdadm primary r1
	[root@node2 ~]# drbdadm role r1
	Primary/Secondary
	掛載設備，然后看文件是否存在
	[root@node2 ~]# mount /dev/drbd0 /data
	[root@node2 ~]# cd /data/
	[root@node2 mnt]# ls
	test
	OK! 到這里已經算是完成了！

同樣，在Node2上建立文件，然后

卸載/mnt/：umount /mnt/

將Node2降級成備用節點：drbdadm secondary r1

在Node1上升級為主機節點：drbdadm primary r1

在Node1上掛載：mount /dev/drbd0 /data

會發現Node2上的文件也同步到了Node1上。

問題

1、umount時，如果提示device is busy，使用下面方法解決： fuser -m /data 顯示：/data: 25023c 然后kill -9 25023 即可

2、"Split-Brain"（腦裂）的情況：

假設把Primary主機的的eth0設備宕掉，然后直接在Secondary主機上進行提權升級為DRBD的主節點，并且mount掛載DRBD，這時會發現之前在Primary主機上寫入的數據文件確實同步過來了。 接著再把Primary主機的eth0設備恢復，看看有沒有自動恢復 主從關系。經過查看，發現DRBD檢測出了Split-Brain的狀況，也就是兩個節點都處于standalone狀態， 故障描述如下：Split-Brain detected,dropping connection! 這就是傳說中的“腦裂”。

DRBD官方推薦的手動恢復方案：

Secondary主機上的操作

	drbdadm secondary r0
	drbdadm disconnect all
	drbdadm --discard-my-data connect r0 //或者"drbdadm -- --discard-my-data connect r0"

Primary主機上的操作

	drbdadm disconnect all
	drbdadm connect r0
	drbdsetup /dev/drbd0 primary

檢查drdb狀態

	[root@kylin-01 ~]# drbdadm status r1
	r1 role:Primary
	disk:UpToDate
	kylin-02 role:Secondary
	peer-disk:UpToDate

安裝keepalived

直接采用yum安裝

yum install -y keepalived

查看keepalived版本

	[root@kylin-02 keepalived]# keepalived -v
	Keepalived v2.0.20 (01/22,2020)
	Copyright(C) 2001-2020 Alexandre Cassen,
	Built with kernel headers for Linux 4.19.90
	Running on Linux 4.19.90-52.25.v2207.ky10.x86_64 #1 SMP Fri Jun 2 1228 CST 2023
	..........

更改keepalived配置

注：本處采用pgsql測試兩個機器數據同步

pg docker-compose的文件(docker的配置安裝請自行百度)

	version: "3"
	services:
	postgresql:
	image: postgres:11.8
	container_name: postgres
	hostname: postgres
	ports:
	- "5432:5432"
	volumes:
	- "/data/pgsql:/var/lib/postgresql/data"
	- "/etc/localtime:/etc/localtime"
	restart: on-failure
	logging:
	driver: "json-file"
	options:
	tag: postgres
	cap_add:
	- ALL
	environment:
	POSTGRES_USER: "root"
	POSTGRES_PASSWORD: "123456"
	ALLOW_IP_RANGE: "0.0.0.0/0"

	cd /etc/keepalived
	vim keepalived.conf

主節點keepalived.conf 文件內容如下：

注意：再執行stop腳本時不能直接執行，需要采用腳本調用腳本的方式執行，否則執行不完就會被kill掉（原因暫時沒弄清楚）

	! Configuration File for keepalived
	global_defs {
	#notification_email {
	# acassen@firewall.loc
	# failover@firewall.loc
	# sysadmin@firewall.loc
	#}
	#notification_email_from Alexandre.Cassen@firewall.loc
	#smtp_server 192.168.200.1
	#smtp_connect_timeout 30
	router_id kylin-02 # 節點標識，主機名
	vrrp_skip_check_adv_addr
	vrrp_strict
	vrrp_garp_interval 0
	vrrp_gna_interval 0
	}
	vrrp_instance VI_1 {
	state BACKUP
	interface ens32 # 網卡
	virtual_router_id 51
	priority 100 # 節點權重，主節點100 備節點小于100，數字越大優先級越高
	mcast_src_ip 192.168.1.240 # 本機IP
	advert_int 1
	authentication {
	auth_type PASS
	auth_pass 1111
	}
	virtual_ipaddress {
	192.168.1.239 # 綁定的虛擬IP
	}
	notify_master "/etc/keepalived/notify.sh" # 節點為master時執行腳本
	notify_backup "/etc/keepalived/notify_back.sh" # 切換為備節點時執行腳本
	notify_stop "/etc/keepalived/notify_back.sh" # stop keepalived時執行的腳本
	}

備節點keepalived.conf 文件內容如下：

	! Configuration File for keepalived
	global_defs {
	#notification_email {
	# acassen@firewall.loc
	# failover@firewall.loc
	# sysadmin@firewall.loc
	#}
	#notification_email_from Alexandre.Cassen@firewall.loc
	#smtp_server 192.168.200.1
	#smtp_connect_timeout 30
	router_id kylin-02 # 節點標識，主機名
	vrrp_skip_check_adv_addr
	vrrp_strict
	vrrp_garp_interval 0
	vrrp_gna_interval 0
	}
	vrrp_instance VI_1 {
	state BACKUP
	interface ens32
	virtual_router_id 51
	priority 99 # 節點權重，主節點100 備節點小于100，數字越大優先級越高
	mcast_src_ip 192.168.1.241 # 本機IP
	advert_int 1
	authentication {
	auth_type PASS
	auth_pass 1111
	}
	virtual_ipaddress {
	192.168.1.239
	}
	notify_master "/etc/keepalived/notify.sh"
	notify_backup "/etc/keepalived/notify_back.sh"
	notify_stop "/etc/keepalived/notify_back.sh"
	}

notify.sh

	#!/bin/bash
	drbdadm primary r1
	while true
	do
	drdbs=$(drbdadm role r1)
	echo "drbd status is $drdbs"
	if [[ "$drdbs" == "Primary" ]];then
	break
	else
	drbdadm primary r1
	sleep 3
	fi
	done
	mount /dev/drbd0 /data
	docker-compose -f /opt/pgsql/docker-compose.yml up -d

stop.sh

	#!/bin/bash
	docker stop postgres
	umount /data/
	drbdadm secondary r1
	while true
	do
	drdbs=$(drbdadm role r1)
	echo "drbd status is $drdbs"
	if [[ "$drdbs"=="Secondary" ]];then
	break
	else
	drbdadm secondary r1
	sleep 3
	fi
	done

notify_back.sh

	#!/bin/bash
	/etc/keepalived/stop.sh

啟動keepalived

systemctl start keepalived

開機啟動

systemctl enable keepalived

驗證

在數據庫里面執行創建刪除，切換keepalived節點后查看數據是否同步

鏈接：https://www.cnblogs.com/pgyLang/p/18124303