2020-10-25

PostgreSQL: Check config file loaded on runtime

1/ Current config loaded to memory at runtime
TABLE pg_file_settings;
SELECT * FROM pg_catalog.pg_file_settings order by name asc;
SELECT * FROM pg_catalog.pg_file_settings order by sourceline asc;



































postgres=# SELECT count (*) FROM pg_catalog.pg_file_settings;
 count 
-------
    77
(1 row)



2/ Ref:
SELECT pg_read_file('pg_hba.conf');
select pg_reload_conf();

2020-10-24

PatroniCluster 3 Node: [postgresql.base.conf]

#/opt/postgres/pgdata/postgresql.base.conf
#LastUpdate: #18:13 2020.10.22
#cat /opt/postgres/pgdata/postgresql.base.conf | egrep "LastUpdate|#10.0.2.*"
################################################################
#scp /opt/postgres/pgdata/postgresql.base.conf root@srv112:/opt/postgres/pgdata/postgresql.base.conf; scp /opt/postgres/pgdata/postgresql.base.conf root@srv113:/opt/postgres/pgdata/postgresql.base.conf
################################################################
#export PATH=$PATH:/opt/postgres/pgscript; postgres-check-running-config.sh
#postgres=# TABLE pg_file_settings;
#10.0.2.111*
#10.0.2.112*
#10.0.2.113*

################################################################
checkpoint_completion_target = '0.9'
cluster_name = 'postgres'
datestyle = 'iso, mdy'
default_statistics_target = '500'
default_text_search_config = 'pg_catalog.english'
dynamic_shared_memory_type = 'posix'
effective_cache_size = '12GB'
effective_io_concurrency = '200'
hot_standby = 'on'
lc_messages = 'en_US.UTF-8'
lc_monetary = 'en_US.UTF-8'
lc_numeric = 'en_US.UTF-8'
lc_time = 'en_US.UTF-8'
listen_addresses = '0.0.0.0'
log_destination = 'syslog'
log_directory = '/opt/postgres/pglog'
log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log'
log_line_prefix = '%t user=%u database=%d client_ip=%h '
log_min_duration_statement = '0'
log_rotation_age = '1d'
log_rotation_size = '100MB'
log_statement = 'ddl'
log_timezone = 'Asia/Saigon'
log_truncate_on_rotation = 'True'
logging_collector = 'True'

maintenance_work_mem = '2GB'
max_connections = '1000'
max_locks_per_transaction = '64'
max_parallel_workers = '8'
max_parallel_workers_per_gather = '4'
max_prepared_transactions = '0'
max_replication_slots = '10'
max_wal_senders = '10'
max_wal_size = '8GB'
max_worker_processes = '8'

min_wal_size = '4GB'
port = '5432'
random_page_cost = '1.1'
shared_buffers = '4GB'
timezone = 'Asia/Saigon'
track_commit_timestamp = 'off'
unix_socket_directories = '.'
wal_buffers = '16MB'
wal_keep_segments = '8'
wal_level = 'replica'
wal_log_hints = 'on'
work_mem = '524kB'
hba_file = '/opt/postgres/pgdata/pg_hba.conf'
ident_file = '/opt/postgres/pgdata/pg_ident.conf'

pg_stat_statements.max = 10000
pg_stat_statements.track = all
shared_preload_libraries = 'pg_stat_statements, pgaudit'
track_activity_query_size = 2048
pg_stat_statements.track = all
#THE_END

PatroniCluster 3 Node: [pg_hba.conf]

#/opt/postgres/pgdata/pg_hba.conf
#LastUpdate: #16:54 2020.10.22
################################################################
#cat /opt/postgres/pgdata/pg_hba.conf | egrep "LastUpdate|#10.0.2.*"
################################################################
#scp /opt/postgres/pgdata/pg_hba.conf root@srv112:/opt/postgres/pgdata/pg_hba.conf;
#scp /opt/postgres/pgdata/pg_hba.conf root@srv113:/opt/postgres/pgdata/pg_hba.conf
################################################################
#10.0.2.111*
#10.0.2.112*
#10.0.2.113*
################################################################
# PostgreSQL Client Authentication Configuration File
# ===================================================
# TYPE DATABASE USER ADDRESS METHOD
# "local" is for Unix domain socket connections only
local all all trust
# IPv4 local connections:
host all all 127.0.0.1/32 trust
# IPv6 local connections:
host all all ::1/128 trust
# Allow replication connections from localhost, by a user with the
# replication privilege.
local replication all trust
host replication all 127.0.0.1/32 trust
host replication all ::1/128 trust
###################################
#/opt/postgres/pgdata/pg_hba.conf
host replication replicator 127.0.0.1/32 md5
host replication replicator 10.0.2.111/32 trust
host replication replicator 10.0.2.112/32 trust
host replication replicator 10.0.2.113/32 trust
host replication replicator all md5
host replication replicator 0.0.0.0/0 md5
host all postgres 127.0.0.1/32 trust
host all postgres 10.0.2.111/32 md5
host all postgres 10.0.2.112/32 md5
host all postgres 10.0.2.113/32 md5
host all postgres 0.0.0.0/0 md5
host all all 0.0.0.0/0 md5

2020-10-23

PatroniCluster 3 Node: [patroni-status.sh]

#/opt/script/patroni-status.sh
#LastUpdate: #14:32 2020.10.22
################################################################
#cat /opt/script/patroni-status.sh | grep LastUpdate
################################################################
#scp /opt/script/patroni-status.sh root@srv112:/opt/script/patroni-status.sh
#scp /opt/script/patroni-status.sh root@srv113:/opt/script/patroni-status.sh
#RESTART PATRONI:
#systemctl daemon-reload; sudo systemctl restart patroni; sleep 5; sudo systemctl status patroni --no-pager;
#10.0.2.111*
#10.0.2.112*
#10.0.2.113*
################################################################
#systemctl status patroni.service
#systemctl enable patroni.service
#sudo systemctl start patroni.service
#sudo systemctl status patroni.service
################################################################
#REF: https://www.cybertec-postgresql.com/en/patroni-setting-up-a-highly-available-postgresql-cluster/
echo "############################################"
echo "DAEMON STATUS:"
sudo systemctl status patroni --no-pager;


echo "--------------------------------------------"
echo "PROCESS STATUS:"
ps aux| egrep "PID|py"
echo "--------------------------------------------"
echo "TCP STATUS:"
netstat -ntlup | egrep "PID|LISTEN" | egrep "PID|python|etcd|postgres|haproxy" | sort -t: -k2 -n
echo "--------------------------------------------"

echo "PATRONI CLUSTER STATUS:"
patronictl -c /etc/patroni.yml list postgres
echo ""
#curl http://10.0.2.111:8008 | json_pp
curl http://$(hostname -I | awk '{print $1}'):8008 | json_pp
echo "---"
echo "DONE"
#THE_END

PatroniCluster 3 Node: [patroni-stop.sh]

#/opt/script/patroni-stop.sh
#LastUpdate: #12:54 2020.10.22
################################################################
#cat /opt/script/patroni-stop.sh | grep LastUpdate
################################################################
#scp /opt/script/patroni-stop.sh root@srv112:/opt/script/patroni-stop.sh
#scp /opt/script/patroni-stop.sh root@srv113:/opt/script/patroni-stop.sh
#RESTART PATRONI:
#systemctl daemon-reload; sudo systemctl restart patroni; sleep 5; sudo systemctl status patroni --no-pager;
#10.0.2.111*
#10.0.2.112*
#10.0.2.113*
################################################################
#systemctl status patroni.service
#systemctl enable patroni.service
#sudo systemctl start patroni.service
#sudo systemctl status patroni.service
#################################################
#REF: https://www.cybertec-postgresql.com/en/patroni-setting-up-a-highly-available-postgresql-cluster/
sudo systemctl stop patroni
sleep 2
echo "############################################"
echo "DAEMON STATUS:"
sudo systemctl status patroni --no-pager;
echo "--------------------------------------------"
echo "PROCESS STATUS:"
ps aux| egrep "PID|py"
echo "--------------------------------------------"
echo "TCP STATUS:"
netstat -ntlup | egrep "PID|LISTEN" | egrep "PID|python|etcd|postgres|haproxy" | sort -t: -k2 -n
#THE_END

PatroniCluster 3 Node: [patroni-restart.sh]

#/opt/script/patroni-restart.sh
#LastUpdate: #12:01 2020.10.22
################################################################
#cat /opt/script/patroni-restart.sh | grep LastUpdate
################################################################
#scp /opt/script/patroni-restart.sh root@srv112:/opt/script/patroni-restart.sh
#scp /opt/script/patroni-restart.sh root@srv113:/opt/script/patroni-restart.sh
#RESTART PATRONI:
#systemctl daemon-reload; sudo systemctl restart patroni; sleep 5; sudo systemctl status patroni --no-pager;
#10.0.2.111*
#10.0.2.112*
#10.0.2.113*
################################################################
#systemctl status patroni.service
#systemctl enable patroni.service
#sudo systemctl start patroni.service
#sudo systemctl status patroni.service
#################################################
#REF: https://www.linode.com/docs/databases/postgresql/create-a-highly-available-postgresql-cluster-using-patroni-and-haproxy/#install-patroni
#https://www.cybertec-postgresql.com/en/patroni-setting-up-a-highly-available-postgresql-cluster/
sudo systemctl restart patroni
sleep 2
echo "############################################"
echo "DAEMON STATUS:"
sudo systemctl status patroni --no-pager;
echo "--------------------------------------------"
echo "PROCESS STATUS:"
ps aux| egrep "PID|py"
echo "--------------------------------------------"
echo "TCP STATUS:"
netstat -ntlup | egrep "PID|LISTEN" | egrep "PID|python|etcd|postgres|haproxy" | sort -t: -k2 -n

#THE_END

PatroniCluster 3 Node: [/etc/patroni.yml]

#/etc/patroni.yml
#LastUpdate: #17:19 2020.10.22, #16:57 2020.10.22
################################################################
#cat /etc/patroni.yml | egrep "LastUpdate|#10.0.2.*"
################################################################
#scp /etc/patroni.yml root@srv112:/etc/patroni.yml; scp /etc/patroni.yml root@srv113:/etc/patroni.yml
################################################################
#RESTART PATRONI:
#systemctl daemon-reload; sudo systemctl restart patroni; sleep 2; sudo systemctl status patroni --no-pager;
#patroni-restart.sh
#10.0.2.111*
#10.0.2.112
#10.0.2.113
################################################################
#REF: https://github.com/zalando/patroni/blob/master/postgres0.yml
scope: postgres
namespace: /db/
name: psql_node01
#name: psql_node02
#name: psql_node03
restapi:
    listen: 10.0.2.111:8008
    #listen: 10.0.2.112:8008
    #listen: 10.0.2.113:8008
    connect_address: 10.0.2.111:8008
    #connect_address: 10.0.2.112:8008
    #connect_address: 10.0.2.113:8008
    
etcd:
    hosts: 10.0.2.111:2379
    #host: 10.0.2.112:2379
    #host: 10.0.2.113:2379
    protocol: http
    
bootstrap:
    dcs:
        ttl: 30
        loop_wait: 10
        retry_timeout: 10
        maximum_lag_on_failover: 1048576
        postgresql:
            use_pg_rewind: true
            parameters:
                checkpoint_completion_target = '0.9'
                cluster_name = 'postgres'
                datestyle = 'iso, mdy'
                default_statistics_target = '500'
                default_text_search_config = 'pg_catalog.english'
                dynamic_shared_memory_type = 'posix'
                effective_cache_size = '12GB'
                effective_io_concurrency = '200'
                hot_standby = 'on'
                lc_messages = 'en_US.UTF-8'
                lc_monetary = 'en_US.UTF-8'
                lc_numeric = 'en_US.UTF-8'
                lc_time = 'en_US.UTF-8'
                listen_addresses = '0.0.0.0'
                log_destination = 'syslog'
                log_directory = '/opt/postgres/pglog'
                log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log'
                log_line_prefix = '%t user=%u database=%d client_ip=%h '
                log_min_duration_statement = '0'
                log_rotation_age = '1d'
                log_rotation_size = '100MB'
                log_statement = 'ddl'
                log_timezone = 'Asia/Saigon'
                log_truncate_on_rotation = 'True'
                logging_collector = 'True'
                maintenance_work_mem = '2GB'
                max_connections = '1000'
                max_locks_per_transaction = '64'
                max_parallel_workers = '8'
                max_parallel_workers_per_gather = '4'
                max_prepared_transactions = '0'
                max_replication_slots = '10'
                max_wal_senders = '10'
                max_wal_size = '8GB'
                max_worker_processes = '8'
                min_wal_size = '4GB'
                port = '5432'
                random_page_cost = '1.1'
                shared_buffers = '4GB'
                timezone = 'Asia/Saigon'
                track_commit_timestamp = 'off'
                unix_socket_directories = '.'
                wal_buffers = '16MB'
                wal_keep_segments = '8'
                wal_level = 'replica'
                wal_log_hints = 'on'
                work_mem = '524kB'
                hba_file = '/opt/postgres/pgdata/pg_hba.conf'
                ident_file = '/opt/postgres/pgdata/pg_ident.conf'                
                pg_stat_statements.max = 10000
                pg_stat_statements.track = all
                shared_preload_libraries = 'pg_stat_statements, pgaudit'
                track_activity_query_size = 2048
                pg_stat_statements.track = all
    initdb:
    - encoding: UTF8
    - data-checksums
    - auth-host: md5
    - auth-local: trust
    ##/opt/postgres/pgdata/pg_hba.conf
    pg_hba:
    - host replication replicator 10.0.2.111/32 trust
    - host replication replicator 10.0.2.112/32 trust
    - host replication replicator 10.0.2.113/32 trust
    - host replication replicator all md5
    - host replication replicator 0.0.0.0/0 md5    
    - host all postgres 127.0.0.1/32 trust
    - host all postgres 10.0.2.111/32 md5
    - host all postgres 10.0.2.112/32 md5
    - host all postgres 10.0.2.113/32 md5    
    - host all postgres 0.0.0.0/0 md5
    - host all all 0.0.0.0/0 md5
    users:
        admin:
            password: admin
            options:
                - createrole
                - createdb
postgresql:
    listen: 0.0.0.0:5432
    connect_address: 10.0.2.111:5432
    #connect_address: 10.0.2.112:5432
    #connect_address: 10.0.2.113:5432
    data_dir: /opt/postgres/pgdata
    bin_dir: /opt/postgres/bin
    pgpass: /tmp/pgpass
    authentication:
        replication:
            username: replicator
            password: xxx
        superuser:
            username: postgres
            password: xxx
    parameters:
        unix_socket_directories: '.'
tags:
    nofailover: false
    noloadbalance: false
    clonefrom: false
    nosync: false
#THE_END

PatroniCluster 3 Node: [patroni.service]

#/etc/systemd/system/patroni.service
#LastUpdate: #11:36 2020.10.22
################################################################
#cat /etc/systemd/system/patroni.service | grep LastUpdate
################################################################
#scp /etc/systemd/system/patroni.service root@srv112:/etc/systemd/system/patroni.service
#scp /etc/systemd/system/patroni.service root@srv113:/etc/systemd/system/patroni.service
#RESTART PATRONI:
#systemctl daemon-reload; sudo systemctl restart patroni; sleep 5; sudo systemctl status patroni --no-pager;
#10.0.2.111*
#10.0.2.112*
#10.0.2.113*
################################################################
[Unit]
Description=High availability PostgreSQL Cluster
After=syslog.target network.target
[Service]
Type=simple
User=postgres
Group=postgres
ExecStart=/usr/local/bin/patroni /etc/patroni.yml
KillMode=process
TimeoutSec=30
Restart=no
[Install]
WantedBy=multi-user.targ

PatroniCluster 3 Node: Completed Guide Index

#Index:

1/ ENV
10.0.2.111 srv111
10.0.2.112 srv112
10.0.2.113 srv113

root@srv111:/# date; cat /etc/lsb-release 
Fri Oct 23 16:19:52 +07 2020
DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=18.04
DISTRIB_CODENAME=bionic
DISTRIB_DESCRIPTION="Ubuntu 18.04.5 LTS"

root@srv111:/# date; uname -a
Fri Oct 23 16:19:46 +07 2020
Linux srv111 4.15.0-118-generic #119-Ubuntu SMP Tue Sep 8 12:30:01 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux


2/ /etc/systemd/system/patroni.service
REF: https://www.mgiay.com/2020/10/patronicluster-3-node-patroniservice.html

3/ /etc/patroni.yml
5/ /opt/script/patroni-stop.sh
7/ /opt/postgres/pgdata/postgresql.base.conf
9/ Patroni Controller:
#9:19 2020.10.23
root@srv078psql101:~# patronictl -c /etc/patroni.yml list postgres
+----------+------------+-----------+--------+---------+----+-----------+
| Cluster  |   Member   |   Host    |  Role  |  State  | TL | Lag in MB |
+----------+------------+-----------+--------+---------+----+-----------+
| postgres | indo_psql1 | 10.0.2.78 | Leader | running |  5 |       0.0 |
| postgres | indo_psql2 | 10.0.2.79 |        | running |  5 |       0.0 |
+----------+------------+-----------+--------+---------+----+-----------+
root@srv078psql101:~# 
root@srv078psql101:~# #patronictl -c /etc/patroni.yml edit-config postgres

How to convert IOPS<->MB

Công thức quy đổi IOPS ra MB/s và ngược lại, MB/s ra IOPS:
1/ IOPS = (MB/s thông lượng / KB vào ra) x 1.024

2/ MB/s = (IOPS x KB vào ra) / 1.024

Theo nội dung trên, ta có IOPS 150.000 và kích thước dữ liệu vào ra 4KB, quy đổi ra MB/s sẽ là:
MB/s = (IOPS x KB vào ra) / 1.024 = (150.000 x 4) / 1.024 = 585,9375
Như vậy, 150.000 IOPS sẽ gần bằng 600 MB/s.

IOPS=1.000.000
->MB/s=3906.25MB/s ~ 3.9GB/s


2020-10-22

Patroni: Installing

1/ Install:
##sudo apt-get update
##sudo apt-get install --fix-missing
##sudo apt-get -y upgrade
##sudo apt -y autoremove
##sudo apt-get -y install python-pip
##sudo apt-get -y install python3-pip

PIP_CACHE="/opt/setup/pip"
mkdir -p $PIP_CACHE
sudo pip --cache-dir=$PIP_CACHE

##sudo pip3 --cache-dir=$PIP_CACHE install --upgrade pip
##sudo pip3 --cache-dir=$PIP_CACHE install psychopy
\\##sudo pip3 --cache-dir=$PIP_CACHE install psycopg2: ERR
##sudo pip3 --cache-dir=$PIP_CACHE install psycopg2-binary

##sudo pip --cache-dir=$PIP_CACHE install --upgrade setuptools

##sudo pip --cache-dir=$PIP_CACHE install patroni
##sudo pip --cache-dir=$PIP_CACHE install wheel


2/ Result:

/usr/local/bin/patronictl version -> "patronictl version 2.0.1"
/usr/bin/patronictl version       -> "patronictl version 1.4.2"    
/usr/local/bin/patroni --version  -> "patroni 2.0.1"   


3/ #/etc/systemd/system/patroni.service
#/etc/systemd/system/patroni.service
#LastUpdate: #11:36 2020.10.22
################################################################
#cat /etc/systemd/system/patroni.service | grep LastUpdate
################################################################
#scp /etc/systemd/system/patroni.service root@srv112:/etc/systemd/system/patroni.service
#scp /etc/systemd/system/patroni.service root@srv113:/etc/systemd/system/patroni.service

#RESTART PATRONI:
#systemctl daemon-reload; sudo systemctl restart patroni; sleep 5; sudo systemctl status patroni --no-pager;
#10.0.2.111*
#10.0.2.112*
#10.0.2.113*
################################################################
[Unit]
Description=High availability PostgreSQL Cluster
After=syslog.target network.target

[Service]
Type=simple
User=postgres
Group=postgres
ExecStart=/usr/local/bin/patroni /etc/patroni.yml
KillMode=process
TimeoutSec=30
Restart=no

[Install]
WantedBy=multi-user.targ






4/ #/opt/script/patroni-restart.sh
#/opt/script/patroni-restart.sh
#LastUpdate: #12:01 2020.10.22
################################################################
#cat /opt/script/patroni-restart.sh | grep LastUpdate
################################################################
#scp /opt/script/patroni-restart.sh root@srv112:/opt/script/patroni-restart.sh
#scp /opt/script/patroni-restart.sh root@srv113:/opt/script/patroni-restart.sh

#RESTART PATRONI:
#systemctl daemon-reload; sudo systemctl restart patroni; sleep 5; sudo systemctl status patroni --no-pager;
#10.0.2.111*
#10.0.2.112*
#10.0.2.113*
################################################################
#systemctl status patroni.service
#systemctl enable patroni.service

#sudo systemctl start patroni.service
#sudo systemctl status patroni.service
#################################################
#REF: https://www.linode.com/docs/databases/postgresql/create-a-highly-available-postgresql-cluster-using-patroni-and-haproxy/#install-patroni
#https://www.cybertec-postgresql.com/en/patroni-setting-up-a-highly-available-postgresql-cluster/
sudo systemctl restart patroni
sleep 2

echo "############################################"
echo "DAEMON STATUS:"
sudo systemctl status patroni --no-pager;

echo "--------------------------------------------"
echo "PROCESS STATUS:"
ps aux| egrep "PID|py"

echo "--------------------------------------------"
echo "TCP STATUS:"
netstat -ntlup | egrep "PID|LISTEN" | egrep "PID|python|etcd|postgres|haproxy" | sort -t: -k2 -n
#THE_END


5/#/opt/script/patroni-status.sh
#LastUpdate: #14:32 2020.10.22
################################################################
#cat /opt/script/patroni-status.sh | grep LastUpdate
################################################################
#scp /opt/script/patroni-status.sh root@srv112:/opt/script/patroni-status.sh
#scp /opt/script/patroni-status.sh root@srv113:/opt/script/patroni-status.sh

#RESTART PATRONI:
#systemctl daemon-reload; sudo systemctl restart patroni; sleep 5; sudo systemctl status patroni --no-pager;
#10.0.2.111*
#10.0.2.112*
#10.0.2.113*
################################################################
#systemctl status patroni.service
#systemctl enable patroni.service

#sudo systemctl start patroni.service
#sudo systemctl status patroni.service
################################################################
#REF: https://www.cybertec-postgresql.com/en/patroni-setting-up-a-highly-available-postgresql-cluster/
echo "############################################"
echo "DAEMON STATUS:"
sudo systemctl status patroni --no-pager;



echo "--------------------------------------------"
echo "PROCESS STATUS:"
ps aux| egrep "PID|py"

echo "--------------------------------------------"
echo "TCP STATUS:"
netstat -ntlup | egrep "PID|LISTEN" | egrep "PID|python|etcd|postgres|haproxy" | sort -t: -k2 -n

echo "--------------------------------------------"
echo "PATRONI CLUSTER STATUS:"
patronictl -c /etc/patroni.yml list postgres
echo ""

#curl http://10.0.2.111:8008 | json_pp
curl http://$(hostname -I | awk '{print $1}'):8008 | json_pp

echo "---"
echo "DONE"
#THE_END


6/ #/opt/script/patroni-stop.sh
#LastUpdate: #12:54 2020.10.22
################################################################
#cat /opt/script/patroni-stop.sh | grep LastUpdate
################################################################
#scp /opt/script/patroni-stop.sh root@srv112:/opt/script/patroni-stop.sh
#scp /opt/script/patroni-stop.sh root@srv113:/opt/script/patroni-stop.sh

#RESTART PATRONI:
#systemctl daemon-reload; sudo systemctl restart patroni; sleep 5; sudo systemctl status patroni --no-pager;
#10.0.2.111*
#10.0.2.112*
#10.0.2.113*
################################################################
#systemctl status patroni.service
#systemctl enable patroni.service

#sudo systemctl start patroni.service
#sudo systemctl status patroni.service
#################################################
#REF: https://www.cybertec-postgresql.com/en/patroni-setting-up-a-highly-available-postgresql-cluster/
sudo systemctl stop patroni
sleep 2

echo "############################################"
echo "DAEMON STATUS:"
sudo systemctl status patroni --no-pager;

echo "--------------------------------------------"
echo "PROCESS STATUS:"
ps aux| egrep "PID|py"

echo "--------------------------------------------"
echo "TCP STATUS:"
netstat -ntlup | egrep "PID|LISTEN" | egrep "PID|python|etcd|postgres|haproxy" | sort -t: -k2 -n

#THE_END


















2020-10-21

How to install Keepalived: 2 node/3 node

#1/ Node:
Node01: 10.0.2.111
Node02: 10.0.2.112
Node03: 10.0.2.113
V.IP: 10.0.2.110


2/ Config:
#/etc/keepalived/keepalived.conf
#LastUpdate: #18:47 2020.10.21, #18:43 2020.10.21
################################################################
#cat /etc/keepalived/keepalived.conf | egrep "LastUpdate|#10.0.2.11"
################################################################
#RESTART KEEPALIVED:
#START:
#systemctl daemon-reload; sudo systemctl restart keepalived; sleep 5; sudo systemctl status keepalived --no-pager;
#STOP:
#systemctl daemon-reload; sudo systemctl stop keepalived; sudo systemctl status keepalived --no-pager;
#10.0.2.111*
#10.0.2.112
#10.0.2.113
################################################################
#LAN INTERFACE: ens160
#LAN MASTER : 10.0.2.111/24
#LAN BACKUP1: 10.0.2.112/24
#LAN BACKUP2: 10.0.2.113/24
#V.I.P      : 10.0.2.110/32
################################################################
global_defs {
    router_id PSQL_HA
}

#LAN, MASTER: highest ID
vrrp_instance VRRP_LAN_ens160 {
    state MASTER
    interface ens160            # interface to monitor
    virtual_router_id 110       # Assign one ID for this route
    priority 9                  #MASTER, highest ID
    #priority 8                 #BACKUP, lowest ID
    #priority 7                 #BACKUP, lowest ID
    advert_int 1
    smtp_alert
    authentication {
        auth_type PASS
        auth_pass 111173LAN
    }
    virtual_ipaddress {
        10.0.2.110
    }
}
########
#THE_END
########


3/ Restart:
systemctl daemon-reload; sudo systemctl restart keepalived; sleep 5; sudo systemctl status keepalived --no-pager;


4/ Monitor:

root@srv111:~# sudo systemctl status keepalived --no-pager;

● keepalived.service - Keepalive Daemon (LVS and VRRP)
   Loaded: loaded (/lib/systemd/system/keepalived.service; enabled; vendor preset: enabled)
   Active: active (running) since Wed 2020-10-21 18:51:07 +07; 5s ago
  Process: 2564 ExecStart=/usr/sbin/keepalived $DAEMON_ARGS (code=exited, status=0/SUCCESS)
 Main PID: 2565 (keepalived)
    Tasks: 3 (limit: 4915)
   CGroup: /system.slice/keepalived.service
           ├─2565 /usr/sbin/keepalived
           ├─2566 /usr/sbin/keepalived
           └─2567 /usr/sbin/keepalived
Oct 21 18:51:07 srv111 Keepalived_healthcheckers[2566]: Opening file '/etc/keepalived/keepalived.conf'.
Oct 21 18:51:07 srv111 Keepalived[2565]: Starting VRRP child process, pid=2567
Oct 21 18:51:07 srv111 Keepalived_vrrp[2567]: Registering Kernel netlink reflector
Oct 21 18:51:07 srv111 Keepalived_vrrp[2567]: Registering Kernel netlink command channel
Oct 21 18:51:07 srv111 Keepalived_vrrp[2567]: Registering gratuitous ARP shared channel
Oct 21 18:51:07 srv111 Keepalived_vrrp[2567]: Opening file '/etc/keepalived/keepalived.conf'.
Oct 21 18:51:07 srv111 Keepalived_vrrp[2567]: Truncating auth_pass to 8 characters
Oct 21 18:51:07 srv111 Keepalived_vrrp[2567]: Using LinkWatch kernel netlink reflector...
Oct 21 18:51:08 srv111 Keepalived_vrrp[2567]: VRRP_Instance(VRRP_LAN_ens160) Transition to MASTER STATE
Oct 21 18:51:09 srv111 Keepalived_vrrp[2567]: VRRP_Instance(VRRP_LAN_ens160) Entering MASTER STATE

ETCD:Dynamically REMOVE a node from the cluster

root@srv113:/# cd /;date; etcd-members.sh; etcdctl cluster-health
Wed Oct 21 08:52:45 +07 2020
..
ETCD MEMBERs: 
1b1776006795f774: name=etcd1 peerURLs=http://10.0.2.111:2380 clientURLs=http://10.0.2.111:2379 isLeader=true
89df995ba1c163a3: name=etcd3 peerURLs=http://10.0.2.113:2380 clientURLs=http://10.0.2.113:2379 isLeader=false
9e01e0d9b238a128: name=etcd2 peerURLs=http://10.0.2.112:2380 clientURLs=http://10.0.2.112:2379 isLeader=false
ea59efe5cd5e204e: name=etcd4 peerURLs=http://etcd4:2380 clientURLs=http://10.0.2.114:2379 isLeader=false
member 1b1776006795f774 is healthy: got healthy result from http://10.0.2.111:2379
member 89df995ba1c163a3 is healthy: got healthy result from http://10.0.2.113:2379
member 9e01e0d9b238a128 is healthy: got healthy result from http://10.0.2.112:2379
member ea59efe5cd5e204e is healthy: got healthy result from http://10.0.2.114:2379
cluster is healthy
root@srv113:/# 



root@srv113:/# etcdctl member remove ea59efe5cd5e204e
Removed member ea59efe5cd5e204e from cluster
root@srv113:/# 


root@srv113:/# cd /;date; etcd-members.sh; etcdctl cluster-health
Wed Oct 21 08:55:27 +07 2020
..
ETCD MEMBERs: 
1b1776006795f774: name=etcd1 peerURLs=http://10.0.2.111:2380 clientURLs=http://10.0.2.111:2379 isLeader=true
89df995ba1c163a3: name=etcd3 peerURLs=http://10.0.2.113:2380 clientURLs=http://10.0.2.113:2379 isLeader=false
9e01e0d9b238a128: name=etcd2 peerURLs=http://10.0.2.112:2380 clientURLs=http://10.0.2.112:2379 isLeader=false
member 1b1776006795f774 is healthy: got healthy result from http://10.0.2.111:2379
member 89df995ba1c163a3 is healthy: got healthy result from http://10.0.2.113:2379
member 9e01e0d9b238a128 is healthy: got healthy result from http://10.0.2.112:2379
cluster is healthy
root@srv113:/# 

ETCD: Dynamically ADDING a node to the cluster

0/ Environment:
Config Host file:
#HOST FILE:
chattr -i -f /etc/hosts
cp -vR /etc/hosts /etc/hosts-
#{
cat <<EOF > /etc/hosts
#!/bin/bash
127.0.0.1 localhost
10.0.2.111 srv111
10.0.2.112 srv112
10.0.2.113 srv113
10.0.2.114 srv114
#
10.0.2.111 etcd1.mgiay.local etcd1
10.0.2.112 etcd2.mgiay.local etcd2
10.0.2.113 etcd3.mgiay.local etcd3
10.0.2.114 etcd4.mgiay.local etcd4
#THE-END
EOF
cat /etc/hosts
chattr +i -f /etc/hosts
#}
#


1/ Install ETCD on new node: node4
REF: https://www.mgiay.com/2020/10/how-to-install-etcd-from-source-code.html



2/ Copy script to new node: node4
REF: https://www.mgiay.com/2020/10/how-to-install-etcd-cluster-3-node-on.html
#Script name: 
/opt/script/etcd-initial.sh
/opt/script/etcd-stop.sh
/opt/script/etcd-log.sh
/opt/script/etcd-members.sh
/opt/script/etcd-status.sh


3/ Update config of "/opt/script/etcd-restart.sh":
..
#_______________________________________________________________ETCD_CONFIG:BEGIN
#REF: https://etcd.io/docs/v3.4.0/op-guide/clustering/
etcd \
--name etcd4 \
--data-dir=$ETCD_DATA \
--initial-advertise-peer-urls http://10.0.2.114:2380 \
--listen-peer-urls http://10.0.2.114:2380 \
--listen-client-urls http://0.0.0.0:2379 \
--advertise-client-urls http://10.0.2.114:2379 \
--initial-cluster-token etcd-cluster-1 \
--initial-cluster etcd1=http://10.0.2.111:2380,etcd2=http://10.0.2.112:2380,etcd3=http://10.0.2.113:2380,etcd4=http://10.0.2.114:2380 \
--initial-cluster-state existing > $ETCD_LOG 2>&1 &
#_______________________________________________________________ETCD_CONFIG:END
..



4/ Add new node (node4) to existed cluster:
root@srv113:~# etcd-members.sh

ETCD MEMBERs: 
1b1776006795f774: name=etcd1 peerURLs=http://10.0.2.111:2380 clientURLs=http://10.0.2.111:2379 isLeader=true
89df995ba1c163a3: name=etcd3 peerURLs=http://10.0.2.113:2380 clientURLs=http://10.0.2.113:2379 isLeader=false
9e01e0d9b238a128: name=etcd2 peerURLs=http://10.0.2.112:2380 clientURLs=http://10.0.2.112:2379 isLeader=false

#ADDING NEW MEMBER TO EXISTED CLUSTER:
root@srv113:~# etcdctl member add etcd4 http://etcd4:2380
Added member named etcd4 with ID ea59efe5cd5e204e to cluster
ETCD_NAME="etcd4"

ETCD_INITIAL_CLUSTER="etcd1=http://10.0.2.111:2380,etcd3=http://10.0.2.113:2380,etcd2=http://10.0.2.112:2380,etcd4=http://etcd4:2380"
ETCD_INITIAL_CLUSTER_STATE="existing"



5/ Result:

root@srv111:/# cd /;date; etcd-members.sh; etcdctl cluster-health
Wed Oct 21 08:32:00 +07 2020
..
ETCD MEMBERs: 
1b1776006795f774: name=etcd1 peerURLs=http://10.0.2.111:2380 clientURLs=http://10.0.2.111:2379 isLeader=true
89df995ba1c163a3: name=etcd3 peerURLs=http://10.0.2.113:2380 clientURLs=http://10.0.2.113:2379 isLeader=false
9e01e0d9b238a128: name=etcd2 peerURLs=http://10.0.2.112:2380 clientURLs=http://10.0.2.112:2379 isLeader=false
ea59efe5cd5e204e: name=etcd4 peerURLs=http://etcd4:2380 clientURLs=http://10.0.2.114:2379 isLeader=false
member 1b1776006795f774 is healthy: got healthy result from http://10.0.2.111:2379
member 89df995ba1c163a3 is healthy: got healthy result from http://10.0.2.113:2379
member 9e01e0d9b238a128 is healthy: got healthy result from http://10.0.2.112:2379
member ea59efe5cd5e204e is healthy: got healthy result from http://10.0.2.114:2379
cluster is healthy
root@srv111:/# 

root@srv112:/var/cache/apt# cd /;date; etcd-members.sh; etcdctl cluster-health
Wed Oct 21 08:32:52 +07 2020
..
ETCD MEMBERs: 
1b1776006795f774: name=etcd1 peerURLs=http://10.0.2.111:2380 clientURLs=http://10.0.2.111:2379 isLeader=true
89df995ba1c163a3: name=etcd3 peerURLs=http://10.0.2.113:2380 clientURLs=http://10.0.2.113:2379 isLeader=false
9e01e0d9b238a128: name=etcd2 peerURLs=http://10.0.2.112:2380 clientURLs=http://10.0.2.112:2379 isLeader=false
ea59efe5cd5e204e: name=etcd4 peerURLs=http://etcd4:2380 clientURLs=http://10.0.2.114:2379 isLeader=false
member 1b1776006795f774 is healthy: got healthy result from http://10.0.2.111:2379
member 89df995ba1c163a3 is healthy: got healthy result from http://10.0.2.113:2379
member 9e01e0d9b238a128 is healthy: got healthy result from http://10.0.2.112:2379
member ea59efe5cd5e204e is healthy: got healthy result from http://10.0.2.114:2379
cluster is healthy
root@srv112:/# 

root@srv113:~# cd /;date; etcd-members.sh; etcdctl cluster-health
Wed Oct 21 08:32:54 +07 2020
..
ETCD MEMBERs: 
1b1776006795f774: name=etcd1 peerURLs=http://10.0.2.111:2380 clientURLs=http://10.0.2.111:2379 isLeader=true
89df995ba1c163a3: name=etcd3 peerURLs=http://10.0.2.113:2380 clientURLs=http://10.0.2.113:2379 isLeader=false
9e01e0d9b238a128: name=etcd2 peerURLs=http://10.0.2.112:2380 clientURLs=http://10.0.2.112:2379 isLeader=false
ea59efe5cd5e204e: name=etcd4 peerURLs=http://etcd4:2380 clientURLs=http://10.0.2.114:2379 isLeader=false
member 1b1776006795f774 is healthy: got healthy result from http://10.0.2.111:2379
member 89df995ba1c163a3 is healthy: got healthy result from http://10.0.2.113:2379
member 9e01e0d9b238a128 is healthy: got healthy result from http://10.0.2.112:2379
member ea59efe5cd5e204e is healthy: got healthy result from http://10.0.2.114:2379
cluster is healthy
root@srv113:/# 

root@srv114 /# cd /;date; etcd-members.sh; etcdctl cluster-health
Wed Oct 21 08:33:00 +07 2020
..
ETCD MEMBERs: 
1b1776006795f774: name=etcd1 peerURLs=http://10.0.2.111:2380 clientURLs=http://10.0.2.111:2379 isLeader=true
89df995ba1c163a3: name=etcd3 peerURLs=http://10.0.2.113:2380 clientURLs=http://10.0.2.113:2379 isLeader=false
9e01e0d9b238a128: name=etcd2 peerURLs=http://10.0.2.112:2380 clientURLs=http://10.0.2.112:2379 isLeader=false
ea59efe5cd5e204e: name=etcd4 peerURLs=http://etcd4:2380 clientURLs=http://10.0.2.114:2379 isLeader=false
member 1b1776006795f774 is healthy: got healthy result from http://10.0.2.111:2379
member 89df995ba1c163a3 is healthy: got healthy result from http://10.0.2.113:2379
member 9e01e0d9b238a128 is healthy: got healthy result from http://10.0.2.112:2379
member ea59efe5cd5e204e is healthy: got healthy result from http://10.0.2.114:2379
cluster is healthy
root@srv114 /# 



How to install ETCD Cluster 3 node on Ubuntu Server 18.04.LTS.x64

1/ Lab Information: 03 node:
10.0.2.111 etcd1.mgiay.local etcd1
10.0.2.112 etcd2.mgiay.local etcd2
10.0.2.113 etcd3.mgiay.local etcd3


2/ Install ETCD on per node:
REF: 
https://www.mgiay.com/2020/10/how-to-install-etcd-from-source-code.html


3/ ETCD Cluster Initial:

#!/bin/bash
#/opt/script/etcd-initial.sh
#Chạy 1 lần DUY NHẤT trên từng NODE.

#LastUpdate: #14:21 2020.10.20
################################################################
#cat /opt/script/etcd-initial.sh | grep LastUpdate
################################################################
#RESTART ETCD: #\rm -rf /var/lib/etcd/*;
#sudo systemctl daemon-reload;sudo systemctl enable etcd;sudo systemctl restart etcd;sudo systemctl status etcd -l --no-pager
#STOP ETCD: sudo systemctl stop etcd;sudo systemctl status etcd -l --no-pager
#etcd1 10.0.2.111*
#etcd2 10.0.2.112
#etcd3 10.0.2.113
################################################################
ETCD_HOME="/opt/etcd"
mkdir -p $ETCD_HOME
/bin/rm -rf $ETCD_HOME
mkdir -p $ETCD_HOME

ETCD_DATA=$ETCD_HOME/data
mkdir -p $ETCD_DATA

ETCD_LOG=$ETCD_HOME/etcd.log

echo "------------------------------------------------------------"
echo "MAKING [ETCD_HOME]:"
sleep 2
echo "[ETCD_HOME: $ETCD_HOME]"
echo "[ETCD_DATA: $ETCD_DATA]"
echo "[ETCD_LOG : $ETCD_LOG]"

echo "------------------------------------------------------------"
echo "CREATING GROUP/USER: ETCD:"
sleep 2
GroupName=etcd
UserName=etcd
GroupID=6000
UserID=6000
userdel $UserName
#groupdel $GroupName
groupadd -g $GroupID $GroupName
useradd  -u $UserID -s "/sbin/nologin" -d "/opt/$UserName" -c "ETCD.SUPER.ADMIN" -g $GroupName $UserName
echo "GROUP: [$(cat /etc/group | grep etcd)]" 
sleep 1
echo "PWD: [$(cat /etc/passwd | grep etcd)]"
sleep 1
echo "------------------------------------------------------------"
echo "RESULT: [$(pwd)]"
sleep 2
chown -R     $GroupName.$UserName $ETCD_HOME
chmod -R 700                      $ETCD_HOME

ls -lh $ETCD_HOME
tree $ETCD_HOME

#THE_END



4/ ETCD Cluster Config on 3 node:
#!/bin/bash
#/opt/script/etcd-restart.sh
#Chú ý các tham số cấu hình của từng node:
#LastUpdate: #13:40 2020.10.20
################################################################
#cat /opt/script/etcd-restart.sh | grep LastUpdate
################################################################
#RESTART ETCD: #\rm -rf /var/lib/etcd/*;
#sudo systemctl daemon-reload;sudo systemctl enable etcd;sudo systemctl restart etcd;sudo systemctl status etcd -l --no-pager
#STOP ETCD: sudo systemctl stop etcd;sudo systemctl status etcd -l --no-pager
#etcd1 10.0.2.111*
#etcd2 10.0.2.112
#etcd3 10.0.2.113

#Line 56:
#--name etcd1 \
#--name etcd2 \
#--name etcd3 \
################################################################
ETCD_HOME="/opt/etcd"
mkdir -p $ETCD_HOME

ETCD_DATA=$ETCD_HOME/data
mkdir -p $ETCD_DATA

ETCD_LOG=$ETCD_HOME/etcd.log

echo "------------------------------------------------------------"
echo "ETCD TCP CONNECTION: CURRENT"
netstat -ntlup | egrep "PID|LISTEN" | sort -t: -k2 -n | egrep "PID|etcd"
sleep 1

fuser -k 2379/tcp
fuser -k 2380/tcp

GroupName=etcd
UserName=etcd
GroupID=6000
UserID=6000

# userdel $UserName
# groupdel $GroupName

# groupadd -g $GroupID $GroupName
# useradd  -u $UserID -s "/sbin/nologin" -d "/opt/$UserName" -c "ETCD.SUPER.ADMIN" -g $GroupName $UserName

chown -R     $GroupName.$UserName $ETCD_HOME
chmod -R 700                      $ETCD_HOME

#check group, user xem da tao duoc chua:
#cat /etc/group  | grep $GroupName; cat /etc/passwd | grep $UserName
#_______________________________________________________________ETCD_CONFIG:BEGIN
#REF: https://etcd.io/docs/v3.4.0/op-guide/clustering/
etcd \
--name etcd1 \
--data-dir=$ETCD_DATA \
--initial-advertise-peer-urls http://10.0.2.111:2380 \
--listen-peer-urls http://10.0.2.111:2380 \
--listen-client-urls http://0.0.0.0:2379 \
--advertise-client-urls http://10.0.2.111:2379 \
--initial-cluster-token etcd-cluster-1 \
--initial-cluster etcd1=http://10.0.2.111:2380,etcd2=http://10.0.2.112:2380,etcd3=http://10.0.2.113:2380 \
--initial-cluster-state new > $ETCD_LOG 2>&1 &
#_______________________________________________________________ETCD_CONFIG:END
sleep 1

echo "------------------------------------------------------------"
echo "ETCD TCP CONNECTION: NEW"
netstat -ntlup | egrep "PID|LISTEN" | sort -t: -k2 -n | egrep "PID|etcd"
sleep 1

echo "------------------------------------------------------------"
echo "ETCD PID:"
ps aux| egrep "PID|etcd"
echo "----"
echo "DONE"

#THE_END



5/ ETCD stop:
#!/bin/bash
#/opt/script/etcd-stop.sh
#LastUpdate: #13:41 2020.10.20
################################################################
#cat /opt/script/etcd-stop.sh | grep LastUpdate
################################################################
#RESTART ETCD: #\rm -rf /var/lib/etcd/*;
#sudo systemctl daemon-reload;sudo systemctl enable etcd;sudo systemctl restart etcd;sudo systemctl status etcd -l --no-pager
#STOP ETCD: #\rm -rf /var/lib/etcd/*;
#sudo systemctl stop etcd;sudo systemctl status etcd -l --no-pager
#etcd1 10.0.2.111*
#etcd2 10.0.2.112
#etcd3 10.0.2.113
################################################################
echo "------------------------------------------------------------"
echo "ETCD TCP CONNECTION: CURRENT"
netstat -ntlup | egrep "PID|LISTEN" | sort -t: -k2 -n | egrep "PID|etcd"
sleep 1

fuser -k 2379/tcp
fuser -k 2380/tcp

GroupName=etcd
UserName=etcd
GroupID=6000
UserID=6000

#chown -R     $GroupName.$UserName $ETCD_HOME
#chmod -R 700                      $ETCD_HOME

echo "------------------------------------------------------------"
echo "ETCD TCP CONNECTION: NEW"
netstat -ntlup | egrep "PID|LISTEN" | sort -t: -k2 -n | egrep "PID|etcd"
sleep 1

echo "------------------------------------------------------------"
echo "ETCD PID:"
ps aux| egrep "PID|etcd"
echo "----"
echo "DONE"

#THE_END



6/ ETCD check log realtime:
#!/bin/bash
#/opt/script/etcd-log.sh
#LastUpdate: #14:31 2020.10.20, #13:40 2020.10.20
################################################################
#cat /opt/script/etcd-log.sh | grep LastUpdate
################################################################
#RESTART ETCD: #\rm -rf /var/lib/etcd/*;
#sudo systemctl daemon-reload;sudo systemctl enable etcd;sudo systemctl restart etcd;sudo systemctl status etcd -l --no-pager
#STOP ETCD: sudo systemctl stop etcd;sudo systemctl status etcd -l --no-pager
#etcd1 10.0.2.111*
#etcd2 10.0.2.112
#etcd3 10.0.2.113
################################################################
ETCD_HOME="/opt/etcd"
#mkdir -p $ETCD_HOME

ETCD_DATA=$ETCD_HOME/data
#mkdir -p $ETCD_DATA

ETCD_LOG=$ETCD_HOME/etcd.log

echo "------------------------------------------------------------"
echo "ETCD TCP CONNECTION: "
netstat -ntlup | egrep "PID|LISTEN" | sort -t: -k2 -n | egrep "PID|etcd"
sleep 1

echo "------------------------------------------------------------"
echo "ETCD PID:"
ps aux| egrep "PID|etcd"
sleep 1

echo "------------------------------------------------------------"
echo "ETCD LOG: "
sleep 3
tail -f $ETCD_LOG

#THE_END



7/ ETCD cluster member list:
#!/bin/bash
#/opt/script/etcd-members.sh
#LastUpdate: #14:14 2020.10.20
################################################################
#cat /opt/script/etcd-members.sh | grep LastUpdate
################################################################
#RESTART ETCD: #\rm -rf /var/lib/etcd/*;
#sudo systemctl daemon-reload;sudo systemctl enable etcd;sudo systemctl restart etcd;sudo systemctl status etcd -l --no-pager
#STOP ETCD: sudo systemctl stop etcd;sudo systemctl status etcd -l --no-pager
#etcd1 10.0.2.111*
#etcd2 10.0.2.112
#etcd3 10.0.2.113
################################################################
ETCD_HOME="/opt/etcd"
echo "------------------------------------------------------------"
echo "ETCD TCP CONNECTION: "
netstat -ntlup | egrep "PID|LISTEN" | sort -t: -k2 -n | egrep "PID|etcd"
sleep 1

echo "------------------------------------------------------------"
echo "ETCD PID:"
ps aux| egrep "PID|etcd"
sleep 1

echo "------------------------------------------------------------"
echo "ETCD MEMBERs: "
sleep 1

etcdctl member list
#THE_END



8/ ETCD cluster status:
#!/bin/bash
#/opt/script/etcd-status.sh
#LastUpdate: #13:40 2020.10.20
################################################################
#cat /opt/script/etcd-status.sh | grep LastUpdate
################################################################
#RESTART ETCD: #\rm -rf /var/lib/etcd/*;
#sudo systemctl daemon-reload;sudo systemctl enable etcd;sudo systemctl restart etcd;sudo systemctl status etcd -l --no-pager
#STOP ETCD: sudo systemctl stop etcd;sudo systemctl status etcd -l --no-pager
#etcd1 10.0.2.111*
#etcd2 10.0.2.112
#etcd3 10.0.2.113
################################################################
ETCD_HOME="/opt/etcd"
#mkdir -p $ETCD_HOME

ETCD_DATA=$ETCD_HOME/data
#mkdir -p $ETCD_DATA

ETCD_LOG=$ETCD_HOME/etcd.log

echo "------------------------------------------------------------"
echo "ETCD TCP CONNECTION: "
netstat -ntlup | egrep "PID|LISTEN" | sort -t: -k2 -n | egrep "PID|etcd"
sleep 1

echo "------------------------------------------------------------"
echo "ETCD PID:"
ps aux| egrep "PID|etcd"
sleep 1

echo "------------------------------------------------------------"
echo "ETCD LOG: "
sleep 2
#tail -n10 -f $ETCD_LOG | more
tail -n 10 $ETCD_LOG
echo "/opt/script/etcd-log.sh"
echo ""

#<<DONE DONE DONE


9/ RESULT:

#


#