pip install pyinstaller
pyinstaller -F report.py
在当前目录会生成dist/report 二进制文件
cp default.docx dist/default.docx
cp config.ini dist/config.ini

/etc/filebeat/filebeat.yml
filebeat.prospectors:
- type: log
  paths:
    - /home/data/log/*
    - /home/data/*.log
  scan_frequency: 20s
  encoding: utf-8
  tail_files: true
  harvester_buffer_size: 5485760
fields:
  ip_address: 192.168.2.2
  env: qa
output.redis:
  hosts: ["192.168.1.1:6379"]
  password: "geekwolf"
  key: "filebeat"
  db: 0
  timeout: 5
  max_retires: 3
  worker: 2
  bulk_max_size: 4096

input {
 #Filebeat
 # beats {
 #   port => 5044
 # }
 #Redis
  redis {
    batch_count => 4096
    data_type => "list"
    key => "filebeat"
    host => "127.0.0.1"
    port => 5044
    password => "geekwolf"
    db => 0
    threads => 2
   }
}
filter {
  ruby {
      code => 'event.set("filename",event.get("source").split("/")[-1])'
  }
}
output {
  if [filename] =~ "nohup" {
    file {
        path => "/data/logs/%{[fields][env]}/%{+YYYY-MM-dd}/%{[fields][ip_address]}/%{filename}"
        flush_interval => 3
        codec => line { format => "%{message}"}
    }
  } else {
    file {
         path => "/data/logs/%{[fields][env]}/%{+YYYY-MM-dd}/%{[fields][ip_address]}/logs/%{filename}"
        flush_interval => 3
        codec => line { format => "%{message}"}
   }
 }
 #stdout { codec => rubydebug }
}

.
├── prod
│   └── 2018-01-13
│       └── 2.2.2.2
│           ├── logs
│           │   ├── rpg_slow_db_.27075
│           └── nohup_service.log
└── qa
    ├── 2018-01-12
    │   ├── 192.168.3.1
    └── 2018-01-13
        ├── 192.168.3.2

# -*- coding: utf-8 -*-
# @Author: Geekwolf
# @Date:   2018-01-29 14:23:04
# @Last Modified by:   Geekwolf
# @Last Modified time: 2018-01-31 10:55:01
#!/usr/bin/env python3
# daemon.py
import os
import sys
import time
import redis
import json
import re
import atexit
import signal
# import collections
class Base(object):
    def __init__(self, *args, **kwargs):
        self.pidfile = '/var/run/plogstash.pid'
        self.service_name = 'Plogstash'
        self.path = '/var/log/plogstash'
        os.makedirs(self.path, exist_ok=True)
        self.logfile = '%s/%s.log' % (self.path, self.service_name)
        self.redis_host = '127.0.0.1'
        self.redis_password = 'geekwolf'
        self.redis_port = 5044
        self.redis_db = 0
        self.redis_key = 'filebeat'
        self.batch_size = 5000
        self.expires = 5  # second
        self.archive_time = 1  # how long time to archive
        self.base_dir = '/data/logs'
        # self._tmp = '/tmp/.%s' % self.service_name
class Daemon(Base):
    def __init__(self, *args, **kwargs):
        super(Daemon, self).__init__(*args, **kwargs)
    def daemonize(self):
        # First fork (detaches from parent)
        try:
            if os.fork() > 0:
                raise SystemExit(0)   # Parent exit
        except OSError as e:
            raise RuntimeError('fork #1 failed.')
        os.chdir('/')
        # set this will 777
        # os.umask(0)
        os.setsid()
        # Second fork (relinquish session leadership)
        try:
            if os.fork() > 0:
                raise SystemExit(0)
        except OSError as e:
            raise RuntimeError('fork #2 failed.')
        # Flush I/O buffers
        sys.stdout.flush()
        sys.stderr.flush()
        # Replace file descriptors for stdin, stdout, and stderr
        with open(self.logfile, 'ab', 0) as f:
            os.dup2(f.fileno(), sys.stdout.fileno())
        with open(self.logfile, 'ab', 0) as f:
            os.dup2(f.fileno(), sys.stderr.fileno())
        with open(self.logfile, 'rb', 0) as f:
            os.dup2(f.fileno(), sys.stdin.fileno())
        # Write the PID file
        print(os.getpid())
        with open(self.pidfile, 'w') as f:
            print(os.getpid(), file=f)
        # Arrange to have the PID file removed on exit/signal
        atexit.register(lambda: os.remove(self.pidfile))
        # Signal handler for termination (required)
        def sigterm_handler(signo, frame):
            raise SystemExit(1)
        signal.signal(signal.SIGTERM, sigterm_handler)
    def get_now_date(self):
        return time.strftime('%Y-%m-%d', time.localtime(time.time()))
    def get_now_timestamp(self):
        return time.time()
    def get_now_time(self):
        return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    def logging(self, msg):
        with open(self.logfile) as f:
            print('%s  %s' % (self.get_now_time(), msg))
    def append_log(self):
        pass
    def start(self):
        if os.path.exists(self.pidfile):
            raise RuntimeError('Already running')
        else:
            try:
                self.daemonize()
                self.append_log()
                self.status()
            except RuntimeError as e:
                print(e, file=sys.stderr)
                raise SystemExit(1)
    def stop(self):
        # f = os.open(self.pipe_path, os.O_RDONLY | os.O_NONBLOCK)
        # ret = os.read(f, 1024).decode('utf-8')
        # print(ret.split('\n'))
        # os.close(f)
        if os.path.exists(self.pidfile):
            # with open(self._tmp) as f:
            #     _data = f.read()
            #     if _data is not None and len(eval(_data)) > 0:
            #         for k, v in eval(_data).items():
            #             v = v['fd'].rstrip('\n')
            #             v.close()
            with open(self.pidfile) as f:
                os.kill(int(f.read()), signal.SIGTERM)
            print('Plogstash is stopped')
        else:
            print('Not running', file=sys.stderr)
            raise SystemExit(1)
    def restart(self):
        self.stop()
        self.start()
    def status(self):
        try:
            with open(self.pidfile, 'r') as f:
                pid = int(f.read().strip())
        except:
            pid = None
        if pid:
            print('%s is running as pid:%s' % (self.service_name, pid))
        else:
            print('%s is not running' % self.service_name)
class Worker(Daemon):
    def __init__(self, *args, **kwargs):
        super(Worker, self).__init__(self, *args, **kwargs)
    def _redis(self):
        pool = redis.ConnectionPool(host=self.redis_host, password=self.redis_password, port=self.redis_port, db=self.redis_db, socket_timeout=10000)
        rc = redis.StrictRedis(connection_pool=pool)
        return rc
    def get_redis_data(self):
        _data = self._redis().lrange(self.redis_key, 0, self.batch_size - 1)
        # 删除数据(可考虑处理完再删除)
        return _data
    def del_redis_data(self):
        _data = self._redis().ltrim(self.redis_key, self.batch_size, -1)
    def append_log(self):
        file_meta = {}
        # file_handler = collections.defaultdict(dict)
        # try:
        #     os.mkfifo(self.pipe_path)
        # except Exception as e:
        #     print(str(e))
        # pipe_ins = os.open(self.pipe_path, os.O_SYNC | os.O_CREAT | os.O_RDWR)
        while True:
            time.sleep(self.archive_time)
            _data = self.get_redis_data()
            if _data:
                for _d in _data:
                    try:
                        _d = json.loads(_d.decode('utf-8'))
                        _path = '%s/%s/%s/%s' % (self.base_dir, _d['fields']['env'], self.get_now_date(), _d['fields']['ip_address'])
                        os.makedirs(_path + '/logs', exist_ok=True)
                        file_name = _d['source'].split('/')[-1]
                       # _path = '%s/%s/%s/%s' % (self.base_dir, _d['fields']['env'],self.get_now_date(), _d['fields']['ip_address'])
                        if re.match('nohup', file_name):
                            file_path = '%s/%s' % (_path, file_name)
                        else:
                            file_path = '%s/logs/%s' % (_path, file_name)
                        with open(file_path, 'a') as f:
                            f.write(_d['message'] + '\n')
                        # if 'fd' not in file_handler[file_path]:
                        #     f = open(file_path, 'a', buffering=1024000)
                        #     file_handler[file_path]['fd'] = str(f)
                        # file_handler[file_path]['time'] = self.get_now_timestamp()
                    except Exception as e:
                        self.logging(str(e))
                self.del_redis_data()
            # with open(self._tmp, 'w') as f:
            #     f.write(json.dumps(file_handler))
if __name__ == '__main__':
    if len(sys.argv) != 2:
        print('Usage: {} [start|stop|restart|status]'.format(sys.argv[0]), file=sys.stderr)
        raise SystemExit(1)
    daemon = Worker()
    if sys.argv[1] == 'start':
        daemon.start()
    elif sys.argv[1] == 'stop':
        daemon.stop()
    elif sys.argv[1] == 'restart':
        print("Restart ...")
        daemon.restart()
    elif sys.argv[1] == 'status':
        daemon.status()
    else:
        print('Unknown command {!r}'.format(sys.argv[1]), file=sys.stderr)
        raise SystemExit(1)

echo "deb http://http.debian.net/debian jessie-backports main" >>/etc/apt/source.list
echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" > /etc/apt/sources.list.d/elastic-6.x.list
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
apt-get -y update
apt-get install -t jessie-backports openjdk-8-jdk
update-java-alternatives -s java-1.8.0-openjdk-amd64
apt-get install -y elasticsearch kibana

vim /etc/elasticsearch/elasticsearch.yml
network.host: 0.0.0.0
vim  /etc/kibana/kibana.yml 
server.host: "0.0.0.0"
elasticsearch.url: "http://localhost:9200"

/etc/init.d/elasticsearch start
/etc/init.d/kibana start

/etc/init.d/zabbix_server stop
tar xf zabbix-3.4.5.tar.gz
cd zabbix-3.4.5
./configure --enable-server --enable-agent --with-mysql --enable-ipv6 --with-net-snmp --with-libcurl --with-libxml2
make -j8
make install

vim /usr/local/etc/zabbix_server.conf
HistoryStorageURL=http://192.168.100.100:9200
HistoryStorageTypes=uint,dbl,str,log,text

修改配置文件vim conf/zabbix.conf.php
1. 如果不同类型使用不同的ES集群，可以按如下进行配置
$HISTORY['url']   = [
    'uint' => 'http://localhost1:9200',
    'text' => 'http://localhost2:9200'
];
$HISTORY['types'] = ['uint', 'text'];
2. 所有类型使用相同ES集群
$HISTORY['url']   = 'http://192.168.100.100:9200';
$HISTORY['types'] = ['str', 'text', 'log', 'uint', 'dbl'];

/etc/init.d/zabbix_server start
此时可以通过观察日志，查看是否连接ES成功

monitor
├── monitor
│   ├── celery.py
│   ├── consumers.py
│   ├── __init__.py
│   ├── routing.py
│   ├── settings.py
│   ├── urls.py
│   └── wsgi.py
├── commands
│   ├── admin.py
│   ├── apps.py
│   ├── __init__.py
│   ├── management
│   │   ├── commands
│   │   │   ├── alert.py
│   │   │   ├── __init__.py
│   │   │   └── __pycache__
│   │   ├── __init__.py
│   │   └── __pycache__
│   ├── migrations
│   │   ├── __init__.py
│   │   └── __pycache__
│   ├── models.py
│   ├── __pycache__
│   ├── tests.py
│   └── views.py
├── static
├── templates
│   ├── base
|        |—— base.html
├── README.md
├── requirements.txt

pip install channels asgi_redis
settings.py添加app和设置CHANNEL_LAYERS
#commands是后面定义Django命令的app
INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'channels',
    'commands'
]
#Redis信息
REDIS_OPTIONS = {
    'HOST': '127.0.0.1',
    'PORT': 6379,
    'PASSWD': 'geekwolf',
    'DB': 0
}
#可以使用内存存储Channels消息
#CHANNEL_LAYERS = {
#    "default": {
#        "BACKEND": "asgiref.inmemory.ChannelLayer",
#        "ROUTING": "channels_example.routing.channel_routing",
#    },
#}
#使用Redis作为消息存储，需安装asgi_redis
CHANNEL_LAYERS = {
    'default': {
        'BACKEND': 'asgi_redis.RedisChannelLayer',
        'CONFIG': {
            'hosts': ['redis://:{0}@{1}:{2}/{3}'.format(REDIS_OPTIONS['PASSWD'], REDIS_OPTIONS['HOST'], REDIS_OPTIONS['PORT'], 1)]
        },
        'ROUTING': 'plonvol.routing.channel_routing'
    }
}
#Redis频道和Channels群组名
GROUP_NAME = 'alarm'

# -*- coding: utf-8 -*-
from channels.routing import route
from .consumers import ws_connect, ws_disconnect, ws_receive
channel_routing = [
    route('websocket.connect', ws_connect),
    route('websocket.disconnect', ws_disconnect),
]

# -*- coding: utf-8 -*-
from channels import Group
from channels.handler import AsgiHandler
from django.conf import settings
def ws_connect(message):
    message.reply_channel.send({'accept': True})
    Group(settings.GROUP_NAME).add(message.reply_channel)
def ws_disconnect(message):
    Group(settings.GROUP_NAME).discard(message.reply_channel)

# -*- coding: utf-8 -*-
import json
import logging
from channels import Group
from django.core.management import BaseCommand
from django.conf import settings
import redis
logger = logging.getLogger(__name__)
class Command(BaseCommand):
    """
    Command to start zabbix alert worker from command line.
    """
    help = 'Subscribe the zabbix alerts channel'
    def handle(self, *args, **options):
        rc = redis.Redis(host=settings.REDIS_OPTIONS['HOST'],
                         password=settings.REDIS_OPTIONS['PASSWD'],
                         port=settings.REDIS_OPTIONS['PORT'],
                         db=settings.REDIS_OPTIONS['DB'])
        rc.delete(settings.GROUP_NAME)
        pubsub = rc.pubsub()
        pubsub.subscribe(settings.GROUP_NAME)
        for item in pubsub.listen():
            if item['type'] == 'message':
                Group(settings.GROUP_NAME).send({'text': bytes.decode(item['data'])})
                logger.debug('send a message %s ' % item)

<!-- 报警声音 -->
<audio id="notify"><source src="/static/notify.ogg" type="audio/ogg">></audio>
<script type="application/javascript">
    var ws_scheme = window.location.protocol == "https:" ? "wss" : "ws";
    var ws = new WebSocket(ws_scheme + '://' + window.location.host + window.location.pathname);
    console.log(ws);
    ws.onmessage = function (message) {
        var data = JSON.parse(message.data);
        if(data['当前状态'] == 'Problem')
        {
           var subject = '<br>故障！,服务器:' + data['告警主机'] + '发生:' + data['告警信息'] + '故障!<br>';
        }
        else{
           var subject = '<br>恢复！,服务器:' + data['告警主机'] + '发生:' + data['告警信息'] + '已经恢复!<br>';
        }
        content = new Array();
        $.each(data,function(k,v){
            content.push("<b>" + k + ':</b>   ' + v)
        })
        var data = subject + content.join("<br>")
        $('#notify')[0].play();
        notify('warning','glyphicon glyphicon-danger-sign',data);
    }
</script>

import redis
import json
rc = redis.Redis(host='127.0.0.1', password='geekwolf', port=6379, db=0)
msg = {"告警主机": "web-server-node1", "告警地址": "192.168.1133.11", "告警时间": "2017-11-11 05:05:22", "告警等级": "严重",
       "告警信息": "Web端口80监控", "问题详情": "80端口连接失败", "当前状态": "Problem", "事件ID": "12345"}
rc.publish('alarm', json.dumps(msg))

启动项目(http/websocket):
python manage.py runserver 0.0.0.0:8000
启动监听报警消息进程:
python manage.py alert

Python: 3.6
Django: 1.11.0
项目地址: git@github.com:geekwolf/fms.git

pip3 install -i https://pypi.douban.com/simple/  -r requirements.txt

python manage.py runserver
http://127.0.0.1:8000
admin admin

wget https://github.com/coreos/etcd/releases/download/v3.2.7/etcd-v3.2.7-linux-arm64.tar.gz
tar xf etcd-v3.2.7-linux-arm64.tar.gz
cd etcd-v3.2.7-linux-amd64
cp etc* /usr/local/bin/
etcd: Etcd服务端文件
etcdctl: 供用户使用的命令客户端

root@a4c8d490:/home/geekwolf# etcd
2017-09-07 15:42:23.957656 I | etcdmain: etcd Version: 3.2.7
2017-09-07 15:42:23.957699 I | etcdmain: Git SHA: bb66589
2017-09-07 15:42:23.957718 I | etcdmain: Go Version: go1.8.3
2017-09-07 15:42:23.957723 I | etcdmain: Go OS/Arch: linux/amd64
2017-09-07 15:42:23.957729 I | etcdmain: setting maximum number of CPUs to 8, total number of available CPUs is 8
2017-09-07 15:42:23.957739 W | etcdmain: no data-dir provided, using default data-dir ./default.etcd
2017-09-07 15:42:23.957764 N | etcdmain: the server is already initialized as member before, starting as etcd member...
2017-09-07 15:42:23.957995 I | embed: listening for peers on http://localhost:2380
2017-09-07 15:42:23.958107 I | embed: listening for client requests on localhost:2379
2017-09-07 15:42:23.964607 I | etcdserver: name = default
2017-09-07 15:42:23.964633 I | etcdserver: data dir = default.etcd
2017-09-07 15:42:23.964652 I | etcdserver: member dir = default.etcd/member
2017-09-07 15:42:23.964657 I | etcdserver: heartbeat = 100ms
2017-09-07 15:42:23.964663 I | etcdserver: election = 1000ms
2017-09-07 15:42:23.964668 I | etcdserver: snapshot count = 100000
2017-09-07 15:42:23.964680 I | etcdserver: advertise client URLs = http://localhost:2379
2017-09-07 15:42:23.973007 I | etcdserver: restarting member 8e9e05c52164694d in cluster cdf818194e3a8c32 at commit index 14
2017-09-07 15:42:23.973041 I | raft: 8e9e05c52164694d became follower at term 2
2017-09-07 15:42:23.973065 I | raft: newRaft 8e9e05c52164694d [peers: [], term: 2, commit: 14, applied: 0, lastindex: 14, lastterm: 2]
2017-09-07 15:42:23.984367 W | auth: simple token is not cryptographically signed
2017-09-07 15:42:23.993237 I | etcdserver: starting server... [version: 3.2.7, cluster version: to_be_decided]
2017-09-07 15:42:23.993659 I | etcdserver/membership: added member 8e9e05c52164694d [http://localhost:2380] to cluster cdf818194e3a8c32
2017-09-07 15:42:23.993754 N | etcdserver/membership: set the initial cluster version to 3.2
2017-09-07 15:42:23.993796 I | etcdserver/api: enabled capabilities for version 3.2
2017-09-07 15:42:24.473288 I | raft: 8e9e05c52164694d is starting a new election at term 2
2017-09-07 15:42:24.473451 I | raft: 8e9e05c52164694d became candidate at term 3
2017-09-07 15:42:24.473519 I | raft: 8e9e05c52164694d received MsgVoteResp from 8e9e05c52164694d at term 3
2017-09-07 15:42:24.473568 I | raft: 8e9e05c52164694d became leader at term 3
2017-09-07 15:42:24.473605 I | raft: raft.node: 8e9e05c52164694d elected leader 8e9e05c52164694d at term 3
2017-09-07 15:42:24.478746 I | etcdserver: published {Name:default ClientURLs:[http://localhost:2379]} to cluster cdf818194e3a8c32
2017-09-07 15:42:24.478824 I | embed: ready to serve client requests
2017-09-07 15:42:24.479116 N | embed: serving insecure client requests on 127.0.0.1:2379, this is strongly discouraged!

查看api版本（默认api版本是v2）
root@a4c8d490:~/k8s/etcd-v3.2.7-linux-amd64# etcdctl  --version
etcdctl version: 3.2.7
API version: 2
使用API V3方法：
Etcd服务端和客户端添加变量 export ETCDCTL_API=3,重新启动etcd服务即可
以下操作在api v3版本:
写入key: etcdctl put foo bar
读取key: etcdctl get foo
多key范围读取: etcdctl get foo foo9(会将foo..foo8的key读取,不包括foo9)
读取过往版本key的值(Etcd键值对的修改都会增加全局修订版本号,--rev为版本号):
 etcdctl get --rev=4 foo foo9
删除key: etcdctl del foo
范围删除(foo->foo9):etcdctl del foo foo9
观察key变化:etcdctl watch foo
观察范围key变化: etcdctl watch foo foo9
从rev=2版本开始观察key变化: etcdctl watch --rev=2 foo
压缩版本5之前的修订版本(压缩后5之前的版本不可能访问): etcdctl compact 5
授予key有效期:
创建租约: 
$ etcdctl lease grant 10
lease 694d5e5b63a74f31 granted with TTL(10s)
附加key foo到租约694d5e5b63a74f31，该租约过期后，会删除附加的所有key
撤销租约(撤销后，附加改租约的所有key被删除): etcdctl lease revoke 32695410dcc0ca06
维持租约(执行后，会一直维持该租约): etcdctl lease keep-alive 32695410dcc0ca0
其他参数可参考 etcdctl --help
通过HTTP操作：
Etcd v2: https://coreos.com/etcd/docs/latest/v2/api.html
Etcd v3: https://coreos.com/etcd/docs/latest/dev-guide/api_grpc_gateway.html

hostnamectl --static  set-hostname etcd1.simlinux.com
hostnamectl --static  set-hostname etcd2.simlinux.com
hostnamectl --static  set-hostname etcd3.simlinux.com

vim /etc/hosts
192.168.234.133 etcd1.simlinux.com 
192.168.234.134 etcd2.simlinux.com 
192.168.234.135 etcd3.simlinux.com

cat  etcd.json 
{
    "CN": "etcd",
    "hosts": [
        "127.0.0.1",
        "192.168.234.133",
        "192.168.234.134",
        "192.168.234.135"
    ],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "CN",
            "L": "ShangHai",
            "ST": "ShangHai",
            "O": "K8s",
            "OU": "System"
        }
    ]
}
生成服务端证书,用于服务端认证客户端(hosts需要包括允许访问ETCD Cluster的IP或者FQDN):
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server server-csr.json | cfssljson -bare server
server-key.pem
server.csr
server.pem
生成对等证书，用于etcd间通信(这里三个节点使用同一个证书,建议分别生成证书):
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer etcd.json | cfssljson -bare etcd
etcd-key.pem
etcd.csr
etcd.pem
将CA和etcd证书拷贝到etcd所有节点:
cp ca.pem  etcd-key.pem etcd.pem server-key.pem server.pem /etc/etcd/ssl/

wget https://github.com/coreos/etcd/releases/download/v3.2.7/etcd-v3.2.7-linux-amd64.tar.gz
tar xf etcd-v3.2.7-linux-amd64.tar.gz
cd etcd-v3.2.7-linux-amd64
chmod +x etcd*
cp etcd* /bin

服务管理(所有节点相同):
vim /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos
[Service]
Type=notify
WorkingDirectory=/data/k8s/etcd/
EnvironmentFile=/etc/etcd/etcd.conf
ExecStart=/bin/etcd 
  --name=${NAME} 
  --cert-file=/etc/etcd/ssl/etcd.pem 
  --key-file=/etc/etcd/ssl/etcd-key.pem 
  --peer-cert-file=/etc/etcd/ssl/etcd.pem 
  --peer-key-file=/etc/etcd/ssl/etcd-key.pem 
  --trusted-ca-file=/etc/etcd/ssl/ca.pem 
  --peer-trusted-ca-file=/etc/etcd/ssl/ca.pem 
  --initial-advertise-peer-urls=${INITIAL_ADVERTISE_PEER_URLS} 
  --listen-peer-urls=${LISTEN_PEER_URLS} 
  --listen-client-urls=${LISTEN_CLIENT_URLS} 
  --advertise-client-urls=${ADVERTISE_CLIENT_URLS} 
  --initial-cluster-token=${INITIAL_CLUSTER_TOKEN} 
  --initial-cluster=${INITIAL_CLUSTER} 
  --initial-cluster-state=new 
  --data-dir=${DATA_DIR}
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

vim /etc/etcd/etcd.conf
#节点名称
NAME="etcd1"
#etcd数据存放目录
DATA_DIR="/data/k8s/etcd"
#etcd节点间通信监听地址
LISTEN_PEER_URLS="https://192.168.234.133:2380"
#对外提供服务的地址
LISTEN_CLIENT_URLS="https://192.168.234.133:2379,https://127.0.0.1:2379"
#通知其他etcd节点本实例地址
INITIAL_ADVERTISE_PEER_URLS="https://192.168.234.133:2380"
#初始化集群内节点地址
INITIAL_CLUSTER="etcd1=https://192.168.234.133:2380,etcd2=https://192.168.234.134:2380,etcd3=https://192.168.234.135:2380"
#初始化状态.new表示新建,已经存在的集群使用existing
INITIAL_CLUSTER_STATE="new"
#创建集群的token,每个集群唯一
INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
#告知其他集群本节点客户端监听地址
ADVERTISE_CLIENT_URLS="https://192.168.234.133:2379"
ETCDCTL_API=3
其中NAME/LISTEN_PEER_URLS/LISTEN_CLIENT_URLS/INITIAL_ADVERTISE_PEER_URLS/ADVERTISE_CLIENT_URLS替换成相应节点名称和地址

yum -y install ntp 
systemctl start ntpd 
systemctl start etcd.service
systemctl stop etcd.service
systemctl status etcd.service(查看服务状态及日志)

[root@etcd1 ~]# export etcd1=192.168.234.133
[root@etcd1 ~]# export etcd2=192.168.234.134
[root@etcd1 ~]# export etcd3=192.168.234.135
[root@etcd1 ~]# export ETCDCTL_API=3
[root@etcd1 ~]# export ENDPOINTS=$etcd1:2379,$etcd2:2379,$etcd3:2379
查看集群成员:
[root@etcd1 etcd]#  etcdctl --write-out=table --endpoints=$ENDPOINTS --cacert=/etc/etcd/ssl/ca.pem  --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem member list
+------------------+---------+-------+------------------------------+------------------------------+
|        ID        | STATUS  | NAME  |          PEER ADDRS          |         CLIENT ADDRS         |
+------------------+---------+-------+------------------------------+------------------------------+
| 1a4a83ef243ff1c9 | started | etcd2 | https://192.168.234.134:2380 | https://192.168.234.134:2379 |
| 68243ef8797bd1ce | started | etcd1 | https://192.168.234.133:2380 | https://192.168.234.133:2379 |
| fa30209a63d949b0 | started | etcd3 | https://192.168.234.135:2380 | https://192.168.234.135:2379 |
+------------------+---------+-------+------------------------------+------------------------------+
查看集群状态:
[root@etcd1 ~]# etcdctl --write-out=table --endpoints=$ENDPOINTS --cacert=/etc/etcd/ssl/ca.pem  --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem endpoint status
+----------------------+------------------+---------+---------+-----------+-----------+------------+
|       ENDPOINT       |        ID        | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX |
+----------------------+------------------+---------+---------+-----------+-----------+------------+
| 192.168.234.133:2379 | 68243ef8797bd1ce |   3.2.7 |   25 kB |     false |        10 |          9 |
| 192.168.234.134:2379 | 1a4a83ef243ff1c9 |   3.2.7 |   25 kB |     false |        10 |          9 |
| 192.168.234.135:2379 | fa30209a63d949b0 |   3.2.7 |   25 kB |      true |        10 |          9 |
+----------------------+------------------+---------+---------+-----------+-----------+------------+
[root@etcd1 ~]# etcdctl --write-out=table --endpoints=$ENDPOINTS --cacert=/etc/etcd/ssl/ca.pem  --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem endpoint health
192.168.234.135:2379 is healthy: successfully committed proposal: took = 1.374345ms
192.168.234.134:2379 is healthy: successfully committed proposal: took = 2.217525ms
192.168.234.133:2379 is healthy: successfully committed proposal: took = 1.996245ms
保存快照:
[root@etcd1 ~]# etcdctl --write-out=table --endpoints=$ENDPOINTS --cacert=/etc/etcd/ssl/ca.pem  --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem snapshot save my.db
Snapshot saved at my.db
查看快照状态:
[root@etcd1 ~]# etcdctl --write-out=table --endpoints=$ENDPOINTS --cacert=/etc/etcd/ssl/ca.pem  --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem snapshot status my.db
+----------+----------+------------+------------+
|   HASH   | REVISION | TOTAL KEYS | TOTAL SIZE |
+----------+----------+------------+------------+
| 9a496339 |        3 |          8 |      25 kB |
+----------+----------+------------+------------+
恢复数据(要先删除原来数据目录,所有节点操作)：
[root@etcd1 ~]# etcdctl  --endpoints=$ENDPOINTS --cacert=/etc/etcd/ssl/ca.pem  --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem snapshot restore my.db  --data-dir=/data/k8s/etcd/
2017-09-09 02:28:52.439616 I | etcdserver/membership: added member 8e9e05c52164694d [http://localhost:2380] to cluster cdf818194e3a8c32
删除节点:
[root@etcd1 ~]# etcdctl --write-out=table --endpoints=$ENDPOINTS --cacert=/etc/etcd/ssl/ca.pem  --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem member remove 68243ef8797bd1ce
更新节点:
[root@etcd1 ~]# etcdctl --write-out=table --endpoints=$ENDPOINTS --cacert=/etc/etcd/ssl/ca.pem  --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem member update 68243ef8797bd1ce https://192.168.234.133:1111(INITIAL_ADVERTISE_PEER_URLS)
添加节点(删除etcd3，添加etcd4):
export etcd4=192.168.234.136
[root@etcd1 ~]#  etcdctl --endpoints=${etcd1}:2379,${etcd2}:2379 --cacert=/etc/etcd/ssl/ca.pem  --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem  member add etcd4 --peer-urls=http://192.168.234.136:2380

curl -s -L -o /bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
curl -s -L -o /bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
curl -s -L -o /bin/cfssl-certinfo https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x /bin/cfssl*

mkdir /opt/ssl
cd /opt/ssl
cfssl print-defaults config > ca-config.json
cfssl print-defaults csr > ca-csr.json

{
"signing": {
    "default": {
        "expiry": "43800h"
    },
    "profiles": {
        "server": {
            "expiry": "43800h",
            "usages": [
                "signing",
                "key encipherment",
                "server auth"
            ]
        },
        "client": {
            "expiry": "43800h",
            "usages": [
                "signing",
                "key encipherment",
                "client auth"
            ]
        },
        "peer": {
            "expiry": "43800h",
            "usages": [
                "signing",
                "key encipherment",
                "server auth",
                "client auth"
            ]
        }
    }
}
}

{
"CN": "Self Signed Ca",
"key": {
    "algo": "rsa",
    "size": 2048
},
"names": [
    {
        "C": "CN",
        "L": "SH",
        "O": "Netease",
        "ST": "SH",            
        "OU": "OT"
    }    ]
}

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
生成ca.pem、ca.csr、ca-key.pem(CA私钥,需妥善保管)

cfssl print-defaults csr > server.json
vim server.json
{
    "CN": "Server",
    "hosts": [
        "192.168.1.1"
       ],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "CN",
            "L": "SH",
            "ST": "SH"
        }
    ]
}
生成服务端证书和私钥
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server server.json | cfssljson -bare server

cfssl print-defaults csr > client.json
vim client.json
{
    "CN": "Client",
    "hosts": [],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "CN",
            "L": "SH",
            "ST": "SH"
        }
    ]
}
生成客户端证书和私钥
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client

cfssl print-defaults csr > member1.json
vim member1.json
{
    "CN": "member1",
    "hosts": [
        "192.168.1.1"
    ],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "CN",
            "L": "SH",
            "ST": "SH"
        }
    ]
}
为节点member1生成证书和私钥:
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer member1.json | cfssljson -bare member1
针对etcd服务,每个etcd节点上按照上述方法生成相应的证书和私钥

openssl x509 -in ca.pem -text -noout
openssl x509 -in server.pem -text -noout
openssl x509 -in client.pem -text -noout