添加服务脚本

添加xz项目用的百度地图反代
更新CDH部署信息
16 changed files with 730 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,7 @@
+BaiduProxy/pkg
+BaiduProxy/bin
+BaiduProxy/src/github.com
+BaiduProxy/src/golang.org
+BaiduProxy/src/h12.io
+BaiduProxy/antiy
+BaiduProxy/.idea
--- a/BaiduProxy/aliyun_baidu_proxy.service
+++ b/BaiduProxy/aliyun_baidu_proxy.service
@ -0,0 +1,13 @@
+# /etc/systemd/system/baidu_proxy.service
+[Unit]
+Description=proxy for api.map.baidu.com and so on
+After=syslog.target  network.target
+Wants=network.target
+Requires=ssh_proxy.service
+ 
+[Service]
+Type=simple
+ExecStart=/usr/local/baiduproxy/antiy 0.0.0.0:7999
+ 
+[Install]
+WantedBy=multi-user.target
--- a/BaiduProxy/aliyun_ssh_proxy.service
+++ b/BaiduProxy/aliyun_ssh_proxy.service
@ -0,0 +1,13 @@
+# /etc/systemd/system/baidu_proxy.service
+[Unit]
+Description=use ssh for remote port redirection.
+After=syslog.target  network.target
+Wants=network.target baidu_proxy.service
+ 
+[Service]
+Type=simple
+WorkingDirectory=/usr/local/baiduproxy
+ExecStart=/usr/bin/autossh -M 0 -NR 0.0.0.0:7999:localhost:7999 -i id_rsa root@220.182.51.94
+ 
+[Install]
+WantedBy=multi-user.target
--- a/BaiduProxy/src/antiy/go.mod
+++ b/BaiduProxy/src/antiy/go.mod
@ -0,0 +1,5 @@
+module antiy
+
+go 1.16
+
+require h12.io/socks v1.0.2
--- a/BaiduProxy/src/antiy/go.sum
+++ b/BaiduProxy/src/antiy/go.sum
@ -0,0 +1,6 @@
+github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364 h1:5XxdakFhqd9dnXoAZy1Mb2R/DZ6D1e+0bGC/JhucGYI=
+github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364/go.mod h1:eDJQioIyy4Yn3MVivT7rv/39gAJTrA7lgmYr8EW950c=
+github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
+github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
+h12.io/socks v1.0.2 h1:cZhhbV8+DE0Y1kotwhr1a3RC3kFO7AtuZ4GLr3qKSc8=
+h12.io/socks v1.0.2/go.mod h1:AIhxy1jOId/XCz9BO+EIgNL2rQiPTBNnOfnVnQ+3Eck=
--- a/BaiduProxy/src/antiy/main.go
+++ b/BaiduProxy/src/antiy/main.go
@ -0,0 +1,129 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+)
+
+var domains = []string{
+	"api.map.baidu.com",
+	"api0.map.bdimg.com",
+	"api1.map.bdimg.com",
+	"api2.map.bdimg.com",
+	"api3.map.bdimg.com",
+	"maponline0.bdimg.com",
+	"maponline1.bdimg.com",
+	"maponline2.bdimg.com",
+	"maponline3.bdimg.com",
+	"dlswbr.baidu.com",
+	"shangetu0.map.bdimg.com",
+	"shangetu1.map.bdimg.com",
+	"shangetu2.map.bdimg.com",
+	"shangetu3.map.bdimg.com",
+	"miao.baidu.com",
+}
+
+func ReplaceHost(contentString, hostname string) string {
+	for _, domain := range domains {
+		contentString = strings.Replace(contentString, domain, hostname+"/"+domain, -1)
+	}
+	contentString = strings.Replace(contentString, "https://", "http://", -1)
+	return contentString
+}
+
+type MyHandler struct {
+}
+
+func (myHandler *MyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	hostname := req.Host
+	getUrl := url.URL{
+		Scheme: "http",
+	}
+	if !strings.HasPrefix(req.URL.Path, "/") {
+		req.URL.Path = "/" + req.URL.Path
+	}
+
+	for _, domain := range domains {
+		if strings.HasPrefix(req.URL.Path, "/"+domain) {
+			getUrl.Host = domain
+			getUrl.Path = strings.Replace(req.URL.Path, domain+"/", "", 1)
+			getUrl.RawQuery = req.URL.RawQuery
+			baiduResp, _ := http.Get(getUrl.String())
+			fmt.Printf("fmt: %s", baiduResp.Header.Get("Content-Type"))
+			fmt.Printf("Get response from %s\n", getUrl.String())
+			content, _ := ioutil.ReadAll(baiduResp.Body)
+			contentString := string(content)
+			if strings.Contains(baiduResp.Header.Get("Content-Type"), "javascript") {
+				contentString = ReplaceHost(contentString, hostname)
+			}
+			resp.Header().Set("Access-Control-Allow-Origin", "*")
+			resp.Header().Set("Access-Control-Allow-Headers", "Origin")
+			resp.Header().Set("Content-Type", baiduResp.Header.Get("Content-Type"))
+			fmt.Fprintf(resp, "%v\n", contentString)
+			return
+		}
+	}
+
+	//fp, _ := os.Open("demo.html")
+	//pageContent, _ := io.ReadAll(fp)
+	pageContent := `
+<!DOCTYPE html>
+<html>
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta name="viewport" content="initial-scale=1.0, user-scalable=no" />
+    <style type="text/css">
+        body, html,#allmap {width: 100%;height: 100%;overflow: hidden;margin:0;font-family:"微软雅黑";}
+    </style>
+    <script type="text/javascript" src="/api.map.baidu.com/api?v=2.0&ak=ADfpGTm3mDV4sZRiA2uDOKv3SbvcPtzG"></script>
+    <title>地图展示</title>
+</head>
+<body>
+<div id="allmap"></div>
+</body>
+</html>
+<script type="text/javascript">
+    // 百度地图API功能
+    var map = new BMap.Map("allmap");    // 创建Map实例
+    map.centerAndZoom(new BMap.Point(116.404, 39.915), 11);  // 初始化地图,设置中心点坐标和地图级别
+    //添加地图类型控件
+    map.addControl(new BMap.MapTypeControl({
+        mapTypes:[
+            BMAP_NORMAL_MAP,
+            BMAP_HYBRID_MAP
+        ]}));
+    map.setCurrentCity("武汉");          // 设置地图显示的城市 此项是必须设置的
+    map.enableScrollWheelZoom(true);     //开启鼠标滚轮缩放
+</script>
+`
+	fmt.Fprint(resp, string(pageContent))
+}
+
+func (myHandler *MyHandler) Handler() {
+
+}
+
+func main() {
+	var port string
+	if len(os.Args) > 1 {
+		port = os.Args[1]
+	} else {
+		port = "127.0.0.1:8080"
+	}
+	m := MyHandler{}
+
+	s := &http.Server{
+		Addr:           port,
+		Handler:        &m,
+		ReadTimeout:    10 * time.Second,
+		WriteTimeout:   10 * time.Second,
+		MaxHeaderBytes: 1 << 20,
+	}
+
+	s.ListenAndServe()
+}
--- a/BaiduProxy/xz_ssh_proxy.service
+++ b/BaiduProxy/xz_ssh_proxy.service
@ -0,0 +1,13 @@
+# /etc/systemd/system/baidu_proxy.service
+[Unit]
+Description=use ssh for remote port redirection.
+After=syslog.target  network.target
+Wants=network.target
+ 
+[Service]
+Type=simple
+WorkingDirectory=/usr/local/baiduproxy
+ExecStart=/usr/bin/autossh -M 0 -NR 0.0.0.0:7999:localhost:7999 -i id_rsa 192.168.55.218
+ 
+[Install]
+WantedBy=multi-user.target
--- a/CDH_Deploy/README.MD
+++ b/CDH_Deploy/README.MD
@ -0,0 +1,25 @@
+# 说明
+
+## HelperScripts
+
+此文件夹主要存放一些帮助脚本
+
+### PartTools.py
+
+用于批量自动分区并挂载的工具. 参数分两种 `-s SIZE` 或 `-d DEVICE,DEVICE,...`
+
+对于 `-s SIZE` , 会将大于此大小(单位为G)的磁盘(排除跟分区所在磁盘)分一个大分区并格式化成Ext4, 然后顺序挂载到系统的 /diskN 目录(如果只有一个磁盘则就是 /disk 目录).
+
+对于 `-d DEV,DEV,...` (例如 `-d sda,sdb,sdc`) 会将列表制定的磁盘(排除根分区所在磁盘)分一个大分区并格式化车成Ext4, 然后和上面一样顺序挂载到系统根目录下.
+
+挂载完成后, 会记录所有挂载上到分区信息到 `/etc/fstab` 文件里面, 以如下格式记录:
+
+``` config
+UUID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx /diskN ext4 defaults 0 0
+```
+
+``` shell
+sudo python2 PartTools.py -s 2000 # 将所有大于2T的磁盘分区&格式化&挂载
+# 或者
+sudo python2 PartTools.py -d sda,sdb,sdc,sdd # 将 /dev/sd{a,b,c,d} 全部格式化并挂载
+```
--- a/CDH_Deploy/ansible.cfg
+++ b/CDH_Deploy/ansible.cfg
@ -0,0 +1,5 @@
+[defaults]
+inventory = hosts
+remote_user = root
+# private_key_file = cdh_private.key
+host_key_checking = False
--- a/CDH_Deploy/cdh_deploy.yml
+++ b/CDH_Deploy/cdh_deploy.yml
@ -0,0 +1,17 @@
+---
+- name: 环境准备
+  hosts: new_cdh_servers
+  # become: true 不需要become，默认就是root用户
+  vars_prompt:
+    - name: mysql_pass
+      prompt: 请输入你想设置的MySQL root账户密码(默认为antiy?918)
+      default: "antiy?918"
+  tasks:
+    - include: tasks/basic_settings.yml
+    - include: tasks/deploy_mysql.yml
+    - include: tasks/deploy_cm.yml
+    - name: 安装完成，请继续配置
+      run_once: yes
+      delegate_to: localhost
+      pause:
+        prompt: "所有安装已经完成，请访问 {{ group['new_cdh_servers'][0] }}:7180 进行下一步配置"
--- a/CDH_Deploy/cdh_private.key
+++ b/CDH_Deploy/cdh_private.key
@ -0,0 +1,38 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAYEAsc5tL7X3uGlVdy98krQ/xxKSvolP5scc0amDi7QhHG2E5RGUxZQR
+y8r1Zjyz1xpVf9cORoNxcdE4U9Fti3GDNHjJjt1ZC+FUx6+TiwBLo7ZmK2UsgXQzSkYhkP
+5MvMBfwNYAMATzrbwehibF/mKAWd6XVSATivWyRuQWFiNBU+pp/KLLLVGAy7KmfypVFfof
+e8p1PSYxRA7IA1smFlBgcgyTwTGuOEO7xY0K/vP21TyzzZqqRA48OaTEDlLtaH+O9ATKEE
+6t/0JhzYlowDf57WpSxRxqP4BvVY9c52VeiHIACZAs1EYdOmRkZgm3jc2+sjfKmf897iEa
+u/yKpKPgQtayPKrE0ro1CnAJ6YEj4TdzAAVdZC4J5365ItyaRQmgWgNWmqJAZc6Iga9pEA
+k3XgqKmxrifUY3epCkcPeeKEPxKmnu2AliQd1J7Nz0PIOAoHulZ/iWO/qN118/B6/TwwDn
+sYWCri851T4DMZO6/yt5uBgIcJTaKdzGh3DIbazZAAAFiBnKPz8Zyj8/AAAAB3NzaC1yc2
+EAAAGBALHObS+197hpVXcvfJK0P8cSkr6JT+bHHNGpg4u0IRxthOURlMWUEcvK9WY8s9ca
+VX/XDkaDcXHROFPRbYtxgzR4yY7dWQvhVMevk4sAS6O2ZitlLIF0M0pGIZD+TLzAX8DWAD
+AE8628HoYmxf5igFnel1UgE4r1skbkFhYjQVPqafyiyy1RgMuypn8qVRX6H3vKdT0mMUQO
+yANbJhZQYHIMk8ExrjhDu8WNCv7z9tU8s82aqkQOPDmkxA5S7Wh/jvQEyhBOrf9CYc2JaM
+A3+e1qUsUcaj+Ab1WPXOdlXohyAAmQLNRGHTpkZGYJt43NvrI3ypn/Pe4hGrv8iqSj4ELW
+sjyqxNK6NQpwCemBI+E3cwAFXWQuCed+uSLcmkUJoFoDVpqiQGXOiIGvaRAJN14Kipsa4n
+1GN3qQpHD3nihD8Spp7tgJYkHdSezc9DyDgKB7pWf4ljv6jddfPwev08MA57GFgq4vOdU+
+AzGTuv8rebgYCHCU2incxodwyG2s2QAAAAMBAAEAAAGBAJhYHgU7DDzdQ3UTItEvz1Vp/I
+F0dRlukKqzaxpQA+6vh6YdIjMHYxgRlaE7cXtj/M3EHV8dUB4M/Q4T6qAHajuTIl9eF+Wt
+804EiY4XereSmIel8DQas7d3z1rECEvRKDm5IgACvialgCHBJvOGwiI+Lbup+BSf31gOJT
+5ihp63wofLk/oIUyo0r6NVSgHChSczmLAEAUImyAjYGBIWXAxb6l0li/h5UHE7H6jCr0KU
+Bm54xjceY2KNtFvDCr03sFpGsAx+kjgWKWoHL9PzCd/8pMGEhnio7caJuutZjTwlOWyEc
+GEEuBM5FiXSLCIe03bshsbxB4CVg3Cfeq4kLdv3gx97ADacwUakJWubftcBIDD/7OcAFQ+
+rHMonhOqznmkR8xRKc8rOylsph+5xTkfOFuWbkjGj7kxz4n4z+p0Rhujl1bPtZvtn8LDix
+hi6qTS/gMk48fTaO9XKVcd3+fRbXp6S6bxq7mnPo6HAhBv28jge6oxv7QtomF46+hMYQAA
+AMEAx91kGZbK8VFDNUk6DUh549LtQNArowzB12A7CqZLMcRZrFGQTtVVjy+Ea0lejp/csp
+KaB63Ut+/GHNqoGRodTinW98FoRVidA6TbpKRthUjj4AGKopRSHI2+3RYR/MCO2H7P0b66
+Jc12deKdxoRGShmhYDF5A8USPhNCO+z90rCTnUY+88cubdYpJdQx6GUVMmRX7kwu+XlKjf
+OGLPcRYwr/mdEUl4nA1vENE/msSVrFeia5QpAdvSnDOQ4hl19mAAAAwQDWuapiZBhQxWd5
+3yvUgqUW0Xg244EaM8e6O+5o+cvjMMw8iJFcvA1PcjlbkTK7Wr0sc3LO3CW6UtWSJTweAA
+zr+Kd+yzK6gRUlSiPMAQMJ5dmodipc+AF9opEnEYR//QI93ttqzlcOVzbqfHIKyZ/b52hs
+GI+P+o9xD8i+iK3f/ZKKEernABZZp40smhLD/x4qpijEmw0MjHQhT5gZN6exoiPJecvZB9
+mIzSDaqvzHZ27h97AsHBAOPoq7XxkNYysAAADBANP8CFTBlBBU7JX36c19CeQAD66uCWyL
+gfWRhEcbueETM8W0FoIy+4J9b2bWwfy3J8ooRw7gR9PH58sn941I66qa+wnmdjLkPE8l33
+njwRfBpJcXkU+qN1Dhg7QQveNalf4mTFp/AyBL+pUoUSkQl8AewAYFA60LpR7VLdomdZL8
+4Afq8wHkExECAKkyE6OQTBqXNyqqNLGRQaLNokVZr4ZndUEKHjCaNzcp+xP09Wc1KRruLK
+uzWwW11nsV0YI+CwAAABBsdW9taW9AbWJwLmxvY2FsAQ==
+-----END OPENSSH PRIVATE KEY-----
--- a/CDH_Deploy/cdh_private.key.pub
+++ b/CDH_Deploy/cdh_private.key.pub
@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxzm0vtfe4aVV3L3yStD/HEpK+iU/mxxzRqYOLtCEcbYTlEZTFlBHLyvVmPLPXGlV/1w5Gg3Fx0ThT0W2LcYM0eMmO3VkL4VTHr5OLAEujtmYrZSyBdDNKRiGQ/ky8wF/A1gAwBPOtvB6GJsX+YoBZ3pdVIBOK9bJG5BYWI0FT6mn8osstUYDLsqZ/KlUV+h97ynU9JjFEDsgDWyYWUGByDJPBMa44Q7vFjQr+8/bVPLPNmqpEDjw5pMQOUu1of470BMoQTq3/QmHNiWjAN/ntalLFHGo/gG9Vj1znZV6IcgAJkCzURh06ZGRmCbeNzb6yN8qZ/z3uIRq7/Iqko+BC1rI8qsTSujUKcAnpgSPhN3MABV1kLgnnfrki3JpFCaBaA1aaokBlzoiBr2kQCTdeCoqbGuJ9Rjd6kKRw954oQ/Eqae7YCWJB3Uns3PQ8g4Cge6Vn+JY7+o3XXz8Hr9PDAOexhYKuLznVPgMxk7r/K3m4GAhwlNop3MaHcMhtrNk= luomio@mbp.local
--- a/CDH_Deploy/hosts
+++ b/CDH_Deploy/hosts
@ -0,0 +1,23 @@
+[cdh_servers]
+cdh01.ghadoop ansible_ssh_host=10.251.24.2 ansible_ssh_port=22
+cdh02.ghadoop ansible_ssh_host=10.251.24.3 ansible_ssh_port=22
+cdh03.ghadoop ansible_ssh_host=10.251.24.4 ansible_ssh_port=22
+cdh04.ghadoop ansible_ssh_host=10.251.24.5 ansible_ssh_port=22
+cdh05.ghadoop ansible_ssh_host=10.251.24.6 ansible_ssh_port=22
+
+[new_cdh_servers]
+cdh01.ghadoop ansible_ssh_host=10.251.24.102 ansible_ssh_port=22
+cdh02.ghadoop ansible_ssh_host=10.251.24.103 ansible_ssh_port=22
+cdh03.ghadoop ansible_ssh_host=10.251.24.104 ansible_ssh_port=22
+cdh04.ghadoop ansible_ssh_host=10.251.24.105 ansible_ssh_port=22
+cdh05.ghadoop ansible_ssh_host=10.251.24.106 ansible_ssh_port=22
+
+[cdh_source]
+server ansible_ssh_host=192.168.0.10 ansible_ssh_port=22
+
+[cdh_control]
+cdh01.ghadoop ansible_ssh_host=10.251.24.102 ansible_ssh_port=22
+
+[new_cdh_servers:vars]
+ansible_ssh_user=root
+ansible_ssh_pass=antiy?918
--- a/CDH_Deploy/tasks/basic_settings.yml
+++ b/CDH_Deploy/tasks/basic_settings.yml
@ -0,0 +1,240 @@
+---
+- name: 上传安装包
+  copy: src=offline_cdh dest=/opt/
+- name: 禁用SELinux和防火墙
+  block:
+    - name: 禁用SELinux
+      ansible.builtin.lineinfile:
+        path: /etc/sysconfig/selinux
+        regexp: '^SELINUX='
+        line: SELINUX=disabled
+    - name: 禁用防火墙
+      ansible.builtin.service:
+        name: firewalld
+        state: stopped
+        enabled: no
+- name: 修改主机名并分发给各主机
+  tags:
+    - host
+  block:
+    - name: 设置主机名
+      ansible.builtin.hostname:
+        name: "{{ inventory_hostname }}"
+    - name: 修改 /etc/sysconfig/network 内的主机名设置
+      ansible.builtin.lineinfile:
+        path: /etc/sysconfig/network
+        regexp: '^HOSTNAME='
+        line: "HOSTNAME={{ inventory_hostname|lower }}"
+    - name: 重启网络服务
+      ansible.builtin.service:
+        name: network
+        state: restarted
+    - name: 分发主机名信息到各个主机
+      blockinfile:
+        path: /etc/hosts
+        block: |
+          {% for h in groups['new_cdh_servers'] | sort %}
+          {{ hostvars[h].ansible_default_ipv4.address }} {{ h }}
+          {% endfor %}
+- name: 安装软件包
+  tags:
+    - pkgs
+  ansible.builtin.yum:
+    name:
+      - sysstat
+      - vim
+      - wget
+      - lrzsz
+      - screen
+      - gcc
+      - python-devel
+      - gcc-c++
+      - ntpdate
+      - libyaml
+      - libyaml-devel
+      - python-setuptools
+      - ntp
+      - libaio # for mysql
+      - expect # for mysql
+      - mariadb # 这是mysql客户端工具，改名了
+      - psmisc # for /opt/cm-5.14.2/etc/init.d/cloudera-scm-server start
+    state: present
+- name: SSH互信配置
+  tags:
+    - dpk
+  block:
+    - name: 生成ssh私钥
+      command:
+        cmd: ssh-keygen -f /root/.ssh/id_rsa -N ""
+        creates: /root/.ssh/id_rsa
+    - name: 将公钥下载到Ansible中控机
+      fetch:
+        src: /root/.ssh/id_rsa.pub
+        dest: /tmp/pubkeys/{{ inventory_hostname }}.pub
+        flat: yes
+    - name: 将ssh指纹信息下载到中控机
+      block:
+        - name: 先cat获取
+          command: cat /etc/ssh/ssh_host_ecdsa_key.pub
+          register: ssh_finger
+        - name: 然后输出到临时文件
+          ansible.builtin.lineinfile:
+            path: /tmp/ssh_fingerprint
+            line: "{{ inventory_hostname }},{{ hostvars[inventory_hostname].ansible_default_ipv4.address }} {{ ssh_finger.stdout_lines[0] }}"
+            insertbefore: BOF
+            create: yes
+        - name: 最后获取到本地
+          fetch:
+            src: /tmp/ssh_fingerprint
+            dest: /tmp/fingerprints/{{ inventory_hostname }}.known_hosts
+            flat: yes
+        - name: 然后删除服务器上的临时文件
+          ansible.builtin.file:
+            path: /tmp/ssh_fingerprint
+            state: absent
+    - name: 删除之前遗留的旧文件
+      run_once: yes
+      delegate_to: localhost
+      file:
+        path: "{{ item }}"
+        state: absent
+      with_items:
+        - /tmp/authorized_keys
+        - /tmp/known_hosts
+    - name: 合并公钥信息
+      run_once: yes
+      delegate_to: localhost
+      ansible.builtin.assemble:
+        src: /tmp/pubkeys/
+        dest: /tmp/authorized_keys
+    - name: 合并指纹信息
+      run_once: yes
+      delegate_to: localhost
+      ansible.builtin.assemble:
+        src: /tmp/fingerprints/
+        dest: /tmp/known_hosts
+    - name: 上传指纹信息和公钥信息到各个主机
+      copy:
+        src: "{{ item }}"
+        dest: "{{ item }}"
+      loop:
+        - /tmp/authorized_keys
+        - /tmp/known_hosts
+    - name: 将公钥复制到用户配置
+      blockinfile:
+        path: /root/.ssh/authorized_keys
+        block: "{{ lookup('file', '/tmp/authorized_keys') }}"
+        create: yes
+        mode: 0600
+    - name: 将指纹信息复制到用户配置
+      blockinfile:
+        path: /root/.ssh/known_hosts
+        block: "{{ lookup('file', '/tmp/known_hosts') }}"
+        create: yes
+        mode: 0600
+    - name: 清楚本地临时文件和目录
+      file:
+        path: "{{ item }}"
+        state: absent
+      with_items:
+        - /tmp/fingerprints/
+        - /tmp/pubkeys/
+        - /tmp/authorized_keys
+        - /tmp/known_hosts
+- name: 移除系统JDK并安装离线包里面的JDK
+  tags:
+    - jdk
+  block:
+    - name: 移除系统JDK包
+      ansible.builtin.yum:
+        name:
+          - java-1.6.0-openjdk
+          - java-1.7.0-openjdk
+          - java-1.8.0-openjdk
+          - java-11-openjdk
+        state: absent
+        autoremove: yes
+    - name: 安装离线包提供的JDK8
+      ansible.builtin.yum:
+        name: /opt/offline_cdh/jdk-8u261-linux-x64.rpm
+        state: present
+- name: 设置时间同步
+  tags:
+    - ntp
+  block:
+    - name: 在主机01上配置NTP服务
+      ansible.builtin.lineinfile:
+        path: /etc/ntp.conf
+        regexp: "^restrict 10.251"
+        line: restrict 10.251.24.0 mask 255.255.255.0 nomodify notrap
+      when: inventory_hostname == groups['new_cdh_servers'][0]
+    - name: 在主机01上启用NTP服务
+      ansible.builtin.service:
+        name: ntpd
+        state: restarted
+        enabled: yes
+    - name: 延时一段时间以等待ntp服务就绪
+      command: sleep 10
+    - name: 从主机01上同步时间
+      command: "ntpdate -u {{ hostvars['cdh01.ghadoop'].ansible_default_ipv4.address }}"
+      when: inventory_hostname != "cdh01.ghadoop"
+- name: 禁用透明大页
+  tags:
+    - nothp
+  block:
+    - name: 临时禁用碎片整理
+      shell: echo 'never' > /sys/kernel/mm/transparent_hugepage/defrag
+      args:
+        executable: /bin/bash
+    - name: 临时禁用透明大页
+      shell: echo 'never' > /sys/kernel/mm/transparent_hugepage/enabled
+      args:
+        executable: /bin/bash
+    - name: 修改Grub配置
+      ansible.builtin.lineinfile:
+        path: /etc/default/grub
+        regexp: "^GRUB_CMDLINE_LINUX="
+        line: GRUB_CMDLINE_LINUX="crashkernel=auto rhgb quiet transparent_hugepage=never"
+    - name: 判断当前是Grub还是Grub2
+      ansible.builtin.stat:
+        path: /etc/{{ item }}
+      register: grub2_cfg
+      with_items:
+        - grub2.cfg
+        - grub2-efi.cfg
+    - debug:
+        var: grub2_cfg
+    - name: 根据是Grub还是Grub2，更新Grub配置
+      command: "grub2-mkconfig -o {{ item.stat.path }}"
+      when: item.stat.readable
+      with_items: "{{ grub2_cfg.results }}"
+- name: 修改系统设置
+  tags:
+    - setconf
+  block:
+    - name: 修改限制文件 /etc/security/limits.conf
+      blockinfile:
+        path: /etc/security/limits.conf
+        block: |
+          * soft nofile 65536
+          * hard nofile 65536
+          root soft nofile 65536
+          root hard nofile 65536
+          * soft memlock unlimited
+          * hard memlock unlimited
+          root soft memlock unlimited
+          root hard memlock unlimited
+          * soft as unlimited
+          * hard as unlimited
+          root soft as unlimited
+          root hard as unlimited
+    - name: 修改 /etc/sysctl.conf
+      ansible.builtin.lineinfile:
+        path: /etc/sysctl.conf
+        regexp: "{{ item.regexp }}"
+        line: "{{ item.line }}"
+        state: present
+        create: yes
+      loop:
+        - { regexp: "^vm.max_map_count=", line: "vm.max_map_count=131072" }
+        - { regexp: "^vm.swappiness=", line: "vm.swappiness=0" }
--- a/CDH_Deploy/tasks/deploy_cm.yml
+++ b/CDH_Deploy/tasks/deploy_cm.yml
@ -0,0 +1,85 @@
+---
+- name: 部署ClouderaManager
+  tags:
+    - cm
+  block:
+    - name: 新建用户
+      ansible.builtin.user:
+        name: cloudera-scm
+        system: true
+        home: /opt/cm-5.14.2/run/cloudera-scm-server
+        create_home: false
+        shell: /bin/false
+        comment: Cloudera SCM User
+    - name: 解压缩CM安装包
+      ansible.builtin.unarchive:
+        src: /opt/offline_cdh/cloudera-manager-centos7-cm5.14.2_x86_64.tar.gz
+        dest: /opt/
+        remote_src: true
+    - name: 复制MySQL连接器
+      ansible.builtin.copy:
+        src: /opt/offline_cdh/mysql-connector-java-5.1.38.jar
+        dest: /opt/cm-5.14.2/share/cmf/lib/
+        remote_src: true
+    - name: 复制Parcel仓库
+      ansible.builtin.copy:
+        src: /opt/offline_cdh/parcel-repo/
+        dest: /opt/cloudera/parcel-repo/
+        remote_src: true
+    - name: 启动ClouderaManagerServer（仅主机执行）
+      when:
+        inventory_hostname == groups['new_cdh_servers'][0]
+      block:
+        - name: 初始化数据库
+          ansible.builtin.shell: |
+            source /opt/cm-5.14.2/etc/default/cloudera-scm-server
+            source /opt/cm-5.14.2/etc/default/cloudera-scm-agent
+            /opt/cm-5.14.2/share/cmf/schema/scm_prepare_database.sh mysql cm -hlocalhost -uroot -p'{{ mysql_pass }}' --scm-host localhost scm scm scm
+        - name: 创建目录
+          ansible.builtin.file:
+            path: /var/lib/cloudera-scm-server
+            state: directory
+        - name: 启动服务
+          ansible.builtin.shell: |
+            source /opt/cm-5.14.2/etc/default/cloudera-scm-server
+            source /opt/cm-5.14.2/etc/default/cloudera-scm-agent
+            /opt/cm-5.14.2/etc/init.d/cloudera-scm-server start
+          args:
+            executable: /bin/bash
+        - name: 查看日志
+          ansible.builtin.command:
+            cmd: tail -n30 /opt/cm-5.14.2/log/cloudera-scm-server/cloudera-scm-server.log
+          register: cm_log
+        - name: Debug输出日志
+          debug:
+            msg: "{{ cm_log.stdout_lines }}"
+        - name: 暂停等待确认
+          pause:
+            prompt: 请确认输出是否正常，按“Ctrl+C c”继续
+    - name: 启动ClouderaManagerAgent（所有主机）
+      block:
+        - name: 配置主节点
+          ansible.builtin.lineinfile:
+            path: /opt/cm-5.14.2/etc/cloudera-scm-agent/config.ini
+            regexp: ^server_host=
+            line: "server_host={{ groups['new_cdh_servers'][0] | lower }}"
+        - name: 启动Agent
+          ansible.builtin.shell: |
+            source /opt/cm-5.14.2/etc/default/cloudera-scm-server
+            source /opt/cm-5.14.2/etc/default/cloudera-scm-agent
+            /opt/cm-5.14.2/etc/init.d/cloudera-scm-agent start
+          args:
+            executable: /bin/bash
+        - name: 查看日志
+          ansible.builtin.command:
+            cmd: tail -n30 /opt/cm-5.14.2/log/cloudera-scm-agent/cloudera-scm-agent.log
+          register: cm_agent_log
+        - name: Debug输出日志
+          debug:
+            msg: "{{ cm_agent_log.stdout_lines }}"
+        - name: 暂停等待确认
+          pause:
+            prompt: |
+              请确认输出是否正常，按“Ctrl+C c”继续
+              注意，以下信息为正常信息：
+              ERROR Failed to connect to previous supervisor.
--- a/CDH_Deploy/tasks/deploy_mysql.yml
+++ b/CDH_Deploy/tasks/deploy_mysql.yml
@ -0,0 +1,110 @@
+---
+- name: 部署MySQL
+  tags:
+    - mysql
+  when:
+    - inventory_hostname == groups['new_cdh_servers'][0]
+  block:
+    - name: 判断程序在不在
+      ansible.builtin.stat:
+        path: /usr/local/mysql/bin/mysqld
+      register: mysqld
+    - name: 解压缩MySQL程序
+      block:
+        - name: 解压缩离线包
+          ansible.builtin.unarchive:
+            src: /opt/offline_cdh/mysql-5.7.31-linux-glibc2.12-x86_64.tar.gz
+            dest: /usr/local/
+            remote_src: yes
+            creates: /usr/local/mysql-5.7.31-linux-glibc2.12-x86_64
+        - name: 重命名文件夹
+          ansible.builtin.command:
+            cmd: mv /usr/local/mysql-5.7.31-linux-glibc2.12-x86_64 /usr/local/mysql
+            creates: /usr/local/mysql
+      when:
+        - not mysqld.stat.exists
+    - name: 创建用户和用户组
+      block:
+        - name: 创建用户组mysql
+          ansible.builtin.group:
+            name: mysql
+            system: yes
+        - name: 创建用户mysql
+          ansible.builtin.user:
+            name: mysql
+            group: mysql
+            system: yes
+            create_home: no
+            home: /usr/local/mysql
+            shell: /sbin/nologin
+    - name: 创建MySQL配置文件
+      ansible.builtin.copy:
+        src: /opt/offline_cdh/my.cnf
+        dest: /etc/my.cnf
+        remote_src: yes
+    - name: 创建MySQL目录并更新权限
+      ansible.builtin.file:
+        path: "{{ item.path }}"
+        state: "{{ item.state }}"
+        recurse: "{{ item.recurse }}"
+        owner: mysql
+        group: mysql
+      loop:
+        - { path: /usr/local/mysql, state: directory, recurse: yes}
+        - { path: /data/mysql, state: directory, recurse: yes }
+        - { path: /data/log/mysql, state: directory, recurse: yes }
+        - { path: /data/log/mysql/mysql.err, state: touch, recurse: no }
+        - { path: /data/log/mysql/slow.log, state: touch, recurse: no }
+    - name: 取得MySQL的密码
+      command: /usr/local/mysql/bin/mysqld --initialize --user=mysql --basedir=/usr/local/mysql --datadir=/data/mysql
+      register: mysqld_output
+    - name: 安装Python-MySQL客户端
+      ansible.builtin.yum:
+        name: MySQL-python
+        state: present
+    - name: 配置MySQL服务
+      block:
+        - name: 修改服务文件
+          ansible.builtin.lineinfile:
+            regexp: "{{ item.regexp }}"
+            line: "{{ item.line }}"
+            path: /usr/local/mysql/support-files/mysql.server
+          loop:
+            - { regexp: "^basedir=", line: basedir=/usr/local/mysql }
+            - { regexp: "^datadir=", line: datadir=/data/mysql }
+        - name: 安装服务脚本到系统目录
+          ansible.builtin.copy:
+            src: /usr/local/mysql/support-files/mysql.server
+            dest: /etc/init.d/mysql
+            remote_src: yes
+            mode: 0755
+        - name: 设置mysql服务默认启动
+          ansible.builtin.service:
+            name: mysql
+            state: restarted
+            sleep: 5
+            enabled: yes
+        - name: 修改MySQL权限表使临时密码生效
+          ansible.builtin.shell: |
+            cd /usr/local/mysql
+            expect <<EOF
+            spawn ./bin/mysql --user=root -p
+            expect "Enter password:"
+            send "{{ mysqld_output.stderr | regex_search('(?<=generated for root@localhost: ).*$', multiline=True) }}\n"
+            expect "mysql>"
+            send "set password=password('{{ mysql_pass }}');grant all privileges on *.* to root@'%' identified by '{{ mysql_pass }}';flush privileges;\n"
+            expect "mysql>"
+            send "quit;\n"
+            exit
+            EOF
+        - name: 创建hive元数据库和cm监控数据库
+          community.mysql.mysql_db:
+            login_user: root
+            login_password: "{{ mysql_pass }}"
+            login_unix_socket: /usr/local/mysql/mysql.sock
+            name: "{{ item }}"
+            encoding: utf8
+            collation: utf8_general_ci
+          loop:
+            - hive_metastore
+            - amon
Author	SHA1	Message	Date
ZhangYueqian	3b3753a918	添加服务脚本	4 years ago
ZhangYueqian	b643a43c99	添加xz项目用的百度地图反代	4 years ago
Zhang Yueqian	8de901aa05	更新CDH部署信息	4 years ago
张月乾	21a172ce63	info: 更新描述信息部分格式	4 years ago
张月乾	3cbe0f8534	info: 更新Readme, 添加脚本使用说明	4 years ago
				`@ -0,0 +1 @@`
				ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxzm0vtfe4aVV3L3yStD/HEpK+iU/mxxzRqYOLtCEcbYTlEZTFlBHLyvVmPLPXGlV/1w5Gg3Fx0ThT0W2LcYM0eMmO3VkL4VTHr5OLAEujtmYrZSyBdDNKRiGQ/ky8wF/A1gAwBPOtvB6GJsX+YoBZ3pdVIBOK9bJG5BYWI0FT6mn8osstUYDLsqZ/KlUV+h97ynU9JjFEDsgDWyYWUGByDJPBMa44Q7vFjQr+8/bVPLPNmqpEDjw5pMQOUu1of470BMoQTq3/QmHNiWjAN/ntalLFHGo/gG9Vj1znZV6IcgAJkCzURh06ZGRmCbeNzb6yN8qZ/z3uIRq7/Iqko+BC1rI8qsTSujUKcAnpgSPhN3MABV1kLgnnfrki3JpFCaBaA1aaokBlzoiBr2kQCTdeCoqbGuJ9Rjd6kKRw954oQ/Eqae7YCWJB3Uns3PQ8g4Cge6Vn+JY7+o3XXz8Hr9PDAOexhYKuLznVPgMxk7r/K3m4GAhwlNop3MaHcMhtrNk= luomio@mbp.local