Compare commits
5 Commits
6f63988ce4
...
3b3753a918
| Author | SHA1 | Date |
|---|---|---|
ZhangYueqian
|
3b3753a918 | 4 years ago |
|
ZhangYueqian
|
b643a43c99 | 4 years ago |
Zhang Yueqian
|
8de901aa05 | 4 years ago |
|
张月乾
|
21a172ce63 | 4 years ago |
|
张月乾
|
3cbe0f8534 | 4 years ago |
16 changed files with 730 additions and 0 deletions
@ -0,0 +1,7 @@ |
|||||||
|
BaiduProxy/pkg |
||||||
|
BaiduProxy/bin |
||||||
|
BaiduProxy/src/github.com |
||||||
|
BaiduProxy/src/golang.org |
||||||
|
BaiduProxy/src/h12.io |
||||||
|
BaiduProxy/antiy |
||||||
|
BaiduProxy/.idea |
||||||
@ -0,0 +1,13 @@ |
|||||||
|
# /etc/systemd/system/baidu_proxy.service |
||||||
|
[Unit] |
||||||
|
Description=proxy for api.map.baidu.com and so on |
||||||
|
After=syslog.target network.target |
||||||
|
Wants=network.target |
||||||
|
Requires=ssh_proxy.service |
||||||
|
|
||||||
|
[Service] |
||||||
|
Type=simple |
||||||
|
ExecStart=/usr/local/baiduproxy/antiy 0.0.0.0:7999 |
||||||
|
|
||||||
|
[Install] |
||||||
|
WantedBy=multi-user.target |
||||||
@ -0,0 +1,13 @@ |
|||||||
|
# /etc/systemd/system/baidu_proxy.service |
||||||
|
[Unit] |
||||||
|
Description=use ssh for remote port redirection. |
||||||
|
After=syslog.target network.target |
||||||
|
Wants=network.target baidu_proxy.service |
||||||
|
|
||||||
|
[Service] |
||||||
|
Type=simple |
||||||
|
WorkingDirectory=/usr/local/baiduproxy |
||||||
|
ExecStart=/usr/bin/autossh -M 0 -NR 0.0.0.0:7999:localhost:7999 -i id_rsa root@220.182.51.94 |
||||||
|
|
||||||
|
[Install] |
||||||
|
WantedBy=multi-user.target |
||||||
@ -0,0 +1,5 @@ |
|||||||
|
module antiy |
||||||
|
|
||||||
|
go 1.16 |
||||||
|
|
||||||
|
require h12.io/socks v1.0.2 |
||||||
@ -0,0 +1,6 @@ |
|||||||
|
github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364 h1:5XxdakFhqd9dnXoAZy1Mb2R/DZ6D1e+0bGC/JhucGYI= |
||||||
|
github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364/go.mod h1:eDJQioIyy4Yn3MVivT7rv/39gAJTrA7lgmYr8EW950c= |
||||||
|
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= |
||||||
|
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= |
||||||
|
h12.io/socks v1.0.2 h1:cZhhbV8+DE0Y1kotwhr1a3RC3kFO7AtuZ4GLr3qKSc8= |
||||||
|
h12.io/socks v1.0.2/go.mod h1:AIhxy1jOId/XCz9BO+EIgNL2rQiPTBNnOfnVnQ+3Eck= |
||||||
@ -0,0 +1,129 @@ |
|||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
"io/ioutil" |
||||||
|
"net/http" |
||||||
|
"net/url" |
||||||
|
"os" |
||||||
|
"strings" |
||||||
|
"time" |
||||||
|
) |
||||||
|
|
||||||
|
var domains = []string{ |
||||||
|
"api.map.baidu.com", |
||||||
|
"api0.map.bdimg.com", |
||||||
|
"api1.map.bdimg.com", |
||||||
|
"api2.map.bdimg.com", |
||||||
|
"api3.map.bdimg.com", |
||||||
|
"maponline0.bdimg.com", |
||||||
|
"maponline1.bdimg.com", |
||||||
|
"maponline2.bdimg.com", |
||||||
|
"maponline3.bdimg.com", |
||||||
|
"dlswbr.baidu.com", |
||||||
|
"shangetu0.map.bdimg.com", |
||||||
|
"shangetu1.map.bdimg.com", |
||||||
|
"shangetu2.map.bdimg.com", |
||||||
|
"shangetu3.map.bdimg.com", |
||||||
|
"miao.baidu.com", |
||||||
|
} |
||||||
|
|
||||||
|
func ReplaceHost(contentString, hostname string) string { |
||||||
|
for _, domain := range domains { |
||||||
|
contentString = strings.Replace(contentString, domain, hostname+"/"+domain, -1) |
||||||
|
} |
||||||
|
contentString = strings.Replace(contentString, "https://", "http://", -1) |
||||||
|
return contentString |
||||||
|
} |
||||||
|
|
||||||
|
type MyHandler struct { |
||||||
|
} |
||||||
|
|
||||||
|
func (myHandler *MyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { |
||||||
|
hostname := req.Host |
||||||
|
getUrl := url.URL{ |
||||||
|
Scheme: "http", |
||||||
|
} |
||||||
|
if !strings.HasPrefix(req.URL.Path, "/") { |
||||||
|
req.URL.Path = "/" + req.URL.Path |
||||||
|
} |
||||||
|
|
||||||
|
for _, domain := range domains { |
||||||
|
if strings.HasPrefix(req.URL.Path, "/"+domain) { |
||||||
|
getUrl.Host = domain |
||||||
|
getUrl.Path = strings.Replace(req.URL.Path, domain+"/", "", 1) |
||||||
|
getUrl.RawQuery = req.URL.RawQuery |
||||||
|
baiduResp, _ := http.Get(getUrl.String()) |
||||||
|
fmt.Printf("fmt: %s", baiduResp.Header.Get("Content-Type")) |
||||||
|
fmt.Printf("Get response from %s\n", getUrl.String()) |
||||||
|
content, _ := ioutil.ReadAll(baiduResp.Body) |
||||||
|
contentString := string(content) |
||||||
|
if strings.Contains(baiduResp.Header.Get("Content-Type"), "javascript") { |
||||||
|
contentString = ReplaceHost(contentString, hostname) |
||||||
|
} |
||||||
|
resp.Header().Set("Access-Control-Allow-Origin", "*") |
||||||
|
resp.Header().Set("Access-Control-Allow-Headers", "Origin") |
||||||
|
resp.Header().Set("Content-Type", baiduResp.Header.Get("Content-Type")) |
||||||
|
fmt.Fprintf(resp, "%v\n", contentString) |
||||||
|
return |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
//fp, _ := os.Open("demo.html")
|
||||||
|
//pageContent, _ := io.ReadAll(fp)
|
||||||
|
pageContent := ` |
||||||
|
<!DOCTYPE html> |
||||||
|
<html> |
||||||
|
<head> |
||||||
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> |
||||||
|
<meta name="viewport" content="initial-scale=1.0, user-scalable=no" /> |
||||||
|
<style type="text/css"> |
||||||
|
body, html,#allmap {width: 100%;height: 100%;overflow: hidden;margin:0;font-family:"微软雅黑";} |
||||||
|
</style> |
||||||
|
<script type="text/javascript" src="/api.map.baidu.com/api?v=2.0&ak=ADfpGTm3mDV4sZRiA2uDOKv3SbvcPtzG"></script> |
||||||
|
<title>地图展示</title> |
||||||
|
</head> |
||||||
|
<body> |
||||||
|
<div id="allmap"></div> |
||||||
|
</body> |
||||||
|
</html> |
||||||
|
<script type="text/javascript"> |
||||||
|
// 百度地图API功能
|
||||||
|
var map = new BMap.Map("allmap"); // 创建Map实例
|
||||||
|
map.centerAndZoom(new BMap.Point(116.404, 39.915), 11); // 初始化地图,设置中心点坐标和地图级别
|
||||||
|
//添加地图类型控件
|
||||||
|
map.addControl(new BMap.MapTypeControl({ |
||||||
|
mapTypes:[ |
||||||
|
BMAP_NORMAL_MAP, |
||||||
|
BMAP_HYBRID_MAP |
||||||
|
]})); |
||||||
|
map.setCurrentCity("武汉"); // 设置地图显示的城市 此项是必须设置的
|
||||||
|
map.enableScrollWheelZoom(true); //开启鼠标滚轮缩放
|
||||||
|
</script> |
||||||
|
` |
||||||
|
fmt.Fprint(resp, string(pageContent)) |
||||||
|
} |
||||||
|
|
||||||
|
func (myHandler *MyHandler) Handler() { |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
func main() { |
||||||
|
var port string |
||||||
|
if len(os.Args) > 1 { |
||||||
|
port = os.Args[1] |
||||||
|
} else { |
||||||
|
port = "127.0.0.1:8080" |
||||||
|
} |
||||||
|
m := MyHandler{} |
||||||
|
|
||||||
|
s := &http.Server{ |
||||||
|
Addr: port, |
||||||
|
Handler: &m, |
||||||
|
ReadTimeout: 10 * time.Second, |
||||||
|
WriteTimeout: 10 * time.Second, |
||||||
|
MaxHeaderBytes: 1 << 20, |
||||||
|
} |
||||||
|
|
||||||
|
s.ListenAndServe() |
||||||
|
} |
||||||
@ -0,0 +1,13 @@ |
|||||||
|
# /etc/systemd/system/baidu_proxy.service |
||||||
|
[Unit] |
||||||
|
Description=use ssh for remote port redirection. |
||||||
|
After=syslog.target network.target |
||||||
|
Wants=network.target |
||||||
|
|
||||||
|
[Service] |
||||||
|
Type=simple |
||||||
|
WorkingDirectory=/usr/local/baiduproxy |
||||||
|
ExecStart=/usr/bin/autossh -M 0 -NR 0.0.0.0:7999:localhost:7999 -i id_rsa 192.168.55.218 |
||||||
|
|
||||||
|
[Install] |
||||||
|
WantedBy=multi-user.target |
||||||
@ -0,0 +1,25 @@ |
|||||||
|
# 说明 |
||||||
|
|
||||||
|
## HelperScripts |
||||||
|
|
||||||
|
此文件夹主要存放一些帮助脚本 |
||||||
|
|
||||||
|
### PartTools.py |
||||||
|
|
||||||
|
用于批量自动分区并挂载的工具. 参数分两种 `-s SIZE` 或 `-d DEVICE,DEVICE,...` |
||||||
|
|
||||||
|
对于 `-s SIZE` , 会将大于此大小(单位为G)的磁盘(排除跟分区所在磁盘)分一个大分区并格式化成Ext4, 然后顺序挂载到系统的 /diskN 目录(如果只有一个磁盘则就是 /disk 目录). |
||||||
|
|
||||||
|
对于 `-d DEV,DEV,...` (例如 `-d sda,sdb,sdc`) 会将列表制定的磁盘(排除根分区所在磁盘)分一个大分区并格式化车成Ext4, 然后和上面一样顺序挂载到系统根目录下. |
||||||
|
|
||||||
|
挂载完成后, 会记录所有挂载上到分区信息到 `/etc/fstab` 文件里面, 以如下格式记录: |
||||||
|
|
||||||
|
``` config |
||||||
|
UUID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx /diskN ext4 defaults 0 0 |
||||||
|
``` |
||||||
|
|
||||||
|
``` shell |
||||||
|
sudo python2 PartTools.py -s 2000 # 将所有大于2T的磁盘分区&格式化&挂载 |
||||||
|
# 或者 |
||||||
|
sudo python2 PartTools.py -d sda,sdb,sdc,sdd # 将 /dev/sd{a,b,c,d} 全部格式化并挂载 |
||||||
|
``` |
||||||
@ -0,0 +1,5 @@ |
|||||||
|
[defaults] |
||||||
|
inventory = hosts |
||||||
|
remote_user = root |
||||||
|
# private_key_file = cdh_private.key |
||||||
|
host_key_checking = False |
||||||
@ -0,0 +1,17 @@ |
|||||||
|
--- |
||||||
|
- name: 环境准备 |
||||||
|
hosts: new_cdh_servers |
||||||
|
# become: true 不需要become，默认就是root用户 |
||||||
|
vars_prompt: |
||||||
|
- name: mysql_pass |
||||||
|
prompt: 请输入你想设置的MySQL root账户密码(默认为antiy?918) |
||||||
|
default: "antiy?918" |
||||||
|
tasks: |
||||||
|
- include: tasks/basic_settings.yml |
||||||
|
- include: tasks/deploy_mysql.yml |
||||||
|
- include: tasks/deploy_cm.yml |
||||||
|
- name: 安装完成，请继续配置 |
||||||
|
run_once: yes |
||||||
|
delegate_to: localhost |
||||||
|
pause: |
||||||
|
prompt: "所有安装已经完成，请访问 {{ group['new_cdh_servers'][0] }}:7180 进行下一步配置" |
||||||
@ -0,0 +1,38 @@ |
|||||||
|
-----BEGIN OPENSSH PRIVATE KEY----- |
||||||
|
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn |
||||||
|
NhAAAAAwEAAQAAAYEAsc5tL7X3uGlVdy98krQ/xxKSvolP5scc0amDi7QhHG2E5RGUxZQR |
||||||
|
y8r1Zjyz1xpVf9cORoNxcdE4U9Fti3GDNHjJjt1ZC+FUx6+TiwBLo7ZmK2UsgXQzSkYhkP |
||||||
|
5MvMBfwNYAMATzrbwehibF/mKAWd6XVSATivWyRuQWFiNBU+pp/KLLLVGAy7KmfypVFfof |
||||||
|
e8p1PSYxRA7IA1smFlBgcgyTwTGuOEO7xY0K/vP21TyzzZqqRA48OaTEDlLtaH+O9ATKEE |
||||||
|
6t/0JhzYlowDf57WpSxRxqP4BvVY9c52VeiHIACZAs1EYdOmRkZgm3jc2+sjfKmf897iEa |
||||||
|
u/yKpKPgQtayPKrE0ro1CnAJ6YEj4TdzAAVdZC4J5365ItyaRQmgWgNWmqJAZc6Iga9pEA |
||||||
|
k3XgqKmxrifUY3epCkcPeeKEPxKmnu2AliQd1J7Nz0PIOAoHulZ/iWO/qN118/B6/TwwDn |
||||||
|
sYWCri851T4DMZO6/yt5uBgIcJTaKdzGh3DIbazZAAAFiBnKPz8Zyj8/AAAAB3NzaC1yc2 |
||||||
|
EAAAGBALHObS+197hpVXcvfJK0P8cSkr6JT+bHHNGpg4u0IRxthOURlMWUEcvK9WY8s9ca |
||||||
|
VX/XDkaDcXHROFPRbYtxgzR4yY7dWQvhVMevk4sAS6O2ZitlLIF0M0pGIZD+TLzAX8DWAD |
||||||
|
AE8628HoYmxf5igFnel1UgE4r1skbkFhYjQVPqafyiyy1RgMuypn8qVRX6H3vKdT0mMUQO |
||||||
|
yANbJhZQYHIMk8ExrjhDu8WNCv7z9tU8s82aqkQOPDmkxA5S7Wh/jvQEyhBOrf9CYc2JaM |
||||||
|
A3+e1qUsUcaj+Ab1WPXOdlXohyAAmQLNRGHTpkZGYJt43NvrI3ypn/Pe4hGrv8iqSj4ELW |
||||||
|
sjyqxNK6NQpwCemBI+E3cwAFXWQuCed+uSLcmkUJoFoDVpqiQGXOiIGvaRAJN14Kipsa4n |
||||||
|
1GN3qQpHD3nihD8Spp7tgJYkHdSezc9DyDgKB7pWf4ljv6jddfPwev08MA57GFgq4vOdU+ |
||||||
|
AzGTuv8rebgYCHCU2incxodwyG2s2QAAAAMBAAEAAAGBAJhYHgU7DDzdQ3UTItEvz1Vp/I |
||||||
|
F0dRlukKqzaxpQA+6vh6YdIjMHYxgRlaE7cXtj/M3EHV8dUB4M/Q4T6qAHajuTIl9eF+Wt |
||||||
|
804EiY4XereSmIel8DQas7d3z1rECEvRKDm5IgACvialgCHBJvOGwiI+Lbup+BSf31gOJT |
||||||
|
5ihp63wofLk/oIUyo0r6NVSgHChSczmLAEAUImyAjYGBIWXAxb6l0li/h5UHE7H6jCr0KU |
||||||
|
+Bm54xjceY2KNtFvDCr03sFpGsAx+kjgWKWoHL9PzCd/8pMGEhnio7caJuutZjTwlOWyEc |
||||||
|
GEEuBM5FiXSLCIe03bshsbxB4CVg3Cfeq4kLdv3gx97ADacwUakJWubftcBIDD/7OcAFQ+ |
||||||
|
rHMonhOqznmkR8xRKc8rOylsph+5xTkfOFuWbkjGj7kxz4n4z+p0Rhujl1bPtZvtn8LDix |
||||||
|
hi6qTS/gMk48fTaO9XKVcd3+fRbXp6S6bxq7mnPo6HAhBv28jge6oxv7QtomF46+hMYQAA |
||||||
|
AMEAx91kGZbK8VFDNUk6DUh549LtQNArowzB12A7CqZLMcRZrFGQTtVVjy+Ea0lejp/csp |
||||||
|
KaB63Ut+/GHNqoGRodTinW98FoRVidA6TbpKRthUjj4AGKopRSHI2+3RYR/MCO2H7P0b66 |
||||||
|
Jc12deKdxoRGShmhYDF5A8USPhNCO+z90rCTnUY+88cubdYpJdQx6GUVMmRX7kwu+XlKjf |
||||||
|
OGLPcRYwr/mdEUl4nA1vENE/msSVrFeia5QpAdvSnDOQ4hl19mAAAAwQDWuapiZBhQxWd5 |
||||||
|
3yvUgqUW0Xg244EaM8e6O+5o+cvjMMw8iJFcvA1PcjlbkTK7Wr0sc3LO3CW6UtWSJTweAA |
||||||
|
zr+Kd+yzK6gRUlSiPMAQMJ5dmodipc+AF9opEnEYR//QI93ttqzlcOVzbqfHIKyZ/b52hs |
||||||
|
GI+P+o9xD8i+iK3f/ZKKEernABZZp40smhLD/x4qpijEmw0MjHQhT5gZN6exoiPJecvZB9 |
||||||
|
mIzSDaqvzHZ27h97AsHBAOPoq7XxkNYysAAADBANP8CFTBlBBU7JX36c19CeQAD66uCWyL |
||||||
|
gfWRhEcbueETM8W0FoIy+4J9b2bWwfy3J8ooRw7gR9PH58sn941I66qa+wnmdjLkPE8l33 |
||||||
|
njwRfBpJcXkU+qN1Dhg7QQveNalf4mTFp/AyBL+pUoUSkQl8AewAYFA60LpR7VLdomdZL8 |
||||||
|
4Afq8wHkExECAKkyE6OQTBqXNyqqNLGRQaLNokVZr4ZndUEKHjCaNzcp+xP09Wc1KRruLK |
||||||
|
uzWwW11nsV0YI+CwAAABBsdW9taW9AbWJwLmxvY2FsAQ== |
||||||
|
-----END OPENSSH PRIVATE KEY----- |
||||||
@ -0,0 +1 @@ |
|||||||
|
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxzm0vtfe4aVV3L3yStD/HEpK+iU/mxxzRqYOLtCEcbYTlEZTFlBHLyvVmPLPXGlV/1w5Gg3Fx0ThT0W2LcYM0eMmO3VkL4VTHr5OLAEujtmYrZSyBdDNKRiGQ/ky8wF/A1gAwBPOtvB6GJsX+YoBZ3pdVIBOK9bJG5BYWI0FT6mn8osstUYDLsqZ/KlUV+h97ynU9JjFEDsgDWyYWUGByDJPBMa44Q7vFjQr+8/bVPLPNmqpEDjw5pMQOUu1of470BMoQTq3/QmHNiWjAN/ntalLFHGo/gG9Vj1znZV6IcgAJkCzURh06ZGRmCbeNzb6yN8qZ/z3uIRq7/Iqko+BC1rI8qsTSujUKcAnpgSPhN3MABV1kLgnnfrki3JpFCaBaA1aaokBlzoiBr2kQCTdeCoqbGuJ9Rjd6kKRw954oQ/Eqae7YCWJB3Uns3PQ8g4Cge6Vn+JY7+o3XXz8Hr9PDAOexhYKuLznVPgMxk7r/K3m4GAhwlNop3MaHcMhtrNk= luomio@mbp.local |
||||||
@ -0,0 +1,23 @@ |
|||||||
|
[cdh_servers] |
||||||
|
cdh01.ghadoop ansible_ssh_host=10.251.24.2 ansible_ssh_port=22 |
||||||
|
cdh02.ghadoop ansible_ssh_host=10.251.24.3 ansible_ssh_port=22 |
||||||
|
cdh03.ghadoop ansible_ssh_host=10.251.24.4 ansible_ssh_port=22 |
||||||
|
cdh04.ghadoop ansible_ssh_host=10.251.24.5 ansible_ssh_port=22 |
||||||
|
cdh05.ghadoop ansible_ssh_host=10.251.24.6 ansible_ssh_port=22 |
||||||
|
|
||||||
|
[new_cdh_servers] |
||||||
|
cdh01.ghadoop ansible_ssh_host=10.251.24.102 ansible_ssh_port=22 |
||||||
|
cdh02.ghadoop ansible_ssh_host=10.251.24.103 ansible_ssh_port=22 |
||||||
|
cdh03.ghadoop ansible_ssh_host=10.251.24.104 ansible_ssh_port=22 |
||||||
|
cdh04.ghadoop ansible_ssh_host=10.251.24.105 ansible_ssh_port=22 |
||||||
|
cdh05.ghadoop ansible_ssh_host=10.251.24.106 ansible_ssh_port=22 |
||||||
|
|
||||||
|
[cdh_source] |
||||||
|
server ansible_ssh_host=192.168.0.10 ansible_ssh_port=22 |
||||||
|
|
||||||
|
[cdh_control] |
||||||
|
cdh01.ghadoop ansible_ssh_host=10.251.24.102 ansible_ssh_port=22 |
||||||
|
|
||||||
|
[new_cdh_servers:vars] |
||||||
|
ansible_ssh_user=root |
||||||
|
ansible_ssh_pass=antiy?918 |
||||||
@ -0,0 +1,240 @@ |
|||||||
|
--- |
||||||
|
- name: 上传安装包 |
||||||
|
copy: src=offline_cdh dest=/opt/ |
||||||
|
- name: 禁用SELinux和防火墙 |
||||||
|
block: |
||||||
|
- name: 禁用SELinux |
||||||
|
ansible.builtin.lineinfile: |
||||||
|
path: /etc/sysconfig/selinux |
||||||
|
regexp: '^SELINUX=' |
||||||
|
line: SELINUX=disabled |
||||||
|
- name: 禁用防火墙 |
||||||
|
ansible.builtin.service: |
||||||
|
name: firewalld |
||||||
|
state: stopped |
||||||
|
enabled: no |
||||||
|
- name: 修改主机名并分发给各主机 |
||||||
|
tags: |
||||||
|
- host |
||||||
|
block: |
||||||
|
- name: 设置主机名 |
||||||
|
ansible.builtin.hostname: |
||||||
|
name: "{{ inventory_hostname }}" |
||||||
|
- name: 修改 /etc/sysconfig/network 内的主机名设置 |
||||||
|
ansible.builtin.lineinfile: |
||||||
|
path: /etc/sysconfig/network |
||||||
|
regexp: '^HOSTNAME=' |
||||||
|
line: "HOSTNAME={{ inventory_hostname|lower }}" |
||||||
|
- name: 重启网络服务 |
||||||
|
ansible.builtin.service: |
||||||
|
name: network |
||||||
|
state: restarted |
||||||
|
- name: 分发主机名信息到各个主机 |
||||||
|
blockinfile: |
||||||
|
path: /etc/hosts |
||||||
|
block: | |
||||||
|
{% for h in groups['new_cdh_servers'] | sort %} |
||||||
|
{{ hostvars[h].ansible_default_ipv4.address }} {{ h }} |
||||||
|
{% endfor %} |
||||||
|
- name: 安装软件包 |
||||||
|
tags: |
||||||
|
- pkgs |
||||||
|
ansible.builtin.yum: |
||||||
|
name: |
||||||
|
- sysstat |
||||||
|
- vim |
||||||
|
- wget |
||||||
|
- lrzsz |
||||||
|
- screen |
||||||
|
- gcc |
||||||
|
- python-devel |
||||||
|
- gcc-c++ |
||||||
|
- ntpdate |
||||||
|
- libyaml |
||||||
|
- libyaml-devel |
||||||
|
- python-setuptools |
||||||
|
- ntp |
||||||
|
- libaio # for mysql |
||||||
|
- expect # for mysql |
||||||
|
- mariadb # 这是mysql客户端工具，改名了 |
||||||
|
- psmisc # for /opt/cm-5.14.2/etc/init.d/cloudera-scm-server start |
||||||
|
state: present |
||||||
|
- name: SSH互信配置 |
||||||
|
tags: |
||||||
|
- dpk |
||||||
|
block: |
||||||
|
- name: 生成ssh私钥 |
||||||
|
command: |
||||||
|
cmd: ssh-keygen -f /root/.ssh/id_rsa -N "" |
||||||
|
creates: /root/.ssh/id_rsa |
||||||
|
- name: 将公钥下载到Ansible中控机 |
||||||
|
fetch: |
||||||
|
src: /root/.ssh/id_rsa.pub |
||||||
|
dest: /tmp/pubkeys/{{ inventory_hostname }}.pub |
||||||
|
flat: yes |
||||||
|
- name: 将ssh指纹信息下载到中控机 |
||||||
|
block: |
||||||
|
- name: 先cat获取 |
||||||
|
command: cat /etc/ssh/ssh_host_ecdsa_key.pub |
||||||
|
register: ssh_finger |
||||||
|
- name: 然后输出到临时文件 |
||||||
|
ansible.builtin.lineinfile: |
||||||
|
path: /tmp/ssh_fingerprint |
||||||
|
line: "{{ inventory_hostname }},{{ hostvars[inventory_hostname].ansible_default_ipv4.address }} {{ ssh_finger.stdout_lines[0] }}" |
||||||
|
insertbefore: BOF |
||||||
|
create: yes |
||||||
|
- name: 最后获取到本地 |
||||||
|
fetch: |
||||||
|
src: /tmp/ssh_fingerprint |
||||||
|
dest: /tmp/fingerprints/{{ inventory_hostname }}.known_hosts |
||||||
|
flat: yes |
||||||
|
- name: 然后删除服务器上的临时文件 |
||||||
|
ansible.builtin.file: |
||||||
|
path: /tmp/ssh_fingerprint |
||||||
|
state: absent |
||||||
|
- name: 删除之前遗留的旧文件 |
||||||
|
run_once: yes |
||||||
|
delegate_to: localhost |
||||||
|
file: |
||||||
|
path: "{{ item }}" |
||||||
|
state: absent |
||||||
|
with_items: |
||||||
|
- /tmp/authorized_keys |
||||||
|
- /tmp/known_hosts |
||||||
|
- name: 合并公钥信息 |
||||||
|
run_once: yes |
||||||
|
delegate_to: localhost |
||||||
|
ansible.builtin.assemble: |
||||||
|
src: /tmp/pubkeys/ |
||||||
|
dest: /tmp/authorized_keys |
||||||
|
- name: 合并指纹信息 |
||||||
|
run_once: yes |
||||||
|
delegate_to: localhost |
||||||
|
ansible.builtin.assemble: |
||||||
|
src: /tmp/fingerprints/ |
||||||
|
dest: /tmp/known_hosts |
||||||
|
- name: 上传指纹信息和公钥信息到各个主机 |
||||||
|
copy: |
||||||
|
src: "{{ item }}" |
||||||
|
dest: "{{ item }}" |
||||||
|
loop: |
||||||
|
- /tmp/authorized_keys |
||||||
|
- /tmp/known_hosts |
||||||
|
- name: 将公钥复制到用户配置 |
||||||
|
blockinfile: |
||||||
|
path: /root/.ssh/authorized_keys |
||||||
|
block: "{{ lookup('file', '/tmp/authorized_keys') }}" |
||||||
|
create: yes |
||||||
|
mode: 0600 |
||||||
|
- name: 将指纹信息复制到用户配置 |
||||||
|
blockinfile: |
||||||
|
path: /root/.ssh/known_hosts |
||||||
|
block: "{{ lookup('file', '/tmp/known_hosts') }}" |
||||||
|
create: yes |
||||||
|
mode: 0600 |
||||||
|
- name: 清楚本地临时文件和目录 |
||||||
|
file: |
||||||
|
path: "{{ item }}" |
||||||
|
state: absent |
||||||
|
with_items: |
||||||
|
- /tmp/fingerprints/ |
||||||
|
- /tmp/pubkeys/ |
||||||
|
- /tmp/authorized_keys |
||||||
|
- /tmp/known_hosts |
||||||
|
- name: 移除系统JDK并安装离线包里面的JDK |
||||||
|
tags: |
||||||
|
- jdk |
||||||
|
block: |
||||||
|
- name: 移除系统JDK包 |
||||||
|
ansible.builtin.yum: |
||||||
|
name: |
||||||
|
- java-1.6.0-openjdk |
||||||
|
- java-1.7.0-openjdk |
||||||
|
- java-1.8.0-openjdk |
||||||
|
- java-11-openjdk |
||||||
|
state: absent |
||||||
|
autoremove: yes |
||||||
|
- name: 安装离线包提供的JDK8 |
||||||
|
ansible.builtin.yum: |
||||||
|
name: /opt/offline_cdh/jdk-8u261-linux-x64.rpm |
||||||
|
state: present |
||||||
|
- name: 设置时间同步 |
||||||
|
tags: |
||||||
|
- ntp |
||||||
|
block: |
||||||
|
- name: 在主机01上配置NTP服务 |
||||||
|
ansible.builtin.lineinfile: |
||||||
|
path: /etc/ntp.conf |
||||||
|
regexp: "^restrict 10.251" |
||||||
|
line: restrict 10.251.24.0 mask 255.255.255.0 nomodify notrap |
||||||
|
when: inventory_hostname == groups['new_cdh_servers'][0] |
||||||
|
- name: 在主机01上启用NTP服务 |
||||||
|
ansible.builtin.service: |
||||||
|
name: ntpd |
||||||
|
state: restarted |
||||||
|
enabled: yes |
||||||
|
- name: 延时一段时间以等待ntp服务就绪 |
||||||
|
command: sleep 10 |
||||||
|
- name: 从主机01上同步时间 |
||||||
|
command: "ntpdate -u {{ hostvars['cdh01.ghadoop'].ansible_default_ipv4.address }}" |
||||||
|
when: inventory_hostname != "cdh01.ghadoop" |
||||||
|
- name: 禁用透明大页 |
||||||
|
tags: |
||||||
|
- nothp |
||||||
|
block: |
||||||
|
- name: 临时禁用碎片整理 |
||||||
|
shell: echo 'never' > /sys/kernel/mm/transparent_hugepage/defrag |
||||||
|
args: |
||||||
|
executable: /bin/bash |
||||||
|
- name: 临时禁用透明大页 |
||||||
|
shell: echo 'never' > /sys/kernel/mm/transparent_hugepage/enabled |
||||||
|
args: |
||||||
|
executable: /bin/bash |
||||||
|
- name: 修改Grub配置 |
||||||
|
ansible.builtin.lineinfile: |
||||||
|
path: /etc/default/grub |
||||||
|
regexp: "^GRUB_CMDLINE_LINUX=" |
||||||
|
line: GRUB_CMDLINE_LINUX="crashkernel=auto rhgb quiet transparent_hugepage=never" |
||||||
|
- name: 判断当前是Grub还是Grub2 |
||||||
|
ansible.builtin.stat: |
||||||
|
path: /etc/{{ item }} |
||||||
|
register: grub2_cfg |
||||||
|
with_items: |
||||||
|
- grub2.cfg |
||||||
|
- grub2-efi.cfg |
||||||
|
- debug: |
||||||
|
var: grub2_cfg |
||||||
|
- name: 根据是Grub还是Grub2，更新Grub配置 |
||||||
|
command: "grub2-mkconfig -o {{ item.stat.path }}" |
||||||
|
when: item.stat.readable |
||||||
|
with_items: "{{ grub2_cfg.results }}" |
||||||
|
- name: 修改系统设置 |
||||||
|
tags: |
||||||
|
- setconf |
||||||
|
block: |
||||||
|
- name: 修改限制文件 /etc/security/limits.conf |
||||||
|
blockinfile: |
||||||
|
path: /etc/security/limits.conf |
||||||
|
block: | |
||||||
|
* soft nofile 65536 |
||||||
|
* hard nofile 65536 |
||||||
|
root soft nofile 65536 |
||||||
|
root hard nofile 65536 |
||||||
|
* soft memlock unlimited |
||||||
|
* hard memlock unlimited |
||||||
|
root soft memlock unlimited |
||||||
|
root hard memlock unlimited |
||||||
|
* soft as unlimited |
||||||
|
* hard as unlimited |
||||||
|
root soft as unlimited |
||||||
|
root hard as unlimited |
||||||
|
- name: 修改 /etc/sysctl.conf |
||||||
|
ansible.builtin.lineinfile: |
||||||
|
path: /etc/sysctl.conf |
||||||
|
regexp: "{{ item.regexp }}" |
||||||
|
line: "{{ item.line }}" |
||||||
|
state: present |
||||||
|
create: yes |
||||||
|
loop: |
||||||
|
- { regexp: "^vm.max_map_count=", line: "vm.max_map_count=131072" } |
||||||
|
- { regexp: "^vm.swappiness=", line: "vm.swappiness=0" } |
||||||
@ -0,0 +1,85 @@ |
|||||||
|
--- |
||||||
|
- name: 部署ClouderaManager |
||||||
|
tags: |
||||||
|
- cm |
||||||
|
block: |
||||||
|
- name: 新建用户 |
||||||
|
ansible.builtin.user: |
||||||
|
name: cloudera-scm |
||||||
|
system: true |
||||||
|
home: /opt/cm-5.14.2/run/cloudera-scm-server |
||||||
|
create_home: false |
||||||
|
shell: /bin/false |
||||||
|
comment: Cloudera SCM User |
||||||
|
- name: 解压缩CM安装包 |
||||||
|
ansible.builtin.unarchive: |
||||||
|
src: /opt/offline_cdh/cloudera-manager-centos7-cm5.14.2_x86_64.tar.gz |
||||||
|
dest: /opt/ |
||||||
|
remote_src: true |
||||||
|
- name: 复制MySQL连接器 |
||||||
|
ansible.builtin.copy: |
||||||
|
src: /opt/offline_cdh/mysql-connector-java-5.1.38.jar |
||||||
|
dest: /opt/cm-5.14.2/share/cmf/lib/ |
||||||
|
remote_src: true |
||||||
|
- name: 复制Parcel仓库 |
||||||
|
ansible.builtin.copy: |
||||||
|
src: /opt/offline_cdh/parcel-repo/ |
||||||
|
dest: /opt/cloudera/parcel-repo/ |
||||||
|
remote_src: true |
||||||
|
- name: 启动ClouderaManagerServer（仅主机执行） |
||||||
|
when: |
||||||
|
inventory_hostname == groups['new_cdh_servers'][0] |
||||||
|
block: |
||||||
|
- name: 初始化数据库 |
||||||
|
ansible.builtin.shell: | |
||||||
|
source /opt/cm-5.14.2/etc/default/cloudera-scm-server |
||||||
|
source /opt/cm-5.14.2/etc/default/cloudera-scm-agent |
||||||
|
/opt/cm-5.14.2/share/cmf/schema/scm_prepare_database.sh mysql cm -hlocalhost -uroot -p'{{ mysql_pass }}' --scm-host localhost scm scm scm |
||||||
|
- name: 创建目录 |
||||||
|
ansible.builtin.file: |
||||||
|
path: /var/lib/cloudera-scm-server |
||||||
|
state: directory |
||||||
|
- name: 启动服务 |
||||||
|
ansible.builtin.shell: | |
||||||
|
source /opt/cm-5.14.2/etc/default/cloudera-scm-server |
||||||
|
source /opt/cm-5.14.2/etc/default/cloudera-scm-agent |
||||||
|
/opt/cm-5.14.2/etc/init.d/cloudera-scm-server start |
||||||
|
args: |
||||||
|
executable: /bin/bash |
||||||
|
- name: 查看日志 |
||||||
|
ansible.builtin.command: |
||||||
|
cmd: tail -n30 /opt/cm-5.14.2/log/cloudera-scm-server/cloudera-scm-server.log |
||||||
|
register: cm_log |
||||||
|
- name: Debug输出日志 |
||||||
|
debug: |
||||||
|
msg: "{{ cm_log.stdout_lines }}" |
||||||
|
- name: 暂停等待确认 |
||||||
|
pause: |
||||||
|
prompt: 请确认输出是否正常，按“Ctrl+C c”继续 |
||||||
|
- name: 启动ClouderaManagerAgent（所有主机） |
||||||
|
block: |
||||||
|
- name: 配置主节点 |
||||||
|
ansible.builtin.lineinfile: |
||||||
|
path: /opt/cm-5.14.2/etc/cloudera-scm-agent/config.ini |
||||||
|
regexp: ^server_host= |
||||||
|
line: "server_host={{ groups['new_cdh_servers'][0] | lower }}" |
||||||
|
- name: 启动Agent |
||||||
|
ansible.builtin.shell: | |
||||||
|
source /opt/cm-5.14.2/etc/default/cloudera-scm-server |
||||||
|
source /opt/cm-5.14.2/etc/default/cloudera-scm-agent |
||||||
|
/opt/cm-5.14.2/etc/init.d/cloudera-scm-agent start |
||||||
|
args: |
||||||
|
executable: /bin/bash |
||||||
|
- name: 查看日志 |
||||||
|
ansible.builtin.command: |
||||||
|
cmd: tail -n30 /opt/cm-5.14.2/log/cloudera-scm-agent/cloudera-scm-agent.log |
||||||
|
register: cm_agent_log |
||||||
|
- name: Debug输出日志 |
||||||
|
debug: |
||||||
|
msg: "{{ cm_agent_log.stdout_lines }}" |
||||||
|
- name: 暂停等待确认 |
||||||
|
pause: |
||||||
|
prompt: | |
||||||
|
请确认输出是否正常，按“Ctrl+C c”继续 |
||||||
|
注意，以下信息为正常信息： |
||||||
|
ERROR Failed to connect to previous supervisor. |
||||||
@ -0,0 +1,110 @@ |
|||||||
|
--- |
||||||
|
- name: 部署MySQL |
||||||
|
tags: |
||||||
|
- mysql |
||||||
|
when: |
||||||
|
- inventory_hostname == groups['new_cdh_servers'][0] |
||||||
|
block: |
||||||
|
- name: 判断程序在不在 |
||||||
|
ansible.builtin.stat: |
||||||
|
path: /usr/local/mysql/bin/mysqld |
||||||
|
register: mysqld |
||||||
|
- name: 解压缩MySQL程序 |
||||||
|
block: |
||||||
|
- name: 解压缩离线包 |
||||||
|
ansible.builtin.unarchive: |
||||||
|
src: /opt/offline_cdh/mysql-5.7.31-linux-glibc2.12-x86_64.tar.gz |
||||||
|
dest: /usr/local/ |
||||||
|
remote_src: yes |
||||||
|
creates: /usr/local/mysql-5.7.31-linux-glibc2.12-x86_64 |
||||||
|
- name: 重命名文件夹 |
||||||
|
ansible.builtin.command: |
||||||
|
cmd: mv /usr/local/mysql-5.7.31-linux-glibc2.12-x86_64 /usr/local/mysql |
||||||
|
creates: /usr/local/mysql |
||||||
|
when: |
||||||
|
- not mysqld.stat.exists |
||||||
|
- name: 创建用户和用户组 |
||||||
|
block: |
||||||
|
- name: 创建用户组mysql |
||||||
|
ansible.builtin.group: |
||||||
|
name: mysql |
||||||
|
system: yes |
||||||
|
- name: 创建用户mysql |
||||||
|
ansible.builtin.user: |
||||||
|
name: mysql |
||||||
|
group: mysql |
||||||
|
system: yes |
||||||
|
create_home: no |
||||||
|
home: /usr/local/mysql |
||||||
|
shell: /sbin/nologin |
||||||
|
- name: 创建MySQL配置文件 |
||||||
|
ansible.builtin.copy: |
||||||
|
src: /opt/offline_cdh/my.cnf |
||||||
|
dest: /etc/my.cnf |
||||||
|
remote_src: yes |
||||||
|
- name: 创建MySQL目录并更新权限 |
||||||
|
ansible.builtin.file: |
||||||
|
path: "{{ item.path }}" |
||||||
|
state: "{{ item.state }}" |
||||||
|
recurse: "{{ item.recurse }}" |
||||||
|
owner: mysql |
||||||
|
group: mysql |
||||||
|
loop: |
||||||
|
- { path: /usr/local/mysql, state: directory, recurse: yes} |
||||||
|
- { path: /data/mysql, state: directory, recurse: yes } |
||||||
|
- { path: /data/log/mysql, state: directory, recurse: yes } |
||||||
|
- { path: /data/log/mysql/mysql.err, state: touch, recurse: no } |
||||||
|
- { path: /data/log/mysql/slow.log, state: touch, recurse: no } |
||||||
|
- name: 取得MySQL的密码 |
||||||
|
command: /usr/local/mysql/bin/mysqld --initialize --user=mysql --basedir=/usr/local/mysql --datadir=/data/mysql |
||||||
|
register: mysqld_output |
||||||
|
- name: 安装Python-MySQL客户端 |
||||||
|
ansible.builtin.yum: |
||||||
|
name: MySQL-python |
||||||
|
state: present |
||||||
|
- name: 配置MySQL服务 |
||||||
|
block: |
||||||
|
- name: 修改服务文件 |
||||||
|
ansible.builtin.lineinfile: |
||||||
|
regexp: "{{ item.regexp }}" |
||||||
|
line: "{{ item.line }}" |
||||||
|
path: /usr/local/mysql/support-files/mysql.server |
||||||
|
loop: |
||||||
|
- { regexp: "^basedir=", line: basedir=/usr/local/mysql } |
||||||
|
- { regexp: "^datadir=", line: datadir=/data/mysql } |
||||||
|
- name: 安装服务脚本到系统目录 |
||||||
|
ansible.builtin.copy: |
||||||
|
src: /usr/local/mysql/support-files/mysql.server |
||||||
|
dest: /etc/init.d/mysql |
||||||
|
remote_src: yes |
||||||
|
mode: 0755 |
||||||
|
- name: 设置mysql服务默认启动 |
||||||
|
ansible.builtin.service: |
||||||
|
name: mysql |
||||||
|
state: restarted |
||||||
|
sleep: 5 |
||||||
|
enabled: yes |
||||||
|
- name: 修改MySQL权限表使临时密码生效 |
||||||
|
ansible.builtin.shell: | |
||||||
|
cd /usr/local/mysql |
||||||
|
expect <<EOF |
||||||
|
spawn ./bin/mysql --user=root -p |
||||||
|
expect "Enter password:" |
||||||
|
send "{{ mysqld_output.stderr | regex_search('(?<=generated for root@localhost: ).*$', multiline=True) }}\n" |
||||||
|
expect "mysql>" |
||||||
|
send "set password=password('{{ mysql_pass }}');grant all privileges on *.* to root@'%' identified by '{{ mysql_pass }}';flush privileges;\n" |
||||||
|
expect "mysql>" |
||||||
|
send "quit;\n" |
||||||
|
exit |
||||||
|
EOF |
||||||
|
- name: 创建hive元数据库和cm监控数据库 |
||||||
|
community.mysql.mysql_db: |
||||||
|
login_user: root |
||||||
|
login_password: "{{ mysql_pass }}" |
||||||
|
login_unix_socket: /usr/local/mysql/mysql.sock |
||||||
|
name: "{{ item }}" |
||||||
|
encoding: utf8 |
||||||
|
collation: utf8_general_ci |
||||||
|
loop: |
||||||
|
- hive_metastore |
||||||
|
- amon |
||||||
Loading…
Reference in new issue