Skip to content

tingweiwang / k8s

Go to a project
Commit f3de3d2e by tingweiwang
Options

优化gpu-k8s脚本

1 parent 7672412f
Inline Side-by-side
Showing with 103 additions and 64 deletions
config/ansible-hosts
[etcd]
192.168.1.32
etcd_ip
[master]
192.168.1.32
[node]
192.168.1.32
master_ip
......
config/daemon.json_gpu
{
"registry-mirrors": ["https://hub-mirror.c.163.com","https://rrkngb5t.mirror.aliyuncs.com"],
"insecure-registries":["harbor_host"],
"default-runtime": "nvidia",
"runtimes": {
......@@ -8,3 +9,4 @@
}
}
}
script/autodl/reset_mysql.sh
#!/bin/bash
user=root
password=seetatech
#echo "删除数据库autodl-core"
#mysql -u $user -p$password -e 'drop database `autodl-core`'
echo "删除数据库autodl-core"
mysql -u $user -p$password -e 'drop database `autodl-core`'
echo "删除数据库autocnn"
mysql -u $user -p$password -e 'drop database `autocnn`'
#################################################
#echo "重新创建数据库autodl-core"
#mysql -u $user -p$password -e 'create database `autodl-core`'
echo "重新创建数据库autodl-core"
mysql -u $user -p$password -e 'create database `autodl-core`'
echo "重新创建数据库autocnn"
mysql -u $user -p$password -e 'create database `autocnn`'
script/harbor/push-autodl-workspace.sh 0 → 100755
#!/bin/bash
#wangtingwei
harbor_host=192.168.1.32:5000
harbor_passwd=admin
image_path=/root/workspace_image
for image in `ls $image_path`
do
echo "开始导入镜像到本地"
docker load -i $image_path/$image
done
echo "开始给镜像打tag"
image_name=(
hb.seetatech.com/autodl-workspace/pytorch:cuda10.0-py3-v1.2.0
hb.seetatech.com/autodl-workspace/pytorch:cpu-py3-v0.4.1
hb.seetatech.com/autodl-workspace/pytorch:cuda9.2-py3-v0.4.1
hb.seetatech.com/autodl-workspace/tensorflow:cpu-py3-v1.5.0
hb.seetatech.com/autodl-workspace/tensorflow:gpu-py3-v1.5.0
hb.seetatech.com/autodl-workspace/tensorflow:cpu-py3-v1.8.0
hb.seetatech.com/autodl-workspace/tensorflow:gpu-py3-v1.8.0
hb.seetatech.com/autodl-workspace/tensorflow:cpu-py3-v1.14.0
hb.seetatech.com/autodl-workspace/tensorflow:gpu-py3-v1.14.0
)
for i in ${image_name[@]}
do
for z in `echo $i |awk -F / '{print $3}'`
do
echo $z
docker tag $i $harbor_host/autodl-workspace/$z
docker push $harbor_host/autodl-workspace/$z
done
done
script/harbor/push_harbor.sh → script/harbor/push-base-images.sh
......@@ -16,7 +16,7 @@ docker tag nvidia/k8s-device-plugin:1.10 $harbor_host/k8s/k8s-device-plugin:1.10
docker tag busybox:1.28.3 $harbor_host/k8s/busybox:1.28.3
docker tag registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0 $harbor_host/k8s/pause-amd64:3.0
docker tag nginx:latest $harbor_host/k8s/nginx:latest
docker tag hb.seetatech.com/k8s/operator:0.0.19 $harbor_host/k8s/operator:0.0.19
#docker tag hb.seetatech.com/k8s/operator:0.0.19 $harbor_host/k8s/operator:0.0.19
echo "开始推送镜像到harbor"
docker push $harbor_host/k8s/image-proxy:v0.9.0
docker push $harbor_host/k8s/ubuntu-basic:16.04
......@@ -25,5 +25,5 @@ docker push $harbor_host/k8s/k8s-device-plugin:1.10
docker push $harbor_host/k8s/busybox:1.28.3
docker push $harbor_host/k8s/pause-amd64:3.0
docker push $harbor_host/k8s/nginx:latest
docker push $harbor_host/k8s/operator:0.0.19
#docker push $harbor_host/k8s/operator:0.0.19
script/k8s/deploy-k8s.sh → script/k8s/deploy-gpu-k8s.sh
#!/bin/bash
#author:王庭威
###########################配置ansible节点无交互##############################
#配置k8s master节点以及gpu节点，都在一台部署。
###########################配置变量########################################################
port=22
password=seetatech
etcd_ip=192.168.1.32
......@@ -9,16 +10,17 @@ harbor_host=192.168.1.32:5000
harbor_passwd=admin #写死的不能改，harbor配置文件中password写死了，当前只能是admin
image_path=/root/k8s/images #写死的不能改
interface=eno1 #本机IP的物理网卡名称，用于flannel的配置。
########################################################################################
echo "开始安装ansbile"
apt update --allow-insecure-repositories
apt install sshpass ansible -y --allow-unauthenticated
apt install sshpass ansible -y --allow-unauthenticated
echo "拷贝ansible-hosts文件"
cp /root/k8s/config/ansible-hosts /etc/ansible/hosts
for ip in 192.168.1.32
do
sshpass -p $password ssh-copy-id -i ~/.ssh/id_rsa.pub -p $port -o StrictHostKeyChecking=no root@$ip
done
ansible all -m ping
cp -a /root/k8s/config/ansible-hosts /etc/ansible/hosts
echo "渲染修改ansible hosts"
sed -i s/etcd_ip/$etcd_ip/g /etc/ansible/hosts
sed -i s/master_ip/$master_ip/g /etc/ansible/hosts
sshpass -p $password ssh-copy-id -i ~/.ssh/id_rsa.pub -p $port -o StrictHostKeyChecking=no root@$master_ip
ansible master -m ping
if [ $? -eq 0 ]
then
echo "ansible 配置成功"
......@@ -90,10 +92,10 @@ echo "+++++++++++++++++++++++++++++change bash++++++++++++++++++++++++++++++"
rm -rf /bin/sh && ln -s /bin/bash /bin/sh
sleep 3
###################################################安装etcd##################################
ansible all -m shell -a "mkdir /opt/kubernetes/{cfg,ssl,bin,log} -pv"
ansible all -m copy -a "src=/root/k8s/ssl/ca.pem dest=/opt/kubernetes/ssl/"
ansible all -m copy -a "src=/root/k8s/ssl/ca-key.pem dest=/opt/kubernetes/ssl/"
ansible all -m copy -a "src=/root/k8s/ssl/ca.csr dest=/opt/kubernetes/ssl/"
ansible master -m shell -a "mkdir /opt/kubernetes/{cfg,ssl,bin,log} -pv"
ansible master -m copy -a "src=/root/k8s/ssl/ca.pem dest=/opt/kubernetes/ssl/"
ansible master -m copy -a "src=/root/k8s/ssl/ca-key.pem dest=/opt/kubernetes/ssl/"
ansible master -m copy -a "src=/root/k8s/ssl/ca.csr dest=/opt/kubernetes/ssl/"
cat >>etcd-csr.json<<EOF
{
"CN": "etcd",
......@@ -165,10 +167,10 @@ rm -rf etcd.*
etcdctl --endpoints=https://$etcd_ip:2379 --ca-file=/opt/kubernetes/ssl/ca.pem --cert-file=/opt/kubernetes/ssl/etcd.pem --key-file=/opt/kubernetes/ssl/etcd-key.pem set /coreos.com/network/config '{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'
cp /root/k8s/bin/mk-docker-opts.sh /opt/kubernetes/bin/
cp /root/k8s/bin/flanneld /opt/kubernetes/bin/
ansible all -m copy -a "src=/opt/kubernetes/bin/mk-docker-opts.sh dest=/opt/kubernetes/bin/ mode=755"
ansible all -m copy -a "src=/opt/kubernetes/bin/flanneld dest=/opt/kubernetes/bin/ mode=755"
ansible all -m copy -a "src=/opt/kubernetes/ssl/etcd.pem dest=/opt/kubernetes/ssl/"
ansible all -m copy -a "src=/opt/kubernetes/ssl/etcd-key.pem dest=/opt/kubernetes/ssl/"
ansible master -m copy -a "src=/opt/kubernetes/bin/mk-docker-opts.sh dest=/opt/kubernetes/bin/ mode=755"
ansible master -m copy -a "src=/opt/kubernetes/bin/flanneld dest=/opt/kubernetes/bin/ mode=755"
ansible master -m copy -a "src=/opt/kubernetes/ssl/etcd.pem dest=/opt/kubernetes/ssl/"
ansible master -m copy -a "src=/opt/kubernetes/ssl/etcd-key.pem dest=/opt/kubernetes/ssl/"
cat >>/opt/kubernetes/cfg/flanneld<<EOF
FLANNEL_OPTIONS="--etcd-endpoints=https://$etcd_ip:2379 \\
-etcd-cafile=/opt/kubernetes/ssl/ca.pem \\
......@@ -176,26 +178,27 @@ FLANNEL_OPTIONS="--etcd-endpoints=https://$etcd_ip:2379 \\
-etcd-keyfile=/opt/kubernetes/ssl/etcd-key.pem \\
--iface=$interface"
EOF
ansible all -m copy -a "src=/opt/kubernetes/cfg/flanneld dest=/opt/kubernetes/cfg/ mode=755"
ansible master -m copy -a "src=/opt/kubernetes/cfg/flanneld dest=/opt/kubernetes/cfg/ mode=755"
cp /root/k8s/service/flanneld.service /lib/systemd/system/
ansible all -m copy -a "src=/lib/systemd/system/flanneld.service dest=/lib/systemd/system/ mode=755"
ansible all -m shell -a "systemctl daemon-reload && service flanneld restart && service flanneld status && systemctl enable flanneld.service"
ansible all -m copy -a "src=/root/k8s/service/docker.service dest=/lib/systemd/system/docker.service mode=755"
###############################所有节点安装docker###############################################
ansible master -m copy -a "src=/lib/systemd/system/flanneld.service dest=/lib/systemd/system/ mode=755"
ansible master -m shell -a "systemctl daemon-reload && service flanneld restart && service flanneld status && systemctl enable flanneld.service"
ansible master -m copy -a "src=/root/k8s/service/docker.service dest=/lib/systemd/system/docker.service mode=755"
###############################安装docker###############################################
echo "当前harbor仓库地址为$harbor_host,"
sed -i s/harbor_host/$harbor_host/g /root/k8s/config/daemon.json
sed -i s/harbor_host/$harbor_host/g /root/k8s/config/daemon.json_gpu
echo "开始安装docker"
ansible all -m shell -a "apt update --allow-insecure-repositories"
ansible all -m shell -a "apt install docker-ce -y --allow-unauthenticated"
ansible master -m shell -a "apt update --allow-insecure-repositories"
ansible master -m shell -a "apt install docker-ce -y --allow-unauthenticated"
echo "分发已修改的daemon.json文件"
ansible all -m copy -a "src=/root/k8s/config/daemon.json dest=/etc/docker/"
ansible all -m shell -a "service docker restart"
ansible all -m shell -a "service docker status"
ansible all -m shell -a "cat /etc/docker/daemon.json"
ansible master -m copy -a "src=/root/k8s/config/daemon.json_gpu dest=/etc/docker/"
ansible master -m shell -a "service docker restart"
ansible master -m shell -a "service docker status"
ansible master -m shell -a "cat /etc/docker/daemon.json"
echo "还原daemon.json模板文件"
sed -i s/$harbor_host/harbor_host/g /root/k8s/config/daemon.json
###############################安装master节点####################################################################
sed -i s/$harbor_host/harbor_host/g /root/k8s/config/daemon.json_gpu
####################################安装nvidia-docker2#####################################
ansible master -m shell -a "apt install nvidia-docker2 --allow-insecure-repositories"
###############################安装master节点###############################################
cat >>kubernetes-csr.json<<EOF
{
"CN": "kubernetes",
......@@ -286,13 +289,13 @@ ansible master -m shell -a "cp /opt/kubernetes/bin/kubectl /usr/bin/"
#echo "设置命令自动补全"
#source <(kubectl completion bash)
#echo "source <(kubectl completion bash)" >> ~/.bashrc
##############################################安装node节点###################################
ansible node -m copy -a "src=/root/k8s/bin/kube-proxy dest=/opt/kubernetes/bin/ mode=777"
ansible node -m copy -a "src=/root/k8s/bin/kubelet dest=/opt/kubernetes/bin/ mode=777"
ansible node -m copy -a "src=/root/k8s/service/kubelet.service dest=/lib/systemd/system/ mode=777"
ansible node -m copy -a "src=/root/k8s/service/kubelet.service dest=/etc/systemd/system/ mode=777"
ansible node -m copy -a "src=/root/k8s/service/kube-proxy.service dest=/lib/systemd/system/ mode=777"
ansible node -m shell -a "systemctl daemon-reload"
##############################################安装master 的node节点###################################
ansible master -m copy -a "src=/root/k8s/bin/kube-proxy dest=/opt/kubernetes/bin/ mode=777"
ansible master -m copy -a "src=/root/k8s/bin/kubelet dest=/opt/kubernetes/bin/ mode=777"
ansible master -m copy -a "src=/root/k8s/service/kubelet.service dest=/lib/systemd/system/ mode=777"
ansible master -m copy -a "src=/root/k8s/service/kubelet.service dest=/etc/systemd/system/ mode=777"
ansible master -m copy -a "src=/root/k8s/service/kube-proxy.service dest=/lib/systemd/system/ mode=777"
ansible master -m shell -a "systemctl daemon-reload"
kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap
#根据你自己的token.csv中的token值进行设定,切记该文件中只有前面内容是token。
BOOTSTRAP_TOKEN=`cat /opt/kubernetes/cfg/token.csv |awk -F ',' '{print $1}'`
......@@ -313,11 +316,11 @@ kubectl config set-context default \
--kubeconfig=bootstrap.kubeconfig
#设置默认上下文
kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
ansible node -m copy -a "src=/root/k8s/script/k8s/bootstrap.kubeconfig dest=/opt/kubernetes/cfg/ mode=644"
ansible node -m copy -a "src=/opt/kubernetes/cfg/token.csv dest=/opt/kubernetes/cfg/"
ansible master -m copy -a "src=/root/k8s/script/k8s/bootstrap.kubeconfig dest=/opt/kubernetes/cfg/ mode=644"
ansible master -m copy -a "src=/opt/kubernetes/cfg/token.csv dest=/opt/kubernetes/cfg/"
##############################################################################################
for node_ip in 192.168.1.32
for node_ip in $master_ip
do
cat >>/opt/kubernetes/cfg/kubelet.$node_ip<<EOF
KUBELET_OPTS="--logtostderr=false \
......@@ -359,8 +362,8 @@ EOF
scp /opt/kubernetes/cfg/kube-proxy.$node_ip $node_ip:/opt/kubernetes/cfg/kube-proxy
done
kubectl create -f /root/k8s/config/auto-cert.yaml
ansible node -m copy -a "src=/root/k8s/ssl/kube-proxy.pem dest=/opt/kubernetes/ssl/"
ansible node -m copy -a "src=/root/k8s/ssl/kube-proxy-key.pem dest=/opt/kubernetes/ssl/"
ansible master -m copy -a "src=/root/k8s/ssl/kube-proxy.pem dest=/opt/kubernetes/ssl/"
ansible master -m copy -a "src=/root/k8s/ssl/kube-proxy-key.pem dest=/opt/kubernetes/ssl/"
kubectl config set-cluster kubernetes \
--certificate-authority=/opt/kubernetes/ssl/ca.pem \
......@@ -380,20 +383,19 @@ kubectl config set-context default \
--kubeconfig=kube-proxy.kubeconfig
kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
ansible node -m copy -a "src=/root/k8s/script/k8s/kube-proxy.kubeconfig dest=/opt/kubernetes/cfg/"
ansible node -m shell -a "service kubelet restart"
ansible master -m copy -a "src=/root/k8s/script/k8s/kube-proxy.kubeconfig dest=/opt/kubernetes/cfg/"
ansible master -m shell -a "service kubelet restart"
#rm -rf bootstrap.kubeconfig kube-proxy.kubeconfig
sleep 5
kubectl get csr
kubectl get csr|grep 'Pending' | awk 'NR>0{print $1}'| xargs kubectl certificate approve
ansible node -m shell -a "service kube-proxy restart"
ansible node -m shell -a "systemctl enable kubelet.service"
ansible node -m shell -a "systemctl enable kube-proxy.service"
ansible master -m shell -a "service kube-proxy restart"
ansible master -m shell -a "systemctl enable kubelet.service"
ansible master -m shell -a "systemctl enable kube-proxy.service"
###########################安装harbor私有仓库##############
echo "解压harbor到/usr/local/目录"
tar -xvzf /root/k8s/package/harbor_aly.tar.gz -C /usr/local/
cd /usr/local/harbor/ && sed -i s/harbor_host/$harbor_host/g harbor.cfg && ./prepare && ./install.sh && echo "harbor安装成功"
###########################推送私有镜像到harbor仓库#######
for image in `ls $image_path`
do
......@@ -409,7 +411,7 @@ docker tag busybox:1.28.3 $harbor_host/k8s/busybox:1.28.3
docker tag registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0 $harbor_host/k8s/pause-amd64:3.0
docker tag nginx:latest $harbor_host/k8s/nginx:latest
echo "开始推送镜像到harbor"
echo "您的harbor服务器访问地址为:$harbor_host,请创建harbor所项目目录:k8s,core,autodl,seetaas，您有90s的时间"
echo "您的harbor服务器访问地址为:$harbor_host,请创建harbor所项目目录:k8s core autodl seetaas autodl-workspacem 您有90s的时间"
sleep 90
docker login $harbor_host -uadmin -p$harbor_passwd
docker push $harbor_host/k8s/image-proxy:v0.9.0
......@@ -419,15 +421,20 @@ docker push $harbor_host/k8s/k8s-device-plugin:1.10
docker push $harbor_host/k8s/busybox:1.28.3
docker push $harbor_host/k8s/pause-amd64:3.0
docker push $harbor_host/k8s/nginx:latest
################################安装coredns######################
################################安装coredns以及nvidia-kubernetes插件##########
sed -i s/harbor_host/$harbor_host/g /root/k8s/config/coredns.yaml
kubectl create -f /root/k8s/config/coredns.yaml
echo "恢复模板配置"
sed -i s/$harbor_host/harbor_host/g /root/k8s/config/coredns.yaml
##############################安装nvidia-kubernetes#################
sed -i s/harbor_host/$harbor_host/g /root/k8s/config/nvidia-device-plugin.yml
kubectl create -f /root/k8s/config/nvidia-device-plugin.yml
echo "恢复模板配置"
sed -i s/$harbor_host/harbor_host/g /root/k8s/config/nvidia-device-plugin.yml
echo "请在master节点 设置命令自动补全"
##############################################################################
echo "设置服务的alias快捷方式,需要手动source 环境变量"
echo "alias k='kubectl -n autodl'" >> ~/.bashrc
echo "alias ks='kubectl -n seetaas'" >> ~/.bashrc
sleep 5
##############################################################################
echo "请手动在master节点 设置命令自动补全，命令在脚本最后一行注释"
#source <(kubectl completion bash) && echo "source <(kubectl completion bash)" >> ~/.bashrc"
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!