TencentBlueKing · abstractmj · May 20, 2024 · Apr 23, 2024
diff --git a/bcs-ops/Makefile b/bcs-ops/Makefile
@@ -9,15 +9,15 @@ clean:
 	-rm MD5SUMS
 
 build:clean
-	find . -not -path "*/.git/*" -a -not -path "*/bcs-ops-offline/*" -a -not -path "*/charts/*" -a -not -path "*/Makefile" -a -not -path "*/functions/*" -a -not -path "*.yaml" -a -not -path "*.md" -type f -print0 | xargs -0 chmod 555
-	find ./functions/ -not -path "*/.git/*" -a -not -path "*/bcs-ops-offline/*" -a -not -path "*/charts/*" -a -not -path "*/Makefile"  -type f -print0 | xargs -0 chmod 444
-	tar -czvf bcs-ops-script-release-$(VER).tar.gz --exclude=bcs-ops-offline --exclude=Makefile --exclude=\..* --exclude=.*tar.gz ./*
+	find . -not -path "*/.git/*" -a -not -path "*/bcs-ops-offline/*" -a -not -path "*/charts/*" -a -not -path "*/functions/*" -a -not -path "*.yaml" -a -not -path "*.md" -type f -print0 | xargs -0 chmod 555
+	find ./functions/ -not -path "*/.git/*" -a -not -path "*/bcs-ops-offline/*" -a -not -path "*/charts/*"  -type f -print0 | xargs -0 chmod 444
+	tar -czvf bcs-ops-script-release-$(VER).tar.gz --exclude=bcs-ops-offline --exclude=\..* --exclude=.*tar.gz ./*
 	md5sum bcs-ops-script-release-$(VER).tar.gz >> MD5SUMS
 
 build_offline_pkg:clean
-	find . -not -path "*/.git/*" -a -not -path "*/bcs-ops-offline/*" -a -not -path "*/charts/*" -a -not -path "*/Makefile" -a -not -path "*/functions/*" -a -not -path "*.yaml" -a -not -path "*.md" -type f -print0 | xargs -0 chmod 555
-	find ./functions/ -not -path "*/.git/*" -a -not -path "*/bcs-ops-offline/*" -a -not -path "*/charts/*" -a -not -path "*/Makefile" -type f -print0 | xargs -0 chmod 444
-	tar -czvf bcs-ops-script-release-$(VER).tar.gz --exclude=bcs-ops-offline --exclude=Makefile --exclude=\..* --exclude=.*tar.gz ./*
+	find . -not -path "*/.git/*" -a -not -path "*/bcs-ops-offline/*" -a -not -path "*/charts/*" -a -not -path "*/functions/*" -a -not -path "*.yaml" -a -not -path "*.md" -type f -print0 | xargs -0 chmod 555
+	find ./functions/ -not -path "*/.git/*" -a -not -path "*/bcs-ops-offline/*" -a -not -path "*/charts/*" -type f -print0 | xargs -0 chmod 444
+	tar -czvf bcs-ops-script-release-$(VER).tar.gz --exclude=bcs-ops-offline --exclude=\..* --exclude=.*tar.gz ./*
 	md5sum bcs-ops-script-release-$(VER).tar.gz >> MD5SUMS
 	./offline_package.sh env/offline-manifest.yaml
 	for f in $(find bcs-ops-offline -iname bcs-ops-offline-*.tgz -type f);do

diff --git a/bcs-ops/clean_node.sh b/bcs-ops/clean_node.sh
@@ -51,7 +51,7 @@ clean_cni() {
 }
 
 clean_vni() {
-  ip l | awk '/cni0|kube-ipvs0/{eth=$2;gsub(":","",eth);print eth}' | xargs -r -n 1 ip l d
+  ip l | awk '/cni0|kube-ipvs0/ && !/veth/ {eth=$2;gsub(":","",eth);print eth}' | xargs -r -n 1 ip l d
   ip l | awk '/veth/{eth=$2;split(eth,a,"@");print a[1]}' | xargs -r -n 1 ip l d
 }
 

diff --git a/bcs-ops/install_master.sh b/bcs-ops/install_master.sh
@@ -41,6 +41,8 @@ safe_source() {
   return 0
 }
 
+# try init host then check host
+"${ROOT_DIR}"/system/init_host.sh -i all
 "${ROOT_DIR}"/system/check_host.sh -c all
 safe_source "${ROOT_DIR}/functions/utils.sh"
 safe_source "${ROOT_DIR}/functions/k8s.sh"

diff --git a/bcs-ops/k8s/install_containerd b/bcs-ops/k8s/install_containerd
@@ -239,6 +239,9 @@ main() {
   # function testing
   local test_img_url
   test_img_url=${BK_PUBLIC_REPO:-"docker.io"}/library/hello-world:latest
+  if [ -n "${PRIVATE_DOCKER_REPO}" ];then
+    test_img_url=${PRIVATE_DOCKER_REPO}/hello-world:latest
+  fi
   utils::log "DEBUG" "hello-world: ${test_img_url}"
 
   if ! ctr -n k8s.io i list|grep ${test_img_url};then

diff --git a/bcs-ops/k8s/install_docker b/bcs-ops/k8s/install_docker
@@ -218,6 +218,9 @@ main() {
   # function testing
   local test_img_url
   test_img_url=${BK_PUBLIC_REPO:-"docker.io"}/library/hello-world:latest
+  if [ -n "${PRIVATE_DOCKER_REPO}" ];then
+    test_img_url=${PRIVATE_DOCKER_REPO}/hello-world:latest
+  fi
   utils::log "DEBUG" "hello-world: ${test_img_url}"
 
   if ! docker run --rm "${test_img_url}"; then

diff --git a/bcs-ops/k8s/operate_completion b/bcs-ops/k8s/operate_completion
@@ -56,6 +56,9 @@ check_completion() {
       "yum")
         _yum_bash-completion
         ;;
+      "curl")
+        _yum_bash-completion
+        ;;
       *)
         utils::log "ERROR" \
           "unkown ${INSTALL_METHOD} to exec download bash_completion"

diff --git a/bcs-ops/k8s/operate_flannel b/bcs-ops/k8s/operate_flannel
@@ -67,7 +67,10 @@ op_flannel() {
   ipv6_status=${K8S_IPv6_STATUS:-"Disable"}
   ipv6_status=${ipv6_status,,}
   action=$1
-  if [[ -z ${BK_PUBLIC_REPO} ]]; then
+  if [[ -n ${PRIVATE_DOCKER_REPO} ]]; then
+      flannel_image="${PRIVATE_DOCKER_REPO}/flannel:${FLANNEL_VER}"
+      flannelcni_image="${PRIVATE_DOCKER_REPO}/flannel-cni-plugin:${FLANNELCNI_VER}"
+  elif [[ -z ${BK_PUBLIC_REPO} ]]; then
     flannel_image="docker.io/flannel/flannel:${FLANNEL_VER}"
     flannelcni_image="docker.io/flannel/flannel-cni-plugin:${FLANNELCNI_VER}"
   else

diff --git a/bcs-ops/k8s/operate_metrics_server b/bcs-ops/k8s/operate_metrics_server
@@ -63,7 +63,9 @@ op_metric() {
 
   local image action
   action=$1
-  if [[ -z ${BK_PUBLIC_REPO} ]]; then
+  if [[ -n ${PRIVATE_DOCKER_REPO} ]]; then
+    image="${PRIVATE_DOCKER_REPO}/metrics-server:${METRICS_VER}"
+  elif [[ -z ${BK_PUBLIC_REPO} ]]; then
     image="registry.k8s.io/metrics-server/metrics-server:${METRICS_VER}"
   else
     image="${BK_PUBLIC_REPO}/registry.k8s.io/metrics-server/metrics-server:${METRICS_VER}"

diff --git a/bcs-ops/k8s/operate_multus b/bcs-ops/k8s/operate_multus
@@ -74,7 +74,9 @@ op_multus() {
   ipv6_status=${K8S_IPv6_STATUS:-"Disable"}
   ipv6_status=${ipv6_status,,}
   action=$1
-  if [[ -z ${BK_PUBLIC_REPO} ]]; then
+  if [[ -n ${PRIVATE_DOCKER_REPO} ]]; then
+    multus_image="${PRIVATE_DOCKER_REPO}/multus-cni:${MULTUS_VERSION}"
+  elif [[ -z ${BK_PUBLIC_REPO} ]]; then
     multus_image="ghcr.io/k8snetworkplumbingwg/multus-cni:${MULTUS_VERSION}"
   else
     multus_image="${BK_PUBLIC_REPO}/registry.k8s.io/multus-cni:${MULTUS_VERSION}"

diff --git a/bcs-ops/k8s/render_kubeadm b/bcs-ops/k8s/render_kubeadm
@@ -43,7 +43,10 @@ version() {
 
 render_cluster() {
   local image_registry config_file=$1
-  if [[ -n ${BK_PUBLIC_REPO:-} ]]; then
+  if [[ -n ${PRIVATE_DOCKER_REPO:-} ]]; then
+      # ToDo: registry.k8s.io replace k8s.gcr.io
+      image_registry=${PRIVATE_DOCKER_REPO}
+  elif [[ -n ${BK_PUBLIC_REPO:-} ]]; then
     # ToDo: registry.k8s.io replace k8s.gcr.io
     image_registry=${BK_PUBLIC_REPO}/registry.k8s.io
   else

diff --git a/bcs-ops/sops/bcs-ops X bk-sops 使用文档.md b/bcs-ops/sops/bcs-ops X bk-sops 使用文档.md
@@ -8,6 +8,8 @@ bcs-ops 借助 bk-sops 的编排能力，实现原生集群的流程化创建、
 
 ```bash
 ✦ ➜  cd bk-bcs/bcs-ops
+# build_image时需要通过apk命令安装软件，需要能够访问apk源；
+# 如果访问失败，可以尝试将镜像内的apk源替换为腾讯云等国内源，详情见https://mirrors.tencent.com/#/index
 ✦ ➜  make build_image
 # check
 ✦ ➜  docker run --rm --entrypoint /bin/ls bcs-ops-upload:test
@@ -19,8 +21,10 @@ upload.py # 导入脚本
 ### 镜像使用说明
 
 注意[导入事项](#导入事项)
+
 env 文件配置见[envfile 示例](#envfile%20示例)
 
+导入bkrepo前需要提前创建好bucket
 ```bash
 # 上传脚本包至 bkrepo, env 文件见下文
 ✦ ➜  docker run --env-file env bcs-ops-upload:test upload bkrepo
@@ -47,9 +51,11 @@ BKAPI_HOST=bkapi.example.com // 环境的bkapi host
 APP_CODE=bk_sops        // 确认 bk_sops 免登陆验证的 app_code
 APP_SECRET=your_secret // 和上面对应 secret
 
+REPO_DOCKER_HOST=docker.example.com //环境的 docker host
 REPO_HOST=bkrepo.example.com //环境的 bkrepo host
 REPO_PROJECT=blueking        // bkrepo的项目，默认 blueking
 REPO_BUCKET=bcs-ops        // bkrepo的bucket，需要在bkrepo上提前创建公开的bucket
+REPO_DOCKER_BUCKET=docker-local // bkrepo的bucket，需要在bkrepo上提前创建公开的docker bucket
 REPO_PATH=scripts        // bkrepo 的路径
 REPO_USER=xxxx        // bkrepo的用户，联系管理员获取，需要拥有上面 bucket 的权限！
 REPO_PASSWD=xxxx // bkrepo的密码，联系管理员获取
@@ -107,18 +113,54 @@ LOG_LEVEL=DEBUG // 日志等级
 
 ### 公共流程模板使用说明
 
+使用标准运维公共流程前，需要确保以下前置条件
+1. 相关节点已经安装gse agent并纳管到了执行流程的业务下
+2. 如果没有外网环境，将离线包内容推送到bkrepo上后可以使用bkrepo下载安装/镜像文件
+```
+# 上传离线包内容至bkrepo
+# 解压离线包后进入解压目录version-xxxxx下执行 
+# 用户也可以根据自己需求推到其它自建仓库中
+source env
+for i in `ls bin-tools/*`;do
+  curl -u ${REPO_USER}:${REPO_PASSWD} -T ${i} http://${REPO_HOST}/generic/${REPO_PROJECT}/${REPO_BUCKET}/
+done
+
+docker login -u ${REPO_USER} -p ${REPO_PASSWD} ${REPO_DOCKER_HOST}
+for i in `ls images/*.tar`; do 
+  image=$(docker load < $i|awk ' { print $NF } ');
+  target_image=${REPO_DOCKER_HOST}/${REPO_PROJECT}/${REPO_DOCKER_BUCKET}/$(echo ${image}|awk -F'/' ' { print $NF } ')
+  docker tag ${image} ${target_image}
+  docker push ${target_image}
+done
+
+#执行标准运维时参数 bcs-env中配置
+REPO_URL=1
+REPO_URL=your_generic_repo_url
+PRIVATE_DOCKER_REPO=your_docker_repo_url
+INSTALL_METHOD=curl
+```
+
+3. 如果没有bkrepo，则需要进行离线安装，需要填写FILE_SERVER和FILE_PATH参数，提前将离线包放在FILE_SERVER上
+```
+FILE_SERVER=1.2.3.4
+FILE_PATH=~/*.tar.gz
+#执行标准运维时参数 bcs-env中配置
+BCS_OFFLINE=1
+```
+
+
 #### id10.【BCS】bcsops distribute <a id="id10"></a>
 
+功能描述
+
+分发存储在 `bkrepo` 上的脚本包至节点机器 `HOST_IP`
+
 参数
 
 1. `WORKSPACE`: bcs-ops 脚本包工作路径，默认 `/data/bcs-ops` (注意，这个路径与后续所有流程的`WORKSPACE`变量一致！如果需要修改，请统一修改！)
 2. `HOST_IP`: 分发节点
 3. `SCRIPT_URL`: 脚本包下载地址，若是通过[镜像上传](#镜像使用说明)的标准运维流程，则路径自动配置为 bkrepo 的下载路径。如果是手动上传的，则需要自己修改这个路径。(依赖这个流程的模板也要对应的更新)
 
-功能描述
-
-分发存储在 `bkrepo` 上的脚本包至节点机器 `HOST_IP`
-
 #### id11. 【BCS】Setup Kubernetes on Linux <a id="id11"></a>
 
 参数

diff --git a/bcs-ops/sops/bcs_bk_sops_common.dat b/bcs-ops/sops/bcs_bk_sops_common.dat
diff --git a/bcs-ops/system/config_envfile.sh b/bcs-ops/system/config_envfile.sh
@@ -118,6 +118,7 @@ init_env() {
   REPO_MIRRORS=${REPO_MIRRORS:-"https://mirror.ccs.tencentyun.com"}
   ### other image
   BK_PUBLIC_REPO=${BK_PUBLIC_REPO:-"hub.bktencent.com"}
+  PRIVATE_DOCKER_REPO=${PRIVATE_DOCKER_REPO:-""}
 
   # helm
   BKREPO_URL=${BKREPO_URL:-"https://hub.bktencent.com/chartrepo/blueking"}
@@ -317,6 +318,7 @@ MIRROR_IP="${MIRROR_IP}"
 REPO_MIRRORS="${REPO_MIRRORS}"
 ### registry.k8s.io
 BK_PUBLIC_REPO="${BK_PUBLIC_REPO}"
+PRIVATE_DOCKER_REPO="${PRIVATE_DOCKER_REPO}"
 
 ## helm
 BKREPO_URL="${BKREPO_URL}"

diff --git a/bcs-ops/system/init_host.sh b/bcs-ops/system/init_host.sh
@@ -201,6 +201,19 @@ set_yum_repo() {
         wget -O /etc/yum.repos.d/CentOS-Base.repo $BASE_YUM_LINK
         mv /etc/yum.repos.d/epel.repo /etc/yum.repos.d/epel.repo.$BACKUPTIME
         wget -O /etc/yum.repos.d/epel.repo $BASE_EPEL_LINK
+
+        if [[ -f /etc/tlinux-release ]];then
+          if grep "TencentOS Server 3.[0-9]*" /etc/tlinux-release;then
+            sed -i "s/\$releasever/8/g" /etc/yum.repos.d/epel.repo
+            sed -i "s/\$releasever/8/g" /etc/yum.repos.d/CentOS-Base.repo
+          elif grep "TencentOS Server 2.[0-9]*" /etc/tlinux-release;then
+            sed -i "s/\$releasever/7/g" /etc/yum.repos.d/epel.repo
+            sed -i "s/\$releasever/7/g" /etc/yum.repos.d/CentOS-Base.repo
+          elif grep "Tencent tlinux release 2.[0-9]*" /etc/tlinux-release;then
+            sed -i "s/\$releasever/7/g" /etc/yum.repos.d/epel.repo
+            sed -i "s/\$releasever/7/g" /etc/yum.repos.d/CentOS-Base.repo
+          fi
+        fi
         yum clean all
         yum makecache