diff --git a/deploy/scripts/9-other-script/migrate_minio_database.sh b/deploy/scripts/9-other-script/migrate_minio_database.sh new file mode 100755 index 0000000000000000000000000000000000000000..26f1128cd0eb396f505e9b122f79f408739d16e8 --- /dev/null +++ b/deploy/scripts/9-other-script/migrate_minio_database.sh @@ -0,0 +1,244 @@ +#!/bin/bash +set -e + +# 颜色定义 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +PURPLE='\033[0;35m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# 日志函数 +log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } +log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; } +log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } +log_step() { echo -e "${PURPLE}[STEP]${NC} $1"; } + +# 全局变量 +PV_NAME="" +POD_NAME="" +STORAGE_DIR="" +NEW_POD_NAME="" +MINIO_PASSWORD="" +MINIO_ROOT_USER="minioadmin" +MINIO_HOST="" +MINIO_PORT="" +source_dir="/home/dump/minio" + +# 打印横幅 +print_banner() { + echo -e "${GREEN}" + echo "====================================================================" + echo " MINIO数据导入脚本" + echo " Euler Copilot 项目专用" + echo "====================================================================" + echo -e "${NC}" +} + +print_completion_banner() { + echo -e "${GREEN}" + echo "====================================================================" + echo " MINIO数据导入已完成!" + echo "====================================================================" + echo -e "${NC}" +} + +# 用户确认 +confirm_execution() { + echo -e "${YELLOW}警告: 此操作将清空现有MinIO数据并导入新数据!${NC}" + read -p "确认执行MinIO数据导入操作?(y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + log_info "操作已取消" + exit 0 + fi +} + +# 清除颜色代码和特殊字符 +clean_output() { + echo "$1" | sed -r "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[mGK]//g" | tr -d '\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' +} + +# 检查命令执行结果 +check_command() { + if [ $? -eq 0 ]; then + log_success "$1" + else + log_error "$2" + exit 1 + fi +} + +# 检查前置条件 +check_prerequisites() { + log_step "检查前置条件..." + + # 检查kubectl是否可用 + kubectl cluster-info &> /dev/null + check_command "kubectl连接正常" "无法连接到Kubernetes集群" + + # 检查命名空间是否存在 + if ! kubectl get namespace euler-copilot &> /dev/null; then + log_error "命名空间 euler-copilot 不存在" + exit 1 + fi + log_success "命名空间 euler-copilot 存在" +} + +# 获取K8s资源信息 +get_kubernetes_resources() { + log_step "获取Kubernetes资源信息..." + + # 获取PV名称 + PV_NAME=$(kubectl get pv -n euler-copilot | grep minio | awk '{print $1}') + if [ -z "$PV_NAME" ]; then + log_error "未找到MinIO的PV" + exit 1 + fi + log_info "PV名称: $PV_NAME" + + # 获取Pod名称 + POD_NAME=$(kubectl get pods -n euler-copilot | grep minio | grep Running | awk '{print $1}') + POD_NAME=$(clean_output "$POD_NAME") + if [ -z "$POD_NAME" ]; then + log_error "未找到运行的MinIO Pod" + exit 1 + fi + log_info "Pod名称: $POD_NAME" + + # 设置存储目录 + STORAGE_DIR="/var/lib/rancher/k3s/storage/${PV_NAME}_euler-copilot_minio-storage/" + log_info "存储目录: $STORAGE_DIR" +} + +# 复制数据 +copy_data() { + log_step "复制数据..." + + if [ -d "$source_dir" ]; then + # 检查源数据 + local source_count=$(find "$source_dir" -type f 2>/dev/null | wc -l) + if [ "$source_count" -eq 0 ]; then + log_warning "源目录为空,跳过复制" + return 0 + fi + + log_info "源目录文件数: $source_count" + cp -r "$source_dir"/* "$STORAGE_DIR" + check_command "数据复制完成" "数据复制失败" + + # 验证复制结果 + local new_count=$(find "$STORAGE_DIR" -type f 2>/dev/null | wc -l) + log_info "复制后文件数量: $new_count" + + if [ "$source_count" -eq "$new_count" ]; then + log_success "文件数量验证成功" + else + log_warning "文件数量不一致: 源=$source_count, 目标=$new_count" + fi + else + log_error "源目录 $source_dir 不存在" + exit 1 + fi +} + +# 重启Pod +restart_pod() { + log_step "重启Pod..." + + kubectl delete pod "$POD_NAME" -n euler-copilot + check_command "Pod删除命令执行成功" "Pod删除失败" + + # 等待Pod重启 + log_info "等待Pod重启..." + local timeout=60 + local counter=0 + + while [ $counter -lt $timeout ]; do + NEW_POD_NAME=$(kubectl get pods -n euler-copilot | grep minio | grep Running | awk '{print $1}') + NEW_POD_NAME=$(clean_output "$NEW_POD_NAME") + + if [ -n "$NEW_POD_NAME" ]; then + log_success "Pod重启成功: $NEW_POD_NAME" + return 0 + fi + + counter=$((counter + 5)) + sleep 5 + echo -n "." + done + + log_error "Pod未在指定时间内重启成功" + return 1 +} + +# 获取MinIO配置 +get_minio_config() { + log_step "获取MinIO配置..." + + MINIO_PASSWORD=$(kubectl get secret euler-copilot-database -n euler-copilot -o jsonpath='{.data.minio-password}' 2>/dev/null | base64 --decode) + if [ $? -ne 0 ] || [ -z "$MINIO_PASSWORD" ]; then + log_error "获取MinIO密码失败" + exit 1 + fi + + MINIO_HOST=$(kubectl get svc -n euler-copilot | grep minio-service | awk '{print $3}') + MINIO_PORT=$(kubectl get svc -n euler-copilot | grep minio-service | awk '{split($5, a, "/"); print a[1]}') + + log_info "MinIO配置:" + log_info " 主机: $MINIO_HOST" + log_info " 端口: $MINIO_PORT" + log_info " 用户: $MINIO_ROOT_USER" +} + +# 验证MinIO数据 +verify_minio_data() { + log_step "验证MinIO数据..." + + # 设置mc客户端 + kubectl exec -it "$NEW_POD_NAME" -n euler-copilot -- mc alias set myminio "http://${MINIO_HOST}:${MINIO_PORT}" "$MINIO_ROOT_USER" "$MINIO_PASSWORD" + check_command "MinIO客户端设置成功" "MinIO客户端设置失败" + + # 列出buckets + log_info "MinIO Buckets列表:" + kubectl exec -it "$NEW_POD_NAME" -n euler-copilot -- mc ls myminio + if [ $? -eq 0 ]; then + log_success "MinIO连接正常" + else + log_error "MinIO连接失败" + exit 1 + fi + + # 检查具体bucket内容 + log_info "检查witchaind-doc bucket:" + kubectl exec -it "$NEW_POD_NAME" -n euler-copilot -- mc ls myminio/witchaind-doc + if [ $? -eq 0 ]; then + log_success "witchaind-doc bucket访问成功" + else + log_warning "witchaind-doc bucket不存在或为空" + fi +} + +# 主函数 +main() { + print_banner + confirm_execution + + # 执行各个步骤 + check_prerequisites + get_kubernetes_resources + copy_data + restart_pod + get_minio_config + verify_minio_data + + print_completion_banner + log_success "MinIO数据导入流程已全部完成" + log_info "请验证业务功能是否正常" +} + +# 执行主函数 +main "$@" diff --git a/deploy/scripts/9-other-script/migrate_mysql_database.sh b/deploy/scripts/9-other-script/migrate_mysql_database.sh new file mode 100755 index 0000000000000000000000000000000000000000..aa45b4d8679dfbc7a4f8163af19a5246258088da --- /dev/null +++ b/deploy/scripts/9-other-script/migrate_mysql_database.sh @@ -0,0 +1,262 @@ +#!/bin/bash + +# MySQL数据库恢复脚本 +# 描述:用于Euler Copilot项目的MySQL数据库恢复 + +set -e # 遇到错误立即退出 + +# 颜色定义 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +PURPLE='\033[0;35m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# 打印函数 +print_color() { + local color=$1 + shift + echo -e "${color}$*${NC}" +} + +log() { + print_color "${GREEN}" "[$(date '+%Y-%m-%d %H:%M:%S')] $1" +} + +info() { + print_color "${BLUE}" "[$(date '+%Y-%m-%d %H:%M:%S')] $1" +} + +warning() { + print_color "${YELLOW}" "[$(date '+%Y-%m-%d %H:%M:%S')] 警告: $1" +} + +error() { + print_color "${RED}" "[$(date '+%Y-%m-%d %H:%M:%S')] 错误: $1" >&2 + exit 1 +} + +step() { + echo + print_color "${PURPLE}" "[$(date '+%Y-%m-%d %H:%M:%S')] === $1 ===" +} + +# 配置变量 +NAMESPACE="euler-copilot" +SECRET_NAME="authhub-secret" +PASSWORD_KEY="mysql-password" +DB_USER="authhub" +DB_NAME="oauth2" +BACKUP_FILE="/home/dump/mysql/mysql.sql" +POD_BACKUP_PATH="/home/mysql.sql" + +# 显示横幅 +show_banner() { + echo + print_color "${PURPLE}" "================================================" + print_color "${PURPLE}" " MySQL 数据库恢复脚本" + print_color "${PURPLE}" " Euler Copilot 项目专用" + print_color "${PURPLE}" "================================================" + echo +} + +# 检查必要工具 +check_dependencies() { + step "检查必要工具" + command -v kubectl >/dev/null 2>&1 || error "kubectl 未安装" + command -v base64 >/dev/null 2>&1 || error "base64 未安装" + log "依赖检查通过" +} + +# 获取MySQL Pod名称 +get_mysql_pod() { + step "查找MySQL Pod" + POD_NAME=$(kubectl get pod -n $NAMESPACE 2>/dev/null | grep mysql | grep Running | awk '{print $1}') + + if [ -z "$POD_NAME" ]; then + error "未找到运行的MySQL Pod" + fi + log "找到Pod: $POD_NAME" +} + +# 获取MySQL密码 +get_mysql_password() { + step "获取MySQL密码" + MYSQL_PASSWORD=$(kubectl get secret $SECRET_NAME -n $NAMESPACE -o jsonpath="{.data.$PASSWORD_KEY}" 2>/dev/null | base64 --decode) + + if [ -z "$MYSQL_PASSWORD" ]; then + error "无法获取MySQL密码,请检查secret是否存在" + fi + log "密码获取成功" +} + +# 检查备份文件是否存在 +check_backup_file() { + step "检查备份文件" + if [ ! -f "$BACKUP_FILE" ]; then + error "备份文件 $BACKUP_FILE 不存在" + fi + + # 显示文件信息 + local file_size=$(du -h "$BACKUP_FILE" | cut -f1) + local file_lines=$(wc -l < "$BACKUP_FILE" 2>/dev/null || echo "未知") + info "文件路径: $BACKUP_FILE" + info "文件大小: $file_size" + info "文件行数: $file_lines" + + log "备份文件检查通过" +} + +# 拷贝备份文件到Pod +copy_backup_to_pod() { + step "拷贝备份文件到Pod" + info "从本地拷贝到Pod: $BACKUP_FILE -> $POD_NAME:$POD_BACKUP_PATH" + + kubectl cp "$BACKUP_FILE" "$POD_NAME:$POD_BACKUP_PATH" -n $NAMESPACE + + if [ $? -eq 0 ]; then + log "备份文件拷贝完成" + else + error "文件拷贝失败" + fi +} + +# 验证数据库连接 +test_database_connection() { + step "测试数据库连接" + info "测试用户 $DB_USER 连接到数据库 $DB_NAME" + + kubectl exec $POD_NAME -n $NAMESPACE -- bash -c " + mysql -u$DB_USER -p'$MYSQL_PASSWORD' -e 'SELECT 1;' $DB_NAME 2>/dev/null + " >/dev/null 2>&1 + + if [ $? -eq 0 ]; then + log "数据库连接测试成功" + else + error "数据库连接失败,请检查密码和网络连接" + fi +} + +# 执行数据库恢复 +restore_database() { + step "执行数据库恢复" + warning "此操作将覆盖现有数据库数据!" + + info "开始恢复数据库..." + kubectl exec $POD_NAME -n $NAMESPACE -- bash -c " + mysql -u$DB_USER -p'$MYSQL_PASSWORD' $DB_NAME < $POD_BACKUP_PATH + " + + local restore_status=$? + + if [ $restore_status -eq 0 ]; then + log "数据库恢复成功" + else + error "数据库恢复失败,退出码: $restore_status" + fi +} + +# 验证恢复结果 +verify_restore() { + step "验证恢复结果" + info "检查数据库表信息..." + + local table_count=$(kubectl exec $POD_NAME -n $NAMESPACE -- bash -c " + mysql -u$DB_USER -p'$MYSQL_PASSWORD' -N -e \\ + \"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '$DB_NAME';\" 2>/dev/null + " 2>/dev/null) + + if [ -n "$table_count" ] && [ "$table_count" -gt 0 ]; then + log "恢复验证成功,数据库包含 $table_count 张表" + + # 显示部分表名 + info "数据库表列表:" + kubectl exec $POD_NAME -n $NAMESPACE -- bash -c " + mysql -u$DB_USER -p'$MYSQL_PASSWORD' -e \\ + \"SELECT table_name FROM information_schema.tables WHERE table_schema = '$DB_NAME' LIMIT 10;\" 2>/dev/null + " 2>/dev/null + else + warning "无法获取表信息,但恢复操作已完成" + fi +} + +# 清理临时文件 +cleanup() { + step "清理临时文件" + info "删除Pod内的备份文件: $POD_BACKUP_PATH" + + kubectl exec $POD_NAME -n $NAMESPACE -- rm -f "$POD_BACKUP_PATH" 2>/dev/null || true + + log "清理完成" +} + +# 显示使用说明 +usage() { + show_banner + print_color "${YELLOW}" "用法: $0" + echo + print_color "${CYAN}" "说明: 该脚本用于恢复Euler Copilot项目的MySQL数据库" + echo + print_color "${CYAN}" "前提条件:" + print_color "${WHITE}" " 1. kubectl已配置并可访问集群" + print_color "${WHITE}" " 2. mysql.sql文件存在于当前目录" + print_color "${WHITE}" " 3. 具有足够的集群权限" + echo + print_color "${RED}" "警告: 此操作将覆盖现有数据库数据!" + echo +} + +# 确认操作 +confirm_operation() { + print_color "${YELLOW}" "警告: 此操作将清空现有数据库数据并导入新数据!" + echo + read -p "$(print_color "${YELLOW}" "确认执行数据库恢复操作?(y/N): ")" confirm + case $confirm in + [yY] | [yY][eE][sS]) + return 0 + ;; + *) + warning "操作已取消" + exit 0 + ;; + esac +} + +# 主函数 +main() { + show_banner + step "开始MySQL数据库恢复流程" + + check_dependencies + get_mysql_pod + get_mysql_password + check_backup_file + test_database_connection + confirm_operation + copy_backup_to_pod + restore_database + verify_restore + cleanup + + echo + print_color "${GREEN}" "================================================" + print_color "${GREEN}" " Mysql数据恢复已完成!" + print_color "${GREEN}" "================================================" + echo + print_color "${GREEN}" "✓ 备份文件检查完成" + print_color "${GREEN}" "✓ 数据库连接测试通过" + print_color "${GREEN}" "✓ 数据恢复执行成功" + print_color "${GREEN}" "✓ 临时文件清理完成" + echo + print_color "${BLUE}" "提示: 建议检查应用运行状态以确保数据恢复成功。" +} + +# 脚本入口 +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + usage + exit 0 +fi + +main diff --git a/deploy/scripts/9-other-script/migrate_opengauss_database.sh b/deploy/scripts/9-other-script/migrate_opengauss_database.sh new file mode 100755 index 0000000000000000000000000000000000000000..bf80d987a70e18fa2549f5d293aa1f845d9aef94 --- /dev/null +++ b/deploy/scripts/9-other-script/migrate_opengauss_database.sh @@ -0,0 +1,210 @@ +#!/bin/bash + +# OpenGauss数据库导入脚本 +# 描述:用于Euler Copilot项目的数据库数据导入 + +set -e # 遇到错误立即退出 + +# 颜色定义 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +PURPLE='\033[0;35m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# 打印函数 +log() { + echo -e "${GREEN}[$(date '+%Y-%m-%d %H:%M:%S')] $1${NC}" +} + +info() { + echo -e "${BLUE}[$(date '+%Y-%m-%d %H:%M:%S')] $1${NC}" +} + +warning() { + echo -e "${YELLOW}[$(date '+%Y-%m-%d %H:%M:%S')] 警告: $1${NC}" +} + +error() { + echo -e "${RED}[$(date '+%Y-%m-%d %H:%M:%S')] 错误: $1${NC}" >&2 + exit 1 +} + +step() { + echo -e "${PURPLE}[$(date '+%Y-%m-%d %H:%M:%S')] === $1 ===${NC}" +} + +# 配置变量 +NAMESPACE="euler-copilot" +SECRET_NAME="euler-copilot-database" +PASSWORD_KEY="gauss-password" +BACKUP_FILE="/home/dump/opengauss/opengauss.sql" +POD_BACKUP_PATH="/home/omm/opengauss.sql" + +# 检查必要工具 +check_dependencies() { + step "检查必要工具" + command -v kubectl >/dev/null 2>&1 || error "kubectl 未安装" + command -v base64 >/dev/null 2>&1 || error "base64 未安装" + log "依赖检查通过" +} + +# 获取数据库密码 +get_database_password() { + step "获取数据库密码" + PASSWORD=$(kubectl get secret $SECRET_NAME -n $NAMESPACE -o jsonpath="{.data.$PASSWORD_KEY}" 2>/dev/null | base64 --decode) + + if [ -z "$PASSWORD" ]; then + error "无法获取数据库密码,请检查secret是否存在" + fi + log "密码获取成功" +} + +# 获取OpenGauss Pod名称 +get_opengauss_pod() { + step "查找OpenGauss Pod" + POD_NAME=$(kubectl get pod -n $NAMESPACE 2>/dev/null | grep opengauss | grep Running | awk '{print $1}') + + if [ -z "$POD_NAME" ]; then + error "未找到运行的OpenGauss Pod" + fi + log "找到Pod: $POD_NAME" +} + +# 检查备份文件是否存在 +check_backup_file() { + step "检查备份文件" + if [ ! -f "$BACKUP_FILE" ]; then + error "备份文件 $BACKUP_FILE 不存在" + fi + log "备份文件检查通过: $BACKUP_FILE" +} + +# 拷贝备份文件到Pod +copy_backup_to_pod() { + step "拷贝备份文件到Pod" + kubectl cp "$BACKUP_FILE" "$POD_NAME:$POD_BACKUP_PATH" -n $NAMESPACE + kubectl exec -it "$POD_NAME" -n $NAMESPACE -- bash -c "chown omm:omm $POD_BACKUP_PATH" + log "备份文件拷贝完成" +} + +# 执行数据库导入 +import_database() { + step "执行数据库导入操作" + + info "步骤1: 禁用外键约束..." + kubectl exec -it "$POD_NAME" -n $NAMESPACE -- su - omm -s /bin/bash -c \ + "gsql -d postgres -U postgres -W '$PASSWORD' -c \"SET session_replication_role = replica;\"" + log "外键约束已禁用" + + info "步骤2: 清空表数据..." + kubectl exec -it "$POD_NAME" -n $NAMESPACE -- su - omm -s /bin/bash -c \ + "gsql -d postgres -U postgres -W '$PASSWORD' -c \" +TRUNCATE TABLE + action, team, knowledge_base, document, chunk, document_type, + role, role_action, users, task, task_report, team_user, user_role, + dataset, dataset_doc, image, qa, task_queue, team_message, + testcase, testing, user_message +CASCADE;\"" + log "表数据清空完成" + + info "步骤3: 导入数据..." + kubectl exec -it "$POD_NAME" -n $NAMESPACE -- su - omm -s /bin/bash -c \ + "gsql -d postgres -U postgres -f $POD_BACKUP_PATH -W '$PASSWORD'" + log "数据导入完成" + + info "步骤4: 启用外键约束..." + kubectl exec -it "$POD_NAME" -n $NAMESPACE -- su - omm -s /bin/bash -c \ + "gsql -d postgres -U postgres -W '$PASSWORD' -c \"SET session_replication_role = origin;\"" + log "外键约束已启用" + + log "数据库导入操作全部完成" +} + +# 清理临时文件 +cleanup() { + step "清理临时文件" + kubectl exec -it "$POD_NAME" -n $NAMESPACE -- rm -f "$POD_BACKUP_PATH" 2>/dev/null || true + log "清理完成" +} + +# 显示横幅 +show_banner() { + echo -e "${PURPLE}" + echo "================================================================" + echo " OpenGauss 数据库导入脚本" + echo " Euler Copilot 项目专用" + echo "================================================================" + echo -e "${NC}" +} + +# 主函数 +main() { + show_banner + step "开始OpenGauss数据库导入流程" + + check_dependencies + get_database_password + get_opengauss_pod + check_backup_file + copy_backup_to_pod + import_database + cleanup + + echo -e "${GREEN}" + echo "================================================================" + echo " OpenGauss数据导入已完成!" + echo "================================================================" + echo -e "${NC}" + + echo -e "${GREEN}✓ 外键约束已禁用${NC}" + echo -e "${GREEN}✓ 表数据已清空${NC}" + echo -e "${GREEN}✓ 新数据已导入${NC}" + echo -e "${GREEN}✓ 外键约束已重新启用${NC}" + echo "" + echo -e "${BLUE}提示: 建议检查应用运行状态以确保数据导入成功。${NC}" +} + +# 显示使用说明 +usage() { + show_banner + echo -e "${YELLOW}用法: $0${NC}" + echo "" + echo -e "说明: 该脚本用于导入Euler Copilot项目的OpenGauss数据库数据" + echo "" + echo -e "${CYAN}前提条件:${NC}" + echo -e " 1. kubectl已配置并可访问集群" + echo -e " 2. opengauss.sql文件存在于当前目录" + echo -e " 3. 具有足够的集群权限" + echo "" + echo -e "${RED}警告: 此操作将清空现有数据库数据并导入新数据!${NC}" +} + +# 脚本入口 +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + usage + exit 0 +fi + +# 修复颜色显示问题 +echo -e "${YELLOW}警告: 此操作将清空现有数据库数据并导入新数据!${NC}" + +# 方法1:使用临时变量 +yellow_text="${YELLOW}确认执行数据库导入操作?(y/N): ${NC}" +read -p "$(echo -e "$yellow_text")" confirm + +# 方法2:或者直接使用echo -e(备选方案) +# echo -e "${YELLOW}确认执行数据库导入操作?(y/N): ${NC}\c" +# read confirm + +case $confirm in + [yY] | [yY][eE][sS]) + main + ;; + *) + warning "操作已取消" + exit 0 + ;; +esac