fix(images): normalize shell scripts to LF line endings
CodeQL Advanced / Analyze (actions) (push) Canceled after 0s
CodeQL Advanced / Analyze (java-kotlin) (push) Canceled after 0s
CodeQL Advanced / Analyze (javascript-typescript) (push) Canceled after 0s
CodeQL Advanced / Analyze (python) (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Python Backend Docker Image CI / Build and Push Backend Python Docker Image (push) Canceled after 0s
Backend Docker Image CI / Build and Push Backend Docker Image (push) Canceled after 0s
Database Docker Image CI / Build and Push Database Docker Image (push) Canceled after 0s
Frontend Docker Image CI / Build and Push Frontend Docker Image (push) Canceled after 0s
Gateway Docker Image CI / Build and Push Gateway Docker Image (push) Canceled after 0s
Runtime Docker Image CI / Build and Push Runtime Docker Image (push) Canceled after 0s
CodeQL Advanced / Analyze (actions) (push) Canceled after 0s
CodeQL Advanced / Analyze (java-kotlin) (push) Canceled after 0s
CodeQL Advanced / Analyze (javascript-typescript) (push) Canceled after 0s
CodeQL Advanced / Analyze (python) (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Python Backend Docker Image CI / Build and Push Backend Python Docker Image (push) Canceled after 0s
Backend Docker Image CI / Build and Push Backend Docker Image (push) Canceled after 0s
Database Docker Image CI / Build and Push Database Docker Image (push) Canceled after 0s
Frontend Docker Image CI / Build and Push Frontend Docker Image (push) Canceled after 0s
Gateway Docker Image CI / Build and Push Gateway Docker Image (push) Canceled after 0s
Runtime Docker Image CI / Build and Push Runtime Docker Image (push) Canceled after 0s
database image crashed at start: CRLF shebang in start.sh made the kernel look for interpreter '/bin/bash\r'. Other image Dockerfiles mask this with dos2unix at build time, but scripts/images/database/Dockerfile does not. Convert all tracked .sh files to LF and pin eol=lf via .gitattributes.
This commit is contained in:
apescale
+21
-21
@@ -1,22 +1,22 @@
|
||||
#!/bin/bash
|
||||
# 遇到错误即退出
|
||||
set -e
|
||||
|
||||
# 使用 -n 判断环境变量 HOME_PAGE_URL 是否非空（即已设置且有值）
|
||||
if [ -n "$HOME_PAGE_URL" ]; then
|
||||
echo "检测到 HOME_PAGE_URL 环境变量已配置，值为: $HOME_PAGE_URL。准备更新数据库..."
|
||||
|
||||
# 只有变量非空时，才会进入这里执行 SQL
|
||||
psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "deepdata" -v my_url="$HOME_PAGE_URL" <<-EOSQL
|
||||
|
||||
UPDATE t_sys_param
|
||||
SET param_value = :'my_url'
|
||||
WHERE id = 'sys.home.page.url';
|
||||
|
||||
EOSQL
|
||||
|
||||
echo "sys.home.page.url 更新完成！"
|
||||
else
|
||||
# 如果变量为空或未设置，打印提示并直接跳过
|
||||
echo "未配置 HOME_PAGE_URL 环境变量或值为空，跳过 sys.home.page.url 的更新操作。"
|
||||
#!/bin/bash
|
||||
# 遇到错误即退出
|
||||
set -e
|
||||
|
||||
# 使用 -n 判断环境变量 HOME_PAGE_URL 是否非空（即已设置且有值）
|
||||
if [ -n "$HOME_PAGE_URL" ]; then
|
||||
echo "检测到 HOME_PAGE_URL 环境变量已配置，值为: $HOME_PAGE_URL。准备更新数据库..."
|
||||
|
||||
# 只有变量非空时，才会进入这里执行 SQL
|
||||
psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "deepdata" -v my_url="$HOME_PAGE_URL" <<-EOSQL
|
||||
|
||||
UPDATE t_sys_param
|
||||
SET param_value = :'my_url'
|
||||
WHERE id = 'sys.home.page.url';
|
||||
|
||||
EOSQL
|
||||
|
||||
echo "sys.home.page.url 更新完成！"
|
||||
else
|
||||
# 如果变量为空或未设置，打印提示并直接跳过
|
||||
echo "未配置 HOME_PAGE_URL 环境变量或值为空，跳过 sys.home.page.url 的更新操作。"
|
||||
fi
|
||||
@@ -1,8 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
umask 0022
|
||||
|
||||
echo "Starting main application..."
|
||||
exec "$@"
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
umask 0022
|
||||
|
||||
echo "Starting main application..."
|
||||
exec "$@"
|
||||
|
||||
@@ -1,97 +1,97 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
run_once=false
|
||||
if [ "${1:-}" = "--once" ]; then
|
||||
run_once=true
|
||||
shift
|
||||
fi
|
||||
|
||||
LOG_ROOT="${1:-${LOG_ROTATION_PATH:-/tmp/ray}}"
|
||||
LOG_ROTATION_MAX_SIZE="${LOG_ROTATION_MAX_SIZE:-100MB}"
|
||||
LOG_ROTATION_BACKUP_COUNT="${LOG_ROTATION_BACKUP_COUNT:-30}"
|
||||
LOG_ROTATION_INTERVAL_SECONDS="${LOG_ROTATION_INTERVAL_SECONDS:-300}"
|
||||
|
||||
parse_size_bytes() {
|
||||
local value
|
||||
value="$(printf '%s' "$1" | tr '[:lower:]' '[:upper:]')"
|
||||
value="${value// /}"
|
||||
case "$value" in
|
||||
*GB) echo $((${value%GB} * 1024 * 1024 * 1024)) ;;
|
||||
*G) echo $((${value%G} * 1024 * 1024 * 1024)) ;;
|
||||
*MB) echo $((${value%MB} * 1024 * 1024)) ;;
|
||||
*M) echo $((${value%M} * 1024 * 1024)) ;;
|
||||
*KB) echo $((${value%KB} * 1024)) ;;
|
||||
*K) echo $((${value%K} * 1024)) ;;
|
||||
*B) echo "${value%B}" ;;
|
||||
*) echo "$value" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
file_size_bytes() {
|
||||
wc -c < "$1" | tr -d ' '
|
||||
}
|
||||
|
||||
backup_path() {
|
||||
local file="$1"
|
||||
local index="$2"
|
||||
local ext="${file##*.}"
|
||||
local stem="${file%.$ext}"
|
||||
echo "${stem}.${index}.${ext}"
|
||||
}
|
||||
|
||||
is_backup_file() {
|
||||
local name
|
||||
name="$(basename "$1")"
|
||||
[[ "$name" =~ \.[0-9]+\.(log|out|err)$ ]]
|
||||
}
|
||||
|
||||
rotate_file() {
|
||||
local file="$1"
|
||||
local max_bytes="$2"
|
||||
local backup_count="$3"
|
||||
|
||||
[ -f "$file" ] || return 0
|
||||
is_backup_file "$file" && return 0
|
||||
|
||||
local current_size
|
||||
current_size="$(file_size_bytes "$file")"
|
||||
[ "$current_size" -lt "$max_bytes" ] && return 0
|
||||
|
||||
rm -f "$(backup_path "$file" "$backup_count")"
|
||||
|
||||
local index
|
||||
for ((index = backup_count - 1; index >= 1; index--)); do
|
||||
local src dst
|
||||
src="$(backup_path "$file" "$index")"
|
||||
dst="$(backup_path "$file" "$((index + 1))")"
|
||||
[ -f "$src" ] && mv "$src" "$dst"
|
||||
done
|
||||
|
||||
cp -p "$file" "$(backup_path "$file" 1)"
|
||||
: > "$file"
|
||||
}
|
||||
|
||||
rotate_once() {
|
||||
[ -d "$LOG_ROOT" ] || return 0
|
||||
|
||||
local max_bytes
|
||||
max_bytes="$(parse_size_bytes "$LOG_ROTATION_MAX_SIZE")"
|
||||
|
||||
while IFS= read -r -d '' file; do
|
||||
rotate_file "$file" "$max_bytes" "$LOG_ROTATION_BACKUP_COUNT"
|
||||
done < <(
|
||||
find "$LOG_ROOT" -type f \( -name '*.log' -o -name '*.out' -o -name '*.err' \) -print0 2>/dev/null || true
|
||||
)
|
||||
}
|
||||
|
||||
if [ "$run_once" = true ]; then
|
||||
rotate_once
|
||||
exit 0
|
||||
fi
|
||||
|
||||
while true; do
|
||||
rotate_once
|
||||
sleep "$LOG_ROTATION_INTERVAL_SECONDS"
|
||||
done
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
run_once=false
|
||||
if [ "${1:-}" = "--once" ]; then
|
||||
run_once=true
|
||||
shift
|
||||
fi
|
||||
|
||||
LOG_ROOT="${1:-${LOG_ROTATION_PATH:-/tmp/ray}}"
|
||||
LOG_ROTATION_MAX_SIZE="${LOG_ROTATION_MAX_SIZE:-100MB}"
|
||||
LOG_ROTATION_BACKUP_COUNT="${LOG_ROTATION_BACKUP_COUNT:-30}"
|
||||
LOG_ROTATION_INTERVAL_SECONDS="${LOG_ROTATION_INTERVAL_SECONDS:-300}"
|
||||
|
||||
parse_size_bytes() {
|
||||
local value
|
||||
value="$(printf '%s' "$1" | tr '[:lower:]' '[:upper:]')"
|
||||
value="${value// /}"
|
||||
case "$value" in
|
||||
*GB) echo $((${value%GB} * 1024 * 1024 * 1024)) ;;
|
||||
*G) echo $((${value%G} * 1024 * 1024 * 1024)) ;;
|
||||
*MB) echo $((${value%MB} * 1024 * 1024)) ;;
|
||||
*M) echo $((${value%M} * 1024 * 1024)) ;;
|
||||
*KB) echo $((${value%KB} * 1024)) ;;
|
||||
*K) echo $((${value%K} * 1024)) ;;
|
||||
*B) echo "${value%B}" ;;
|
||||
*) echo "$value" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
file_size_bytes() {
|
||||
wc -c < "$1" | tr -d ' '
|
||||
}
|
||||
|
||||
backup_path() {
|
||||
local file="$1"
|
||||
local index="$2"
|
||||
local ext="${file##*.}"
|
||||
local stem="${file%.$ext}"
|
||||
echo "${stem}.${index}.${ext}"
|
||||
}
|
||||
|
||||
is_backup_file() {
|
||||
local name
|
||||
name="$(basename "$1")"
|
||||
[[ "$name" =~ \.[0-9]+\.(log|out|err)$ ]]
|
||||
}
|
||||
|
||||
rotate_file() {
|
||||
local file="$1"
|
||||
local max_bytes="$2"
|
||||
local backup_count="$3"
|
||||
|
||||
[ -f "$file" ] || return 0
|
||||
is_backup_file "$file" && return 0
|
||||
|
||||
local current_size
|
||||
current_size="$(file_size_bytes "$file")"
|
||||
[ "$current_size" -lt "$max_bytes" ] && return 0
|
||||
|
||||
rm -f "$(backup_path "$file" "$backup_count")"
|
||||
|
||||
local index
|
||||
for ((index = backup_count - 1; index >= 1; index--)); do
|
||||
local src dst
|
||||
src="$(backup_path "$file" "$index")"
|
||||
dst="$(backup_path "$file" "$((index + 1))")"
|
||||
[ -f "$src" ] && mv "$src" "$dst"
|
||||
done
|
||||
|
||||
cp -p "$file" "$(backup_path "$file" 1)"
|
||||
: > "$file"
|
||||
}
|
||||
|
||||
rotate_once() {
|
||||
[ -d "$LOG_ROOT" ] || return 0
|
||||
|
||||
local max_bytes
|
||||
max_bytes="$(parse_size_bytes "$LOG_ROTATION_MAX_SIZE")"
|
||||
|
||||
while IFS= read -r -d '' file; do
|
||||
rotate_file "$file" "$max_bytes" "$LOG_ROTATION_BACKUP_COUNT"
|
||||
done < <(
|
||||
find "$LOG_ROOT" -type f \( -name '*.log' -o -name '*.out' -o -name '*.err' \) -print0 2>/dev/null || true
|
||||
)
|
||||
}
|
||||
|
||||
if [ "$run_once" = true ]; then
|
||||
rotate_once
|
||||
exit 0
|
||||
fi
|
||||
|
||||
while true; do
|
||||
rotate_once
|
||||
sleep "$LOG_ROTATION_INTERVAL_SECONDS"
|
||||
done
|
||||
|
||||
@@ -1,32 +1,32 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
LOG_DIR="${POSTGRES_LOG_DIR:-/var/log/deepdata/database}"
|
||||
LOG_ROTATION_BACKUP_COUNT="${LOG_ROTATION_BACKUP_COUNT:-30}"
|
||||
LOG_ROTATION_INTERVAL_SECONDS="${LOG_ROTATION_INTERVAL_SECONDS:-300}"
|
||||
|
||||
prune_once() {
|
||||
[ -d "$LOG_DIR" ] || return 0
|
||||
|
||||
mapfile -t files < <(
|
||||
find "$LOG_DIR" -maxdepth 1 -type f -name 'postgresql-*.log' -printf '%T@ %p\n' 2>/dev/null \
|
||||
| sort -rn \
|
||||
| sed 's/^[^ ]* //'
|
||||
)
|
||||
|
||||
local index
|
||||
for ((index = LOG_ROTATION_BACKUP_COUNT; index < ${#files[@]}; index++)); do
|
||||
rm -f "${files[$index]}"
|
||||
done
|
||||
}
|
||||
|
||||
if [ "${1:-}" = "--once" ]; then
|
||||
prune_once
|
||||
exit 0
|
||||
fi
|
||||
|
||||
while true; do
|
||||
prune_once
|
||||
sleep "$LOG_ROTATION_INTERVAL_SECONDS"
|
||||
done
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
LOG_DIR="${POSTGRES_LOG_DIR:-/var/log/deepdata/database}"
|
||||
LOG_ROTATION_BACKUP_COUNT="${LOG_ROTATION_BACKUP_COUNT:-30}"
|
||||
LOG_ROTATION_INTERVAL_SECONDS="${LOG_ROTATION_INTERVAL_SECONDS:-300}"
|
||||
|
||||
prune_once() {
|
||||
[ -d "$LOG_DIR" ] || return 0
|
||||
|
||||
mapfile -t files < <(
|
||||
find "$LOG_DIR" -maxdepth 1 -type f -name 'postgresql-*.log' -printf '%T@ %p\n' 2>/dev/null \
|
||||
| sort -rn \
|
||||
| sed 's/^[^ ]* //'
|
||||
)
|
||||
|
||||
local index
|
||||
for ((index = LOG_ROTATION_BACKUP_COUNT; index < ${#files[@]}; index++)); do
|
||||
rm -f "${files[$index]}"
|
||||
done
|
||||
}
|
||||
|
||||
if [ "${1:-}" = "--once" ]; then
|
||||
prune_once
|
||||
exit 0
|
||||
fi
|
||||
|
||||
while true; do
|
||||
prune_once
|
||||
sleep "$LOG_ROTATION_INTERVAL_SECONDS"
|
||||
done
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
/usr/local/bin/prune-postgres-logs.sh &
|
||||
|
||||
exec docker-entrypoint.sh "$@"
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
/usr/local/bin/prune-postgres-logs.sh &
|
||||
|
||||
exec docker-entrypoint.sh "$@"
|
||||
|
||||
@@ -1,92 +1,92 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
LOG_DIR="${NGINX_LOG_DIR:-/var/log/deepdata/frontend}"
|
||||
LOG_ROTATION_MAX_SIZE="${LOG_ROTATION_MAX_SIZE:-100MB}"
|
||||
LOG_ROTATION_BACKUP_COUNT="${LOG_ROTATION_BACKUP_COUNT:-30}"
|
||||
LOG_ROTATION_INTERVAL_SECONDS="${LOG_ROTATION_INTERVAL_SECONDS:-300}"
|
||||
|
||||
parse_size_bytes() {
|
||||
local value
|
||||
value="$(printf '%s' "$1" | tr '[:lower:]' '[:upper:]')"
|
||||
value="${value// /}"
|
||||
case "$value" in
|
||||
*GB) echo $((${value%GB} * 1024 * 1024 * 1024)) ;;
|
||||
*G) echo $((${value%G} * 1024 * 1024 * 1024)) ;;
|
||||
*MB) echo $((${value%MB} * 1024 * 1024)) ;;
|
||||
*M) echo $((${value%M} * 1024 * 1024)) ;;
|
||||
*KB) echo $((${value%KB} * 1024)) ;;
|
||||
*K) echo $((${value%K} * 1024)) ;;
|
||||
*B) echo "${value%B}" ;;
|
||||
*) echo "$value" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
file_size_bytes() {
|
||||
wc -c < "$1" | tr -d ' '
|
||||
}
|
||||
|
||||
backup_path() {
|
||||
local file="$1"
|
||||
local index="$2"
|
||||
local ext="${file##*.}"
|
||||
local stem="${file%.$ext}"
|
||||
echo "${stem}.${index}.${ext}"
|
||||
}
|
||||
|
||||
rotate_file() {
|
||||
local file="$1"
|
||||
local max_bytes="$2"
|
||||
local backup_count="$3"
|
||||
|
||||
[ -f "$file" ] || return 1
|
||||
|
||||
local current_size
|
||||
current_size="$(file_size_bytes "$file")"
|
||||
[ "$current_size" -lt "$max_bytes" ] && return 1
|
||||
|
||||
rm -f "$(backup_path "$file" "$backup_count")"
|
||||
|
||||
local index
|
||||
for ((index = backup_count - 1; index >= 1; index--)); do
|
||||
local src dst
|
||||
src="$(backup_path "$file" "$index")"
|
||||
dst="$(backup_path "$file" "$((index + 1))")"
|
||||
[ -f "$src" ] && mv "$src" "$dst"
|
||||
done
|
||||
|
||||
mv "$file" "$(backup_path "$file" 1)"
|
||||
install -o nginx -g nginx -m 0644 /dev/null "$file"
|
||||
return 0
|
||||
}
|
||||
|
||||
rotate_once() {
|
||||
mkdir -p "$LOG_DIR"
|
||||
touch "$LOG_DIR/access.log" "$LOG_DIR/error.log"
|
||||
chown -R nginx:nginx "$LOG_DIR" 2>/dev/null || true
|
||||
|
||||
local max_bytes
|
||||
max_bytes="$(parse_size_bytes "$LOG_ROTATION_MAX_SIZE")"
|
||||
|
||||
local rotated=false
|
||||
for file in "$LOG_DIR/access.log" "$LOG_DIR/error.log"; do
|
||||
if rotate_file "$file" "$max_bytes" "$LOG_ROTATION_BACKUP_COUNT"; then
|
||||
rotated=true
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$rotated" = true ]; then
|
||||
nginx -s reopen || true
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "${1:-}" = "--once" ]; then
|
||||
rotate_once
|
||||
exit 0
|
||||
fi
|
||||
|
||||
while true; do
|
||||
rotate_once
|
||||
sleep "$LOG_ROTATION_INTERVAL_SECONDS"
|
||||
done
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
LOG_DIR="${NGINX_LOG_DIR:-/var/log/deepdata/frontend}"
|
||||
LOG_ROTATION_MAX_SIZE="${LOG_ROTATION_MAX_SIZE:-100MB}"
|
||||
LOG_ROTATION_BACKUP_COUNT="${LOG_ROTATION_BACKUP_COUNT:-30}"
|
||||
LOG_ROTATION_INTERVAL_SECONDS="${LOG_ROTATION_INTERVAL_SECONDS:-300}"
|
||||
|
||||
parse_size_bytes() {
|
||||
local value
|
||||
value="$(printf '%s' "$1" | tr '[:lower:]' '[:upper:]')"
|
||||
value="${value// /}"
|
||||
case "$value" in
|
||||
*GB) echo $((${value%GB} * 1024 * 1024 * 1024)) ;;
|
||||
*G) echo $((${value%G} * 1024 * 1024 * 1024)) ;;
|
||||
*MB) echo $((${value%MB} * 1024 * 1024)) ;;
|
||||
*M) echo $((${value%M} * 1024 * 1024)) ;;
|
||||
*KB) echo $((${value%KB} * 1024)) ;;
|
||||
*K) echo $((${value%K} * 1024)) ;;
|
||||
*B) echo "${value%B}" ;;
|
||||
*) echo "$value" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
file_size_bytes() {
|
||||
wc -c < "$1" | tr -d ' '
|
||||
}
|
||||
|
||||
backup_path() {
|
||||
local file="$1"
|
||||
local index="$2"
|
||||
local ext="${file##*.}"
|
||||
local stem="${file%.$ext}"
|
||||
echo "${stem}.${index}.${ext}"
|
||||
}
|
||||
|
||||
rotate_file() {
|
||||
local file="$1"
|
||||
local max_bytes="$2"
|
||||
local backup_count="$3"
|
||||
|
||||
[ -f "$file" ] || return 1
|
||||
|
||||
local current_size
|
||||
current_size="$(file_size_bytes "$file")"
|
||||
[ "$current_size" -lt "$max_bytes" ] && return 1
|
||||
|
||||
rm -f "$(backup_path "$file" "$backup_count")"
|
||||
|
||||
local index
|
||||
for ((index = backup_count - 1; index >= 1; index--)); do
|
||||
local src dst
|
||||
src="$(backup_path "$file" "$index")"
|
||||
dst="$(backup_path "$file" "$((index + 1))")"
|
||||
[ -f "$src" ] && mv "$src" "$dst"
|
||||
done
|
||||
|
||||
mv "$file" "$(backup_path "$file" 1)"
|
||||
install -o nginx -g nginx -m 0644 /dev/null "$file"
|
||||
return 0
|
||||
}
|
||||
|
||||
rotate_once() {
|
||||
mkdir -p "$LOG_DIR"
|
||||
touch "$LOG_DIR/access.log" "$LOG_DIR/error.log"
|
||||
chown -R nginx:nginx "$LOG_DIR" 2>/dev/null || true
|
||||
|
||||
local max_bytes
|
||||
max_bytes="$(parse_size_bytes "$LOG_ROTATION_MAX_SIZE")"
|
||||
|
||||
local rotated=false
|
||||
for file in "$LOG_DIR/access.log" "$LOG_DIR/error.log"; do
|
||||
if rotate_file "$file" "$max_bytes" "$LOG_ROTATION_BACKUP_COUNT"; then
|
||||
rotated=true
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$rotated" = true ]; then
|
||||
nginx -s reopen || true
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "${1:-}" = "--once" ]; then
|
||||
rotate_once
|
||||
exit 0
|
||||
fi
|
||||
|
||||
while true; do
|
||||
rotate_once
|
||||
sleep "$LOG_ROTATION_INTERVAL_SECONDS"
|
||||
done
|
||||
|
||||
@@ -1,62 +1,62 @@
|
||||
#!/bin/bash
|
||||
|
||||
function set_proxy_protocol() {
|
||||
if [ "${REAL_IP_MODE}" != "proxy_protocol" ]; then
|
||||
echo "REAL_IP_MODE is ${REAL_IP_MODE}, no need to update nginx configuration file."
|
||||
return 0
|
||||
fi
|
||||
sed -i 's/listen 3000;/listen 3000 proxy_protocol;/' /opt/frontend/http_backend.conf
|
||||
sed -i 's/listen 3000 ssl;/listen 3000 ssl proxy_protocol;/' /opt/frontend/https_backend.conf
|
||||
sed -i '/access_log.*main/a\ set_real_ip_from 0.0.0.0/0;' /opt/frontend/nginx.conf
|
||||
sed -i '/access_log.*main/a\ real_ip_header proxy_protocol;' /opt/frontend/nginx.conf
|
||||
echo "Nginx configuration file updated."
|
||||
}
|
||||
|
||||
mkdir -p /var/log/deepdata/frontend
|
||||
touch /var/log/deepdata/frontend/access.log /var/log/deepdata/frontend/error.log
|
||||
chown -R nginx:nginx /var/log/deepdata/frontend 2>/dev/null || true
|
||||
/usr/local/bin/rotate-nginx-logs.sh &
|
||||
|
||||
if [ -f "/cert/server.pem" ]; then
|
||||
cp /cert/server.pem /etc/nginx/cert/server.pem
|
||||
chown nginx:nginx /etc/nginx/cert/server.pem
|
||||
fi
|
||||
|
||||
if [ -f "/cert/server.key" ]; then
|
||||
# Private key MUST be encrypted. Plaintext keys are not accepted.
|
||||
if grep -q "ENCRYPTED" /cert/server.key 2>/dev/null; then
|
||||
if [ -z "$CERT_PASS" ]; then
|
||||
echo "ERROR: CERT_PASS is required to decrypt the private key"
|
||||
exit 1
|
||||
fi
|
||||
echo "$CERT_PASS" | openssl pkey -in /cert/server.key -out /etc/nginx/cert/server.key -passin stdin
|
||||
chmod 600 /etc/nginx/cert/server.key
|
||||
else
|
||||
echo "ERROR: /cert/server.key is not encrypted. Please encrypt it before deployment."
|
||||
exit 1
|
||||
fi
|
||||
chown nginx:nginx /etc/nginx/cert/server.key
|
||||
fi
|
||||
|
||||
set_proxy_protocol
|
||||
cp /opt/frontend/nginx.conf /etc/nginx/nginx.conf
|
||||
if [ -f "/etc/nginx/cert/server.pem" ]; then
|
||||
cp /opt/frontend/https_backend.conf /etc/nginx/conf.d/default.conf
|
||||
cp /opt/frontend/routes.inc /etc/nginx/conf.d/routes.inc
|
||||
echo "Switching to HTTPS config"
|
||||
else
|
||||
cp /opt/frontend/http_backend.conf /etc/nginx/conf.d/default.conf
|
||||
cp /opt/frontend/routes.inc /etc/nginx/conf.d/routes.inc
|
||||
|
||||
if [ -n "$DOMAIN" ]; then
|
||||
cron
|
||||
certbot --nginx "-d ${DOMAIN//,/ -d }"
|
||||
echo "Switching to HTTPS config, Domain: $DOMAIN"
|
||||
echo "0 0 1 * * root /usr/local/bin/certbot renew --quiet" | tee /etc/cron.d/certbot-renew
|
||||
chmod 0644 /etc/cron.d/certbot-renew
|
||||
else
|
||||
echo "Switching to HTTP config"
|
||||
fi
|
||||
fi
|
||||
|
||||
exec nginx -g "daemon off;"
|
||||
#!/bin/bash
|
||||
|
||||
function set_proxy_protocol() {
|
||||
if [ "${REAL_IP_MODE}" != "proxy_protocol" ]; then
|
||||
echo "REAL_IP_MODE is ${REAL_IP_MODE}, no need to update nginx configuration file."
|
||||
return 0
|
||||
fi
|
||||
sed -i 's/listen 3000;/listen 3000 proxy_protocol;/' /opt/frontend/http_backend.conf
|
||||
sed -i 's/listen 3000 ssl;/listen 3000 ssl proxy_protocol;/' /opt/frontend/https_backend.conf
|
||||
sed -i '/access_log.*main/a\ set_real_ip_from 0.0.0.0/0;' /opt/frontend/nginx.conf
|
||||
sed -i '/access_log.*main/a\ real_ip_header proxy_protocol;' /opt/frontend/nginx.conf
|
||||
echo "Nginx configuration file updated."
|
||||
}
|
||||
|
||||
mkdir -p /var/log/deepdata/frontend
|
||||
touch /var/log/deepdata/frontend/access.log /var/log/deepdata/frontend/error.log
|
||||
chown -R nginx:nginx /var/log/deepdata/frontend 2>/dev/null || true
|
||||
/usr/local/bin/rotate-nginx-logs.sh &
|
||||
|
||||
if [ -f "/cert/server.pem" ]; then
|
||||
cp /cert/server.pem /etc/nginx/cert/server.pem
|
||||
chown nginx:nginx /etc/nginx/cert/server.pem
|
||||
fi
|
||||
|
||||
if [ -f "/cert/server.key" ]; then
|
||||
# Private key MUST be encrypted. Plaintext keys are not accepted.
|
||||
if grep -q "ENCRYPTED" /cert/server.key 2>/dev/null; then
|
||||
if [ -z "$CERT_PASS" ]; then
|
||||
echo "ERROR: CERT_PASS is required to decrypt the private key"
|
||||
exit 1
|
||||
fi
|
||||
echo "$CERT_PASS" | openssl pkey -in /cert/server.key -out /etc/nginx/cert/server.key -passin stdin
|
||||
chmod 600 /etc/nginx/cert/server.key
|
||||
else
|
||||
echo "ERROR: /cert/server.key is not encrypted. Please encrypt it before deployment."
|
||||
exit 1
|
||||
fi
|
||||
chown nginx:nginx /etc/nginx/cert/server.key
|
||||
fi
|
||||
|
||||
set_proxy_protocol
|
||||
cp /opt/frontend/nginx.conf /etc/nginx/nginx.conf
|
||||
if [ -f "/etc/nginx/cert/server.pem" ]; then
|
||||
cp /opt/frontend/https_backend.conf /etc/nginx/conf.d/default.conf
|
||||
cp /opt/frontend/routes.inc /etc/nginx/conf.d/routes.inc
|
||||
echo "Switching to HTTPS config"
|
||||
else
|
||||
cp /opt/frontend/http_backend.conf /etc/nginx/conf.d/default.conf
|
||||
cp /opt/frontend/routes.inc /etc/nginx/conf.d/routes.inc
|
||||
|
||||
if [ -n "$DOMAIN" ]; then
|
||||
cron
|
||||
certbot --nginx "-d ${DOMAIN//,/ -d }"
|
||||
echo "Switching to HTTPS config, Domain: $DOMAIN"
|
||||
echo "0 0 1 * * root /usr/local/bin/certbot renew --quiet" | tee /etc/cron.d/certbot-renew
|
||||
chmod 0644 /etc/cron.d/certbot-renew
|
||||
else
|
||||
echo "Switching to HTTP config"
|
||||
fi
|
||||
fi
|
||||
|
||||
exec nginx -g "daemon off;"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
echo "Starting main application..."
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
echo "Starting main application..."
|
||||
exec "$@"
|
||||
@@ -1,35 +1,35 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# 1. 初始化环境 (原有的逻辑：注入用户自定义算子)
|
||||
if [ -d "/opt/runtime/user" ]; then
|
||||
cp -r /opt/runtime/user/* /opt/runtime/deepdata/ops/user 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# 2. 根据 RAY_MODE 环境变量决定启动模式
|
||||
if [ "${RAY_MODE}" = "head" ]; then
|
||||
echo "🚀 Starting Ray Head..."
|
||||
# ✅ 修复 Head 分支：
|
||||
# Ray 2.x 已废弃 CLI 参数 --redis-address，改为自动读取环境变量 RAY_REDIS_ADDRESS。
|
||||
# 只要确保 docker-compose.yml 中配置了 RAY_REDIS_ADDRESS 环境变量即可。
|
||||
exec ray start --head \
|
||||
--port=${RAY_HEAD_PORT:-6379} \
|
||||
--redis-password=${RAY_REDIS_PASSWORD} \
|
||||
--dashboard-host=0.0.0.0 \
|
||||
--block
|
||||
|
||||
elif [ "${RAY_MODE}" = "worker" ]; then
|
||||
echo "🚀 Starting Ray Worker connecting to ${RAY_HEAD_ADDRESS}..."
|
||||
# ✅ 修复 Worker 分支：
|
||||
# Ray 2.x 移除了 --worker 参数。只要不传 --head，默认即为 Worker 模式。
|
||||
exec ray start \
|
||||
--address=${RAY_HEAD_ADDRESS} \
|
||||
--redis-password=${RAY_REDIS_PASSWORD} \
|
||||
--num-cpus=${NUM_CPUS:-4} \
|
||||
--block
|
||||
|
||||
else
|
||||
echo "🔧 Starting Runtime (Legacy Mode / Job Execution)..."
|
||||
# 默认行为：执行 docker-compose.yml command 传入的指令，或 Ray Job 的入口点
|
||||
exec "$@"
|
||||
fi
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# 1. 初始化环境 (原有的逻辑：注入用户自定义算子)
|
||||
if [ -d "/opt/runtime/user" ]; then
|
||||
cp -r /opt/runtime/user/* /opt/runtime/deepdata/ops/user 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# 2. 根据 RAY_MODE 环境变量决定启动模式
|
||||
if [ "${RAY_MODE}" = "head" ]; then
|
||||
echo "🚀 Starting Ray Head..."
|
||||
# ✅ 修复 Head 分支：
|
||||
# Ray 2.x 已废弃 CLI 参数 --redis-address，改为自动读取环境变量 RAY_REDIS_ADDRESS。
|
||||
# 只要确保 docker-compose.yml 中配置了 RAY_REDIS_ADDRESS 环境变量即可。
|
||||
exec ray start --head \
|
||||
--port=${RAY_HEAD_PORT:-6379} \
|
||||
--redis-password=${RAY_REDIS_PASSWORD} \
|
||||
--dashboard-host=0.0.0.0 \
|
||||
--block
|
||||
|
||||
elif [ "${RAY_MODE}" = "worker" ]; then
|
||||
echo "🚀 Starting Ray Worker connecting to ${RAY_HEAD_ADDRESS}..."
|
||||
# ✅ 修复 Worker 分支：
|
||||
# Ray 2.x 移除了 --worker 参数。只要不传 --head，默认即为 Worker 模式。
|
||||
exec ray start \
|
||||
--address=${RAY_HEAD_ADDRESS} \
|
||||
--redis-password=${RAY_REDIS_PASSWORD} \
|
||||
--num-cpus=${NUM_CPUS:-4} \
|
||||
--block
|
||||
|
||||
else
|
||||
echo "🔧 Starting Runtime (Legacy Mode / Job Execution)..."
|
||||
# 默认行为：执行 docker-compose.yml command 传入的指令，或 Ray Job 的入口点
|
||||
exec "$@"
|
||||
fi
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
cp -r /opt/runtime/user/* /opt/runtime/deepdata/ops/user
|
||||
|
||||
if [ "${LOG_ROTATION_ENABLED:-true}" = "true" ]; then
|
||||
LOG_ROTATION_PATH="${LOG_ROTATION_PATH:-/tmp/ray}" /usr/local/bin/log-rotate-copytruncate.sh "${LOG_ROTATION_PATH:-/tmp/ray}" &
|
||||
fi
|
||||
|
||||
echo "Starting main application..."
|
||||
exec "$@"
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
cp -r /opt/runtime/user/* /opt/runtime/deepdata/ops/user
|
||||
|
||||
if [ "${LOG_ROTATION_ENABLED:-true}" = "true" ]; then
|
||||
LOG_ROTATION_PATH="${LOG_ROTATION_PATH:-/tmp/ray}" /usr/local/bin/log-rotate-copytruncate.sh "${LOG_ROTATION_PATH:-/tmp/ray}" &
|
||||
fi
|
||||
|
||||
echo "Starting main application..."
|
||||
exec "$@"
|
||||
|
||||
+248
-248
@@ -1,248 +1,248 @@
|
||||
#!/bin/bash
|
||||
### Collect secrets for DeepData installation.
|
||||
###
|
||||
### Usage:
|
||||
### scripts/k8s/collect-secrets.sh [--component deepdata|milvus] [--namespace <ns>]
|
||||
###
|
||||
### Modes (auto-detected):
|
||||
### 1. Sealed-secrets mode: controller running + kubeseal available → encrypt → apply SealedSecret
|
||||
### 2. Plain mode: no controller → Helm creates Secret with base64 (dev mode)
|
||||
###
|
||||
### Password priority: .env file values > interactive prompt > auto-generated
|
||||
###
|
||||
### Output: writes shell script variables to stdout (eval by caller)
|
||||
|
||||
# NOTE: NOT using 'set -e' because interactive prompts (read) can fail
|
||||
# when stdin is not a TTY. We validate critical values explicitly instead.
|
||||
|
||||
COMPONENT="${1:-deepdata}"
|
||||
NAMESPACE="${NAMESPACE:-deepdata}"
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
|
||||
# Parse --component and --namespace from args if passed differently
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
case $1 in
|
||||
--component) COMPONENT="$2"; shift 2 ;;
|
||||
--namespace|-n) NAMESPACE="$2"; shift 2 ;;
|
||||
*) shift ;;
|
||||
esac
|
||||
done
|
||||
|
||||
TMP_DIR=$(mktemp -d)
|
||||
trap "rm -rf $TMP_DIR" EXIT
|
||||
|
||||
# ========== Detect Mode ==========
|
||||
detect_sealed_secrets() {
|
||||
kubectl get deployment -n "$NAMESPACE" sealed-secrets >/dev/null 2>&1 && return 0
|
||||
kubectl get deployment -n kube-system sealed-secrets >/dev/null 2>&1 && return 0
|
||||
kubectl get pod -n "$NAMESPACE" -l app.kubernetes.io/name=sealed-secrets --no-headers 2>/dev/null | grep -q Running && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
KUBESEAL="$(command -v kubeseal 2>/dev/null || echo "")"
|
||||
[ -z "$KUBESEAL" ] && KUBESEAL="$HOME/bin/kubeseal"
|
||||
[ ! -x "$KUBESEAL" ] && [ -x "./tools/bin/kubeseal" ] && KUBESEAL="./tools/bin/kubeseal"
|
||||
[ ! -x "$KUBESEAL" ] && [ -x "../tools/bin/kubeseal" ] && KUBESEAL="../tools/bin/kubeseal"
|
||||
|
||||
# Auto-download kubeseal if controller exists but binary is missing
|
||||
if detect_sealed_secrets && [ ! -x "$KUBESEAL" ]; then
|
||||
echo "[INFO] Sealed Secrets controller detected, but kubeseal not found. Downloading..." >&2
|
||||
KUBESEAL_URL="https://github.com/bitnami-labs/sealed-secrets/releases/download/v0.27.2/kubeseal-0.27.2-$(uname -s | tr '[:upper:]' '[:lower:]')-$(dpkg --print-architecture 2>/dev/null || uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')"
|
||||
KUBESEAL_DIR="$HOME/bin"
|
||||
mkdir -p "$KUBESEAL_DIR"
|
||||
curl -sSL "$KUBESEAL_URL" -o "$KUBESEAL_DIR/kubeseal" && chmod +x "$KUBESEAL_DIR/kubeseal"
|
||||
KUBESEAL="$KUBESEAL_DIR/kubeseal"
|
||||
echo "[INFO] Installed kubeseal to $KUBESEAL" >&2
|
||||
fi
|
||||
|
||||
if detect_sealed_secrets && [ -x "$KUBESEAL" ]; then
|
||||
MODE="sealed"
|
||||
else
|
||||
MODE="plain"
|
||||
fi
|
||||
|
||||
echo "# DeepData secrets collection — component: $COMPONENT, mode: $MODE" >&2
|
||||
|
||||
# ========== Helper ==========
|
||||
random_hex() {
|
||||
openssl rand -hex 16 2>/dev/null || python3 -c "import secrets;print(secrets.token_hex(16))"
|
||||
}
|
||||
|
||||
seal_secret() {
|
||||
local name="$1" output="$2"
|
||||
shift 2
|
||||
local raw="${TMP_DIR}/${name}-raw.yaml"
|
||||
{
|
||||
echo "apiVersion: v1"
|
||||
echo "kind: Secret"
|
||||
echo "metadata:"
|
||||
echo " name: ${name}"
|
||||
echo " namespace: ${NAMESPACE}"
|
||||
echo "type: Opaque"
|
||||
echo "stringData:"
|
||||
for pair in "$@"; do
|
||||
key="${pair%%=*}"
|
||||
value="${pair#*=}"
|
||||
echo " ${key}: \"${value}\""
|
||||
done
|
||||
} > "$raw"
|
||||
"$KUBESEAL" --controller-name=sealed-secrets --namespace="${NAMESPACE}" -o yaml -f "$raw" > "$output"
|
||||
echo "[INFO] Created SealedSecret: $output" >&2
|
||||
}
|
||||
|
||||
# ========== Load .env (targeted: only reads known secret keys) ==========
|
||||
if [ -f "$ENV_FILE" ]; then
|
||||
echo "[INFO] Loading secrets from ${ENV_FILE}..." >&2
|
||||
|
||||
# Read .env line by line, only extract vars matching our known keys.
|
||||
# Ignores comments, blank lines, and unrelated vars like pgsql_host/shell vars.
|
||||
while IFS='=' read -r key value; do
|
||||
# Skip blank lines and comments
|
||||
[ -z "$key" ] && continue
|
||||
case "$key" in
|
||||
\#*) continue ;;
|
||||
esac
|
||||
# Strip leading/trailing whitespace from key
|
||||
key="$(echo "$key" | xargs)"
|
||||
|
||||
case "$key" in
|
||||
DB_PASSWORD|CERT_PASS|DOMAIN|HOME_PAGE_URL|JWT_SECRET|\
|
||||
LABEL_STUDIO_PASSWORD|LABEL_STUDIO_USER_TOKEN|POSTGRE_PASSWORD|\
|
||||
MINIO_ACCESS_KEY|MINIO_SECRET_KEY)
|
||||
val="$value"
|
||||
case "$val" in
|
||||
\'*|\"*) val="${val:1}" ;;
|
||||
esac
|
||||
case "$val" in
|
||||
*\'|*\") val="${val:0:-1}" ;;
|
||||
esac
|
||||
printf -v "$key" '%s' "$val"
|
||||
;;
|
||||
esac
|
||||
done < "$ENV_FILE"
|
||||
|
||||
[ -n "$DB_PASSWORD" ] && echo "[INFO] Loaded DB_PASSWORD from $ENV_FILE" >&2
|
||||
[ -n "$CERT_PASS" ] && echo "[INFO] Loaded CERT_PASS from $ENV_FILE" >&2
|
||||
[ -n "$JWT_SECRET" ] && echo "[INFO] Loaded JWT_SECRET from $ENV_FILE" >&2
|
||||
else
|
||||
echo "[WARN] No .env file found at $(pwd)/${ENV_FILE} — will prompt for secrets" >&2
|
||||
fi
|
||||
|
||||
# ========== DeepData ==========
|
||||
if [ "$COMPONENT" = "deepdata" ]; then
|
||||
# Collect
|
||||
[ -z "$DB_PASSWORD" ] && read -rsp "Enter DB_PASSWORD: " DB_PASSWORD && echo "" >&2
|
||||
[ -z "$JWT_SECRET" ] && JWT_SECRET=$(openssl rand -hex 32 2>/dev/null || python3 -c "import secrets;print(secrets.token_hex(32))") && echo "[INFO] Auto-generated JWT_SECRET" >&2
|
||||
[ -z "$DOMAIN" ] && read -rp "Enter DOMAIN (enter to skip): " DOMAIN >&2
|
||||
[ -z "$CERT_PASS" ] && read -rsp "Enter CERT_PASS (enter to skip): " CERT_PASS && echo "" >&2
|
||||
HOME_PAGE_URL="${HOME_PAGE_URL:-/data/management}"
|
||||
[ -z "$LABEL_STUDIO_PASSWORD" ] && read -rsp "Enter LABEL_STUDIO_PASSWORD (enter to skip): " LABEL_STUDIO_PASSWORD && echo "" >&2
|
||||
[ -z "$LABEL_STUDIO_USER_TOKEN" ] && LABEL_STUDIO_USER_TOKEN=$(random_hex)$(random_hex) && echo "[INFO] Auto-generated LABEL_STUDIO_USER_TOKEN" >&2
|
||||
|
||||
# ===== MANDATORY CHECK: DB_PASSWORD must not be empty =====
|
||||
if [ -z "$DB_PASSWORD" ]; then
|
||||
echo "[FATAL] DB_PASSWORD is empty! Cannot install without a database password." >&2
|
||||
echo "[FATAL] Set DB_PASSWORD in .env or ensure interactive prompts work." >&2
|
||||
echo "SECRETS_CREATE=SKIP"
|
||||
echo "HELM_VALUES_FILE="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$MODE" = "sealed" ]; then
|
||||
# Clean up any old stale Secret (prevents Helm conflict)
|
||||
if kubectl get secret deepdata-conf -n "$NAMESPACE" >/dev/null 2>&1; then
|
||||
echo "[INFO] Removing old deepdata-conf Secret..." >&2
|
||||
kubectl delete secret deepdata-conf -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1
|
||||
fi
|
||||
# Clean up old SealedSecret (apply is idempotent, but belt-and-suspenders)
|
||||
kubectl delete sealedsecret deepdata-conf -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true
|
||||
|
||||
# DeepData-conf
|
||||
seal_secret "deepdata-conf" "${TMP_DIR}/deepdata-sealed.yaml" \
|
||||
"DB_PASSWORD=${DB_PASSWORD}" \
|
||||
"CERT_PASS=${CERT_PASS}" \
|
||||
"DOMAIN=${DOMAIN}" \
|
||||
"HOME_PAGE_URL=${HOME_PAGE_URL}" \
|
||||
"JWT_SECRET=${JWT_SECRET}" \
|
||||
"LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD}" \
|
||||
"LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN}"
|
||||
kubectl apply -f "${TMP_DIR}/deepdata-sealed.yaml" -n "$NAMESPACE" >/dev/null
|
||||
# Add Helm ownership labels so 'helm install --force' accepts the Secret
|
||||
kubectl wait --for=jsonpath='{.data.DB_PASSWORD}' --timeout=30s secret/deepdata-conf -n "$NAMESPACE" >/dev/null 2>&1 || true
|
||||
kubectl annotate secret deepdata-conf -n "$NAMESPACE" \
|
||||
meta.helm.sh/release-name=deepdata \
|
||||
meta.helm.sh/release-namespace="${NAMESPACE}" \
|
||||
--overwrite >/dev/null 2>&1
|
||||
kubectl label secret deepdata-conf -n "$NAMESPACE" \
|
||||
app.kubernetes.io/managed-by=Helm \
|
||||
--overwrite >/dev/null 2>&1
|
||||
|
||||
# Label Studio
|
||||
if [ -n "$LABEL_STUDIO_PASSWORD" ]; then
|
||||
POSTGRE_PASSWORD="${POSTGRE_PASSWORD:-$DB_PASSWORD}"
|
||||
seal_secret "label-studio-env" "${TMP_DIR}/label-studio-sealed.yaml" \
|
||||
"POSTGRE_PASSWORD=${POSTGRE_PASSWORD}" \
|
||||
"LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD}" \
|
||||
"LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN}"
|
||||
kubectl apply -f "${TMP_DIR}/label-studio-sealed.yaml" -n "$NAMESPACE" >/dev/null
|
||||
fi
|
||||
|
||||
echo "SECRETS_CREATE=false"
|
||||
echo "HELM_VALUES_FILE="
|
||||
else
|
||||
# Plain — delete any old SealedSecret and stale Secret first
|
||||
kubectl delete sealedsecret deepdata-conf -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true
|
||||
if kubectl get secret deepdata-conf -n "$NAMESPACE" >/dev/null 2>&1; then
|
||||
echo "[INFO] Removing old deepdata-conf Secret..." >&2
|
||||
kubectl delete secret deepdata-conf -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
# Write extra values to /tmp (survives script exit for Helm to read)
|
||||
values_file="/tmp/deepdata-secret-values-$$.yaml"
|
||||
cat > "$values_file" <<VALUES_EOF
|
||||
public:
|
||||
secrets:
|
||||
data:
|
||||
DB_PASSWORD: "${DB_PASSWORD}"
|
||||
JWT_SECRET: "${JWT_SECRET}"
|
||||
CERT_PASS: "${CERT_PASS}"
|
||||
DOMAIN: "${DOMAIN}"
|
||||
HOME_PAGE_URL: "${HOME_PAGE_URL}"
|
||||
LABEL_STUDIO_PASSWORD: "${LABEL_STUDIO_PASSWORD}"
|
||||
LABEL_STUDIO_USER_TOKEN: "${LABEL_STUDIO_USER_TOKEN}"
|
||||
VALUES_EOF
|
||||
|
||||
echo "[INFO] Wrote plain secret values to ${values_file}" >&2
|
||||
echo "SECRETS_CREATE=true"
|
||||
echo "HELM_VALUES_FILE=${values_file}"
|
||||
fi
|
||||
|
||||
# ========== Milvus ==========
|
||||
elif [ "$COMPONENT" = "milvus" ]; then
|
||||
# Only needed if secret doesn't already exist
|
||||
if kubectl get secret milvus-minio-secret -n "$NAMESPACE" >/dev/null 2>&1; then
|
||||
echo "# milvus-minio-secret already exists — skipping" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[ -z "$MINIO_ACCESS_KEY" ] && MINIO_ACCESS_KEY=$(random_hex) && echo "[INFO] Auto-generated MINIO_ACCESS_KEY" >&2
|
||||
[ -z "$MINIO_SECRET_KEY" ] && MINIO_SECRET_KEY=$(random_hex)$(random_hex) && echo "[INFO] Auto-generated MINIO_SECRET_KEY" >&2
|
||||
|
||||
if [ "$MODE" = "sealed" ]; then
|
||||
seal_secret "milvus-minio-secret" "${TMP_DIR}/milvus-sealed.yaml" \
|
||||
"accesskey=${MINIO_ACCESS_KEY}" \
|
||||
"secretkey=${MINIO_SECRET_KEY}"
|
||||
kubectl apply -f "${TMP_DIR}/milvus-sealed.yaml" -n "$NAMESPACE" >/dev/null
|
||||
kubectl wait --for=jsonpath='{.data.accesskey}' --timeout=30s secret/milvus-minio-secret -n "$NAMESPACE" >/dev/null 2>&1 || true
|
||||
kubectl annotate secret milvus-minio-secret -n "$NAMESPACE" \
|
||||
meta.helm.sh/release-name=milvus-minio \
|
||||
meta.helm.sh/release-namespace="${NAMESPACE}" \
|
||||
--overwrite >/dev/null 2>&1
|
||||
else
|
||||
kubectl create secret generic milvus-minio-secret \
|
||||
--from-literal=accesskey="$MINIO_ACCESS_KEY" \
|
||||
--from-literal=secretkey="$MINIO_SECRET_KEY" \
|
||||
-n "$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
|
||||
echo "[INFO] Created milvus-minio-secret (plain)" >&2
|
||||
fi
|
||||
fi
|
||||
#!/bin/bash
|
||||
### Collect secrets for DeepData installation.
|
||||
###
|
||||
### Usage:
|
||||
### scripts/k8s/collect-secrets.sh [--component deepdata|milvus] [--namespace <ns>]
|
||||
###
|
||||
### Modes (auto-detected):
|
||||
### 1. Sealed-secrets mode: controller running + kubeseal available → encrypt → apply SealedSecret
|
||||
### 2. Plain mode: no controller → Helm creates Secret with base64 (dev mode)
|
||||
###
|
||||
### Password priority: .env file values > interactive prompt > auto-generated
|
||||
###
|
||||
### Output: writes shell script variables to stdout (eval by caller)
|
||||
|
||||
# NOTE: NOT using 'set -e' because interactive prompts (read) can fail
|
||||
# when stdin is not a TTY. We validate critical values explicitly instead.
|
||||
|
||||
COMPONENT="${1:-deepdata}"
|
||||
NAMESPACE="${NAMESPACE:-deepdata}"
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
|
||||
# Parse --component and --namespace from args if passed differently
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
case $1 in
|
||||
--component) COMPONENT="$2"; shift 2 ;;
|
||||
--namespace|-n) NAMESPACE="$2"; shift 2 ;;
|
||||
*) shift ;;
|
||||
esac
|
||||
done
|
||||
|
||||
TMP_DIR=$(mktemp -d)
|
||||
trap "rm -rf $TMP_DIR" EXIT
|
||||
|
||||
# ========== Detect Mode ==========
|
||||
detect_sealed_secrets() {
|
||||
kubectl get deployment -n "$NAMESPACE" sealed-secrets >/dev/null 2>&1 && return 0
|
||||
kubectl get deployment -n kube-system sealed-secrets >/dev/null 2>&1 && return 0
|
||||
kubectl get pod -n "$NAMESPACE" -l app.kubernetes.io/name=sealed-secrets --no-headers 2>/dev/null | grep -q Running && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
KUBESEAL="$(command -v kubeseal 2>/dev/null || echo "")"
|
||||
[ -z "$KUBESEAL" ] && KUBESEAL="$HOME/bin/kubeseal"
|
||||
[ ! -x "$KUBESEAL" ] && [ -x "./tools/bin/kubeseal" ] && KUBESEAL="./tools/bin/kubeseal"
|
||||
[ ! -x "$KUBESEAL" ] && [ -x "../tools/bin/kubeseal" ] && KUBESEAL="../tools/bin/kubeseal"
|
||||
|
||||
# Auto-download kubeseal if controller exists but binary is missing
|
||||
if detect_sealed_secrets && [ ! -x "$KUBESEAL" ]; then
|
||||
echo "[INFO] Sealed Secrets controller detected, but kubeseal not found. Downloading..." >&2
|
||||
KUBESEAL_URL="https://github.com/bitnami-labs/sealed-secrets/releases/download/v0.27.2/kubeseal-0.27.2-$(uname -s | tr '[:upper:]' '[:lower:]')-$(dpkg --print-architecture 2>/dev/null || uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')"
|
||||
KUBESEAL_DIR="$HOME/bin"
|
||||
mkdir -p "$KUBESEAL_DIR"
|
||||
curl -sSL "$KUBESEAL_URL" -o "$KUBESEAL_DIR/kubeseal" && chmod +x "$KUBESEAL_DIR/kubeseal"
|
||||
KUBESEAL="$KUBESEAL_DIR/kubeseal"
|
||||
echo "[INFO] Installed kubeseal to $KUBESEAL" >&2
|
||||
fi
|
||||
|
||||
if detect_sealed_secrets && [ -x "$KUBESEAL" ]; then
|
||||
MODE="sealed"
|
||||
else
|
||||
MODE="plain"
|
||||
fi
|
||||
|
||||
echo "# DeepData secrets collection — component: $COMPONENT, mode: $MODE" >&2
|
||||
|
||||
# ========== Helper ==========
|
||||
random_hex() {
|
||||
openssl rand -hex 16 2>/dev/null || python3 -c "import secrets;print(secrets.token_hex(16))"
|
||||
}
|
||||
|
||||
seal_secret() {
|
||||
local name="$1" output="$2"
|
||||
shift 2
|
||||
local raw="${TMP_DIR}/${name}-raw.yaml"
|
||||
{
|
||||
echo "apiVersion: v1"
|
||||
echo "kind: Secret"
|
||||
echo "metadata:"
|
||||
echo " name: ${name}"
|
||||
echo " namespace: ${NAMESPACE}"
|
||||
echo "type: Opaque"
|
||||
echo "stringData:"
|
||||
for pair in "$@"; do
|
||||
key="${pair%%=*}"
|
||||
value="${pair#*=}"
|
||||
echo " ${key}: \"${value}\""
|
||||
done
|
||||
} > "$raw"
|
||||
"$KUBESEAL" --controller-name=sealed-secrets --namespace="${NAMESPACE}" -o yaml -f "$raw" > "$output"
|
||||
echo "[INFO] Created SealedSecret: $output" >&2
|
||||
}
|
||||
|
||||
# ========== Load .env (targeted: only reads known secret keys) ==========
|
||||
if [ -f "$ENV_FILE" ]; then
|
||||
echo "[INFO] Loading secrets from ${ENV_FILE}..." >&2
|
||||
|
||||
# Read .env line by line, only extract vars matching our known keys.
|
||||
# Ignores comments, blank lines, and unrelated vars like pgsql_host/shell vars.
|
||||
while IFS='=' read -r key value; do
|
||||
# Skip blank lines and comments
|
||||
[ -z "$key" ] && continue
|
||||
case "$key" in
|
||||
\#*) continue ;;
|
||||
esac
|
||||
# Strip leading/trailing whitespace from key
|
||||
key="$(echo "$key" | xargs)"
|
||||
|
||||
case "$key" in
|
||||
DB_PASSWORD|CERT_PASS|DOMAIN|HOME_PAGE_URL|JWT_SECRET|\
|
||||
LABEL_STUDIO_PASSWORD|LABEL_STUDIO_USER_TOKEN|POSTGRE_PASSWORD|\
|
||||
MINIO_ACCESS_KEY|MINIO_SECRET_KEY)
|
||||
val="$value"
|
||||
case "$val" in
|
||||
\'*|\"*) val="${val:1}" ;;
|
||||
esac
|
||||
case "$val" in
|
||||
*\'|*\") val="${val:0:-1}" ;;
|
||||
esac
|
||||
printf -v "$key" '%s' "$val"
|
||||
;;
|
||||
esac
|
||||
done < "$ENV_FILE"
|
||||
|
||||
[ -n "$DB_PASSWORD" ] && echo "[INFO] Loaded DB_PASSWORD from $ENV_FILE" >&2
|
||||
[ -n "$CERT_PASS" ] && echo "[INFO] Loaded CERT_PASS from $ENV_FILE" >&2
|
||||
[ -n "$JWT_SECRET" ] && echo "[INFO] Loaded JWT_SECRET from $ENV_FILE" >&2
|
||||
else
|
||||
echo "[WARN] No .env file found at $(pwd)/${ENV_FILE} — will prompt for secrets" >&2
|
||||
fi
|
||||
|
||||
# ========== DeepData ==========
|
||||
if [ "$COMPONENT" = "deepdata" ]; then
|
||||
# Collect
|
||||
[ -z "$DB_PASSWORD" ] && read -rsp "Enter DB_PASSWORD: " DB_PASSWORD && echo "" >&2
|
||||
[ -z "$JWT_SECRET" ] && JWT_SECRET=$(openssl rand -hex 32 2>/dev/null || python3 -c "import secrets;print(secrets.token_hex(32))") && echo "[INFO] Auto-generated JWT_SECRET" >&2
|
||||
[ -z "$DOMAIN" ] && read -rp "Enter DOMAIN (enter to skip): " DOMAIN >&2
|
||||
[ -z "$CERT_PASS" ] && read -rsp "Enter CERT_PASS (enter to skip): " CERT_PASS && echo "" >&2
|
||||
HOME_PAGE_URL="${HOME_PAGE_URL:-/data/management}"
|
||||
[ -z "$LABEL_STUDIO_PASSWORD" ] && read -rsp "Enter LABEL_STUDIO_PASSWORD (enter to skip): " LABEL_STUDIO_PASSWORD && echo "" >&2
|
||||
[ -z "$LABEL_STUDIO_USER_TOKEN" ] && LABEL_STUDIO_USER_TOKEN=$(random_hex)$(random_hex) && echo "[INFO] Auto-generated LABEL_STUDIO_USER_TOKEN" >&2
|
||||
|
||||
# ===== MANDATORY CHECK: DB_PASSWORD must not be empty =====
|
||||
if [ -z "$DB_PASSWORD" ]; then
|
||||
echo "[FATAL] DB_PASSWORD is empty! Cannot install without a database password." >&2
|
||||
echo "[FATAL] Set DB_PASSWORD in .env or ensure interactive prompts work." >&2
|
||||
echo "SECRETS_CREATE=SKIP"
|
||||
echo "HELM_VALUES_FILE="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$MODE" = "sealed" ]; then
|
||||
# Clean up any old stale Secret (prevents Helm conflict)
|
||||
if kubectl get secret deepdata-conf -n "$NAMESPACE" >/dev/null 2>&1; then
|
||||
echo "[INFO] Removing old deepdata-conf Secret..." >&2
|
||||
kubectl delete secret deepdata-conf -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1
|
||||
fi
|
||||
# Clean up old SealedSecret (apply is idempotent, but belt-and-suspenders)
|
||||
kubectl delete sealedsecret deepdata-conf -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true
|
||||
|
||||
# DeepData-conf
|
||||
seal_secret "deepdata-conf" "${TMP_DIR}/deepdata-sealed.yaml" \
|
||||
"DB_PASSWORD=${DB_PASSWORD}" \
|
||||
"CERT_PASS=${CERT_PASS}" \
|
||||
"DOMAIN=${DOMAIN}" \
|
||||
"HOME_PAGE_URL=${HOME_PAGE_URL}" \
|
||||
"JWT_SECRET=${JWT_SECRET}" \
|
||||
"LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD}" \
|
||||
"LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN}"
|
||||
kubectl apply -f "${TMP_DIR}/deepdata-sealed.yaml" -n "$NAMESPACE" >/dev/null
|
||||
# Add Helm ownership labels so 'helm install --force' accepts the Secret
|
||||
kubectl wait --for=jsonpath='{.data.DB_PASSWORD}' --timeout=30s secret/deepdata-conf -n "$NAMESPACE" >/dev/null 2>&1 || true
|
||||
kubectl annotate secret deepdata-conf -n "$NAMESPACE" \
|
||||
meta.helm.sh/release-name=deepdata \
|
||||
meta.helm.sh/release-namespace="${NAMESPACE}" \
|
||||
--overwrite >/dev/null 2>&1
|
||||
kubectl label secret deepdata-conf -n "$NAMESPACE" \
|
||||
app.kubernetes.io/managed-by=Helm \
|
||||
--overwrite >/dev/null 2>&1
|
||||
|
||||
# Label Studio
|
||||
if [ -n "$LABEL_STUDIO_PASSWORD" ]; then
|
||||
POSTGRE_PASSWORD="${POSTGRE_PASSWORD:-$DB_PASSWORD}"
|
||||
seal_secret "label-studio-env" "${TMP_DIR}/label-studio-sealed.yaml" \
|
||||
"POSTGRE_PASSWORD=${POSTGRE_PASSWORD}" \
|
||||
"LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD}" \
|
||||
"LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN}"
|
||||
kubectl apply -f "${TMP_DIR}/label-studio-sealed.yaml" -n "$NAMESPACE" >/dev/null
|
||||
fi
|
||||
|
||||
echo "SECRETS_CREATE=false"
|
||||
echo "HELM_VALUES_FILE="
|
||||
else
|
||||
# Plain — delete any old SealedSecret and stale Secret first
|
||||
kubectl delete sealedsecret deepdata-conf -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1 || true
|
||||
if kubectl get secret deepdata-conf -n "$NAMESPACE" >/dev/null 2>&1; then
|
||||
echo "[INFO] Removing old deepdata-conf Secret..." >&2
|
||||
kubectl delete secret deepdata-conf -n "$NAMESPACE" --ignore-not-found >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
# Write extra values to /tmp (survives script exit for Helm to read)
|
||||
values_file="/tmp/deepdata-secret-values-$$.yaml"
|
||||
cat > "$values_file" <<VALUES_EOF
|
||||
public:
|
||||
secrets:
|
||||
data:
|
||||
DB_PASSWORD: "${DB_PASSWORD}"
|
||||
JWT_SECRET: "${JWT_SECRET}"
|
||||
CERT_PASS: "${CERT_PASS}"
|
||||
DOMAIN: "${DOMAIN}"
|
||||
HOME_PAGE_URL: "${HOME_PAGE_URL}"
|
||||
LABEL_STUDIO_PASSWORD: "${LABEL_STUDIO_PASSWORD}"
|
||||
LABEL_STUDIO_USER_TOKEN: "${LABEL_STUDIO_USER_TOKEN}"
|
||||
VALUES_EOF
|
||||
|
||||
echo "[INFO] Wrote plain secret values to ${values_file}" >&2
|
||||
echo "SECRETS_CREATE=true"
|
||||
echo "HELM_VALUES_FILE=${values_file}"
|
||||
fi
|
||||
|
||||
# ========== Milvus ==========
|
||||
elif [ "$COMPONENT" = "milvus" ]; then
|
||||
# Only needed if secret doesn't already exist
|
||||
if kubectl get secret milvus-minio-secret -n "$NAMESPACE" >/dev/null 2>&1; then
|
||||
echo "# milvus-minio-secret already exists — skipping" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[ -z "$MINIO_ACCESS_KEY" ] && MINIO_ACCESS_KEY=$(random_hex) && echo "[INFO] Auto-generated MINIO_ACCESS_KEY" >&2
|
||||
[ -z "$MINIO_SECRET_KEY" ] && MINIO_SECRET_KEY=$(random_hex)$(random_hex) && echo "[INFO] Auto-generated MINIO_SECRET_KEY" >&2
|
||||
|
||||
if [ "$MODE" = "sealed" ]; then
|
||||
seal_secret "milvus-minio-secret" "${TMP_DIR}/milvus-sealed.yaml" \
|
||||
"accesskey=${MINIO_ACCESS_KEY}" \
|
||||
"secretkey=${MINIO_SECRET_KEY}"
|
||||
kubectl apply -f "${TMP_DIR}/milvus-sealed.yaml" -n "$NAMESPACE" >/dev/null
|
||||
kubectl wait --for=jsonpath='{.data.accesskey}' --timeout=30s secret/milvus-minio-secret -n "$NAMESPACE" >/dev/null 2>&1 || true
|
||||
kubectl annotate secret milvus-minio-secret -n "$NAMESPACE" \
|
||||
meta.helm.sh/release-name=milvus-minio \
|
||||
meta.helm.sh/release-namespace="${NAMESPACE}" \
|
||||
--overwrite >/dev/null 2>&1
|
||||
else
|
||||
kubectl create secret generic milvus-minio-secret \
|
||||
--from-literal=accesskey="$MINIO_ACCESS_KEY" \
|
||||
--from-literal=secretkey="$MINIO_SECRET_KEY" \
|
||||
-n "$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
|
||||
echo "[INFO] Created milvus-minio-secret (plain)" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
+167
-167
@@ -1,167 +1,167 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# DeepData Node Cleanup Script
|
||||
# Remove labels and taints from nodes that were configured for DeepData deployment
|
||||
#
|
||||
# Usage: ./node-cleanup.sh [--dry-run] [--nodes NODE1,NODE2]
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Default values
|
||||
DRY_RUN=false
|
||||
NAMESPACE="deepdata"
|
||||
LABEL_KEY="node-role.kubernetes.io/deepdata"
|
||||
LABEL_VALUE="true"
|
||||
TAINT_EFFECT="NoSchedule"
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--namespace)
|
||||
NAMESPACE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--dry-run)
|
||||
DRY_RUN=true
|
||||
shift
|
||||
;;
|
||||
--nodes)
|
||||
PROVIDED_NODES="$2"
|
||||
shift 2
|
||||
;;
|
||||
--label-key)
|
||||
LABEL_KEY="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo -e "${BLUE}=====================================${NC}"
|
||||
echo -e "${BLUE} DeepData Node Cleanup${NC}"
|
||||
echo -e "${BLUE}=====================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if kubectl is available
|
||||
if ! command -v kubectl &> /dev/null; then
|
||||
echo -e "${RED}Error: kubectl is not installed${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if connected to cluster
|
||||
if ! kubectl cluster-info &> /dev/null; then
|
||||
echo -e "${RED}Error: Cannot connect to Kubernetes cluster${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Determine nodes to clean up
|
||||
if [ "$PROVIDED_NODES" != "" ]; then
|
||||
# Use provided nodes
|
||||
IFS=',' read -ra SELECTED_NODES <<< "$PROVIDED_NODES"
|
||||
else
|
||||
# Find nodes with the deepdata label directly from Kubernetes
|
||||
echo -e "${YELLOW}Finding nodes with $LABEL_KEY=$LABEL_VALUE label...${NC}"
|
||||
NODES=$(kubectl get nodes -l "$LABEL_KEY=$LABEL_VALUE" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||
|
||||
if [ -z "$NODES" ]; then
|
||||
echo -e "${GREEN}No nodes found with $LABEL_KEY=$LABEL_VALUE label.${NC}"
|
||||
echo -e "${YELLOW}Cleanup not needed - no nodes were labeled.${NC}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
SELECTED_NODES=()
|
||||
while IFS= read -r NODE; do
|
||||
if [ -n "$NODE" ]; then
|
||||
SELECTED_NODES+=("$NODE")
|
||||
fi
|
||||
done <<< "$NODES"
|
||||
fi
|
||||
|
||||
if [ ${#SELECTED_NODES[@]} -eq 0 ]; then
|
||||
echo -e "${YELLOW}No nodes to clean up.${NC}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}Nodes to clean up:${NC}"
|
||||
for NODE in "${SELECTED_NODES[@]}"; do
|
||||
echo " - $NODE"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo -e "${BLUE}Summary:${NC}"
|
||||
echo " Label to remove: $LABEL_KEY"
|
||||
|
||||
# Check if any node has the taint
|
||||
HAS_TAINTS=false
|
||||
for NODE in "${SELECTED_NODES[@]}"; do
|
||||
TAINT_COUNT=$(kubectl get node "$NODE" -o jsonpath='{range .spec.taints[*]}{.key}{"\n"}{end}' | grep -c "^${LABEL_KEY}$" || echo "0")
|
||||
if [ "$TAINT_COUNT" -gt 0 ]; then
|
||||
HAS_TAINTS=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$HAS_TAINTS" = true ]; then
|
||||
echo " Taint to remove: $LABEL_KEY=$LABEL_VALUE:$TAINT_EFFECT"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
read -p "Remove labels and taints? (y/n) [y]: " CONFIRM
|
||||
CONFIRM=${CONFIRM:-y}
|
||||
|
||||
if [ "$CONFIRM" != "y" ] && [ "$CONFIRM" != "Y" ]; then
|
||||
echo -e "${YELLOW}Cancelled.${NC}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}Removing configuration...${NC}"
|
||||
|
||||
# Remove labels from selected nodes
|
||||
for NODE in "${SELECTED_NODES[@]}"; do
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
echo "[DRY-RUN] kubectl label node $NODE $LABEL_KEY-"
|
||||
else
|
||||
kubectl label node "$NODE" "$LABEL_KEY-" --overwrite
|
||||
echo -e " ${GREEN}✓${NC} Removed label from $NODE"
|
||||
fi
|
||||
done
|
||||
|
||||
# Remove taints (check if node has the taint)
|
||||
for NODE in "${SELECTED_NODES[@]}"; do
|
||||
# Check if node has the taint
|
||||
HAS_TAINT=$(kubectl get node "$NODE" -o jsonpath='{range .spec.taints[*]}{.key}{"\n"}{end}' | grep -c "^${LABEL_KEY}$" || echo "0")
|
||||
|
||||
if [ "$HAS_TAINT" -gt 0 ]; then
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
echo "[DRY-RUN] kubectl taint node $NODE $LABEL_KEY=$LABEL_VALUE:$TAINT_EFFECT-"
|
||||
else
|
||||
kubectl taint node "$NODE" "$LABEL_KEY=$LABEL_VALUE:$TAINT_EFFECT-" --overwrite || true
|
||||
echo -e " ${GREEN}✓${NC} Removed taint from $NODE"
|
||||
fi
|
||||
else
|
||||
echo -e " ${YELLOW}○${NC} No taint to remove from $NODE"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}Cleanup complete!${NC}"
|
||||
|
||||
# Clean up Helm args temp file if it exists
|
||||
if [ -f /tmp/deepdata-helm-args.sh ]; then
|
||||
rm /tmp/deepdata-helm-args.sh
|
||||
echo -e "${GREEN}Removed /tmp/deepdata-helm-args.sh${NC}"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
#!/bin/bash
|
||||
#
|
||||
# DeepData Node Cleanup Script
|
||||
# Remove labels and taints from nodes that were configured for DeepData deployment
|
||||
#
|
||||
# Usage: ./node-cleanup.sh [--dry-run] [--nodes NODE1,NODE2]
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Default values
|
||||
DRY_RUN=false
|
||||
NAMESPACE="deepdata"
|
||||
LABEL_KEY="node-role.kubernetes.io/deepdata"
|
||||
LABEL_VALUE="true"
|
||||
TAINT_EFFECT="NoSchedule"
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--namespace)
|
||||
NAMESPACE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--dry-run)
|
||||
DRY_RUN=true
|
||||
shift
|
||||
;;
|
||||
--nodes)
|
||||
PROVIDED_NODES="$2"
|
||||
shift 2
|
||||
;;
|
||||
--label-key)
|
||||
LABEL_KEY="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo -e "${BLUE}=====================================${NC}"
|
||||
echo -e "${BLUE} DeepData Node Cleanup${NC}"
|
||||
echo -e "${BLUE}=====================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if kubectl is available
|
||||
if ! command -v kubectl &> /dev/null; then
|
||||
echo -e "${RED}Error: kubectl is not installed${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if connected to cluster
|
||||
if ! kubectl cluster-info &> /dev/null; then
|
||||
echo -e "${RED}Error: Cannot connect to Kubernetes cluster${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Determine nodes to clean up
|
||||
if [ "$PROVIDED_NODES" != "" ]; then
|
||||
# Use provided nodes
|
||||
IFS=',' read -ra SELECTED_NODES <<< "$PROVIDED_NODES"
|
||||
else
|
||||
# Find nodes with the deepdata label directly from Kubernetes
|
||||
echo -e "${YELLOW}Finding nodes with $LABEL_KEY=$LABEL_VALUE label...${NC}"
|
||||
NODES=$(kubectl get nodes -l "$LABEL_KEY=$LABEL_VALUE" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||
|
||||
if [ -z "$NODES" ]; then
|
||||
echo -e "${GREEN}No nodes found with $LABEL_KEY=$LABEL_VALUE label.${NC}"
|
||||
echo -e "${YELLOW}Cleanup not needed - no nodes were labeled.${NC}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
SELECTED_NODES=()
|
||||
while IFS= read -r NODE; do
|
||||
if [ -n "$NODE" ]; then
|
||||
SELECTED_NODES+=("$NODE")
|
||||
fi
|
||||
done <<< "$NODES"
|
||||
fi
|
||||
|
||||
if [ ${#SELECTED_NODES[@]} -eq 0 ]; then
|
||||
echo -e "${YELLOW}No nodes to clean up.${NC}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}Nodes to clean up:${NC}"
|
||||
for NODE in "${SELECTED_NODES[@]}"; do
|
||||
echo " - $NODE"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo -e "${BLUE}Summary:${NC}"
|
||||
echo " Label to remove: $LABEL_KEY"
|
||||
|
||||
# Check if any node has the taint
|
||||
HAS_TAINTS=false
|
||||
for NODE in "${SELECTED_NODES[@]}"; do
|
||||
TAINT_COUNT=$(kubectl get node "$NODE" -o jsonpath='{range .spec.taints[*]}{.key}{"\n"}{end}' | grep -c "^${LABEL_KEY}$" || echo "0")
|
||||
if [ "$TAINT_COUNT" -gt 0 ]; then
|
||||
HAS_TAINTS=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$HAS_TAINTS" = true ]; then
|
||||
echo " Taint to remove: $LABEL_KEY=$LABEL_VALUE:$TAINT_EFFECT"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
read -p "Remove labels and taints? (y/n) [y]: " CONFIRM
|
||||
CONFIRM=${CONFIRM:-y}
|
||||
|
||||
if [ "$CONFIRM" != "y" ] && [ "$CONFIRM" != "Y" ]; then
|
||||
echo -e "${YELLOW}Cancelled.${NC}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}Removing configuration...${NC}"
|
||||
|
||||
# Remove labels from selected nodes
|
||||
for NODE in "${SELECTED_NODES[@]}"; do
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
echo "[DRY-RUN] kubectl label node $NODE $LABEL_KEY-"
|
||||
else
|
||||
kubectl label node "$NODE" "$LABEL_KEY-" --overwrite
|
||||
echo -e " ${GREEN}✓${NC} Removed label from $NODE"
|
||||
fi
|
||||
done
|
||||
|
||||
# Remove taints (check if node has the taint)
|
||||
for NODE in "${SELECTED_NODES[@]}"; do
|
||||
# Check if node has the taint
|
||||
HAS_TAINT=$(kubectl get node "$NODE" -o jsonpath='{range .spec.taints[*]}{.key}{"\n"}{end}' | grep -c "^${LABEL_KEY}$" || echo "0")
|
||||
|
||||
if [ "$HAS_TAINT" -gt 0 ]; then
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
echo "[DRY-RUN] kubectl taint node $NODE $LABEL_KEY=$LABEL_VALUE:$TAINT_EFFECT-"
|
||||
else
|
||||
kubectl taint node "$NODE" "$LABEL_KEY=$LABEL_VALUE:$TAINT_EFFECT-" --overwrite || true
|
||||
echo -e " ${GREEN}✓${NC} Removed taint from $NODE"
|
||||
fi
|
||||
else
|
||||
echo -e " ${YELLOW}○${NC} No taint to remove from $NODE"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}Cleanup complete!${NC}"
|
||||
|
||||
# Clean up Helm args temp file if it exists
|
||||
if [ -f /tmp/deepdata-helm-args.sh ]; then
|
||||
rm /tmp/deepdata-helm-args.sh
|
||||
echo -e "${GREEN}Removed /tmp/deepdata-helm-args.sh${NC}"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
|
||||
+504
-504
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user