-
Notifications
You must be signed in to change notification settings - Fork 183
Expand file tree
/
Copy pathdev
More file actions
executable file
·376 lines (348 loc) · 10.3 KB
/
Copy pathdev
File metadata and controls
executable file
·376 lines (348 loc) · 10.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
#!/usr/bin/env bash
set -euo pipefail
TMUX_SESSION="backend-ai-session"
_service_cmd() {
case $1 in
mgr) echo "./backend.ai mgr start-server --debug" ;;
agent) echo "./backend.ai ag start-server --debug" ;;
sftp-agent) echo "./backend.ai ag start-server -f agent-sftp.toml --debug" ;;
storage) echo "./backend.ai storage start-server --debug" ;;
web) echo "./backend.ai web start-server --debug" ;;
proxy-coordinator) echo "./backend.ai app-proxy-coordinator start-server --debug" ;;
proxy-worker) echo "./backend.ai app-proxy-worker start-server --debug" ;;
*) return 1 ;;
esac
}
_service_process() {
case $1 in
mgr) echo "backend.ai: manager" ;;
agent) echo "backend.ai: agent" ;;
sftp-agent) echo "agent-sftp.toml" ;;
storage) echo "backend.ai: storage-proxy" ;;
web) echo "backend.ai: webserver" ;;
proxy-coordinator) echo "backend.ai: proxy-coordinator" ;;
proxy-worker) echo "backend.ai: proxy-worker" ;;
*) return 1 ;;
esac
}
_tmux_window_name() {
case $1 in
proxy-coordinator) echo "app-proxy-coordinator" ;;
proxy-worker) echo "app-proxy-worker" ;;
*) echo "$1" ;;
esac
}
# Services started by `./dev start all` and included in `./dev status`.
# Intentionally excludes sftp-agent since it is an optional dev addon that
# must be started explicitly via `./dev start sftp-agent`.
ALL_SERVICES="mgr agent storage web proxy-coordinator proxy-worker"
OPTIONAL_SERVICES="sftp-agent"
_color() {
local color=$1; shift
case $color in
green) printf "\033[32m%s\033[0m" "$*" ;;
red) printf "\033[31m%s\033[0m" "$*" ;;
yellow) printf "\033[33m%s\033[0m" "$*" ;;
bold) printf "\033[1m%s\033[0m" "$*" ;;
*) printf "%s" "$*" ;;
esac
}
# Path to the pid-file managed by the agent process itself.
# Since setproctitle overwrites the argv of agent workers to just
# "backend.ai: agent", there is no way to distinguish the primary compute
# agent from the optional SFTP agent using `pgrep -f` alone. Both agents
# write their own pid-file (configured in agent.toml / agent-sftp.toml),
# and we use those files as the source of truth for process tracking.
_service_pidfile() {
case $1 in
agent) echo "./agent.pid" ;;
sftp-agent) echo "./agent-sftp.pid" ;;
*) return 1 ;;
esac
}
# Check whether the pid recorded in $1 is still a live process.
_pid_alive() {
local pid=$1
[ -n "$pid" ] && kill -0 "$pid" 2>/dev/null
}
_is_running() {
local svc=$1
case "$svc" in
agent|sftp-agent)
local pidfile
pidfile=$(_service_pidfile "$svc")
if [ -f "$pidfile" ]; then
local pid
pid=$(cat "$pidfile" 2>/dev/null || true)
_pid_alive "$pid"
return $?
fi
return 1
;;
*)
pgrep -f "$(_service_process "$svc")" > /dev/null 2>&1
;;
esac
}
_stop_service() {
local svc=$1
_kill_by_pidfile() {
local signal=$1
local pidfile
pidfile=$(_service_pidfile "$svc")
if [ -f "$pidfile" ]; then
local pid
pid=$(cat "$pidfile" 2>/dev/null || true)
if [ -n "$pid" ]; then
kill "-${signal}" "$pid" 2>/dev/null || true
fi
fi
}
_kill_matching() {
local signal=$1
case "$svc" in
agent|sftp-agent)
_kill_by_pidfile "$signal"
;;
*)
pkill "-${signal}" -f "$(_service_process "$svc")" 2>/dev/null || true
;;
esac
}
if _is_running "$svc"; then
echo " Stopping $svc..."
_kill_matching TERM
for _ in $(seq 1 10); do
_is_running "$svc" || break
sleep 0.5
done
if _is_running "$svc"; then
_kill_matching KILL
sleep 1
fi
# Remove stale pid file on stop so the next status check is accurate.
case "$svc" in
agent|sftp-agent)
rm -f "$(_service_pidfile "$svc")"
;;
esac
fi
}
_start_service() {
local svc=$1
if _is_running "$svc"; then
echo " $(_color yellow "$svc is already running")"
return
fi
local cmd
# Raise the soft open-files limit before exec'ing the service. The long-lived
# tmux server inherits launchd's low default (256 on macOS), so services
# started here would otherwise hit "Too many open files". Best-effort: a
# failure on hosts with a low hard limit must not block startup.
cmd="ulimit -n 65536 2>/dev/null; $(_service_cmd "$svc")"
if ! tmux has-session -t "$TMUX_SESSION" 2>/dev/null; then
local winname
winname=$(_tmux_window_name "$svc")
tmux new-session -d -s "$TMUX_SESSION" -n "$winname" "$cmd"
else
local winname
winname=$(_tmux_window_name "$svc")
tmux new-window -t "$TMUX_SESSION" -n "$winname" "$cmd"
fi
echo " Started $svc"
}
_status_services() {
local services="$*"
for svc in $services; do
if _is_running "$svc"; then
local pid
case "$svc" in
agent|sftp-agent)
pid=$(cat "$(_service_pidfile "$svc")" 2>/dev/null || true)
;;
*)
pid=$(pgrep -f "$(_service_process "$svc")" | head -1)
;;
esac
printf " %-20s %s (pid: %s)\n" "$svc" "$(_color green running)" "$pid"
else
printf " %-20s %s\n" "$svc" "$(_color red stopped)"
fi
done
}
cmd_status() {
echo ""
printf " %-20s %s\n" "SERVICE" "STATUS"
printf " %-20s %s\n" "-------" "------"
_status_services $ALL_SERVICES
# Only show optional services that are actually installed (config exists).
if [ -f "./agent-sftp.toml" ]; then
_status_services sftp-agent
fi
echo ""
}
cmd_stop() {
local svc=$1
if [ "$svc" = "all" ]; then
echo "$(_color bold "Stopping all services...")"
for s in $ALL_SERVICES; do _stop_service "$s"; done
echo "$(_color green "All services stopped.")"
else
echo "$(_color bold "Stopping $svc...")"
_stop_service "$svc"
echo "$(_color green "$svc stopped.")"
fi
}
cmd_start() {
local svc=$1
if [ "$svc" = "all" ]; then
echo "$(_color bold "Starting all services...")"
for s in $ALL_SERVICES; do _start_service "$s"; done
echo "$(_color green "All services started.")"
else
echo "$(_color bold "Starting $svc...")"
_start_service "$svc"
echo "$(_color green "$svc started.")"
fi
}
cmd_restart() {
local svc=$1
if [ "$svc" = "all" ]; then
echo "$(_color bold "Restarting all services...")"
for s in $ALL_SERVICES; do _stop_service "$s"; done
sleep 1
for s in $ALL_SERVICES; do _start_service "$s"; done
echo "$(_color green "All services restarted.")"
else
echo "$(_color bold "Restarting $svc...")"
_stop_service "$svc"
sleep 1
_start_service "$svc"
echo "$(_color green "$svc restarted.")"
fi
}
cmd_log() {
local svc=$1
local follow=${2:-}
local winname
winname=$(_tmux_window_name "$svc")
local win
win=$(tmux list-windows -t "$TMUX_SESSION" -F "#{window_name}" 2>/dev/null | grep "^${winname}$" | head -1) || true
if [ -z "$win" ]; then
echo "$(_color red "No tmux window found for $svc")"
return 1
fi
if [ "$follow" = "-f" ]; then
local last_hash=""
trap 'exit 0' INT
while true; do
local output
output=$(tmux capture-pane -t "$TMUX_SESSION:$win" -p -S -50 2>/dev/null)
local cur_hash
cur_hash=$(echo "$output" | md5sum | cut -d' ' -f1)
if [ "$cur_hash" != "$last_hash" ]; then
clear
echo "$output"
last_hash=$cur_hash
fi
sleep 1
done
else
tmux capture-pane -t "$TMUX_SESSION:$win" -p -S -50
fi
}
cmd_harbor() {
local action=${1:-}
local harbor_dir="./harbor"
if [ ! -d "$harbor_dir" ] || [ ! -f "$harbor_dir/docker-compose.yml" ]; then
echo "$(_color red "Harbor is not installed.")"
echo "Install it by running: ./backendai-install install --with-harbor"
exit 1
fi
case "$action" in
start)
echo "$(_color bold "Starting Harbor...")"
(cd "$harbor_dir" && docker compose up -d)
echo "$(_color green "Harbor started.")"
;;
stop)
echo "$(_color bold "Stopping Harbor...")"
(cd "$harbor_dir" && docker compose down)
echo "$(_color green "Harbor stopped.")"
;;
restart)
echo "$(_color bold "Restarting Harbor...")"
(cd "$harbor_dir" && docker compose down && docker compose up -d)
echo "$(_color green "Harbor restarted.")"
;;
status)
(cd "$harbor_dir" && docker compose ps)
;;
logs)
(cd "$harbor_dir" && docker compose logs -f --tail=200)
;;
*)
echo "$(_color red "Usage: ./dev harbor <start|stop|restart|status|logs>")"
exit 1
;;
esac
}
usage() {
cat <<EOF
Usage: ./dev <command> [service]
Commands:
status Show status of all services
start <service|all> Start a service
stop <service|all> Stop a service
restart <service|all> Restart a service
log <service> [-f] Show recent log output (-f to follow)
harbor <start|stop|restart|status|logs>
Manage the optional Harbor registry
Services:
mgr, agent, storage, web, proxy-coordinator, proxy-worker, all
Optional services (must be started explicitly, not included in 'all'):
sftp-agent Dedicated SFTP upload agent (needs agent-sftp.toml)
Examples:
./dev status
./dev restart mgr
./dev restart all
./dev log mgr
./dev harbor start
./dev start sftp-agent
EOF
}
_validate_service() {
local svc=$1
[ "$svc" = "all" ] && return 0
_service_cmd "$svc" > /dev/null 2>&1 || {
echo "$(_color red "Unknown service: $svc")"
echo "Valid services: $ALL_SERVICES $OPTIONAL_SERVICES all"
exit 1
}
# Fail fast if sftp-agent config is missing
if [ "$svc" = "sftp-agent" ] && [ ! -f "./agent-sftp.toml" ]; then
echo "$(_color red "agent-sftp.toml not found. Run the installer with --with-sftp-agent first.")"
exit 1
fi
}
if [ $# -lt 1 ]; then
usage
exit 0
fi
case "$1" in
status) cmd_status ;;
start|stop|restart)
[ $# -lt 2 ] && { echo "$(_color red "Usage: ./dev $1 <service|all>")"; exit 1; }
_validate_service "$2"
"cmd_$1" "$2"
;;
log)
[ $# -lt 2 ] && { echo "$(_color red "Usage: ./dev log <service> [-f]")"; exit 1; }
_validate_service "$2"
cmd_log "$2" "${3:-}"
;;
harbor)
cmd_harbor "${2:-}"
;;
*) echo "$(_color red "Unknown command: $1")"; usage; exit 1 ;;
esac