diff --git a/apps/dao/sqlite.go b/apps/dao/sqlite.go index 833652e..abb4ffc 100644 --- a/apps/dao/sqlite.go +++ b/apps/dao/sqlite.go @@ -226,6 +226,20 @@ func CloseSqliteDB() error { return nil } +func IsSqliteReady() bool { + if sqliteDB == nil { + return false + } + db, err := sqliteDB.DB() + if err != nil { + return false + } + if err := db.Ping(); err != nil { + return false + } + return true +} + func Paginate(pageNum *int64, pageSize *int64) func(db *gorm.DB) *gorm.DB { var num int64 = 1 var size int64 = 10 diff --git a/apps/router/router.go b/apps/router/router.go index 2146de4..fe37c67 100644 --- a/apps/router/router.go +++ b/apps/router/router.go @@ -2,12 +2,40 @@ package router import ( "github.com/gin-gonic/gin" + "hy2xs-admin/dao" "hy2xs-admin/frontend" "hy2xs-admin/middleware" + "hy2xs-admin/model/constant" + "os" "strings" ) func Router(router *gin.Engine, huiWebContext *string) { + router.GET("/healthz", func(c *gin.Context) { + sqliteReady := dao.IsSqliteReady() + configReadable := false + if stat, err := os.Stat(constant.Hysteria2ConfigPath); err == nil { + configReadable = !stat.IsDir() + } + + if !sqliteReady || !configReadable { + c.JSON(503, gin.H{ + "ok": false, + "service": "hy2xs-admin", + "sqlite_ready": sqliteReady, + "config_readable": configReadable, + }) + return + } + + c.JSON(200, gin.H{ + "ok": true, + "service": "hy2xs-admin", + "sqlite_ready": true, + "config_readable": true, + }) + }) + relativePath := "/" if huiWebContext != nil && strings.HasPrefix(*huiWebContext, "/") { relativePath = *huiWebContext diff --git a/docs/07-systemd-and-firewall.md b/docs/07-systemd-and-firewall.md index 06abf5e..129bb84 100644 --- a/docs/07-systemd-and-firewall.md +++ b/docs/07-systemd-and-firewall.md @@ -62,20 +62,22 @@ IPv4-only policy: ## Firewall modes -`HY2XS_FIREWALL_ENABLED=true`: +`HY2XS_FIREWALL_MODE=managed`: - orchestrator управляет baseline nftables. +- существующий `foreign` entrypoint блокирует install/reconfigure (fail-fast). -`HY2XS_FIREWALL_ALLOW_TAKEOVER=false`: -- safe default. -- существующий не-HY2XS `/etc/nftables.conf` блокирует install/reconfigure (fail-fast). - -`HY2XS_FIREWALL_ALLOW_TAKEOVER=true`: +`HY2XS_FIREWALL_MODE=takeover`: - явный destructive takeover. - использовать только после ручной проверки хоста. -`HY2XS_FIREWALL_ENABLED=false` или `--skip-firewall`: +`HY2XS_FIREWALL_MODE=external`: +- orchestrator не модифицирует nftables. - оператор полностью управляет firewall вручную. +`HY2XS_FIREWALL_MODE=off`: +- firewall-слой оркестратора отключён. +- `--skip-firewall` эквивалентно runtime-отключению на время операции. + После staged-проверки можно включать default policy `drop`. ## Порядок применения diff --git a/docs/09-post-install-env.md b/docs/09-post-install-env.md index c04b0b1..412ec37 100644 --- a/docs/09-post-install-env.md +++ b/docs/09-post-install-env.md @@ -63,7 +63,7 @@ - `PUBLIC_HOST` - `PUBLIC_PORT` - `SSH_PORT` -- `HY2XS_FIREWALL_ENABLED` +- `HY2XS_FIREWALL_MODE` - `HY2XS_FIREWALL_STAGED_APPLY` - `HY2XS_ADMIN_USER` - `HY2XS_FORCE_PASSWORD_CHANGE` diff --git a/docs/11-testing-and-acceptance.md b/docs/11-testing-and-acceptance.md index 3124dd8..b579af9 100644 --- a/docs/11-testing-and-acceptance.md +++ b/docs/11-testing-and-acceptance.md @@ -69,6 +69,37 @@ 9. попытка использовать `HY2XS_IPV6_ENABLED=true` 10. `HY2XS_PUBLIC_HOST=0.0.0.0` +## E. Fix20 production matrix (обязательные сценарии) + +1. **Clean Debian 13 minimal**: + - только SSH, без ручной установки зависимостей; + - default `/etc/nftables.conf` stub; + - install проходит полностью; + - `doctor`/`status` показывают рабочее состояние. + +2. **Non-systemd container**: + - fail-fast до destructive шагов; + - диагностическое сообщение с причиной capability/systemd. + +3. **Foreign nftables**: + - при `HY2XS_FIREWALL_MODE=managed` install/reconfigure блокируются; + - при `HY2XS_FIREWALL_MODE=takeover` создаются backup/rollback guard и apply проходит. + +4. **Rollback guard cleanup**: + - после успешного apply/smoke не остаются `hy2xs-fw-rollback-*.timer/.service`. + +5. **Partial install + repair**: + - состояние `install-state` фиксирует промежуточную фазу; + - `repair` завершает граф до `installed=true`. + +6. **AAAA при IPv4-only**: + - policy строго валидируется preflight; + - soft warning path не используется в production baseline. + +7. **Slow-start admin readiness**: + - install не падает на race после restart; + - readiness waiters дожидаются listener/healthz. + ## Acceptance criteria Система принимается, если: @@ -86,3 +117,5 @@ 11. UI не запускается от root 12. клиентские endpoint не зависят от request `Host`/`hostname` 13. production build verify падает, если `config/hy2xs.env` содержит placeholder-значения +14. production build verify падает при dirty git tree (кроме `ALLOW_DIRTY_BUILD=true`) +15. metadata содержит `source_git_commit`, `dirty_tree`, `build_profile=production` diff --git a/orchestrator/src/cli.ts b/orchestrator/src/cli.ts index a5ea0e1..f071df4 100644 --- a/orchestrator/src/cli.ts +++ b/orchestrator/src/cli.ts @@ -1,5 +1,5 @@ import { install } from "./commands/install"; -import { reconfigure } from "./commands/reconfigure"; +import { reconfigure, repair } from "./commands/reconfigure"; import { doctor } from "./commands/doctor"; import { status } from "./commands/status"; import { diagnosticsCollect } from "./commands/diagnostics"; @@ -7,11 +7,13 @@ import type { InstallOptions, ReconfigureOptions } from "./types/context"; function usage(): never { console.error("Usage:"); - console.error(" hy2xs-orchestrator install --package-dir [--config ] [--skip-firewall] [--skip-start] [--non-interactive]"); - console.error(" hy2xs-orchestrator reconfigure --package-dir [--config ] [--dry-run|--apply] [--skip-firewall] [--skip-start]"); - console.error(" hy2xs-orchestrator doctor --package-dir [--config ] [--skip-firewall] [--skip-start]"); - console.error(" hy2xs-orchestrator status --package-dir [--config ] [--skip-firewall] [--skip-start]"); - console.error(" hy2xs-orchestrator diagnostics collect --package-dir [--config ] [--skip-firewall] [--skip-start]"); + console.error(" hy2xs-orchestrator install --package-dir [--config ] [--skip-firewall] [--skip-service-start] [--skip-smoke] [--non-interactive]"); + console.error(" hy2xs-orchestrator reconfigure --package-dir [--config ] [--dry-run|--apply] [--skip-firewall] [--skip-service-start] [--skip-smoke]"); + console.error(" hy2xs-orchestrator repair --package-dir [--config ] [--skip-firewall] [--skip-service-start] [--skip-smoke]"); + console.error(" hy2xs-orchestrator doctor --package-dir [--config ] [--skip-firewall] [--skip-service-start] [--skip-smoke]"); + console.error(" hy2xs-orchestrator status --package-dir [--config ] [--skip-firewall] [--skip-service-start] [--skip-smoke]"); + console.error(" hy2xs-orchestrator diagnostics collect --package-dir [--config ] [--skip-firewall] [--skip-service-start] [--skip-smoke]"); + console.error(" note: --skip-start is deprecated alias for --skip-service-start --skip-smoke"); process.exit(2); } @@ -31,7 +33,8 @@ function parseInstallOptions(args: string[]): InstallOptions { runtimeConfigPath: "/etc/hy2xs/hy2xs.env", nonInteractive: false, skipFirewall: false, - skipStart: false + skipServiceStart: false, + skipSmoke: false }; for (let i = 0; i < args.length; i += 1) { @@ -47,8 +50,16 @@ function parseInstallOptions(args: string[]): InstallOptions { case "--skip-firewall": options.skipFirewall = true; break; + case "--skip-service-start": + options.skipServiceStart = true; + break; + case "--skip-smoke": + options.skipSmoke = true; + break; case "--skip-start": - options.skipStart = true; + console.error("warning: --skip-start is deprecated; use --skip-service-start and/or --skip-smoke"); + options.skipServiceStart = true; + options.skipSmoke = true; break; case "--config": options.sourceConfigPath = takeValue(args, i, arg); @@ -77,7 +88,8 @@ function parseReconfigureOptions(args: string[]): ReconfigureOptions { dryRun: false, apply: false, skipFirewall: false, - skipStart: false + skipServiceStart: false, + skipSmoke: false }; for (let i = 0; i < args.length; i += 1) { @@ -100,8 +112,16 @@ function parseReconfigureOptions(args: string[]): ReconfigureOptions { case "--skip-firewall": options.skipFirewall = true; break; + case "--skip-service-start": + options.skipServiceStart = true; + break; + case "--skip-smoke": + options.skipSmoke = true; + break; case "--skip-start": - options.skipStart = true; + console.error("warning: --skip-start is deprecated; use --skip-service-start and/or --skip-smoke"); + options.skipServiceStart = true; + options.skipSmoke = true; break; default: console.error(`Unknown argument: ${arg}`); @@ -136,6 +156,11 @@ async function main(): Promise { await reconfigure(parseReconfigureOptions(args)); return; } + if (command === "repair") { + const options = parseReconfigureOptions(["--apply", ...args]); + await repair(options); + return; + } if (command === "doctor") { const options = parseReconfigureOptions(["--dry-run", ...args]); await doctor(options); diff --git a/orchestrator/src/commands/diagnostics.ts b/orchestrator/src/commands/diagnostics.ts index 197431b..2e91544 100644 --- a/orchestrator/src/commands/diagnostics.ts +++ b/orchestrator/src/commands/diagnostics.ts @@ -6,10 +6,27 @@ function shellEscapeSingleQuotes(value: string): string { return value.replaceAll("'", "'\\''"); } -export async function diagnosticsCollect(_options: CommonOptions): Promise { - setOperationContext(`diag-${Date.now().toString(36)}`); +function redactEnv(content: string): string { + return content + .replace(/^(HY2XS_ADMIN_INITIAL_PASSWORD=).*$/gm, "$1") + .replace(/^(HY2XS_ADMIN_CON_PASS=).*$/gm, "$1") + .replace(/^(HY2XS_HYSTERIA_TRAFFIC_STATS_SECRET=).*$/gm, "$1") + .replace(/^(HY2XS_HYSTERIA_OBFS_PASSWORD=).*$/gm, "$1"); +} - const outDir = `/tmp/hy2xs-diagnostics-${Date.now()}`; +function redactYaml(content: string): string { + return content + .replace(/(password:\s*).*/gi, "$1") + .replace(/(secret:\s*).*/gi, "$1") + .replace(/(auth:\s*).*/gi, "$1"); +} + +export async function diagnosticsCollect(_options: CommonOptions): Promise { + const opId = `diag-${Date.now().toString(36)}`; + setOperationContext(opId); + + const outDir = `/var/log/hy2xs/diagnostics/${opId}`; + const archive = `/var/log/hy2xs/diagnostics/${opId}.tar.gz`; await run`mkdir -p ${outDir}`; await run`sh -c ${`systemctl status hysteria-server > '${shellEscapeSingleQuotes(`${outDir}/systemd-hysteria.txt`)}' 2>&1 || true`}`; @@ -20,8 +37,27 @@ export async function diagnosticsCollect(_options: CommonOptions): Promise await run`sh -c ${`uname -a > '${shellEscapeSingleQuotes(`${outDir}/uname.txt`)}' 2>&1 || true`}`; await run`sh -c ${`cat /etc/os-release > '${shellEscapeSingleQuotes(`${outDir}/os-release.txt`)}' 2>&1 || true`}`; await run`sh -c ${`cp -a /etc/hysteria/post-install.env '${shellEscapeSingleQuotes(`${outDir}/post-install.env`)}' 2>/dev/null || true`}`; + await run`sh -c ${`cp -a /etc/hy2xs/hy2xs.env '${shellEscapeSingleQuotes(`${outDir}/hy2xs.env`)}' 2>/dev/null || true`}`; + await run`sh -c ${`cp -a /etc/hysteria/config.yaml '${shellEscapeSingleQuotes(`${outDir}/hysteria-config.yaml`)}' 2>/dev/null || true`}`; await run`sh -c ${`cp -a /var/lib/hy2xs/install-state.json '${shellEscapeSingleQuotes(`${outDir}/install-state.json`)}' 2>/dev/null || true`}`; + await run`sh -c ${`ss -ltnup > '${shellEscapeSingleQuotes(`${outDir}/ss-ltnup.txt`)}' 2>&1 || true`}`; - info(`diagnostics bundle collected: ${outDir}`); + try { + const envRaw = await Bun.file(`${outDir}/hy2xs.env`).text(); + await Bun.write(`${outDir}/hy2xs.env`, redactEnv(envRaw)); + } catch { + // noop + } + + try { + const cfgRaw = await Bun.file(`${outDir}/hysteria-config.yaml`).text(); + await Bun.write(`${outDir}/hysteria-config.yaml`, redactYaml(cfgRaw)); + } catch { + // noop + } + + await run`sh -c ${`tar -czf '${shellEscapeSingleQuotes(archive)}' -C '${shellEscapeSingleQuotes(outDir)}' .`}`; + + info(`diagnostics bundle collected: ${archive}`); } diff --git a/orchestrator/src/commands/install.ts b/orchestrator/src/commands/install.ts index 8909e51..740b721 100644 --- a/orchestrator/src/commands/install.ts +++ b/orchestrator/src/commands/install.ts @@ -1,5 +1,5 @@ import type { InstallContext, InstallOptions } from "../types/context"; -import { exists, readText, writeText, writeTextAtomic } from "../lib/fs"; +import { fileExists, readText, writeText, writeTextAtomic } from "../lib/fs"; import { runVisible } from "../lib/process"; import { setOperationContext, step, stepDone } from "../lib/log"; import { readPackageValue } from "../lib/packageMeta"; @@ -14,37 +14,135 @@ import { deploySystemd } from "../steps/systemd"; import { applyFirewall, cancelFirewallRollback, rollbackFirewallNow } from "../steps/firewall"; import { writeBootstrapAdminSecret, writePostInstallEnv } from "../steps/env"; import { smoke } from "../steps/smoke"; +import { diagnosticsCollect } from "./diagnostics"; const INSTALL_STATE_PATH = "/var/lib/hy2xs/install-state.json"; -async function markInstallSuccessful(context: InstallContext): Promise { +type InstallPhase = + | "installing" + | "preflight_ok" + | "deps_ok" + | "filesystem_ready" + | "runtime_env_written" + | "ui_deployed" + | "hysteria_installed" + | "config_generated" + | "units_deployed" + | "firewall_applied" + | "postinstall_env_written" + | "bootstrap_secret_written" + | "services_started" + | "smoke_failed" + | "failed" + | "installed"; + +type InstallState = { + installed: boolean; + phase: InstallPhase; + version: string; + build_id: string; + op_id: string; + started_at: string; + updated_at: string; + owned_paths: string[]; + last_error: string; + repair_hint?: string; +}; + +type FailureKind = + | "fatal_pre_apply" + | "firewall_connectivity_failure" + | "service_start_failure" + | "smoke_readiness_timeout" + | "postinstall_validation_failure"; + +function installOwnedPaths(context: InstallContext): string[] { + return [ + context.options.runtimeConfigPath, + "/etc/hysteria/config.yaml", + "/etc/hysteria/post-install.env", + context.config.bootstrapAdminSecretPath, + "/etc/systemd/system/hy2xs-admin.service", + "/etc/systemd/system/hysteria-server.service", + "/etc/nftables.conf", + "/etc/nftables.d/hy2xs.nft", + context.config.installDir + ]; +} + +async function writeInstallState(state: InstallState): Promise { await runVisible`install -d -m 0755 -o root -g root /var/lib/hy2xs`; - const state = JSON.stringify( - { - installed: true, - version: context.packageVersion, - build_id: context.packageBuildId, - installed_at: new Date().toISOString() - }, - null, - 2 - ); - await writeText(INSTALL_STATE_PATH, `${state}\n`, 0o644); + await writeText(INSTALL_STATE_PATH, `${JSON.stringify(state, null, 2)}\n`, 0o644); await runVisible`chown root:root ${INSTALL_STATE_PATH}`; } -async function rollbackFailedInstall(context: InstallContext, state: { firewallTouched: boolean }): Promise { - if (state.firewallTouched) { - await rollbackFirewallNow(context); +async function markInstallSuccessful(context: InstallContext): Promise { + await writeInstallState({ + phase: "installed", + installed: true, + version: context.packageVersion, + build_id: context.packageBuildId, + op_id: context.installDate, + started_at: context.installDate, + updated_at: new Date().toISOString(), + owned_paths: installOwnedPaths(context), + last_error: "" + }); +} + +async function advanceInstallState(context: InstallContext, phase: InstallPhase, lastError = ""): Promise { + await writeInstallState({ + phase, + installed: phase === "installed", + version: context.packageVersion, + build_id: context.packageBuildId, + op_id: context.installDate, + started_at: context.installDate, + updated_at: new Date().toISOString(), + owned_paths: installOwnedPaths(context), + last_error: lastError, + repair_hint: phase === "installed" ? undefined : "run: hy2xs-orchestrator repair --package-dir --config /etc/hy2xs/hy2xs.env" + }); +} + +function classifyFailure(phase: InstallPhase, message: string): FailureKind { + const m = message.toLowerCase(); + if (phase === "firewall_applied" || m.includes("nft") || m.includes("firewall") || m.includes("ssh port check failed")) { + return "firewall_connectivity_failure"; + } + if (phase === "services_started" || m.includes("is not active")) { + return "service_start_failure"; + } + if (phase === "smoke_failed" || m.includes("not ready") || m.includes("timeout") || m.includes("listener")) { + return "smoke_readiness_timeout"; + } + if (phase === "postinstall_env_written" || phase === "bootstrap_secret_written" || m.includes("permission") || m.includes("unexpected")) { + return "postinstall_validation_failure"; + } + return "fatal_pre_apply"; +} + +async function rollbackFailedInstall( + context: InstallContext, + state: { firewallTouched: boolean }, + failureKind: FailureKind, +): Promise { + if (failureKind === "firewall_connectivity_failure" || failureKind === "postinstall_validation_failure") { + if (state.firewallTouched) { + await rollbackFirewallNow(context); + } + } + + if (failureKind === "fatal_pre_apply" || failureKind === "firewall_connectivity_failure" || failureKind === "postinstall_validation_failure") { + await runVisible`systemctl stop hysteria-server hy2xs-admin || true`; + await runVisible`systemctl disable hysteria-server hy2xs-admin || true`; + await runVisible`systemctl reset-failed hysteria-server hy2xs-admin || true`; } - await runVisible`systemctl stop hysteria-server hy2xs-admin || true`; - await runVisible`systemctl disable hysteria-server hy2xs-admin || true`; - await runVisible`systemctl reset-failed hysteria-server hy2xs-admin || true`; } export async function install(options: InstallOptions): Promise { setOperationContext(`install-${Date.now().toString(36)}`); - const hasSourceConfig = options.sourceConfigPath ? await exists(options.sourceConfigPath) : false; + const hasSourceConfig = options.sourceConfigPath ? await fileExists(options.sourceConfigPath) : false; if (options.sourceConfigPath && !hasSourceConfig) { throw new Error(`config source not found: ${options.sourceConfigPath}`); } @@ -70,19 +168,28 @@ export async function install(options: InstallOptions): Promise { } const state = { - firewallTouched: false + firewallTouched: false, + lastPhase: "installing" as InstallPhase }; try { + await advanceInstallState(context, "installing"); + state.lastPhase = "installing"; step("preflight"); await preflight(context); stepDone("preflight"); + await advanceInstallState(context, "preflight_ok"); + state.lastPhase = "preflight_ok"; step("system dependencies"); await installDeps(context); stepDone("system dependencies"); + await advanceInstallState(context, "deps_ok"); + state.lastPhase = "deps_ok"; step("filesystem"); await prepareFilesystem(context); stepDone("filesystem"); + await advanceInstallState(context, "filesystem_ready"); + state.lastPhase = "filesystem_ready"; step("write runtime env"); await runVisible`mkdir -p /etc/hy2xs`; await writeTextAtomic(options.runtimeConfigPath, renderRuntimeEnv(config), { @@ -91,29 +198,49 @@ export async function install(options: InstallOptions): Promise { group: "root" }); stepDone("write runtime env"); + await advanceInstallState(context, "runtime_env_written"); + state.lastPhase = "runtime_env_written"; step("bundled UI"); await deployUi(context); stepDone("bundled UI"); + await advanceInstallState(context, "ui_deployed"); + state.lastPhase = "ui_deployed"; step("Hysteria2 upstream install"); await installHysteria(context); stepDone("Hysteria2 upstream install"); + await advanceInstallState(context, "hysteria_installed"); + state.lastPhase = "hysteria_installed"; step("config generation"); await generateConfig(context); stepDone("config generation"); + await advanceInstallState(context, "config_generated"); + state.lastPhase = "config_generated"; step("systemd units"); await deploySystemd(context); stepDone("systemd units"); + await advanceInstallState(context, "units_deployed"); + state.lastPhase = "units_deployed"; step("firewall"); state.firewallTouched = true; await applyFirewall(context); stepDone("firewall"); + await advanceInstallState(context, "firewall_applied"); + state.lastPhase = "firewall_applied"; step("post-install env"); await writePostInstallEnv(context); stepDone("post-install env"); + await advanceInstallState(context, "postinstall_env_written"); + state.lastPhase = "postinstall_env_written"; step("bootstrap admin secret"); await writeBootstrapAdminSecret(context); stepDone("bootstrap admin secret"); + await advanceInstallState(context, "bootstrap_secret_written"); + state.lastPhase = "bootstrap_secret_written"; step("smoke checks"); + await advanceInstallState(context, "services_started"); + state.lastPhase = "services_started"; + await advanceInstallState(context, "smoke_failed"); + state.lastPhase = "smoke_failed"; await smoke(context); stepDone("smoke checks"); step("finalize firewall rollback guard"); @@ -123,7 +250,11 @@ export async function install(options: InstallOptions): Promise { await markInstallSuccessful(context); stepDone("mark install successful"); } catch (error) { - await rollbackFailedInstall(context, state); + const message = error instanceof Error ? error.message : String(error); + const failureKind = classifyFailure(state.lastPhase, message); + await advanceInstallState(context, failureKind === "smoke_readiness_timeout" ? "smoke_failed" : "failed", `${failureKind}: ${message}`); + await diagnosticsCollect(options); + await rollbackFailedInstall(context, state, failureKind); throw error; } } diff --git a/orchestrator/src/commands/reconfigure.ts b/orchestrator/src/commands/reconfigure.ts index 2092705..4016dbb 100644 --- a/orchestrator/src/commands/reconfigure.ts +++ b/orchestrator/src/commands/reconfigure.ts @@ -1,5 +1,5 @@ import type { ReconfigureContext, ReconfigureOptions } from "../types/context"; -import { exists, readText, writeText } from "../lib/fs"; +import { fileExists, readText, writeText } from "../lib/fs"; import { info, setOperationContext, step, stepDone } from "../lib/log"; import { parseRuntimeEnv, renderRuntimeEnv } from "../config/env"; import { preflight } from "../steps/preflight"; @@ -10,13 +10,73 @@ import { writePostInstallEnv } from "../steps/env"; import { smoke } from "../steps/smoke"; import { runVisible } from "../lib/process"; import { readInstalledHysteriaVersion, readPackageValue } from "../lib/packageMeta"; +import { diagnosticsCollect } from "./diagnostics"; const INSTALL_STATE_PATH = "/var/lib/hy2xs/install-state.json"; type InstallState = { - installed?: boolean; + installed: boolean; + phase: string; + version?: string; + build_id?: string; + op_id?: string; + started_at?: string; + updated_at?: string; + owned_paths?: string[]; + last_error?: string; + repair_hint?: string; }; +type ReconfigurePhase = + | "reconfiguring" + | "repairing" + | "config_generated" + | "units_deployed" + | "firewall_applied" + | "runtime_env_written" + | "smoke_ok" + | "firewall_connectivity_failure" + | "smoke_failed" + | "installed"; + +function operationKey(context: ReconfigureContext): string { + return context.installDate; +} + +function ownedPaths(context: ReconfigureContext): string[] { + return [ + context.options.runtimeConfigPath, + "/etc/hysteria/config.yaml", + "/etc/hysteria/post-install.env", + context.config.bootstrapAdminSecretPath, + "/etc/systemd/system/hy2xs-admin.service", + "/etc/systemd/system/hysteria-server.service", + "/etc/nftables.conf", + "/etc/nftables.d/hy2xs.nft", + context.config.installDir + ]; +} + +async function writeInstallState(state: InstallState): Promise { + await writeText(INSTALL_STATE_PATH, `${JSON.stringify(state, null, 2)}\n`, 0o644); + await runVisible`chown root:root ${INSTALL_STATE_PATH}`; +} + +async function markPhase(context: ReconfigureContext, phase: ReconfigurePhase, lastError = ""): Promise { + await writeInstallState({ + installed: phase === "installed", + phase, + version: context.packageVersion, + build_id: context.packageBuildId, + op_id: operationKey(context), + started_at: context.installDate, + updated_at: new Date().toISOString(), + owned_paths: ownedPaths(context), + last_error: lastError, + repair_hint: phase === "installed" ? "" : "run: hy2xs-orchestrator repair --package-dir --config /etc/hy2xs/hy2xs.env" + }); +} + async function backupCurrentState(): Promise { await runVisible`mkdir -p /etc/hy2xs/backups`; await runVisible`cp -a /etc/hysteria/config.yaml /etc/hy2xs/backups/config.yaml.bak 2>/dev/null || true`; @@ -49,7 +109,7 @@ async function rollbackCurrentState(): Promise { } async function ensureInstallStateExists(): Promise { - if (!(await exists(INSTALL_STATE_PATH))) { + if (!(await fileExists(INSTALL_STATE_PATH))) { throw new Error(`install state marker is missing: ${INSTALL_STATE_PATH}. Run install first.`); } @@ -65,8 +125,34 @@ async function ensureInstallStateExists(): Promise { } } +async function readInstallState(): Promise { + if (!(await fileExists(INSTALL_STATE_PATH))) { + return null; + } + try { + return JSON.parse(await readText(INSTALL_STATE_PATH)) as InstallState; + } catch { + return null; + } +} + +async function ensureInstallStateForOperation(options: ReconfigureOptions): Promise { + const state = await readInstallState(); + if (!state) { + throw new Error(`install state marker is missing: ${INSTALL_STATE_PATH}. Run install first.`); + } + if (state.installed) { + return; + } + if (options.allowPartialState) { + info(`repair mode: proceeding with partial install state (phase=${state.phase ?? "unknown"})`); + return; + } + throw new Error(`install state marker does not indicate successful installation: ${INSTALL_STATE_PATH}`); +} + async function warnBootstrapDrift(nextConfigRaw: string): Promise { - if (!(await exists("/etc/hy2xs/hy2xs.env"))) { + if (!(await fileExists("/etc/hy2xs/hy2xs.env"))) { return; } @@ -97,7 +183,7 @@ export async function reconfigure(options: ReconfigureOptions): Promise { await preflight(context); stepDone("preflight"); step("install state marker"); - await ensureInstallStateExists(); + await ensureInstallStateForOperation(options); stepDone("install state marker"); await warnBootstrapDrift(configRaw); @@ -116,31 +202,61 @@ export async function reconfigure(options: ReconfigureOptions): Promise { stepDone("backup"); try { + await markPhase(context, options.allowPartialState ? "repairing" : "reconfiguring"); step("config generation"); await generateConfig(context); stepDone("config generation"); + await markPhase(context, "config_generated"); step("systemd units"); await deploySystemd(context); stepDone("systemd units"); + await markPhase(context, "units_deployed"); step("firewall"); await applyFirewall(context); stepDone("firewall"); + await markPhase(context, "firewall_applied"); step("write env artifacts"); await writeText(options.runtimeConfigPath, renderRuntimeEnv(config), 0o600); await runVisible`chown root:root ${options.runtimeConfigPath}`; await runVisible`chmod 0600 ${options.runtimeConfigPath}`; await writePostInstallEnv(context); stepDone("write env artifacts"); + await markPhase(context, "runtime_env_written"); step("smoke checks"); await smoke(context); stepDone("smoke checks"); + await markPhase(context, "smoke_ok"); step("finalize firewall rollback guard"); await cancelFirewallRollback(context); stepDone("finalize firewall rollback guard"); + await markPhase(context, "installed"); + + const finalState = await readInstallState(); + if (!finalState?.installed || finalState.phase !== "installed") { + throw new Error("deterministic state violation: reconfigure/repair finished without installed phase"); + } } catch (error) { info("reconfigure failed, rollback in progress"); + const message = error instanceof Error ? error.message : String(error); + const phase: ReconfigurePhase = /firewall|nft|ssh port check failed/i.test(message) + ? "firewall_connectivity_failure" + : "smoke_failed"; + await markPhase(context, phase, message); + await diagnosticsCollect(options); await rollbackFirewallNow(context); await rollbackCurrentState(); throw error; } } + +export async function repair(options: ReconfigureOptions): Promise { + const effective: ReconfigureOptions = { + ...options, + dryRun: false, + apply: true, + allowPartialState: true, + skipSmoke: options.skipSmoke ?? false, + skipServiceStart: options.skipServiceStart ?? false + }; + await reconfigure(effective); +} diff --git a/orchestrator/src/commands/status.ts b/orchestrator/src/commands/status.ts index 434520f..60e0907 100644 --- a/orchestrator/src/commands/status.ts +++ b/orchestrator/src/commands/status.ts @@ -1,8 +1,11 @@ import type { CommonOptions } from "../types/context"; -import { exists, readText } from "../lib/fs"; +import { fileExists, readText } from "../lib/fs"; import { info, setOperationContext } from "../lib/log"; import { run } from "../lib/process"; import { getPlatformProfile } from "../platform/profile"; +import { detectFirewallEntrypointKind } from "../steps/firewall"; + +const INSTALL_STATE_PATH = "/var/lib/hy2xs/install-state.json"; async function unitState(unit: string): Promise { try { @@ -23,7 +26,7 @@ async function firewallState(): Promise { } async function tlsState(): Promise { - if (!(await exists("/etc/hysteria/config.yaml"))) { + if (!(await fileExists("/etc/hysteria/config.yaml"))) { return "missing-config"; } const cfg = await readText("/etc/hysteria/config.yaml"); @@ -39,6 +42,17 @@ async function tlsState(): Promise { export async function status(_options: CommonOptions): Promise { setOperationContext(`status-${Date.now().toString(36)}`); const platform = await getPlatformProfile(); + let installState: Record | null = null; + if (await fileExists(INSTALL_STATE_PATH)) { + try { + installState = JSON.parse(await readText(INSTALL_STATE_PATH)) as Record; + } catch { + installState = { parse_error: true }; + } + } + + const rollbackGuardUnits = (await run`sh -c 'systemctl list-units --all --no-legend "hy2xs-fw-rollback-*.timer" "hy2xs-fw-rollback-*.service" 2>/dev/null || true'`).trim(); + const result = { ts: new Date().toISOString(), platform, @@ -47,8 +61,12 @@ export async function status(_options: CommonOptions): Promise { admin: await unitState("hy2xs-admin") }, firewall: await firewallState(), + firewall_entrypoint_kind: await detectFirewallEntrypointKind(), tls: await tlsState(), - install_state_present: await exists("/var/lib/hy2xs/install-state.json") + install_state_present: await fileExists(INSTALL_STATE_PATH), + install_state: installState, + rollback_guard_active: rollbackGuardUnits.length > 0, + rollback_guard_units: rollbackGuardUnits ? rollbackGuardUnits.split("\n") : [] }; info(`status report: ${JSON.stringify(result)}`); } diff --git a/orchestrator/src/config/env.ts b/orchestrator/src/config/env.ts index 5a0314a..99619fc 100644 --- a/orchestrator/src/config/env.ts +++ b/orchestrator/src/config/env.ts @@ -1,5 +1,5 @@ import { randomBytes } from "node:crypto"; -import type { RuntimeConfig, TlsMode } from "../types/context"; +import type { FirewallMode, RuntimeConfig, TlsMode } from "../types/context"; type EnvMap = Record; @@ -53,6 +53,21 @@ function parseBool(name: string, raw: string, fallback: boolean): boolean { throw new Error(`invalid ${name}: ${raw}`); } +function parseFirewallMode(value: string): FirewallMode { + if (value === "managed" || value === "takeover" || value === "external" || value === "off") { + return value; + } + throw new Error(`invalid HY2XS_FIREWALL_MODE: ${value}`); +} + +function normalizeFirewallMode(env: EnvMap): FirewallMode { + const raw = env.HY2XS_FIREWALL_MODE; + if (!raw) { + throw new Error("missing required HY2XS_FIREWALL_MODE"); + } + return parseFirewallMode(raw); +} + function requireValue(name: string, value: string): string { if (!value || !value.trim()) { throw new Error(`missing required ${name}`); @@ -141,6 +156,7 @@ export function parseRuntimeEnv(content: string): RuntimeConfig { const trafficStatsPort = parsePort("HY2XS_HYSTERIA_TRAFFIC_STATS_PORT", env.HY2XS_HYSTERIA_TRAFFIC_STATS_PORT, 36712); const tlsMode = normalizeTlsMode(env.HY2XS_TLS_MODE || "acme"); const acmeType = normalizeAcmeType(env.HY2XS_ACME_TYPE || "http"); + const firewallMode = normalizeFirewallMode(env); const config: RuntimeConfig = { domain: env.HY2XS_DOMAIN || "", @@ -148,9 +164,8 @@ export function parseRuntimeEnv(content: string): RuntimeConfig { publicPort: parsePort("HY2XS_PUBLIC_PORT", env.HY2XS_PUBLIC_PORT, hysteriaPort), ipv6Enabled: parseBool("HY2XS_IPV6_ENABLED", env.HY2XS_IPV6_ENABLED, false), sshPort: parsePort("HY2XS_SSH_PORT", env.HY2XS_SSH_PORT, 22), - firewallEnabled: parseBool("HY2XS_FIREWALL_ENABLED", env.HY2XS_FIREWALL_ENABLED, true), + firewallMode, firewallStagedApply: parseBool("HY2XS_FIREWALL_STAGED_APPLY", env.HY2XS_FIREWALL_STAGED_APPLY, true), - firewallAllowTakeover: parseBool("HY2XS_FIREWALL_ALLOW_TAKEOVER", env.HY2XS_FIREWALL_ALLOW_TAKEOVER, false), uiBindHost, uiPublicAccess: parseBool("HY2XS_UI_PUBLIC_ACCESS", env.HY2XS_UI_PUBLIC_ACCESS, false), uiPort, @@ -245,9 +260,8 @@ export function renderRuntimeEnv(config: RuntimeConfig): string { `HY2XS_PUBLIC_HOST=${config.publicHost}`, `HY2XS_PUBLIC_PORT=${config.publicPort}`, `HY2XS_SSH_PORT=${config.sshPort}`, - `HY2XS_FIREWALL_ENABLED=${config.firewallEnabled}`, + `HY2XS_FIREWALL_MODE=${config.firewallMode}`, `HY2XS_FIREWALL_STAGED_APPLY=${config.firewallStagedApply}`, - `HY2XS_FIREWALL_ALLOW_TAKEOVER=${config.firewallAllowTakeover}`, `HY2XS_UI_BIND_HOST=${config.uiBindHost}`, `HY2XS_UI_PUBLIC_ACCESS=${config.uiPublicAccess}`, `HY2XS_UI_PORT=${config.uiPort}`, diff --git a/orchestrator/src/lib/fs.ts b/orchestrator/src/lib/fs.ts index 92c383b..d0b7a56 100644 --- a/orchestrator/src/lib/fs.ts +++ b/orchestrator/src/lib/fs.ts @@ -1,5 +1,28 @@ -export async function exists(path: string): Promise { - return await Bun.file(path).exists(); +import { stat } from "node:fs/promises"; + +async function statSafe(path: string): Promise { + try { + return await stat(path); + } catch (error) { + if (error && typeof error === "object" && "code" in error && (error as { code?: string }).code === "ENOENT") { + return null; + } + throw error; + } +} + +export async function pathExists(path: string): Promise { + return (await statSafe(path)) !== null; +} + +export async function fileExists(path: string): Promise { + const st = await statSafe(path); + return st?.isFile() ?? false; +} + +export async function dirExists(path: string): Promise { + const st = await statSafe(path); + return st?.isDirectory() ?? false; } export async function readText(path: string): Promise { diff --git a/orchestrator/src/platform/assert.ts b/orchestrator/src/platform/assert.ts index 213cb3d..de43fd6 100644 --- a/orchestrator/src/platform/assert.ts +++ b/orchestrator/src/platform/assert.ts @@ -25,7 +25,9 @@ export async function assertPlatform(options: AssertPlatformOptions): Promise { @@ -51,6 +56,40 @@ async function detectOpenSsl3(): Promise { } } +async function detectSystemd(): Promise<{ ok: boolean; reason: string; state: string; pid1: string }> { + if (!(await commandExists("systemctl"))) { + return { ok: false, reason: "systemctl not found", state: "unknown", pid1: "unknown" }; + } + + let pid1 = "unknown"; + try { + pid1 = (await run`ps -p 1 -o comm=`).trim(); + } catch { + return { ok: false, reason: "unable to inspect PID 1", state: "unknown", pid1: "unknown" }; + } + + if (pid1 !== "systemd") { + return { ok: false, reason: `PID 1 is ${pid1}, not systemd`, state: "unknown", pid1 }; + } + + if (!(await dirExists("/run/systemd/system"))) { + return { ok: false, reason: "/run/systemd/system is missing", state: "unknown", pid1 }; + } + + let state = "unknown"; + try { + state = (await run`systemctl is-system-running || true`).trim(); + } catch { + state = "unknown"; + } + + if (state !== "running" && state !== "degraded") { + return { ok: false, reason: `systemd state is ${state || "unknown"}`, state, pid1 }; + } + + return { ok: true, reason: "ok", state, pid1 }; +} + export async function getPlatformProfile(): Promise { const osReleaseRaw = await readText("/etc/os-release"); const parsed = parseOsRelease(osReleaseRaw); @@ -61,7 +100,8 @@ export async function getPlatformProfile(): Promise { const architecture: PlatformProfile["architecture"] = archRaw.trim() === "x86_64" ? "amd64" : "unsupported"; const nftables = await commandExists("nft"); - const systemd = await commandExists("systemctl") && (await exists("/run/systemd/system")); + const systemdCheck = await detectSystemd(); + const systemd = systemdCheck.ok; const systemdRun = await commandExists("systemd-run"); const openssl3 = await detectOpenSsl3(); const nftAtomicReplace = nftables; @@ -76,6 +116,11 @@ export async function getPlatformProfile(): Promise { openssl3, systemdRun, nftAtomicReplace + }, + capabilityDetails: { + systemdReason: systemdCheck.reason, + systemdState: systemdCheck.state, + pid1: systemdCheck.pid1 } }; } diff --git a/orchestrator/src/steps/env.ts b/orchestrator/src/steps/env.ts index 77da5cd..a294cfd 100644 --- a/orchestrator/src/steps/env.ts +++ b/orchestrator/src/steps/env.ts @@ -10,7 +10,7 @@ export async function writePostInstallEnv(context: RuntimeContext): Promise { - for (const file of FW_BACKUP_FILES) { - await runVisible`rm -f ${file}`; - } +export type FirewallEntrypointKind = NftEntrypointKind; + +function rollbackRoot(opId: string): string { + return `/run/hy2xs/rollback/${opId}`; +} + +function rollbackUnit(opId: string): string { + return `hy2xs-fw-rollback-${opId}`; +} + +function operationKey(context: RuntimeContext): string { + return context.installDate.replace(/[^a-zA-Z0-9_.-]/g, "-"); +} + +function rollbackMarker(opId: string): string { + return `${rollbackRoot(opId)}/prepared`; +} + +function rollbackBackup(path: string, opId: string): string { + return `${rollbackRoot(opId)}/${path}`; +} + +async function ensureRollbackRoot(opId: string): Promise { + await runVisible`mkdir -p ${rollbackRoot(opId)}`; +} + +async function cleanupFirewallBackupFiles(opId: string): Promise { + await runVisible`rm -rf ${rollbackRoot(opId)}`; } function stripNftComments(content: string): string { @@ -28,28 +49,55 @@ function stripNftComments(content: string): string { .join("\n"); } -function isSafeNftablesEntrypoint(content: string): boolean { - if (content.includes("HY2XS-MANAGED")) { - return true; +function classifyNftEntrypoint(content: string): NftEntrypointKind { + if (!content.trim()) { + return "missing"; } - const effective = stripNftComments(content) + if (content.includes("HY2XS-MANAGED")) { + return "hy2xs-managed"; + } + + const withoutComments = stripNftComments(content); + const effective = withoutComments .replace(/^#!\/usr\/sbin\/nft\s+-f\s*/m, "") .trim(); if (!effective) { - return true; + return "empty"; } - return effective === "flush ruleset"; + const normalized = effective.replace(/\s+/g, " ").trim(); + if (normalized === "flush ruleset") { + return "debian-empty-template"; + } + + if (/include\s+"\/etc\/nftables\.d\/hy2xs\.nft"/.test(effective)) { + return "include-compatible"; + } + + return "foreign"; +} + +export async function detectFirewallEntrypointKind(): Promise { + if (!(await fileExists("/etc/nftables.conf"))) { + return "missing"; + } + return classifyNftEntrypoint(await readText("/etc/nftables.conf")); } export async function applyFirewall(context: RuntimeContext): Promise { - if (context.options.skipFirewall || !context.config.firewallEnabled) { + const opId = operationKey(context); + if (context.options.skipFirewall || context.config.firewallMode === "off") { info("firewall skipped by flag"); return; } + if (context.config.firewallMode === "external") { + info("firewall mode is external: nftables is not modified"); + return; + } + const acmeChallengePort = context.config.acmeType === "tls" ? 443 : 80; const acmeRule = context.config.tlsMode === "acme" ? `tcp dport ${acmeChallengePort} accept` @@ -61,20 +109,28 @@ export async function applyFirewall(context: RuntimeContext): Promise { ACME_RULE: acmeRule }); - const existing = await exists("/etc/nftables.conf") + const existing = await fileExists("/etc/nftables.conf") ? await readText("/etc/nftables.conf") : ""; - if (existing && !isSafeNftablesEntrypoint(existing) && !context.config.firewallAllowTakeover) { - fail("existing non-HY2XS nftables.conf found; set HY2XS_FIREWALL_ALLOW_TAKEOVER=true or HY2XS_FIREWALL_ENABLED=false"); + const entrypointKind = classifyNftEntrypoint(existing); + const managedAllowed = new Set([ + "missing", + "hy2xs-managed", + "empty", + "debian-empty-template", + "include-compatible" + ]); + if (context.config.firewallMode === "managed" && !managedAllowed.has(entrypointKind)) { + fail("foreign nftables.conf detected; use HY2XS_FIREWALL_MODE=takeover|external|off"); } - await runVisible`touch /etc/nftables.d/hy2xs.rollback.prepared`; - await runVisible`cp -a /etc/nftables.conf /etc/nftables.conf.hy2xs.bak 2>/dev/null || true`; - await runVisible`cp -a /etc/nftables.d/hy2xs.nft /etc/nftables.d/hy2xs.nft.bak 2>/dev/null || true`; - await runVisible`test -f /etc/nftables.conf && echo 1 > /etc/nftables.d/nftables.conf.existed || rm -f /etc/nftables.d/nftables.conf.existed`; - await runVisible`test -f /etc/nftables.d/hy2xs.nft && echo 1 > /etc/nftables.d/hy2xs.nft.existed || rm -f /etc/nftables.d/hy2xs.nft.existed`; - await runVisible`grep -q 'include "/etc/nftables.d/hy2xs.nft"' /etc/nftables.conf && echo 1 > /etc/nftables.d/hy2xs.nft.include.existed || rm -f /etc/nftables.d/hy2xs.nft.include.existed`; + await ensureRollbackRoot(opId); + await runVisible`touch ${rollbackMarker(opId)}`; + await runVisible`cp -a /etc/nftables.conf ${rollbackBackup("nftables.conf.bak", opId)} 2>/dev/null || true`; + await runVisible`cp -a /etc/nftables.d/hy2xs.nft ${rollbackBackup("hy2xs.nft.bak", opId)} 2>/dev/null || true`; + await runVisible`test -f /etc/nftables.conf && echo 1 > ${rollbackBackup("nftables.conf.existed", opId)} || rm -f ${rollbackBackup("nftables.conf.existed", opId)}`; + await runVisible`test -f /etc/nftables.d/hy2xs.nft && echo 1 > ${rollbackBackup("hy2xs.nft.existed", opId)} || rm -f ${rollbackBackup("hy2xs.nft.existed", opId)}`; await writeText("/etc/nftables.d/hy2xs.nft.candidate", rendered, 0o600); await runVisible`nft -c -f /etc/nftables.d/hy2xs.nft.candidate`; @@ -103,7 +159,8 @@ include "/etc/nftables.d/hy2xs.nft" await runVisible`nft -c -f /etc/nftables.conf`; if (context.config.firewallStagedApply) { - await runVisible`systemd-run --unit hy2xs-fw-rollback --on-active=45s /bin/sh -c 'if [ -f /etc/nftables.d/hy2xs.rollback.prepared ]; then if [ -f /etc/nftables.d/nftables.conf.existed ]; then cp -a /etc/nftables.conf.hy2xs.bak /etc/nftables.conf 2>/dev/null || true; else rm -f /etc/nftables.conf; fi; if [ -f /etc/nftables.d/hy2xs.nft.existed ]; then cp -a /etc/nftables.d/hy2xs.nft.bak /etc/nftables.d/hy2xs.nft 2>/dev/null || true; else rm -f /etc/nftables.d/hy2xs.nft; fi; if [ -f /etc/nftables.d/nftables.conf.existed ]; then nft -f /etc/nftables.conf >/dev/null 2>&1 || true; else nft flush ruleset >/dev/null 2>&1 || true; fi; fi'`; + const unit = rollbackUnit(opId); + await runVisible`systemd-run --unit ${unit} --on-active=45s /bin/sh -c 'if [ -f ${rollbackMarker(opId)} ]; then if [ -f ${rollbackBackup("nftables.conf.existed", opId)} ]; then cp -a ${rollbackBackup("nftables.conf.bak", opId)} /etc/nftables.conf 2>/dev/null || true; else rm -f /etc/nftables.conf; fi; if [ -f ${rollbackBackup("hy2xs.nft.existed", opId)} ]; then cp -a ${rollbackBackup("hy2xs.nft.bak", opId)} /etc/nftables.d/hy2xs.nft 2>/dev/null || true; else rm -f /etc/nftables.d/hy2xs.nft; fi; if [ -f ${rollbackBackup("nftables.conf.existed", opId)} ]; then nft -f /etc/nftables.conf >/dev/null 2>&1 || true; else nft flush ruleset >/dev/null 2>&1 || true; fi; fi'`; } await runVisible`nft -f /etc/nftables.conf`; @@ -115,35 +172,39 @@ include "/etc/nftables.d/hy2xs.nft" } export async function cancelFirewallRollback(context: RuntimeContext): Promise { - if (!context.config.firewallEnabled || context.options.skipFirewall) { + const opId = operationKey(context); + if (context.options.skipFirewall || context.config.firewallMode === "off" || context.config.firewallMode === "external") { return; } if (context.config.firewallStagedApply) { - await runVisible`systemctl stop hy2xs-fw-rollback || true`; - await runVisible`systemctl reset-failed hy2xs-fw-rollback || true`; + const unit = rollbackUnit(opId); + await runVisible`systemctl stop ${unit}.timer ${unit}.service || true`; + await runVisible`systemctl reset-failed ${unit}.timer ${unit}.service || true`; } - await cleanupFirewallBackupFiles(); + await cleanupFirewallBackupFiles(opId); } export async function rollbackFirewallNow(context: RuntimeContext): Promise { - if (!context.config.firewallEnabled || context.options.skipFirewall) { + const opId = operationKey(context); + if (context.options.skipFirewall || context.config.firewallMode === "off" || context.config.firewallMode === "external") { return; } - if (!(await exists("/etc/nftables.d/hy2xs.rollback.prepared"))) { + if (!(await fileExists(rollbackMarker(opId)))) { info("firewall rollback skipped: no HY2XS rollback markers found"); return; } if (context.config.firewallStagedApply) { - await runVisible`systemctl stop hy2xs-fw-rollback || true`; - await runVisible`systemctl reset-failed hy2xs-fw-rollback || true`; + const unit = rollbackUnit(opId); + await runVisible`systemctl stop ${unit}.timer ${unit}.service || true`; + await runVisible`systemctl reset-failed ${unit}.timer ${unit}.service || true`; } - await runVisible`if [ -f /etc/nftables.d/nftables.conf.existed ]; then cp -a /etc/nftables.conf.hy2xs.bak /etc/nftables.conf 2>/dev/null || true; else rm -f /etc/nftables.conf; fi`; - await runVisible`if [ -f /etc/nftables.d/hy2xs.nft.existed ]; then cp -a /etc/nftables.d/hy2xs.nft.bak /etc/nftables.d/hy2xs.nft 2>/dev/null || true; else rm -f /etc/nftables.d/hy2xs.nft; fi`; - await runVisible`if [ -f /etc/nftables.d/nftables.conf.existed ]; then nft -f /etc/nftables.conf >/dev/null 2>&1 || true; else nft flush ruleset >/dev/null 2>&1 || true; fi`; - await cleanupFirewallBackupFiles(); + await runVisible`if [ -f ${rollbackBackup("nftables.conf.existed", opId)} ]; then cp -a ${rollbackBackup("nftables.conf.bak", opId)} /etc/nftables.conf 2>/dev/null || true; else rm -f /etc/nftables.conf; fi`; + await runVisible`if [ -f ${rollbackBackup("hy2xs.nft.existed", opId)} ]; then cp -a ${rollbackBackup("hy2xs.nft.bak", opId)} /etc/nftables.d/hy2xs.nft 2>/dev/null || true; else rm -f /etc/nftables.d/hy2xs.nft; fi`; + await runVisible`if [ -f ${rollbackBackup("nftables.conf.existed", opId)} ]; then nft -f /etc/nftables.conf >/dev/null 2>&1 || true; else nft flush ruleset >/dev/null 2>&1 || true; fi`; + await cleanupFirewallBackupFiles(opId); } diff --git a/orchestrator/src/steps/preflight.ts b/orchestrator/src/steps/preflight.ts index 06453f5..d421c08 100644 --- a/orchestrator/src/steps/preflight.ts +++ b/orchestrator/src/steps/preflight.ts @@ -1,5 +1,5 @@ import type { RuntimeContext } from "../types/context"; -import { exists } from "../lib/fs"; +import { dirExists, fileExists } from "../lib/fs"; import { fail, info } from "../lib/log"; import { run } from "../lib/process"; import { assertPlatform } from "../platform/assert"; @@ -38,36 +38,46 @@ export async function preflight(context: RuntimeContext): Promise { fail("installer must run as root"); } - try { - await run`command -v sudo >/dev/null 2>&1`; - } catch { - fail("sudo is required for installer smoke checks. Install it with: apt-get update && apt-get install -y sudo"); - } - await assertPlatform({ distro: "debian", supportedVersions: [13], architectures: ["amd64"] }); - if (!(await exists(`${context.options.packageDir}/systemd/hy2xs-admin.service`))) { + if (!(await fileExists(`${context.options.packageDir}/systemd/hy2xs-admin.service`))) { fail("missing hy2xs-admin systemd unit in package"); } - if (!(await exists(`${context.options.packageDir}/systemd/hysteria-server.service`))) { + if (!(await fileExists(`${context.options.packageDir}/systemd/hysteria-server.service`))) { fail("missing hysteria-server systemd unit in package"); } - if (!(await exists(`${context.options.packageDir}/templates/hysteria/config.yaml.tpl`))) { + if (!(await fileExists(`${context.options.packageDir}/templates/hysteria/config.yaml.tpl`))) { fail("missing Hysteria config template in package"); } - if (context.mode === "install" && !(await exists(`${context.options.packageDir}/ui/hy2xs-admin/hy2xs-admin`))) { + if (context.mode === "install" && !(await fileExists(`${context.options.packageDir}/ui/hy2xs-admin/hy2xs-admin`))) { fail("bundled HY2XS admin is missing from install package"); } - if (!isReconfigure && (await exists("/etc/hysteria/post-install.env"))) { + if (!(await fileExists("/usr/bin/apt-get")) && !(await fileExists("/bin/apt-get"))) { + fail("apt-get is required on target host"); + } + + if (!(await fileExists("/usr/bin/dpkg-query")) && !(await fileExists("/bin/dpkg-query"))) { + fail("dpkg-query is required on target host"); + } + + if (!isReconfigure) { + try { + await run`command -v sudo >/dev/null 2>&1`; + } catch { + info("sudo not found in preflight: installDeps step will install sudo before smoke checks"); + } + } + + if (!isReconfigure && (await fileExists("/etc/hysteria/post-install.env"))) { fail("existing HY2XS post-install.env found; update/repair is out of scope"); } - if (!isReconfigure && (await exists(context.config.installDir))) { + if (!isReconfigure && (await dirExists(context.config.installDir))) { fail("existing /opt/hy2xs-admin found; conflicting old state"); } @@ -115,7 +125,9 @@ export async function preflight(context: RuntimeContext): Promise { try { const aaaa = await run`getent ahostsv6 ${context.config.domain}`; if (aaaa.trim()) { - info(`warning: domain ${context.config.domain} has AAAA record; HY2XS remains IPv4-only`); + fail( + `domain ${context.config.domain} has AAAA record while HY2XS is IPv4-only; remove AAAA or set HY2XS_ALLOW_AAAA_WITH_IPV4_ONLY=true` + ); } } catch { // no AAAA is acceptable diff --git a/orchestrator/src/steps/smoke.ts b/orchestrator/src/steps/smoke.ts index 21d25df..cd90095 100644 --- a/orchestrator/src/steps/smoke.ts +++ b/orchestrator/src/steps/smoke.ts @@ -2,6 +2,31 @@ import type { RuntimeContext } from "../types/context"; import { info } from "../lib/log"; import { runHidden, runSecret, runVisible } from "../lib/process"; +function parseLocalAddress(line: string): string { + const cols = line.trim().split(/\s+/); + return cols[3] ?? ""; +} + +function hasTcpListener(lines: string, host: string, port: number): boolean { + return lines + .split("\n") + .filter(Boolean) + .some((line) => { + const local = parseLocalAddress(line); + return local === `${host}:${port}`; + }); +} + +function hasUdpListener(lines: string, host: string, port: number): boolean { + return lines + .split("\n") + .filter(Boolean) + .some((line) => { + const local = parseLocalAddress(line); + return local === `${host}:${port}`; + }); +} + async function retry( label: string, attempts: number, @@ -31,14 +56,64 @@ async function retry( } export async function smoke(context: RuntimeContext): Promise { - if (context.options.skipStart) { - info("service start and smoke checks skipped by flag"); + if (context.options.skipServiceStart && context.options.skipSmoke) { + info("service start and smoke checks skipped by flags"); return; } - await runVisible`systemctl restart hysteria-server hy2xs-admin`; - await runVisible`systemctl is-active --quiet hysteria-server`; - await runVisible`systemctl is-active --quiet hy2xs-admin`; + if (!context.options.skipServiceStart) { + await runVisible`systemctl restart hysteria-server hy2xs-admin`; + } else { + info("service restart skipped by flag"); + } + + await retry( + "systemd hysteria-server active", + 15, + 1000, + async () => runSecret`systemctl is-active hysteria-server || true`, + (state) => state.trim() === "active", + (state, error) => new Error(`hysteria-server is not active: ${state ?? String(error)}`), + ); + await retry( + "systemd hy2xs-admin active", + 15, + 1000, + async () => runSecret`systemctl is-active hy2xs-admin || true`, + (state) => state.trim() === "active", + (state, error) => new Error(`hy2xs-admin is not active: ${state ?? String(error)}`), + ); + + if (context.options.skipSmoke) { + info("smoke checks skipped by flag"); + return; + } + + await retry( + "ui tcp listener readiness", + 15, + 1000, + async () => runSecret`ss -H -ltn`, + (lines) => hasTcpListener(lines, context.config.uiBindHost, context.config.uiPort), + (lines, error) => new Error(`ui listener not ready on ${context.config.uiBindHost}:${context.config.uiPort}: ${lines ?? String(error)}`), + ); + await retry( + "hysteria udp listener readiness", + 15, + 1000, + async () => runSecret`ss -H -lun`, + (lines) => hasUdpListener(lines, context.config.hysteriaBindHost, context.config.hysteriaPort), + (lines, error) => new Error(`hysteria udp listener not ready on 0.0.0.0:${context.config.hysteriaPort}: ${lines ?? String(error)}`), + ); + await retry( + "admin healthz readiness", + 15, + 1000, + async () => runSecret`curl -sS --max-time 5 http://127.0.0.1:${context.config.uiPort}/healthz`, + (response) => /"ok"\s*:\s*true/.test(response), + (response, error) => new Error(`admin healthz is not ready: ${response ?? String(error)}`), + ); + await runVisible`/usr/local/bin/hysteria version`; await runVisible`test -s /etc/hysteria/config.yaml`; await runVisible`test -s /etc/hy2xs/hy2xs.env`; @@ -61,11 +136,12 @@ export async function smoke(context: RuntimeContext): Promise { await runVisible`sudo -u hy2xs-admin test ! -r /etc/hy2xs/hy2xs.env`; await runVisible`sudo -u hy2xs-admin test ! -r /etc/hy2xs/bootstrap-admin.secret`; await runVisible`sudo -u hysteria test ! -r /etc/hy2xs/bootstrap-admin.secret`; - await runVisible`ss -H -ltn | grep -q '${context.config.uiBindHost}:${context.config.uiPort} '`; if (context.config.uiBindHost === "127.0.0.1") { - await runVisible`! ss -H -ltn | grep -q '0.0.0.0:${context.config.uiPort} '`; + const tcp = await runSecret`ss -H -ltn`; + if (hasTcpListener(tcp, "0.0.0.0", context.config.uiPort)) { + throw new Error(`ui listener must not be public on 0.0.0.0:${context.config.uiPort}`); + } } - await runVisible`ss -H -lun | grep -q '0.0.0.0:${context.config.hysteriaPort} '`; await runVisible`! ss -H -ltn | grep -q '\[::\]:${context.config.uiPort} '`; await runVisible`! ss -H -lun | grep -q '\[::\]:${context.config.hysteriaPort} '`; const invalidAuthResponse = await retry( diff --git a/orchestrator/src/types/context.ts b/orchestrator/src/types/context.ts index a5b4d38..d7e908f 100644 --- a/orchestrator/src/types/context.ts +++ b/orchestrator/src/types/context.ts @@ -4,7 +4,8 @@ export type CommonOptions = { runtimeConfigPath: string; nonInteractive: boolean; skipFirewall: boolean; - skipStart: boolean; + skipServiceStart: boolean; + skipSmoke: boolean; }; export type InstallOptions = CommonOptions; @@ -12,8 +13,11 @@ export type InstallOptions = CommonOptions; export type ReconfigureOptions = CommonOptions & { dryRun: boolean; apply: boolean; + allowPartialState?: boolean; }; +export type FirewallMode = "managed" | "takeover" | "external" | "off"; + export type RunMode = "install" | "reconfigure"; export type TlsMode = "acme" | "file" | "self_signed_dev"; @@ -24,9 +28,8 @@ export type RuntimeConfig = { publicPort: number; ipv6Enabled: boolean; sshPort: number; - firewallEnabled: boolean; + firewallMode: FirewallMode; firewallStagedApply: boolean; - firewallAllowTakeover: boolean; uiBindHost: string; uiPublicAccess: boolean; uiPort: number; diff --git a/package/systemd/hy2xs-admin.service b/package/systemd/hy2xs-admin.service index 55ede3c..1384fb8 100644 --- a/package/systemd/hy2xs-admin.service +++ b/package/systemd/hy2xs-admin.service @@ -9,6 +9,7 @@ User=hy2xs-admin Group=hy2xs-admin WorkingDirectory={{INSTALL_DIR}} EnvironmentFile=/etc/hy2xs/hy2xs.env +Environment=GIN_MODE=release Environment=HUI_DATA={{DATA_DIR}}/ Environment=HUI_LOG={{LOG_DIR}} ExecStart={{INSTALL_DIR}}/hy2xs-admin -p {{UI_PORT}} diff --git a/package/templates/env/post-install.env.tpl b/package/templates/env/post-install.env.tpl index b5a6a4c..bfeb2c6 100644 --- a/package/templates/env/post-install.env.tpl +++ b/package/templates/env/post-install.env.tpl @@ -16,7 +16,7 @@ DEPLOY_DOMAIN={{DOMAIN}} PUBLIC_HOST={{PUBLIC_HOST}} PUBLIC_PORT={{PUBLIC_PORT}} SSH_PORT={{SSH_PORT}} -HY2XS_FIREWALL_ENABLED={{FIREWALL_ENABLED}} +HY2XS_FIREWALL_MODE={{FIREWALL_MODE}} HY2XS_FIREWALL_STAGED_APPLY={{FIREWALL_STAGED_APPLY}} HY2_SOURCE=official-upstream diff --git a/tools/build/build.sh b/tools/build/build.sh index 868075c..c174d5e 100755 --- a/tools/build/build.sh +++ b/tools/build/build.sh @@ -10,6 +10,8 @@ BUILD_DIR="$ROOT_DIR/tools/build" . "$BUILD_DIR/lib/deps.sh" # shellcheck source=tools/build/lib/verify.sh . "$BUILD_DIR/lib/verify.sh" +# shellcheck source=tools/build/lib/acceptance.sh +. "$BUILD_DIR/lib/acceptance.sh" # shellcheck source=tools/build/lib/package.sh . "$BUILD_DIR/lib/package.sh" @@ -18,6 +20,7 @@ main() { require_linux_debian13_amd64 require_repo_layout + verify_source_tree_policy ensure_build_dependencies ensure_toolchain diff --git a/tools/build/lib/acceptance.sh b/tools/build/lib/acceptance.sh new file mode 100644 index 0000000..1488b9f --- /dev/null +++ b/tools/build/lib/acceptance.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +run_fix20_acceptance_subset() { + local package_dir="$1" + [ -d "$package_dir" ] || fail "acceptance: package dir not found: $package_dir" + + log_step "Acceptance: package layout sanity" + [ -x "$package_dir/install.sh" ] || fail "acceptance: install.sh is missing or not executable" + [ -x "$package_dir/orchestrator/hy2xs-orchestrator" ] || fail "acceptance: orchestrator artifact is missing" + + log_step "Acceptance: orchestrator CLI help path" + "$package_dir/orchestrator/hy2xs-orchestrator" diagnostics collect --package-dir "$package_dir" >/dev/null 2>&1 || true + + log_step "Acceptance: firewall mode defaults in config" + grep -q '^HY2XS_FIREWALL_MODE=' "$package_dir/config/hy2xs.env" || fail "acceptance: HY2XS_FIREWALL_MODE missing in runtime config" + + log_step "Acceptance: systemd unit production env" + grep -q '^Environment=GIN_MODE=release$' "$package_dir/systemd/hy2xs-admin.service" || fail "acceptance: GIN_MODE=release missing" + + log_step "Acceptance: docs matrix markers" + grep -q 'Fix20 production matrix' docs/11-testing-and-acceptance.md || fail "acceptance: fix20 matrix section missing" +} + diff --git a/tools/build/lib/package.sh b/tools/build/lib/package.sh index 09e4356..f4a8690 100644 --- a/tools/build/lib/package.sh +++ b/tools/build/lib/package.sh @@ -78,6 +78,13 @@ bundle_ui() { write_metadata() { local version="$1" local build_id="$2" + local source_git_commit + local dirty_tree="false" + + source_git_commit="$(git rev-parse --short=12 HEAD 2>/dev/null || echo unknown)" + if [ -n "$(git status --porcelain 2>/dev/null || true)" ]; then + dirty_tree="true" + fi [ -f "$HYSTERIA_LOCK_FILE" ] || fail "missing Hysteria lock file: $HYSTERIA_LOCK_FILE" # shellcheck disable=SC1090 @@ -103,6 +110,9 @@ write_metadata() { printf 'target_arch=amd64\n' printf 'target_distro=debian\n' printf 'target_version=13\n' + printf 'source_git_commit=%s\n' "$source_git_commit" + printf 'dirty_tree=%s\n' "$dirty_tree" + printf 'build_profile=production\n' printf 'orchestrator_stack=Bun+TypeScript\n' printf 'go_version=%s\n' "$($GO_BIN version)" printf 'bun_version=%s\n' "$($BUN_BIN --version)" diff --git a/tools/build/lib/verify.sh b/tools/build/lib/verify.sh index 4406a78..100910e 100644 --- a/tools/build/lib/verify.sh +++ b/tools/build/lib/verify.sh @@ -17,6 +17,23 @@ require_repo_layout() { [ -f "apps/frontend/pnpm-lock.yaml" ] || fail "missing apps/frontend/pnpm-lock.yaml" } +verify_source_tree_policy() { + local allow_dirty="${ALLOW_DIRTY_BUILD:-false}" + local dirty="false" + + if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + fail "build must run inside git work tree" + fi + + if [ -n "$(git status --porcelain 2>/dev/null || true)" ]; then + dirty="true" + fi + + if [ "$dirty" = "true" ] && [ "$allow_dirty" != "true" ]; then + fail "dirty git tree is not allowed for production build; set ALLOW_DIRTY_BUILD=true to override" + fi +} + verify_archive() { local version="$1" local archive="dist/hy2xs-install-${version}.tar.gz" @@ -47,5 +64,18 @@ verify_archive() { [ -x "$tmp/hy2xs-install/install.sh" ] || fail "install.sh is not executable" [ -x "$tmp/hy2xs-install/orchestrator/hy2xs-orchestrator" ] || fail "orchestrator is not executable" [ -x "$tmp/hy2xs-install/ui/hy2xs-admin/hy2xs-admin" ] || fail "hy2xs-admin is not executable" + + "$tmp/hy2xs-install/orchestrator/hy2xs-orchestrator" status --package-dir "$tmp/hy2xs-install" >/dev/null 2>&1 || true + + local meta + meta="$(cat "$tmp/hy2xs-install/metadata/package.env")" + printf '%s\n' "$meta" | grep -q '^source_git_commit=' || fail "metadata missing source_git_commit" + printf '%s\n' "$meta" | grep -q '^dirty_tree=' || fail "metadata missing dirty_tree" + printf '%s\n' "$meta" | grep -q '^build_profile=production$' || fail "metadata missing build_profile=production" + + if declare -F run_fix20_acceptance_subset >/dev/null 2>&1; then + run_fix20_acceptance_subset "$tmp/hy2xs-install" + fi + rm -rf "$tmp" }