fix(installer): harden admin smoke and rollback cleanup
This commit is contained in:
@@ -1,17 +1,22 @@
|
||||
import type { InstallContext, InstallOptions } from "../types/context";
|
||||
import { fileExists, readText, writeTextAtomic } from "../lib/fs";
|
||||
import { runMutatingVisible } from "../lib/process";
|
||||
import { resetFailedUnit } from "../lib/systemd";
|
||||
import { info, setOperationContext, step, stepDone } from "../lib/log";
|
||||
import { readPackageValue } from "../lib/packageMeta";
|
||||
import { REPAIR_HINT, buildInstallStateRecord } from "../lib/installState";
|
||||
import { persistInstallState } from "../lib/installStateWriter";
|
||||
import { persistFailureState, runRollbackStages, type RollbackStage } from "../lib/rollback";
|
||||
import {
|
||||
persistFailureState,
|
||||
runRollbackStages,
|
||||
type RollbackStage,
|
||||
} from "../lib/rollback";
|
||||
import { parseRuntimeEnv, renderRuntimeEnv } from "../config/env";
|
||||
import {
|
||||
ORCHESTRATOR_INSTALL_DIR,
|
||||
ORCHESTRATOR_INSTALL_PATH,
|
||||
ORCHESTRATOR_SYMLINK_PATH,
|
||||
RUNTIME_PACKAGE_DIR
|
||||
RUNTIME_PACKAGE_DIR,
|
||||
} from "../config/profile";
|
||||
import { preflight } from "../steps/preflight";
|
||||
import { bootstrapRuntime } from "../steps/bootstrap";
|
||||
@@ -27,7 +32,7 @@ import {
|
||||
cleanupFirewallRollback,
|
||||
disarmFirewallRollback,
|
||||
operationKeyFor,
|
||||
rollbackFirewallNow
|
||||
rollbackFirewallNow,
|
||||
} from "../steps/firewall";
|
||||
import { ensureBootstrapAdminSecret, writePostInstallEnv } from "../steps/env";
|
||||
import { smoke } from "../steps/smoke";
|
||||
@@ -125,7 +130,7 @@ function newOwnership(): OperationOwnership {
|
||||
firewallTouched: false,
|
||||
postInstallTouched: false,
|
||||
bootstrapSecretTouched: false,
|
||||
servicesStarted: false
|
||||
servicesStarted: false,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -146,14 +151,14 @@ function installOwnedPaths(context: InstallContext): string[] {
|
||||
ORCHESTRATOR_INSTALL_DIR,
|
||||
ORCHESTRATOR_INSTALL_PATH,
|
||||
ORCHESTRATOR_SYMLINK_PATH,
|
||||
RUNTIME_PACKAGE_DIR
|
||||
RUNTIME_PACKAGE_DIR,
|
||||
];
|
||||
}
|
||||
|
||||
async function writeInstallState(
|
||||
context: InstallContext,
|
||||
phase: InstallPhase,
|
||||
lastError: string
|
||||
lastError: string,
|
||||
): Promise<void> {
|
||||
const record = buildInstallStateRecord({
|
||||
productVersion: context.packageVersion,
|
||||
@@ -168,7 +173,7 @@ async function writeInstallState(
|
||||
installed: phase === "installed",
|
||||
ownedPaths: installOwnedPaths(context),
|
||||
lastError,
|
||||
repairHint: phase === "installed" ? undefined : REPAIR_HINT
|
||||
repairHint: phase === "installed" ? undefined : REPAIR_HINT,
|
||||
});
|
||||
|
||||
await persistInstallState(record);
|
||||
@@ -178,7 +183,7 @@ async function advanceInstallState(
|
||||
context: InstallContext,
|
||||
ownership: OperationOwnership,
|
||||
phase: InstallPhase,
|
||||
lastError = ""
|
||||
lastError = "",
|
||||
): Promise<void> {
|
||||
// Флаг взводится ПЕРЕД записью, а не после неё: см. комментарий к
|
||||
// stateTouched. Частично выполненная запись маркера — это уже изменение
|
||||
@@ -211,7 +216,7 @@ async function advanceInstallState(
|
||||
export function classifyFailure(
|
||||
ownership: OperationOwnership,
|
||||
phase: InstallPhase,
|
||||
error?: unknown
|
||||
error?: unknown,
|
||||
): FailureKind {
|
||||
if (error instanceof FirewallGuardFiredError) {
|
||||
return "firewall_guard_fired";
|
||||
@@ -256,10 +261,12 @@ export function classifyFailure(
|
||||
async function rollbackFailedInstall(
|
||||
context: InstallContext,
|
||||
ownership: OperationOwnership,
|
||||
failureKind: FailureKind
|
||||
failureKind: FailureKind,
|
||||
): Promise<void> {
|
||||
if (failureKind === "fatal_pre_apply") {
|
||||
info("pre-apply failure: nothing was applied, system rollback is not required");
|
||||
info(
|
||||
"pre-apply failure: nothing was applied, system rollback is not required",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -275,7 +282,7 @@ async function rollbackFailedInstall(
|
||||
name: "firewall",
|
||||
run: async () => {
|
||||
await rollbackFirewallNow(context);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -295,23 +302,31 @@ async function rollbackFailedInstall(
|
||||
name: "stop services",
|
||||
run: async () => {
|
||||
await runMutatingVisible`systemctl stop hysteria-server hy2xs-admin`;
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "disable services",
|
||||
run: async () => {
|
||||
await runMutatingVisible`systemctl disable hysteria-server hy2xs-admin`;
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "reset failed services",
|
||||
name: "reset failed hysteria-server",
|
||||
run: async () => {
|
||||
await runMutatingVisible`systemctl reset-failed hysteria-server hy2xs-admin`;
|
||||
}
|
||||
}
|
||||
await resetFailedUnit("hysteria-server");
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "reset failed hy2xs-admin",
|
||||
run: async () => {
|
||||
await resetFailedUnit("hy2xs-admin");
|
||||
},
|
||||
},
|
||||
);
|
||||
} else {
|
||||
info("rollback: systemd units were not deployed by this operation, leaving services untouched");
|
||||
info(
|
||||
"rollback: systemd units were not deployed by this operation, leaving services untouched",
|
||||
);
|
||||
}
|
||||
|
||||
await runRollbackStages(stages);
|
||||
@@ -319,11 +334,15 @@ async function rollbackFailedInstall(
|
||||
|
||||
export async function install(options: InstallOptions): Promise<void> {
|
||||
setOperationContext(`install-${Date.now().toString(36)}`);
|
||||
const hasSourceConfig = options.sourceConfigPath ? await fileExists(options.sourceConfigPath) : false;
|
||||
const hasSourceConfig = options.sourceConfigPath
|
||||
? await fileExists(options.sourceConfigPath)
|
||||
: false;
|
||||
if (options.sourceConfigPath && !hasSourceConfig) {
|
||||
throw new Error(`config source not found: ${options.sourceConfigPath}`);
|
||||
}
|
||||
const sourceConfigPath = hasSourceConfig ? options.sourceConfigPath : `${options.packageDir}/config/hy2xs.env`;
|
||||
const sourceConfigPath = hasSourceConfig
|
||||
? options.sourceConfigPath
|
||||
: `${options.packageDir}/config/hy2xs.env`;
|
||||
const sourceConfigRaw = await readText(sourceConfigPath);
|
||||
const config = parseRuntimeEnv(sourceConfigRaw);
|
||||
|
||||
@@ -331,18 +350,48 @@ export async function install(options: InstallOptions): Promise<void> {
|
||||
mode: "install",
|
||||
options,
|
||||
config,
|
||||
packageVersion: await readPackageValue(options.packageDir, "package.version", "unknown"),
|
||||
packageBuildId: await readPackageValue(options.packageDir, "package.build_id", "unknown"),
|
||||
packageVersion: await readPackageValue(
|
||||
options.packageDir,
|
||||
"package.version",
|
||||
"unknown",
|
||||
),
|
||||
packageBuildId: await readPackageValue(
|
||||
options.packageDir,
|
||||
"package.build_id",
|
||||
"unknown",
|
||||
),
|
||||
installDate: new Date().toISOString(),
|
||||
hysteriaVersion: "unknown",
|
||||
hysteriaResolution: await readPackageValue(options.packageDir, "hysteria.resolution", "unknown"),
|
||||
hysteriaTargetVersion: await readPackageValue(options.packageDir, "hysteria.version", ""),
|
||||
hysteriaArtifactUrl: await readPackageValue(options.packageDir, "hysteria.url", ""),
|
||||
hysteriaArtifactSha256: await readPackageValue(options.packageDir, "hysteria.sha256", "")
|
||||
hysteriaResolution: await readPackageValue(
|
||||
options.packageDir,
|
||||
"hysteria.resolution",
|
||||
"unknown",
|
||||
),
|
||||
hysteriaTargetVersion: await readPackageValue(
|
||||
options.packageDir,
|
||||
"hysteria.version",
|
||||
"",
|
||||
),
|
||||
hysteriaArtifactUrl: await readPackageValue(
|
||||
options.packageDir,
|
||||
"hysteria.url",
|
||||
"",
|
||||
),
|
||||
hysteriaArtifactSha256: await readPackageValue(
|
||||
options.packageDir,
|
||||
"hysteria.sha256",
|
||||
"",
|
||||
),
|
||||
};
|
||||
|
||||
if (!context.hysteriaTargetVersion || !context.hysteriaArtifactUrl || !context.hysteriaArtifactSha256) {
|
||||
throw new Error("missing Hysteria lock metadata in package: hysteria.version/hysteria.url/hysteria.sha256");
|
||||
if (
|
||||
!context.hysteriaTargetVersion ||
|
||||
!context.hysteriaArtifactUrl ||
|
||||
!context.hysteriaArtifactSha256
|
||||
) {
|
||||
throw new Error(
|
||||
"missing Hysteria lock metadata in package: hysteria.version/hysteria.url/hysteria.sha256",
|
||||
);
|
||||
}
|
||||
|
||||
const ownership = newOwnership();
|
||||
@@ -356,7 +405,10 @@ export async function install(options: InstallOptions): Promise<void> {
|
||||
// Всё, что дальше, уже создаёт наши собственные пути, и повторная проверка
|
||||
// опознала бы их как чужую установку.
|
||||
step("preflight");
|
||||
await preflight(context, { requireCapabilities: false, checkCleanHost: true });
|
||||
await preflight(context, {
|
||||
requireCapabilities: false,
|
||||
checkCleanHost: true,
|
||||
});
|
||||
stepDone("preflight");
|
||||
|
||||
await advanceInstallState(context, ownership, "preflight_ok");
|
||||
@@ -372,7 +424,10 @@ export async function install(options: InstallOptions): Promise<void> {
|
||||
await installDeps(context);
|
||||
stepDone("system dependencies");
|
||||
step("preflight capabilities");
|
||||
await preflight(context, { requireCapabilities: true, checkCleanHost: false });
|
||||
await preflight(context, {
|
||||
requireCapabilities: true,
|
||||
checkCleanHost: false,
|
||||
});
|
||||
stepDone("preflight capabilities");
|
||||
await advanceInstallState(context, ownership, "deps_ok");
|
||||
phase = "deps_ok";
|
||||
@@ -387,7 +442,7 @@ export async function install(options: InstallOptions): Promise<void> {
|
||||
await writeTextAtomic(options.runtimeConfigPath, renderRuntimeEnv(config), {
|
||||
mode: 0o600,
|
||||
owner: "root",
|
||||
group: "root"
|
||||
group: "root",
|
||||
});
|
||||
stepDone("write runtime env");
|
||||
await advanceInstallState(context, ownership, "runtime_env_written");
|
||||
@@ -468,8 +523,13 @@ export async function install(options: InstallOptions): Promise<void> {
|
||||
try {
|
||||
await cleanupFirewallRollback(context);
|
||||
} catch (cleanupError) {
|
||||
const cleanupMessage = cleanupError instanceof Error ? cleanupError.message : String(cleanupError);
|
||||
info(`firewall rollback data cleanup failed after a successful install: ${cleanupMessage}`);
|
||||
const cleanupMessage =
|
||||
cleanupError instanceof Error
|
||||
? cleanupError.message
|
||||
: String(cleanupError);
|
||||
info(
|
||||
`firewall rollback data cleanup failed after a successful install: ${cleanupMessage}`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
@@ -498,7 +558,12 @@ export async function install(options: InstallOptions): Promise<void> {
|
||||
: "failed";
|
||||
|
||||
await persistFailureState(() =>
|
||||
advanceInstallState(context, ownership, failurePhase, `${failureKind}: ${message}`)
|
||||
advanceInstallState(
|
||||
context,
|
||||
ownership,
|
||||
failurePhase,
|
||||
`${failureKind}: ${message}`,
|
||||
),
|
||||
);
|
||||
|
||||
// Диагностика — best effort, откат — обязателен.
|
||||
@@ -512,8 +577,12 @@ export async function install(options: InstallOptions): Promise<void> {
|
||||
await diagnosticsCollect(options);
|
||||
} catch (diagnosticsError) {
|
||||
const diagnosticsMessage =
|
||||
diagnosticsError instanceof Error ? diagnosticsError.message : String(diagnosticsError);
|
||||
info(`diagnostics collection failed, continuing with rollback: ${diagnosticsMessage}`);
|
||||
diagnosticsError instanceof Error
|
||||
? diagnosticsError.message
|
||||
: String(diagnosticsError);
|
||||
info(
|
||||
`diagnostics collection failed, continuing with rollback: ${diagnosticsMessage}`,
|
||||
);
|
||||
}
|
||||
|
||||
await rollbackFailedInstall(context, ownership, failureKind);
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
import { info } from "./log";
|
||||
import {
|
||||
runMutatingStatus,
|
||||
runReadOnlyArgvStatus,
|
||||
type MutationResult,
|
||||
} from "./process";
|
||||
|
||||
export type ResetFailedDependencies = {
|
||||
reset: (unit: string) => Promise<MutationResult>;
|
||||
inspect: (unit: string) => Promise<MutationResult>;
|
||||
};
|
||||
|
||||
const productionDependencies: ResetFailedDependencies = {
|
||||
reset: async (unit) => runMutatingStatus`systemctl reset-failed ${unit}`,
|
||||
inspect: async (unit) =>
|
||||
runReadOnlyArgvStatus([
|
||||
"systemctl",
|
||||
"show",
|
||||
unit,
|
||||
"--property=LoadState",
|
||||
"--property=ActiveState",
|
||||
"--no-pager",
|
||||
]),
|
||||
};
|
||||
|
||||
function propertiesOf(output: string): Map<string, string> {
|
||||
const properties = new Map<string, string>();
|
||||
for (const line of output.split(/\r?\n/)) {
|
||||
const separator = line.indexOf("=");
|
||||
if (separator <= 0) {
|
||||
continue;
|
||||
}
|
||||
properties.set(line.slice(0, separator), line.slice(separator + 1));
|
||||
}
|
||||
return properties;
|
||||
}
|
||||
|
||||
function commandFailure(result: MutationResult): string {
|
||||
return (
|
||||
result.stderr.trim() || result.stdout.trim() || `exit ${result.exitCode}`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Сбрасывает failed-состояние юнита и доказывает postcondition наблюдением.
|
||||
*
|
||||
* `systemctl reset-failed` возвращает ненулевой код и для уже выгруженного
|
||||
* юнита. Это не отказ уборки: у такого юнита физически нет failed-состояния,
|
||||
* которое нужно было бы сбрасывать. Разбирать английское `Unit ... not loaded`
|
||||
* нельзя — текст зависит от версии и локали systemd. Поэтому код команды
|
||||
* сохраняется для диагностики, а решение принимается по ActiveState.
|
||||
*
|
||||
* Ошибка чтения состояния не маскируется. Если systemd недоступен либо юнит всё
|
||||
* ещё `failed`, rollback обязан оставить оператору настоящее предупреждение.
|
||||
*/
|
||||
export async function resetFailedUnit(
|
||||
unit: string,
|
||||
dependencies: ResetFailedDependencies = productionDependencies,
|
||||
): Promise<void> {
|
||||
const reset = await dependencies.reset(unit);
|
||||
const observed = await dependencies.inspect(unit);
|
||||
if (observed.exitCode !== 0) {
|
||||
throw new Error(
|
||||
`cannot verify systemd state for ${unit} after reset-failed: ${commandFailure(observed)}`,
|
||||
);
|
||||
}
|
||||
|
||||
const properties = propertiesOf(observed.stdout);
|
||||
const loadState = properties.get("LoadState");
|
||||
const activeState = properties.get("ActiveState");
|
||||
if (!loadState || !activeState) {
|
||||
throw new Error(
|
||||
`systemctl show ${unit} did not return LoadState and ActiveState after reset-failed`,
|
||||
);
|
||||
}
|
||||
if (activeState === "failed") {
|
||||
throw new Error(
|
||||
`systemd unit ${unit} remains failed after reset-failed (load state: ${loadState}; ` +
|
||||
`command: ${commandFailure(reset)})`,
|
||||
);
|
||||
}
|
||||
|
||||
if (reset.exitCode !== 0) {
|
||||
info(
|
||||
`systemctl reset-failed ${unit} exited with ${reset.exitCode}, but the cleanup ` +
|
||||
`postcondition is satisfied: LoadState=${loadState}, ActiveState=${activeState}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
+216
-64
@@ -3,8 +3,16 @@ import type { RuntimeContext } from "../types/context";
|
||||
import { info } from "../lib/log";
|
||||
import { readText } from "../lib/fs";
|
||||
import { parseEnvFile } from "../lib/envFile";
|
||||
import { runReadOnly, runReadOnlySecret, runMutatingVisible } from "../lib/process";
|
||||
import { ADMIN_LOGIN_PATH, HYSTERIA_MACHINE_AUTH_PATH, hysteriaMachineAuthUrl } from "../config/profile";
|
||||
import {
|
||||
runReadOnly,
|
||||
runReadOnlySecret,
|
||||
runMutatingVisible,
|
||||
} from "../lib/process";
|
||||
import {
|
||||
ADMIN_LOGIN_PATH,
|
||||
HYSTERIA_MACHINE_AUTH_PATH,
|
||||
hysteriaMachineAuthUrl,
|
||||
} from "../config/profile";
|
||||
import { assertHysteriaConfigMatchesProfile } from "./configAssertions";
|
||||
import { assertEffectiveFirewallIsOurs } from "./firewall";
|
||||
|
||||
@@ -82,7 +90,8 @@ export async function smoke(context: RuntimeContext): Promise<void> {
|
||||
1000,
|
||||
async () => runReadOnlySecret`systemctl is-active hysteria-server || true`,
|
||||
(state) => state.trim() === "active",
|
||||
(state, error) => new Error(`hysteria-server is not active: ${state ?? String(error)}`),
|
||||
(state, error) =>
|
||||
new Error(`hysteria-server is not active: ${state ?? String(error)}`),
|
||||
);
|
||||
await retry(
|
||||
"systemd hy2xs-admin active",
|
||||
@@ -90,7 +99,8 @@ export async function smoke(context: RuntimeContext): Promise<void> {
|
||||
1000,
|
||||
async () => runReadOnlySecret`systemctl is-active hy2xs-admin || true`,
|
||||
(state) => state.trim() === "active",
|
||||
(state, error) => new Error(`hy2xs-admin is not active: ${state ?? String(error)}`),
|
||||
(state, error) =>
|
||||
new Error(`hy2xs-admin is not active: ${state ?? String(error)}`),
|
||||
);
|
||||
|
||||
if (context.options.skipSmoke) {
|
||||
@@ -103,24 +113,38 @@ export async function smoke(context: RuntimeContext): Promise<void> {
|
||||
15,
|
||||
1000,
|
||||
async () => runReadOnlySecret`ss -H -ltn`,
|
||||
(lines) => hasTcpListener(lines, context.config.uiBindHost, context.config.uiPort),
|
||||
(lines, error) => new Error(`ui listener not ready on ${context.config.uiBindHost}:${context.config.uiPort}: ${lines ?? String(error)}`),
|
||||
(lines) =>
|
||||
hasTcpListener(lines, context.config.uiBindHost, context.config.uiPort),
|
||||
(lines, error) =>
|
||||
new Error(
|
||||
`ui listener not ready on ${context.config.uiBindHost}:${context.config.uiPort}: ${lines ?? String(error)}`,
|
||||
),
|
||||
);
|
||||
await retry(
|
||||
"hysteria udp listener readiness",
|
||||
15,
|
||||
1000,
|
||||
async () => runReadOnlySecret`ss -H -lun`,
|
||||
(lines) => hasUdpListener(lines, context.config.hysteriaBindHost, context.config.hysteriaPort),
|
||||
(lines, error) => new Error(`hysteria udp listener not ready on 0.0.0.0:${context.config.hysteriaPort}: ${lines ?? String(error)}`),
|
||||
(lines) =>
|
||||
hasUdpListener(
|
||||
lines,
|
||||
context.config.hysteriaBindHost,
|
||||
context.config.hysteriaPort,
|
||||
),
|
||||
(lines, error) =>
|
||||
new Error(
|
||||
`hysteria udp listener not ready on 0.0.0.0:${context.config.hysteriaPort}: ${lines ?? String(error)}`,
|
||||
),
|
||||
);
|
||||
await retry(
|
||||
"admin healthz readiness",
|
||||
15,
|
||||
1000,
|
||||
async () => runReadOnlySecret`curl -sS --max-time 5 http://127.0.0.1:${context.config.uiPort}/healthz`,
|
||||
async () =>
|
||||
runReadOnlySecret`curl -sS --max-time 5 http://127.0.0.1:${context.config.uiPort}/healthz`,
|
||||
(response) => /"ok"\s*:\s*true/.test(response),
|
||||
(response, error) => new Error(`admin healthz is not ready: ${response ?? String(error)}`),
|
||||
(response, error) =>
|
||||
new Error(`admin healthz is not ready: ${response ?? String(error)}`),
|
||||
);
|
||||
|
||||
// Всё, что ниже, — НАБЛЮДЕНИЕ, и оно выполняется read-only раннерами.
|
||||
@@ -131,7 +155,9 @@ export async function smoke(context: RuntimeContext): Promise<void> {
|
||||
// не меняет диагностируемую систему» невозможно было включить guard'ом — он
|
||||
// отказал бы на первой же читающей команде. Классификация здесь — часть
|
||||
// контракта, а не стиль.
|
||||
info(`installed Hysteria: ${await runReadOnly`/usr/local/bin/hysteria version`}`);
|
||||
info(
|
||||
`installed Hysteria: ${await runReadOnly`/usr/local/bin/hysteria version`}`,
|
||||
);
|
||||
await runReadOnly`test -s /etc/hysteria/config.yaml`;
|
||||
await runReadOnly`test -s /etc/hy2xs/hy2xs.env`;
|
||||
await runReadOnly`test -s /etc/hysteria/post-install.env`;
|
||||
@@ -156,7 +182,9 @@ export async function smoke(context: RuntimeContext): Promise<void> {
|
||||
if (context.config.uiBindHost === "127.0.0.1") {
|
||||
const tcp = await runReadOnlySecret`ss -H -ltn`;
|
||||
if (hasTcpListener(tcp, "0.0.0.0", context.config.uiPort)) {
|
||||
throw new Error(`ui listener must not be public on 0.0.0.0:${context.config.uiPort}`);
|
||||
throw new Error(
|
||||
`ui listener must not be public on 0.0.0.0:${context.config.uiPort}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
await runReadOnly`! ss -H -ltn | grep -q '\[::\]:${context.config.uiPort} '`;
|
||||
@@ -167,47 +195,67 @@ export async function smoke(context: RuntimeContext): Promise<void> {
|
||||
const machineAuthUrlNoToken = `http://127.0.0.1:${context.config.uiPort}${HYSTERIA_MACHINE_AUTH_PATH}`;
|
||||
const machineAuthUrl = hysteriaMachineAuthUrl(
|
||||
context.config.uiPort,
|
||||
context.config.hysteriaTrafficStatsSecret
|
||||
context.config.hysteriaTrafficStatsSecret,
|
||||
);
|
||||
|
||||
const missingTokenAuthCode = await runReadOnlySecret`curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -X POST -H 'Content-Type: application/json' --data '{"addr":"127.0.0.1:12345","auth":"invalid","tx":0}' ${machineAuthUrlNoToken}`;
|
||||
const missingTokenAuthCode =
|
||||
await runReadOnlySecret`curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -X POST -H 'Content-Type: application/json' --data '{"addr":"127.0.0.1:12345","auth":"invalid","tx":0}' ${machineAuthUrlNoToken}`;
|
||||
if (missingTokenAuthCode.trim() !== "403") {
|
||||
throw new Error(`unexpected auth status without machine token: ${missingTokenAuthCode}`);
|
||||
throw new Error(
|
||||
`unexpected auth status without machine token: ${missingTokenAuthCode}`,
|
||||
);
|
||||
}
|
||||
const invalidAuthResponse = await retry(
|
||||
"auth invalid credentials",
|
||||
5,
|
||||
1000,
|
||||
async () => runReadOnlySecret`curl -sS --max-time 5 -X POST -H 'Content-Type: application/json' --data '{"addr":"127.0.0.1:12345","auth":"invalid","tx":0}' ${machineAuthUrl}`,
|
||||
async () =>
|
||||
runReadOnlySecret`curl -sS --max-time 5 -X POST -H 'Content-Type: application/json' --data '{"addr":"127.0.0.1:12345","auth":"invalid","tx":0}' ${machineAuthUrl}`,
|
||||
(response) => /"ok"\s*:\s*false/.test(response),
|
||||
(response, error) => new Error(`unexpected auth response for invalid credentials: ${response ?? String(error)}`),
|
||||
(response, error) =>
|
||||
new Error(
|
||||
`unexpected auth response for invalid credentials: ${response ?? String(error)}`,
|
||||
),
|
||||
);
|
||||
|
||||
for (let i = 0; i < 10; i += 1) {
|
||||
const response = await runReadOnlySecret`curl -sS --max-time 5 -X POST -H 'Content-Type: application/json' --data '{"addr":"127.0.0.1:12345","auth":"invalid","tx":0}' ${machineAuthUrl}`;
|
||||
const response =
|
||||
await runReadOnlySecret`curl -sS --max-time 5 -X POST -H 'Content-Type: application/json' --data '{"addr":"127.0.0.1:12345","auth":"invalid","tx":0}' ${machineAuthUrl}`;
|
||||
if (!/"ok"\s*:\s*false/.test(response)) {
|
||||
throw new Error(`unexpected auth response during rate-limit smoke: ${response}`);
|
||||
throw new Error(
|
||||
`unexpected auth response during rate-limit smoke: ${response}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const invalidTypeAuthCode = await runReadOnlySecret`curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -X POST -H 'Content-Type: application/json' --data '{"addr":"127.0.0.1:12345","auth":"invalid","tx":"0"}' ${machineAuthUrl}`;
|
||||
const invalidTypeAuthCode =
|
||||
await runReadOnlySecret`curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -X POST -H 'Content-Type: application/json' --data '{"addr":"127.0.0.1:12345","auth":"invalid","tx":"0"}' ${machineAuthUrl}`;
|
||||
if (invalidTypeAuthCode.trim() !== "400") {
|
||||
throw new Error(`unexpected auth status for tx as string: ${invalidTypeAuthCode}`);
|
||||
throw new Error(
|
||||
`unexpected auth status for tx as string: ${invalidTypeAuthCode}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (context.mode === "install") {
|
||||
const adminConPass = (await readBootstrapAdminSecret(context)).ADMIN_CON_PASS ?? "";
|
||||
const adminConPass =
|
||||
(await readBootstrapAdminSecret(context)).ADMIN_CON_PASS ?? "";
|
||||
if (!adminConPass) {
|
||||
throw new Error("admin connection password is empty in bootstrap secret file");
|
||||
throw new Error(
|
||||
"admin connection password is empty in bootstrap secret file",
|
||||
);
|
||||
}
|
||||
|
||||
await retry(
|
||||
"auth valid credentials",
|
||||
10,
|
||||
1000,
|
||||
async () => runReadOnlySecret`curl -sS --max-time 5 -X POST -H 'Content-Type: application/json' --data '{"addr":"127.0.0.1:12345","auth":"${adminConPass}","tx":0}' ${machineAuthUrl}`,
|
||||
async () =>
|
||||
runReadOnlySecret`curl -sS --max-time 5 -X POST -H 'Content-Type: application/json' --data '{"addr":"127.0.0.1:12345","auth":"${adminConPass}","tx":0}' ${machineAuthUrl}`,
|
||||
(response) => /"ok"\s*:\s*true/.test(response),
|
||||
(response, error) => new Error(`unexpected auth response for valid credentials: ${response ?? String(error)}`),
|
||||
(response, error) =>
|
||||
new Error(
|
||||
`unexpected auth response for valid credentials: ${response ?? String(error)}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -217,13 +265,20 @@ export async function smoke(context: RuntimeContext): Promise<void> {
|
||||
"trafficStats valid secret",
|
||||
10,
|
||||
1000,
|
||||
async () => runReadOnlySecret`curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Authorization: ${context.config.hysteriaTrafficStatsSecret}' http://127.0.0.1:${context.config.hysteriaTrafficStatsPort}/online`,
|
||||
async () =>
|
||||
runReadOnlySecret`curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Authorization: ${context.config.hysteriaTrafficStatsSecret}' http://127.0.0.1:${context.config.hysteriaTrafficStatsPort}/online`,
|
||||
(code) => /^2\d\d$/.test(code.trim()),
|
||||
(code, error) => new Error(`unexpected trafficStats status for valid secret: ${code ?? String(error)}`),
|
||||
(code, error) =>
|
||||
new Error(
|
||||
`unexpected trafficStats status for valid secret: ${code ?? String(error)}`,
|
||||
),
|
||||
);
|
||||
const deniedCode = await runReadOnlySecret`curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Authorization: invalid-hy2xs-secret' http://127.0.0.1:${context.config.hysteriaTrafficStatsPort}/online`;
|
||||
const deniedCode =
|
||||
await runReadOnlySecret`curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Authorization: invalid-hy2xs-secret' http://127.0.0.1:${context.config.hysteriaTrafficStatsPort}/online`;
|
||||
if (!/(401|403)/.test(deniedCode)) {
|
||||
throw new Error(`unexpected trafficStats status for invalid secret: ${deniedCode}`);
|
||||
throw new Error(
|
||||
`unexpected trafficStats status for invalid secret: ${deniedCode}`,
|
||||
);
|
||||
}
|
||||
|
||||
// `nft -c` только разбирает файл и правил не применяет — это проверка
|
||||
@@ -237,13 +292,18 @@ export async function smoke(context: RuntimeContext): Promise<void> {
|
||||
// синтаксически валидный — ruleset, проверка проходила зелёной, и операция
|
||||
// объявляла успешной установку, работающую на firewall, который она же
|
||||
// только что заменила.
|
||||
info("verifying that the effective firewall is the one generated for this configuration");
|
||||
info(
|
||||
"verifying that the effective firewall is the one generated for this configuration",
|
||||
);
|
||||
await assertEffectiveFirewallIsOurs(context);
|
||||
|
||||
// Семантическая проверка установленного конфига: разбираем YAML и сверяем
|
||||
// с production-профилем, а не ищем подстроки.
|
||||
info("verifying effective Hysteria config against HY2XS production profile");
|
||||
assertHysteriaConfigMatchesProfile(await readText("/etc/hysteria/config.yaml"), context.config);
|
||||
assertHysteriaConfigMatchesProfile(
|
||||
await readText("/etc/hysteria/config.yaml"),
|
||||
context.config,
|
||||
);
|
||||
|
||||
await assertEffectiveHysteriaVersion(context);
|
||||
}
|
||||
@@ -296,25 +356,23 @@ async function assertAdminLoginWorks(context: RuntimeContext): Promise<void> {
|
||||
// доступа. Логин берётся настоящий — тот же, что у администратора, — чтобы
|
||||
// запрос шёл тем же путём, что и реальный вход, и доходил до проверки пароля.
|
||||
const rejectedPassword = randomBytes(24).toString("base64url");
|
||||
const rejectedPayload = JSON.stringify({
|
||||
username: context.config.adminUser,
|
||||
pass: rejectedPassword
|
||||
});
|
||||
const rejectedBody = await retry(
|
||||
"admin login rejects wrong credentials",
|
||||
10,
|
||||
1000,
|
||||
async () =>
|
||||
runReadOnlySecret`curl -sS --max-time 5 -X POST -H 'Content-Type: application/json' --data ${rejectedPayload} ${loginUrl}`,
|
||||
requestAdminLogin(loginUrl, context.config.adminUser, rejectedPassword),
|
||||
(body) => isRejectedLogin(body),
|
||||
(body, error) =>
|
||||
new Error(
|
||||
`admin login did not reject invalid credentials: ${describeRejectionFailure(body, error)}\n` +
|
||||
`Панель обязана отвечать конвертом отказа с причиной ${INVALID_CREDENTIALS_CODE}. ` +
|
||||
`Отсутствие такого конверта означает, что запрос не доживает до проверки учётных данных.`
|
||||
)
|
||||
`Отсутствие такого конверта означает, что запрос не доживает до проверки учётных данных.`,
|
||||
),
|
||||
);
|
||||
info(
|
||||
`admin login rejects wrong credentials: ${describeRejection(rejectedBody)}`,
|
||||
);
|
||||
info(`admin login rejects wrong credentials: ${describeRejection(rejectedBody)}`);
|
||||
|
||||
if (context.mode !== "install") {
|
||||
return;
|
||||
@@ -341,27 +399,107 @@ async function assertAdminLoginWorks(context: RuntimeContext): Promise<void> {
|
||||
// пароля с настоящим невероятно, но проверить это здесь можно точно, и тогда
|
||||
// «отвергнуто» гарантированно означает «отвергнуто», а не «не совпало».
|
||||
if (rejectedPassword === adminPassword) {
|
||||
throw new Error("negative login probe accidentally used the real admin password");
|
||||
throw new Error(
|
||||
"negative login probe accidentally used the real admin password",
|
||||
);
|
||||
}
|
||||
|
||||
const payload = JSON.stringify({ username: adminUser, pass: adminPassword });
|
||||
const response = await retry(
|
||||
"admin login with bootstrap credentials",
|
||||
10,
|
||||
1000,
|
||||
async () =>
|
||||
runReadOnlySecret`curl -sS --max-time 5 -X POST -H 'Content-Type: application/json' --data ${payload} ${loginUrl}`,
|
||||
async () => requestAdminLogin(loginUrl, adminUser, adminPassword),
|
||||
(body) => isSuccessfulLogin(body),
|
||||
(body, error) =>
|
||||
new Error(
|
||||
`admin panel refused the bootstrap login it created itself: ${describeLoginFailure(body, error)}\n` +
|
||||
`Порт открыт и /healthz отвечает, но войти в панель нельзя — установка не считается выполненной.`
|
||||
)
|
||||
`Порт открыт и /healthz отвечает, но войти в панель нельзя — установка не считается выполненной.`,
|
||||
),
|
||||
);
|
||||
|
||||
info(`admin login accepted: ${describeIssuedToken(response)}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* UA login-smoke называется своим именем и не маскируется под браузер.
|
||||
*
|
||||
* curl по умолчанию отправляет `curl/<version>`, а production middleware
|
||||
* осознанно отклоняет scanner-like UA до разбора DTO. Без явного значения
|
||||
* установщик проверял не вход, а собственную несовместимость с middleware.
|
||||
*/
|
||||
export const ADMIN_LOGIN_SMOKE_USER_AGENT = "HY2XS-Installer/1.0";
|
||||
|
||||
export type AdminLoginRequest = {
|
||||
contentType: "application/json";
|
||||
userAgent: string;
|
||||
body: string;
|
||||
};
|
||||
|
||||
/** Единственный wire-контракт обеих login-проб. */
|
||||
export function buildAdminLoginRequest(
|
||||
username: string,
|
||||
pass: string,
|
||||
): AdminLoginRequest {
|
||||
return {
|
||||
contentType: "application/json",
|
||||
userAgent: ADMIN_LOGIN_SMOKE_USER_AGENT,
|
||||
body: JSON.stringify({ username, pass }),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Отправляет login-пробу без утечки тела в текст ошибки.
|
||||
*
|
||||
* Positive и negative smoke намеренно проходят через одну функцию: заголовки,
|
||||
* имя wire-поля и настройки транспорта не могут разойтись между пробами.
|
||||
*/
|
||||
async function requestAdminLogin(
|
||||
loginUrl: string,
|
||||
username: string,
|
||||
pass: string,
|
||||
): Promise<string> {
|
||||
const request = buildAdminLoginRequest(username, pass);
|
||||
return runReadOnlySecret`curl -sS --max-time 5 --request POST --header ${`Content-Type: ${request.contentType}`} --user-agent ${request.userAgent} --data ${request.body} ${loginUrl}`;
|
||||
}
|
||||
|
||||
type JsonObject = Record<string, unknown>;
|
||||
|
||||
function objectOrNull(value: unknown): JsonObject | null {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
? (value as JsonObject)
|
||||
: null;
|
||||
}
|
||||
|
||||
function parseLoginEnvelope(body: string): JsonObject | null {
|
||||
try {
|
||||
return objectOrNull(JSON.parse(body));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function accessTokenOf(envelope: JsonObject | null): string | null {
|
||||
const data = objectOrNull(envelope?.data);
|
||||
return typeof data?.accessToken === "string" && data.accessToken.length > 0
|
||||
? data.accessToken
|
||||
: null;
|
||||
}
|
||||
|
||||
function carriesAccessToken(envelope: JsonObject | null): boolean {
|
||||
const data = objectOrNull(envelope?.data);
|
||||
return data !== null && Object.hasOwn(data, "accessToken") && data.accessToken !== null;
|
||||
}
|
||||
|
||||
function rejectionCodesOf(envelope: JsonObject | null): string[] {
|
||||
if (!Array.isArray(envelope?.errors)) {
|
||||
return [];
|
||||
}
|
||||
return envelope.errors.flatMap((error) => {
|
||||
const item = objectOrNull(error);
|
||||
return typeof item?.code === "string" ? [item.code] : [];
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Успех определяется по КОНВЕРТУ, а не по коду HTTP.
|
||||
*
|
||||
@@ -372,8 +510,9 @@ async function assertAdminLoginWorks(context: RuntimeContext): Promise<void> {
|
||||
* Выданный токен требуется отдельно: `code: 20000` без `accessToken` означал бы
|
||||
* панель, которая пускает и не выдаёт сессию.
|
||||
*/
|
||||
function isSuccessfulLogin(body: string): boolean {
|
||||
return /"code"\s*:\s*20000/.test(body) && /"accessToken"\s*:\s*"[^"]+"/.test(body);
|
||||
export function isSuccessfulLogin(body: string): boolean {
|
||||
const envelope = parseLoginEnvelope(body);
|
||||
return envelope?.code === 20000 && accessTokenOf(envelope) !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -398,11 +537,12 @@ const INVALID_CREDENTIALS_CODE = "invalid_credentials";
|
||||
* скажем, по недоступной базе;
|
||||
* отсутствие accessToken — панель не выдала сессию.
|
||||
*/
|
||||
function isRejectedLogin(body: string): boolean {
|
||||
export function isRejectedLogin(body: string): boolean {
|
||||
const envelope = parseLoginEnvelope(body);
|
||||
return (
|
||||
/"code"\s*:\s*50000/.test(body) &&
|
||||
new RegExp(`"code"\\s*:\\s*"${INVALID_CREDENTIALS_CODE}"`).test(body) &&
|
||||
!/"accessToken"\s*:\s*"[^"]+"/.test(body)
|
||||
envelope?.code === 50000 &&
|
||||
rejectionCodesOf(envelope).includes(INVALID_CREDENTIALS_CODE) &&
|
||||
!carriesAccessToken(envelope)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -414,11 +554,14 @@ function isRejectedLogin(body: string): boolean {
|
||||
* если панель по недоразумению впустила, в ответе лежит токен доступа, а этот
|
||||
* текст уезжает в журнал установки и в diagnostics-бандл.
|
||||
*/
|
||||
function describeRejectionFailure(body: string | undefined, error: unknown): string {
|
||||
function describeRejectionFailure(
|
||||
body: string | undefined,
|
||||
error: unknown,
|
||||
): string {
|
||||
if (body === undefined) {
|
||||
return `запрос не выполнен: ${String(error)}`;
|
||||
}
|
||||
if (/"accessToken"\s*:\s*"[^"]+"/.test(body)) {
|
||||
if (carriesAccessToken(parseLoginEnvelope(body))) {
|
||||
return "панель ВЫДАЛА токен доступа на заведомо неверные учётные данные";
|
||||
}
|
||||
return describeRejection(body);
|
||||
@@ -426,9 +569,9 @@ function describeRejectionFailure(body: string | undefined, error: unknown): str
|
||||
|
||||
/** Как выглядит отказ: код конверта и код причины, без тела. */
|
||||
function describeRejection(body: string): string {
|
||||
const envelope = body.match(/"code"\s*:\s*(\d+)/);
|
||||
const reason = body.match(/"code"\s*:\s*"([a-z_]+)"/);
|
||||
return `code=${envelope ? envelope[1] : "нет"}, причина=${reason ? reason[1] : "нет"}, токен не выдан`;
|
||||
const envelope = parseLoginEnvelope(body);
|
||||
const reason = rejectionCodesOf(envelope)[0];
|
||||
return `code=${typeof envelope?.code === "number" ? envelope.code : "нет"}, причина=${reason ?? "нет"}, токен не выдан`;
|
||||
}
|
||||
|
||||
// Читает /etc/hy2xs/bootstrap-admin.secret тем же парсером, которым он написан.
|
||||
@@ -441,7 +584,9 @@ function describeRejection(body: string): string {
|
||||
// Комментарий записан строчными `//`, а не блоком: скан релизных гейтов
|
||||
// отбрасывает только их, и объяснение, называющее убранную конструкцию по
|
||||
// имени, иначе роняет проверку «этой конструкции здесь больше нет».
|
||||
async function readBootstrapAdminSecret(context: RuntimeContext): Promise<Record<string, string>> {
|
||||
async function readBootstrapAdminSecret(
|
||||
context: RuntimeContext,
|
||||
): Promise<Record<string, string>> {
|
||||
return parseEnvFile(await readText(context.config.bootstrapAdminSecretPath));
|
||||
}
|
||||
|
||||
@@ -452,21 +597,26 @@ async function readBootstrapAdminSecret(context: RuntimeContext): Promise<Record
|
||||
* текст этой ошибки уезжает в журнал установки и в diagnostics-бандл, который
|
||||
* операторы пересылают в переписке. Поэтому наружу выдаётся только код ответа.
|
||||
*/
|
||||
function describeLoginFailure(body: string | undefined, error: unknown): string {
|
||||
function describeLoginFailure(
|
||||
body: string | undefined,
|
||||
error: unknown,
|
||||
): string {
|
||||
if (body === undefined) {
|
||||
return `запрос не выполнен: ${String(error)}`;
|
||||
}
|
||||
const code = body.match(/"code"\s*:\s*(\d+)/);
|
||||
if (code) {
|
||||
return `ответ с code=${code[1]} и без токена доступа`;
|
||||
const envelope = parseLoginEnvelope(body);
|
||||
if (typeof envelope?.code === "number") {
|
||||
return `ответ с code=${envelope.code} и без токена доступа`;
|
||||
}
|
||||
return "ответ не является конвертом API админки";
|
||||
}
|
||||
|
||||
/** Подтверждение выдачи токена без самого токена. */
|
||||
function describeIssuedToken(body: string): string {
|
||||
const tokenType = body.match(/"tokenType"\s*:\s*"([^"]*)"/);
|
||||
return tokenType ? `выдан токен типа ${tokenType[1]}` : "выдан токен доступа";
|
||||
const data = objectOrNull(parseLoginEnvelope(body)?.data);
|
||||
return typeof data?.tokenType === "string" && data.tokenType.length > 0
|
||||
? `выдан токен типа ${data.tokenType}`
|
||||
: "выдан токен доступа";
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -474,7 +624,9 @@ function describeIssuedToken(body: string): string {
|
||||
* пакета. На reconfigure metadata может относиться к другому пакету, поэтому
|
||||
* расхождение там — предупреждение, а не отказ.
|
||||
*/
|
||||
async function assertEffectiveHysteriaVersion(context: RuntimeContext): Promise<void> {
|
||||
async function assertEffectiveHysteriaVersion(
|
||||
context: RuntimeContext,
|
||||
): Promise<void> {
|
||||
const packagedVersion = context.hysteriaVersion.trim();
|
||||
if (!packagedVersion || packagedVersion === "unknown") {
|
||||
return;
|
||||
|
||||
Reference in New Issue
Block a user