Skip to content

Commit 71cfc59

Browse files
myasnikovdaniillexfrei
authored andcommitted
fix(e2e): fail closed on unknown Chainsaw suites
Validate each direct Chainsaw suite name against the runnable inventory before unioning selections. This prevents valid direct, graph, or broad neighbours from hiding deleted or renamed suite paths. Assisted-By: LLM Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent 0ed9194 commit 71cfc59

2 files changed

Lines changed: 75 additions & 9 deletions

File tree

‎hack/select-e2e.sh‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,11 @@
1515
# tier: every suite except kubernetes-previous, for the build
1616
# inputs in broad_suite_pattern
1717
# - full list any path that affects all tests, OR an unrecognised
18-
# packages/* path, OR a changed package that reaches no
19-
# runnable suite through the graph, OR a path matching NEITHER
20-
# the full-suite nor the inert list, OR a yq that failed to
21-
# build the dependency graph (conservative fallbacks)
18+
# Chainsaw suite, OR an unrecognised packages/* path, OR a
19+
# changed package that reaches no runnable suite through the
20+
# graph, OR a path matching NEITHER the full-suite nor the
21+
# inert list, OR a yq that failed to build the dependency
22+
# graph (conservative fallbacks)
2223
# - nothing, and the suite list itself is unavailable: find failed, or
2324
# a non-zero hack/e2e-chainsaw holds no suite file. Unlike the
2425
# exit yq case there is no fallback left to take — an empty list

‎hack/select-e2e_test.bats‎

Lines changed: 70 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -441,17 +441,16 @@ assert_broad_tier() {
441441
rm -rf "$tmp"
442442
}
443443

444-
# Regression, found in independent review of this change: the broad tier used to
445-
# resolve BEFORE the unresolved-suite backstop, so an unrelated go.mod bump in
446-
# the same diff swallowed that per-path escalation and its stderr line. That is
447-
# the merge-before-escalate shape #3330 removed from the graph walk.
444+
# A broad path must not weaken an unknown path's per-path escalation. This small
445+
# case keeps the original regression trigger visible beside the full mixed-path
446+
# matrix below.
448447
@test "an unresolved suite still escalates when a broad path is in the diff" {
449448
tmp=$(mktemp -d)
450449
cp -r packages/core/platform/sources "$tmp/sources"
451450
printf 'go.mod\nhack/e2e-chainsaw/nosuch/a.yaml\n' > "$tmp/diff"
452451
output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources" 2>"$tmp/err")
453452
assert_full_suite "$output"
454-
grep -q 'no runnable suite is named' "$tmp/err" \
453+
grep -Fq "'hack/e2e-chainsaw/nosuch/a.yaml' names no runnable suite ('nosuch')" "$tmp/err" \
455454
|| { echo "expected the unresolved-suite reason on stderr, got: $(cat "$tmp/err")" >&2; exit 1; }
456455
rm -rf "$tmp"
457456
}
@@ -1298,6 +1297,72 @@ assert_broad_tier() {
12981297
rm -rf "$tmp"
12991298
}
13001299

1300+
@test "an unknown Chainsaw suite path escalates independently of valid neighbours" {
1301+
# A deleted suite path and the old side of a suite rename both name a
1302+
# directory absent from the runnable inventory. Each must force a full run
1303+
# on its own account: merging names before checking membership lets a valid
1304+
# direct or graph-selected neighbour hide the unknown one. A broad path is
1305+
# another neighbour, and its 20-suite answer must not pre-empt the full run.
1306+
tmp=$(mktemp -d)
1307+
cp -r packages/core/platform/sources "$tmp/sources"
1308+
[ ! -d hack/e2e-chainsaw/deleted-suite ]
1309+
[ ! -d hack/e2e-chainsaw/renamed-postgres ]
1310+
1311+
for neighbour in \
1312+
hack/e2e-chainsaw/postgres/chainsaw-test.yaml \
1313+
packages/apps/postgres/Chart.yaml; do
1314+
for broad in '' go.mod; do
1315+
for order in unknown-first unknown-last; do
1316+
unknown=hack/e2e-chainsaw/deleted-suite/chainsaw-test.yaml
1317+
if [ "$order" = unknown-first ]; then
1318+
printf '%s\n' "$unknown" "$neighbour" "$broad" > "$tmp/diff"
1319+
else
1320+
printf '%s\n' "$broad" "$neighbour" "$unknown" > "$tmp/diff"
1321+
fi
1322+
output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources" 2>"$tmp/err")
1323+
assert_full_suite "$output"
1324+
if ! grep -Fq "'$unknown' names no runnable suite ('deleted-suite')" "$tmp/err"; then
1325+
echo "unknown path must explain the full run ($neighbour, ${broad:-no broad}, $order); stderr was:" >&2
1326+
cat "$tmp/err" >&2
1327+
exit 1
1328+
fi
1329+
done
1330+
done
1331+
done
1332+
1333+
# A rename diff contains both the old unknown directory and the new valid
1334+
# one. The old side still escalates even though the new side selects a suite.
1335+
old=hack/e2e-chainsaw/renamed-postgres/chainsaw-test.yaml
1336+
printf '%s\n' "$old" hack/e2e-chainsaw/postgres/chainsaw-test.yaml > "$tmp/diff"
1337+
output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources" 2>"$tmp/err")
1338+
assert_full_suite "$output"
1339+
grep -Fq "'$old' names no runnable suite ('renamed-postgres')" "$tmp/err"
1340+
rm -rf "$tmp"
1341+
}
1342+
1343+
@test "known and disabled Chainsaw paths keep their scoped behavior" {
1344+
tmp=$(mktemp -d)
1345+
cp -r packages/core/platform/sources "$tmp/sources"
1346+
1347+
echo hack/e2e-chainsaw/postgres/chainsaw-test.yaml > "$tmp/diff"
1348+
output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources" 2>"$tmp/err")
1349+
assert_selection "a known direct path must stay scoped" "$output" postgres
1350+
[ ! -s "$tmp/err" ]
1351+
1352+
# Disabled files are inert even when their directory names no runnable
1353+
# suite, alone and beside a valid direct path.
1354+
disabled=hack/e2e-chainsaw/no-such-suite/chainsaw-test.yaml.disabled
1355+
echo "$disabled" > "$tmp/diff"
1356+
output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources" 2>"$tmp/err")
1357+
[ -z "$output" ]
1358+
[ ! -s "$tmp/err" ]
1359+
printf '%s\n' "$disabled" hack/e2e-chainsaw/postgres/chainsaw-test.yaml > "$tmp/diff"
1360+
output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources" 2>"$tmp/err")
1361+
assert_selection "a disabled neighbour must stay inert" "$output" postgres
1362+
[ ! -s "$tmp/err" ]
1363+
rm -rf "$tmp"
1364+
}
1365+
13011366
@test "a packages/ path with no graph entry escalates and says which" {
13021367
# The fourth branch that used to escalate in silence: a path under
13031368
# packages/(apps|system|extra)/<name>/ that no PackageSource lists as a

0 commit comments

Comments
 (0)