285 lines
8.4 KiB
Go
285 lines
8.4 KiB
Go
// Copyright (c) 2026 Storm Dragon
|
|
// Authors: Storm Dragon, Angus Kola
|
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
package remoteinput
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
type supervisorTestTarget struct{ mode string }
|
|
|
|
func (target supervisorTestTarget) Apply(string, bool) error {
|
|
switch target.mode {
|
|
case "exit":
|
|
os.Exit(0)
|
|
case "blocked":
|
|
// A timer models a native call that remains blocked without causing
|
|
// Go's non-race runtime to terminate the synthetic child as deadlocked.
|
|
time.Sleep(time.Hour)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (target supervisorTestTarget) Close() error {
|
|
if target.mode == "cleanup-failure" {
|
|
return ErrCleanup
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func TestSupervisorProcess(t *testing.T) {
|
|
mode := os.Getenv("CTHULHU_SUPERVISOR_TEST")
|
|
if mode == "" {
|
|
return
|
|
}
|
|
epoch, _ := strconv.ParseUint(os.Getenv("CTHULHU_SUPERVISOR_EPOCH"), 10, 64)
|
|
control, err := AdoptPipe(os.Stdin, false)
|
|
if err != nil {
|
|
os.Exit(2)
|
|
}
|
|
output, err := AdoptPipe(os.Stdout, true)
|
|
if err != nil {
|
|
os.Exit(2)
|
|
}
|
|
life, err := AdoptPipe(os.NewFile(3, "lifetime"), false)
|
|
if err != nil {
|
|
os.Exit(2)
|
|
}
|
|
if mode == "no-ready" || mode == "bad-ready" {
|
|
if mode == "bad-ready" {
|
|
frame, _ := EncodeAck(Ack{Kind: Ready, Epoch: epoch + 1})
|
|
writeAll(output, frame)
|
|
}
|
|
io.Copy(io.Discard, life)
|
|
frame, _ := EncodeAck(Ack{Kind: Closed, Reason: ReasonEOF, Epoch: epoch})
|
|
writeAll(output, frame)
|
|
os.Exit(0)
|
|
}
|
|
if mode == "wrong-epoch" || mode == "wrong-sequence" || mode == "duplicate-ready" || mode == "withheld" || mode == "bad-exit" {
|
|
ready, _ := EncodeAck(Ack{Kind: Ready, Epoch: epoch})
|
|
writeAll(output, ready)
|
|
var frame [OperationSize]byte
|
|
io.ReadFull(control, frame[:])
|
|
operation, _ := DecodeOperation(frame[:])
|
|
ack := Ack{Kind: Accepted, Epoch: epoch, Sequence: operation.Sequence}
|
|
switch mode {
|
|
case "wrong-epoch":
|
|
ack.Epoch++
|
|
case "wrong-sequence":
|
|
ack.Sequence++
|
|
case "duplicate-ready":
|
|
ack = Ack{Kind: Ready, Epoch: epoch}
|
|
}
|
|
if mode == "withheld" {
|
|
if os.WriteFile(os.Getenv("CTHULHU_SUPERVISOR_MARKER"), []byte("pending"), 0600) != nil {
|
|
os.Exit(2)
|
|
}
|
|
} else {
|
|
encoded, _ := EncodeAck(ack)
|
|
writeAll(output, encoded)
|
|
}
|
|
io.Copy(io.Discard, life)
|
|
closed, _ := EncodeAck(Ack{Kind: Closed, Reason: ReasonEOF, Epoch: epoch, Sequence: operation.Sequence})
|
|
writeAll(output, closed)
|
|
if mode == "bad-exit" {
|
|
os.Exit(1)
|
|
}
|
|
os.Exit(0)
|
|
}
|
|
err = Run(context.Background(), epoch, control, life, output, supervisorTestTarget{mode})
|
|
if err != nil {
|
|
os.Exit(1)
|
|
}
|
|
os.Exit(0)
|
|
}
|
|
|
|
func testSupervisor(t *testing.T, mode string) (*Supervisor, func() *exec.Cmd) {
|
|
t.Helper()
|
|
var command *exec.Cmd
|
|
supervisor := &Supervisor{command: func(epoch uint64) *exec.Cmd {
|
|
cmd := exec.Command(os.Args[0], "-test.run=^TestSupervisorProcess$")
|
|
cmd.Env = append(os.Environ(), "CTHULHU_SUPERVISOR_TEST="+mode,
|
|
"CTHULHU_SUPERVISOR_EPOCH="+strconv.FormatUint(epoch, 10))
|
|
t.Cleanup(func() {
|
|
if cmd.Process != nil {
|
|
cmd.Process.Kill()
|
|
}
|
|
})
|
|
command = cmd
|
|
return cmd
|
|
}}
|
|
return supervisor, func() *exec.Cmd { return command }
|
|
}
|
|
|
|
func startSupervisor(t *testing.T, supervisor *Supervisor, epoch uint64) *Client {
|
|
t.Helper()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
t.Cleanup(cancel)
|
|
client, err := supervisor.Start(ctx, epoch)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { client.Cancel(); client.Wait() })
|
|
return client
|
|
}
|
|
|
|
func TestSupervisorRetirementAndReplacement(t *testing.T) {
|
|
supervisor, command := testSupervisor(t, "normal")
|
|
client := startSupervisor(t, supervisor, 41)
|
|
group, err := syscall.Getpgid(command().Process.Pid)
|
|
if err != nil || group != command().Process.Pid || group == syscall.Getpgrp() {
|
|
t.Fatal("helper shares its writer's process group", err)
|
|
}
|
|
if _, err := supervisor.Start(context.Background(), 42); !errors.Is(err, ErrUnavailable) {
|
|
t.Fatal("overlapping helper admitted", err)
|
|
}
|
|
for _, pressed := range []bool{true, true, false} {
|
|
if err := client.Apply(t.Context(), "Shift_L", pressed); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
// Heartbeats must retain an otherwise idle helper beyond its watchdog.
|
|
select {
|
|
case <-client.Done():
|
|
t.Fatal("idle helper retired", client.Wait())
|
|
case <-time.After(Watchdog + 100*time.Millisecond):
|
|
}
|
|
client.Cancel()
|
|
client.Cancel()
|
|
if err := client.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := client.Apply(t.Context(), "a", true); !errors.Is(err, ErrUnavailable) {
|
|
t.Fatal("retired helper accepted a key", err)
|
|
}
|
|
replacement := startSupervisor(t, supervisor, 42)
|
|
client.Cancel()
|
|
if err := replacement.Apply(t.Context(), "Control_R", true); err != nil {
|
|
t.Fatal("old cleanup affected replacement", err)
|
|
}
|
|
replacement.Cancel()
|
|
if err := replacement.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestSupervisorUncertainRetirementBlocksReplacement(t *testing.T) {
|
|
for _, mode := range []string{"cleanup-failure", "exit", "wrong-epoch", "wrong-sequence", "duplicate-ready", "blocked", "bad-exit"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
supervisor, command := testSupervisor(t, mode)
|
|
client := startSupervisor(t, supervisor, 1)
|
|
_ = client.Apply(t.Context(), "Shift_L", true)
|
|
client.Cancel()
|
|
if err := client.Wait(); !errors.Is(err, ErrUncertain) {
|
|
t.Fatal("missing cleanup barrier", err)
|
|
}
|
|
if _, err := supervisor.Start(t.Context(), 2); !errors.Is(err, ErrUnavailable) {
|
|
t.Fatal("uncertain input automatically replaced", err)
|
|
}
|
|
if mode == "blocked" && command().Process.Signal(syscall.Signal(0)) != nil {
|
|
t.Fatal("supervisor killed helper before native cleanup")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSupervisorCancelledAdmittedCall(t *testing.T) {
|
|
marker := filepath.Join(t.TempDir(), "pending")
|
|
t.Setenv("CTHULHU_SUPERVISOR_MARKER", marker)
|
|
supervisor, _ := testSupervisor(t, "withheld")
|
|
client := startSupervisor(t, supervisor, 31)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
result := make(chan error, 1)
|
|
go func() { result <- client.Apply(ctx, "Shift_L", true) }()
|
|
deadline := time.NewTimer(operationTimeout)
|
|
defer deadline.Stop()
|
|
tick := time.NewTicker(5 * time.Millisecond)
|
|
defer tick.Stop()
|
|
pending:
|
|
for {
|
|
select {
|
|
case <-tick.C:
|
|
if _, err := os.Stat(marker); err == nil {
|
|
break pending
|
|
}
|
|
case <-deadline.C:
|
|
t.Fatal("helper did not receive pending key")
|
|
}
|
|
}
|
|
if _, err := supervisor.Start(t.Context(), 32); !errors.Is(err, ErrUnavailable) {
|
|
t.Fatal("pending key owner replaced", err)
|
|
}
|
|
cancel()
|
|
if err := <-result; !errors.Is(err, ErrUnavailable) {
|
|
t.Fatal("cancelled uncertain key acknowledged", err)
|
|
}
|
|
if err := client.Wait(); err != nil {
|
|
t.Fatal("independent lifeline failed to retire pending call", err)
|
|
}
|
|
}
|
|
|
|
func TestSupervisorLaunchValidationAndFailure(t *testing.T) {
|
|
if _, err := NewSupervisor("relative-helper"); !errors.Is(err, ErrUnavailable) {
|
|
t.Fatal("relative executable accepted", err)
|
|
}
|
|
supervisor, _ := NewSupervisor(filepath.Join(t.TempDir(), "absent-helper"))
|
|
for _, epoch := range []uint64{0, 1, 2} {
|
|
if _, err := supervisor.Start(t.Context(), epoch); !errors.Is(err, ErrUnavailable) {
|
|
t.Fatal("launch failure not private", err)
|
|
}
|
|
}
|
|
if supervisor.active || supervisor.uncertain {
|
|
t.Fatal("failed exec poisoned unused target")
|
|
}
|
|
}
|
|
|
|
func TestSupervisorStartupBarrier(t *testing.T) {
|
|
for _, mode := range []string{"no-ready", "bad-ready"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
supervisor, _ := testSupervisor(t, mode)
|
|
_, err := supervisor.Start(t.Context(), 8)
|
|
if mode == "no-ready" {
|
|
if !errors.Is(err, ErrUnavailable) || supervisor.uncertain {
|
|
t.Fatal("confirmed startup retirement poisoned replacement", err)
|
|
}
|
|
} else if !errors.Is(err, ErrUncertain) || !supervisor.uncertain {
|
|
t.Fatal("invalid readiness bypassed retirement barrier", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSupervisorContextAndRepresentation(t *testing.T) {
|
|
supervisor, _ := testSupervisor(t, "normal")
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
client, err := supervisor.Start(ctx, 11)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { cancel(); client.Wait() })
|
|
for _, name := range []string{"", "private unsupported name", "A"} {
|
|
if err := client.Apply(t.Context(), name, true); !errors.Is(err, ErrUnavailable) {
|
|
t.Fatal("invalid key admitted", err)
|
|
}
|
|
}
|
|
if err := client.Apply(t.Context(), "a", true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cancel()
|
|
if err := client.Wait(); err != nil {
|
|
t.Fatal("context cancellation prevented clean EOF retirement", err)
|
|
}
|
|
}
|