// Faults as a library (RFD0012). The core has no faults: a message is lost because a step on it
// at its receiver drops it, an instance restarts because a step resets it, and a budget is a
// model variable that a step spends. Each factory here returns an ordinary step definition (or
// a message, for defineCall), which a spec could write itself; faults are tagged `fault`, which
// reports read and the core never does.
// Imported from the parts, not the entry, which exports this library too.
import { defineMessage, defineVariable, stepVariable, type AnyMessage, type Ctx, type Delivery, type Message, type ModelView, type Part, type PayloadOf, type ReplyCtx, type Variable } from "../spec/dsl.ts";
import type { Fields, TypeOf } from "../spec/schema.ts";
/** What reports count: a step tagged `fault` is one. */
export const FAULT = "fault";
// Calls ---------------------------------------------------------------------------------------------
/** What a call's failure carries: the payload of the call that failed, and whom it was to. */
export interface Failure
{
readonly payload: P;
readonly to: { readonly id: string };
}
/** A call: a message whose loss, or its reply's, reaches the caller as `failed(call)`. */
export interface Call extends Message {
/** Phantom: marks a call, so `failed` takes only calls. */
readonly __call?: true;
}
/** A call's failure: a plain message, `Charge.failed`, the caller has a step on. */
export type Failed = Message<`${Id}.failed`, Failure>;
/** Each call's failure, by the call itself: two specs may each have a message named Charge. */
const failures = new WeakMap>();
/** The keys failures have (`message:Charge.failed`): only this library makes keys with `.failed`. */
const failureKeys = new Set();
/**
* A call (RFD0009), as a message and its failure: `lose` on the call, or on its reply, sends
* the caller `failed(Charge)`, as an RPC or a `fetch` throws. Its replies carry the request
* they answer, so a lost reply knows which call it fails.
*/
export function defineCall(id: Id, schema?: F, options?: { name?: string; over?: string }): Call> {
const call = defineMessage(id, schema, { ...options, correlate: true });
const failure = defineMessage(`${id}.failed`, { payload: (schema ?? {}) as F, to: { id: String } }, { name: `${call.name} failed`, tone: "refused" });
failures.set(call, failure as Message);
failureKeys.add(failure.key);
return call as Call>;
}
/** The failure of a call from `defineCall`: what the caller's step is `on`. */
export function failed(call: Call): Failed {
const failure = call && typeof call === "object" ? failures.get(call) : undefined;
if (!failure) {
const what = call && typeof call === "object" && "name" in call ? `${call.name} is a message` : `${String(call)} is not a message`;
throw new Error(`failed(${call?.id ?? String(call)}): ${what}, not a call; define it with defineCall from the faults library`);
}
return failure as unknown as Failed;
}
/** Whether a message key is a call's failure, which happens in the caller's own process. */
function isFailure(key: string): boolean {
return failureKeys.has(key);
}
// Budgets ---------------------------------------------------------------------------------------------
/** A budget: a model variable a fault spends. `--set faults=2` raises it. */
export type Budget = Variable;
/** `budget("faults", 1)`: at most one fault on any path, across every step that spends it. */
export function budget(name: string, n: number): Budget {
if (!Number.isInteger(n) || n < 0) throw new Error(`budget("${name}", …) takes a whole number, not ${n}`);
return defineVariable(name, n);
}
/** A part that spends a budget: the step is possible only while it is above zero, and takes one. */
export function spend(budget: Budget): Part {
return {
uses: [budget],
when: (model) => model.get(budget) > 0,
run: (ctx) => ctx.set(budget, ctx.get(budget) - 1),
};
}
/**
* A part bounding a step to at most `limit` times per instance on any path: an action a person
* might take forever, bounded for the checker as TLA+ bounds it with a counter. Each instance
* counts its own, in a model variable named after the step (`Client.pay`). The limit is a
* number, or a variable several steps share, so `--set repeats=3` raises them all.
*/
export function atMost(limit: number | Variable): Part {
if (typeof limit === "number" && (!Number.isInteger(limit) || limit < 0)) throw new Error(`atMost(limit) takes a whole number, not ${limit}`);
const taken = stepVariable(0, { each: true });
const most = (model: ModelView) => (typeof limit === "number" ? limit : model.get(limit));
return {
uses: typeof limit === "number" ? [taken] : [taken, limit],
when: (model) => model.get(taken) < most(model),
run: (ctx) => ctx.set(taken, ctx.get(taken) + 1),
};
}
// Faults ------------------------------------------------------------------------------------------------
interface FaultOptions {
/** The budget it spends; without one, it may happen on every path, any number of times. */
spend?: Budget;
/** In everyday words, instead of the default. */
description?: string;
}
/** What a fault step is: unfair (it may never happen), on a message or on nothing, tagged `fault`. */
interface FaultStep {
on?: On;
fair: false;
tags: readonly string[];
with: readonly Part[];
sends?: readonly AnyMessage[];
when?: (state: any, model: ModelView) => boolean;
description: string;
run(state: any, received: any, ctx: any): void;
}
function fault(step: Omit, "fair" | "tags" | "with">, budget: Budget | undefined): FaultStep {
return { ...step, fair: false, tags: [FAULT], with: budget ? [spend(budget)] : [] };
}
/**
* The receiver loses `message` on the way: a step on it that does nothing with it. For a call,
* the caller's call fails; for a reply, pass `fails` with the call it answers (or the calls it
* may answer), and the caller's call fails when the reply is lost: the callee did the work, and
* the caller cannot tell.
*/
export function lose(message: M, options?: FaultOptions & { fails?: Call | readonly Call[] }): FaultStep {
const failure = message && typeof message === "object" ? failures.get(message) : undefined;
const calls = options?.fails === undefined ? [] : Array.isArray(options.fails) ? options.fails : [options.fails as Call];
const failing = new Map(calls.map((call) => [call.key as string, failed(call) as unknown as Message]));
const sends = [...(failure ? [failure] : []), ...failing.values()];
return fault(
{
on: message,
sends,
description: options?.description ?? `${capital(message.name)} is lost on the way.`,
run(_state: unknown, payload: PayloadOf, ctx: ReplyCtx) {
if (failure) ctx.send(ctx.from, failure, { payload, to: { id: ctx.id } });
const answers = ctx.delivery.answers;
const fails = answers && failing.get(answers.message);
// The failure reaches the caller from the callee, as one from a lost call does.
if (fails) ctx.forward({ message: fails, payload: { payload: answers.payload, to: { id: ctx.from.id } } });
},
},
options?.spend,
);
}
/** The network delivers `message` twice: a step on it that puts it back in flight twice. */
export function duplicate(message: M, options?: FaultOptions): FaultStep {
return fault(
{
on: message,
description: options?.description ?? `${capital(message.name)} arrives twice.`,
run(_state: unknown, _payload: unknown, ctx: ReplyCtx) {
ctx.forward();
ctx.forward();
},
},
options?.spend,
);
}
/**
* The instance crashes and starts again: its state is its initial state but for the fields in
* `keeps` (what it wrote to disk), its timers are gone, and so are the failures on their way
* to it, which would have thrown in code that is no longer running.
*/
export function crash(options?: FaultOptions & { keeps?: readonly string[] }): FaultStep {
const keeps = options?.keeps ?? [];
return fault(
{
when: () => true,
description: options?.description ?? (keeps.length ? `Crashes and restarts, keeping ${keeps.join(", ")}.` : "Crashes and restarts."),
run(_state: unknown, _received: unknown, ctx: Ctx) {
ctx.reset({ keeps, drops: (d: Delivery) => isFailure(d.message) });
},
},
options?.spend,
);
}
function capital(text: string): string {
return text.charAt(0).toUpperCase() + text.slice(1);
}