import { z } from "zod"; import { buildPayloadScript, parsePythonReport } from "../pythonReport.js"; import { errorResult, guardTd, jsonResult } from "../result.js"; import type { ToolContext, ToolRegistrar } from "../types.js"; export const createPhoneGestureSchema = z.object({ name: z.string().default("phone_gesture").describe("Child operator base name inside parent."), parent: z .string() .default("/project1") .describe("COMP that will host the Web Server DAT + Script CHOP."), port: z.coerce .number() .int() .min(1) .max(65535) .default(9982) .describe( "TCP port for the gesture web server (distinct from bridge:9980 and phone_remote:9981).", ), enableImu: z .boolean() .default(true) .describe( "Enable tilt_*, gyro_*, shake channels (iOS Safari requires HTTPS + permission tap).", ), enableMultitouch: z.boolean().default(true).describe("Enable touch0..3_x/y/active channels."), shakeThreshold: z.coerce .number() .min(0.5) .max(50) .default(15) .describe("Acceleration magnitude (m/s^2) above which `shake` fires."), }); type CreatePhoneGestureArgs = z.infer; interface GestureReport { parent: string; server?: string; out?: string; port?: number; url?: string; warnings: string[]; fatal?: string; } // Served HTML page + WebSocket client. Streams DeviceOrientation/DeviceMotion + TouchEvent at ~30 Hz. // iOS Safari requires HTTPS for DeviceMotionEvent.requestPermission; on plain LAN HTTP it silently // no-ops and the page falls back to touch-only. Android/desktop Chrome auto-grants. const HTML_TEMPLATE = ` tdmcp gesture

tdmcp gesture — __PARENT_NAME__

ws: connecting

tilt x 0.0 y 0.0 z 0.0

gyro x 0.0 y 0.0 z 0.0

shake 0.00

`; const CALLBACKS_TEMPLATE = `# Web Server DAT callbacks for tdmcp create_phone_gesture. HTML_PAGE = """__HTML__""" def _state_dat(webServerDAT): return webServerDAT.parent().op("__STATE_NAME__") def onHTTPRequest(webServerDAT, request, response): response["statusCode"] = 200 response["content-type"] = "text/html" response["data"] = HTML_PAGE return response def onWebSocketOpen(webServerDAT, client, uri): s = webServerDAT.fetch("clients", None) if not isinstance(s, set): s = set() s.add(str(client)) webServerDAT.store("clients", s) def onWebSocketClose(webServerDAT, client): s = webServerDAT.fetch("clients", None) if isinstance(s, set): s.discard(str(client)) webServerDAT.store("clients", s) def onWebSocketReceiveText(webServerDAT, client, data): d = _state_dat(webServerDAT) if d is not None: try: d.text = data except Exception: pass def onWebSocketReceiveBinary(webServerDAT, client, data): return def onWebSocketReceivePing(webServerDAT, client, data): webServerDAT.webSocketSendPong(client, data=data) def onWebSocketReceivePong(webServerDAT, client, data): return def onServerStart(webServerDAT): return def onServerStop(webServerDAT): return `; const SCRIPT_CHOP_CALLBACK = `# Script CHOP callback for tdmcp create_phone_gesture. # Reads the most recent JSON frame from the state textDAT and emits channels. import json def _zero(scriptOp): names = ["tilt_x","tilt_y","tilt_z","gyro_x","gyro_y","gyro_z","shake"] for i in range(4): names += ["touch%d_x" % i, "touch%d_y" % i, "touch%d_active" % i] names += ["clients"] for n in names: c = scriptOp.appendChan(n) c[0] = 0.0 scriptOp.numSamples = 1 def onCook(scriptOp): scriptOp.clear() _zero(scriptOp) state_dat = scriptOp.parent().op("__STATE_NAME__") server_dat = scriptOp.parent().op("__SERVER_NAME__") if state_dat is None: return txt = state_dat.text or "" if txt.strip(): try: d = json.loads(txt) t = d.get("tilt") or [0,0,0] g = d.get("gyro") or [0,0,0] scriptOp["tilt_x"][0] = float(t[0] if len(t)>0 else 0) scriptOp["tilt_y"][0] = float(t[1] if len(t)>1 else 0) scriptOp["tilt_z"][0] = float(t[2] if len(t)>2 else 0) scriptOp["gyro_x"][0] = float(g[0] if len(g)>0 else 0) scriptOp["gyro_y"][0] = float(g[1] if len(g)>1 else 0) scriptOp["gyro_z"][0] = float(g[2] if len(g)>2 else 0) scriptOp["shake"][0] = float(d.get("shake") or 0) touches = d.get("touches") or [] for i in range(min(len(touches), 4)): tt = touches[i] or {} scriptOp["touch%d_x" % i][0] = float(tt.get("x") or 0) scriptOp["touch%d_y" % i][0] = float(tt.get("y") or 0) scriptOp["touch%d_active" % i][0] = 1.0 except Exception: pass if server_dat is not None: s = server_dat.fetch("clients", None) if isinstance(s, set): scriptOp["clients"][0] = float(len(s)) return `; const BUILD_SCRIPT = ` import json, base64, traceback, socket import td _p = json.loads(base64.b64decode("__PAYLOAD_B64__").decode("utf-8")) report = {"parent": _p["parent"], "warnings": []} _c = op(_p["parent"]) try: if _c is None: report["fatal"] = "COMP not found: " + _p["parent"] elif not hasattr(_c, "create"): report["fatal"] = _p["parent"] + " is not a COMP." else: name = _p["name"] cb_name = name + "_callbacks" state_name = name + "_state" server_name = name + "_server" script_name = name + "_script" scriptcb_n = name + "_script_cb" out_name = name + "_out" url_name = name + "_url" cb = _c.op(cb_name) or _c.create(td.textDAT, cb_name) state = _c.op(state_name) or _c.create(td.textDAT, state_name) server = _c.op(server_name) or _c.create(td.webserverDAT, server_name) scriptcb = _c.op(scriptcb_n) or _c.create(td.textDAT, scriptcb_n) scrop = _c.op(script_name) or _c.create(td.scriptCHOP, script_name) outop = _c.op(out_name) or _c.create(td.nullCHOP, out_name) urlop = _c.op(url_name) or _c.create(td.textDAT, url_name) cb.text = _p["callbacks"] scriptcb.text = _p["script_callback"] state.text = "" try: scrop.par.callbacks = scriptcb except Exception as _e: report["warnings"].append("Script CHOP callbacks param: " + str(_e)[:160]) try: scrop.cook(force=True) except Exception: pass try: outop.inputConnectors[0].connect(scrop) except Exception as _e: report["warnings"].append("Could not wire script->null: " + str(_e)[:160]) try: server.par.port = _p["port"] server.par.callbacks = cb server.par.active = True except Exception as _e: report["warnings"].append("Web Server DAT params: " + str(_e)[:160]) try: _ip = socket.gethostbyname(socket.gethostname()) except Exception: _ip = "" url = "http://%s:%s/" % (_ip, _p["port"]) urlop.text = url try: server.cook(force=True) except Exception: pass report["server"] = server.path report["out"] = outop.path report["port"] = _p["port"] report["url"] = url try: _err = server.errors(recurse=False) or "" if _err: report["warnings"].append("Web Server DAT: " + _err[:160]) except Exception: pass except Exception: report["fatal"] = traceback.format_exc().splitlines()[-1] print(json.dumps(report)) `; export function buildPhoneGestureScript(payload: { parent: string; name: string; port: number; enableImu: boolean; enableMultitouch: boolean; shakeThreshold: number; }): string { const stateName = `${payload.name}_state`; const serverName = `${payload.name}_server`; const html = HTML_TEMPLATE.replaceAll("__PARENT_NAME__", payload.parent) .replaceAll("__IMU_ENABLED__", payload.enableImu ? "true" : "false") .replaceAll("__TOUCH_ENABLED__", payload.enableMultitouch ? "true" : "false") .replaceAll("__SHAKE_THRESHOLD__", String(payload.shakeThreshold)); // Escape triple-quotes in the HTML so the Python triple-quoted string stays valid. const safeHtml = html.replaceAll('"""', '\\"\\"\\"'); const callbacks = CALLBACKS_TEMPLATE.replaceAll("__HTML__", safeHtml).replaceAll( "__STATE_NAME__", stateName, ); const scriptCallback = SCRIPT_CHOP_CALLBACK.replaceAll("__STATE_NAME__", stateName).replaceAll( "__SERVER_NAME__", serverName, ); return buildPayloadScript(BUILD_SCRIPT, { parent: payload.parent, name: payload.name, port: payload.port, callbacks, script_callback: scriptCallback, }); } export async function createPhoneGestureImpl(ctx: ToolContext, args: CreatePhoneGestureArgs) { return guardTd( async () => { const script = buildPhoneGestureScript({ parent: args.parent, name: args.name, port: args.port, enableImu: args.enableImu, enableMultitouch: args.enableMultitouch, shakeThreshold: args.shakeThreshold, }); const exec = await ctx.client.executePythonScript(script, true); return parsePythonReport(exec.stdout); }, (report) => { if (report.fatal) { return errorResult(`Could not build phone gesture: ${report.fatal}`, report); } const summary = `Phone gesture '${args.name}' streaming on ${report.url} → ${report.out}${ report.warnings.length ? `, ${report.warnings.length} warning(s)` : "" }.`; return jsonResult(summary, report); }, ); } export const registerCreatePhoneGesture: ToolRegistrar = (server, ctx) => { server.registerTool( "create_phone_gesture", { title: "Create phone gesture", description: "Stream a phone's IMU (tilt + gyro + shake) and multitouch into TouchDesigner as CHOP channels you can bind to anything. Builds a Web Server DAT page the phone opens (any browser, no app) and a Null CHOP exposing tilt_x/y/z, gyro_x/y/z, shake, touch0..3_x/y/active, clients. Composable with create_phone_remote on the same COMP (different port). SECURITY: listens on all interfaces with no auth — trusted networks only. iOS Safari needs HTTPS for motion permission; falls back to touch-only on plain HTTP.", inputSchema: createPhoneGestureSchema.shape, annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true }, }, (args) => createPhoneGestureImpl(ctx, args), ); };