Deliveries on master 71 what went out for master · last 48 hours · all time (1532) · for you (0 unread)
Every message X-ray produced for an alert — delivered in X-ray, sent to a channel, rendered in dry-run, held or dropped — and why. One row per message: a group of a hundred findings is one row. Every row carries its message number (#N), the same one printed on the card and the bell row — click it to see the message on its alert's page, with what it was about. external channels dry-run
| When | Alert | Message | Delivered to | Event | Why |
|---|---|---|---|---|---|
| #52143 2026-11-17 10:47:59 UTC |
channel test | warning Error in schedsvc (tilda) on master
show the payload{
"title": "Error in schedsvc (tilda) on master",
"severity": "warning",
"headline": "Error in schedsvc (tilda) on master",
"text": "Error in schedsvc (tilda) on master\nerror parsing time periods",
"message": "error parsing time periods",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=schedsvc&namespace=tilda&trace=ccbb0e14e729833398b26f30b0353c8a&at=1794912478&msg=error+parsing+time+periods",
"log": {
"severity": "ERROR",
"service": "schedsvc",
"cluster": "master",
"namespace": "tilda",
"message": "error parsing time periods",
"trace": "ccbb0e14e729833398b26f30b0353c8a",
"log": "stdout",
"ts": "2026-11-17T10:47:58.466445485Z"
},
"log_key": "383df22466f28d9307b23fa6ffc5e8048ba0976f608f8d339e427d4b7b05b389",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794912509,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52143,
"message_url": "https://xray.tilda.pizza/alerts/messages/52143"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52141 2026-11-17 10:46:09 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nRequest completed",
"message": "Request completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=71a8ac8646af2c3e14e5a5380c42696c&at=1794912357&msg=Request+completed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Request completed",
"trace": "71a8ac8646af2c3e14e5a5380c42696c",
"log": "stdout",
"ts": "2026-11-17T10:45:57.326429667Z"
},
"log_key": "520c14762a99a5be7aefae37d3d55b28e8805375174fc9968943ebb288bd249c",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794912399,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": [],
"reserved_at": 1794912369
},
"message_id": 52141,
"message_url": "/alerts/messages/52141"
} |
#in-app delivered#dev-notifications sent |
opened | in-app: 1 inbox row(s) for [email protected] ok |
| #52139 2026-11-17 10:46:08 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nError retrieving CRF document",
"message": "Error retrieving CRF document",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=71a8ac8646af2c3e14e5a5380c42696c&at=1794912357&msg=Error+retrieving+CRF+document",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Error retrieving CRF document",
"trace": "71a8ac8646af2c3e14e5a5380c42696c",
"log": "stdout",
"ts": "2026-11-17T10:45:57.326163011Z"
},
"log_key": "f245e71b17cb74fb7e8ca75222d2b33f9337d8eff5a8f6754a79cb6dd1d6f98a",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794912399,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52139,
"message_url": "https://xray.tilda.pizza/alerts/messages/52139"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52137 2026-11-17 10:46:08 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nError downloading file",
"message": "Error downloading file",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=71a8ac8646af2c3e14e5a5380c42696c&at=1794912357&msg=Error+downloading+file",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Error downloading file",
"trace": "71a8ac8646af2c3e14e5a5380c42696c",
"log": "stdout",
"ts": "2026-11-17T10:45:57.316985588Z"
},
"log_key": "f7a5008a2692e7f6c2d00abaee03d4c4daac7424425a874c1807a48ac7d2aeef",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794912398,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52137,
"message_url": "https://xray.tilda.pizza/alerts/messages/52137"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52135 2026-11-17 10:46:03 UTC |
channel test | warning Error in exportsvc (tilda) on master
show the payload{
"title": "Error in exportsvc (tilda) on master",
"severity": "warning",
"headline": "Error in exportsvc (tilda) on master",
"text": "Error in exportsvc (tilda) on master\nError processing activity update: Failed to fetch study metadata: API request failed: 500 Server Error: Internal Server Error for url: https://api.master.tilda.pizza/api/trialexec/v2/study/9cfcafa0-9782-4448-8250-ae8288b92a5a/protocol/f4f461b2-29a8-45a0-a114-a27762fac91b/crf",
"message": "Error processing activity update: Failed to fetch study metadata: API request failed: 500 Server Error: Internal Server Error for url: https://api.master.tilda.pizza/api/trialexec/v2/study/9cfcafa0-9782-4448-8250-ae8288b92a5a/protocol/f4f461b2-29a8-45a0-a114-a27762fac91b/crf",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=exportsvc&namespace=tilda&trace=71a8ac8646af2c3e14e5a5380c42696c&at=1794912357&msg=Error+processing+activity+update%3A+Failed+to+fetch+study+metadata%3A+API+request+failed%3A+500+Server+Error%3A+Internal+Server+Error+for+url%3A+https%3A%2F%2Fapi.master.tilda.pizza%2Fapi%2Ftrialexec%2Fv2%2Fstudy%2F9cfcafa0-9782-4448-8250-ae8288b92a5a%2Fprotocol%2Ff4f461b2-29a8-45a0-a114-a27762fac91b%2Fcrf",
"log": {
"severity": "ERROR",
"service": "exportsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Error processing activity update: Failed to fetch study metadata: API request failed: 500 Server Error: Internal Server Error for url: https://api.master.tilda.pizza/api/trialexec/v2/study/9cfcafa0-9782-4448-8250-ae8288b92a5a/protocol/f4f461b2-29a8-45a0-a114-a27762fac91b/crf",
"trace": "71a8ac8646af2c3e14e5a5380c42696c",
"log": "stdout",
"ts": "2026-11-17T10:45:57.36543118Z"
},
"log_key": "72a76c602bf2998e18f2948b8a3845ca6878c0ca26a8107a64f07a90fae9d1e6",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794912393,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52135,
"message_url": "https://xray.tilda.pizza/alerts/messages/52135"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52132 2026-11-17 10:40:57 UTC |
channel test | warning Error in chat-agent (tilda) on master
show the payload{
"title": "Error in chat-agent (tilda) on master",
"severity": "warning",
"headline": "Error in chat-agent (tilda) on master",
"text": "Error in chat-agent (tilda) on master\nhitl_tool: pre-approval card generation failed",
"message": "hitl_tool: pre-approval card generation failed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=chat-agent&namespace=tilda&trace=39833992f95a94666c5b27bd26d74896&at=1794912054&msg=hitl_tool%3A+pre-approval+card+generation+failed",
"log": {
"severity": "ERROR",
"service": "chat-agent",
"cluster": "master",
"namespace": "tilda",
"message": "hitl_tool: pre-approval card generation failed",
"trace": "39833992f95a94666c5b27bd26d74896",
"log": "stdout",
"ts": "2026-11-17T10:40:54.301233838Z"
},
"log_key": "ca87dfe8d32966c60dc52fd9c7c4a653b86eb156dda4e4937a2e62ebac8c46b7",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794912087,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52132,
"message_url": "https://xray.tilda.pizza/alerts/messages/52132"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52130 2026-11-17 10:38:48 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794911917",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T10:38:37.786915Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794911958,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52130,
"message_url": "https://xray.tilda.pizza/alerts/messages/52130"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52128 2026-11-17 10:33:36 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794911611",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T10:33:31.315218Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794911646,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52128,
"message_url": "https://xray.tilda.pizza/alerts/messages/52128"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52126 2026-11-17 10:19:19 UTC |
channel test | warning Any error → notify me (master): 2 more messages held by the rate limit in the last hour
show the payload{
"title": "Any error \u2192 notify me (master): 2 more messages held by the rate limit in the last hour",
"text": "Any error \u2192 notify me (master): 2 more messages held by the rate limit in the last hour\nThe alert allows 20 an hour. Most recent held: Error in schedsvc (tilda) on master; Error in tilda on master",
"body": "The alert allows 20 an hour. Most recent held: Error in schedsvc (tilda) on master; Error in tilda on master",
"url": "https://xray.tilda.pizza/alerts/8",
"severity": "warning",
"_scope": {},
"_job": {
"finding": null,
"force": true,
"retry_at": 1794910789,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52126,
"message_url": "https://xray.tilda.pizza/alerts/messages/52126"
} |
#dev-notifications sent#in-app delivered |
rollup | ok in-app: 1 inbox row(s) for [email protected] |
| #52124 2026-11-17 10:11:38 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=5189cbe86b9f73d7ba71083aba382497&at=1794910294&msg=%5Breminder%5D+wo%3D97bcca47-6e42-49be-bcf8-715ee3a73d53+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ko%27-script+text+%28%27%ED%8C%80%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "5189cbe86b9f73d7ba71083aba382497",
"log": "stderr",
"ts": "2026-11-17T10:11:34.929941666Z"
},
"log_key": "a21b1a376c74f5195c8af08090b36f6360bb500c051ade1192a1ebd1eec933bd",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794910328,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52124,
"message_url": "https://xray.tilda.pizza/alerts/messages/52124"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52122 2026-11-17 10:11:23 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=f8959b8a063c66152dd52e027d24204d&at=1794910274&msg=%5Breminder%5D+wo%3D48275f5e-57af-4670-bd15-2b08e179b69b+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ja%27-script+text+%28%27%E6%A7%98%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "f8959b8a063c66152dd52e027d24204d",
"log": "stderr",
"ts": "2026-11-17T10:11:14.554572887Z"
},
"log_key": "db8c99fb601ac7cbb971b15f6bd7a430dcf42331e2810535259637448bfb7406",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794910313,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52122,
"message_url": "https://xray.tilda.pizza/alerts/messages/52122"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52120 2026-11-17 10:10:58 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=f8959b8a063c66152dd52e027d24204d&at=1794910254&msg=%5Breminder%5D+wo%3D850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27zh-tw%27-script+text+%28%27%E6%82%A8%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "f8959b8a063c66152dd52e027d24204d",
"log": "stderr",
"ts": "2026-11-17T10:10:54.930425341Z"
},
"log_key": "df998a0f4c524270d6d64977e8c1d486629224b4953daef9c122c60f8cfc74ad",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794910288,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52120,
"message_url": "https://xray.tilda.pizza/alerts/messages/52120"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52118 2026-11-17 10:09:08 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794910139",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T10:08:59.462961Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794910178,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52118,
"message_url": "https://xray.tilda.pizza/alerts/messages/52118"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52116 2026-11-17 10:04:52 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\nHTTP request failed: Token refresh failed: status=403 error=invalid_grant error_description=Unknown or invalid refresh token.",
"message": "HTTP request failed: Token refresh failed: status=403 error=invalid_grant error_description=Unknown or invalid refresh token.",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=267ce0983d1ace3747730a30ed6a62d9&at=1794909889&msg=HTTP+request+failed%3A+Token+refresh+failed%3A+status%3D403+error%3Dinvalid_grant+error_description%3DUnknown+or+invalid+refresh+token.",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "HTTP request failed: Token refresh failed: status=403 error=invalid_grant error_description=Unknown or invalid refresh token.",
"trace": "267ce0983d1ace3747730a30ed6a62d9",
"log": "stderr",
"ts": "2026-11-17T10:04:49.79592372Z"
},
"log_key": "0e39edcfbd613b1d0e1fe9dc4ed8d9a19340d971265dd27d9309b644575c137a",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794909924,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52116,
"message_url": "https://xray.tilda.pizza/alerts/messages/52116"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52114 2026-11-17 10:04:52 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\nToken refresh failed: Token refresh failed: status=403 error=invalid_grant error_description=Unknown or invalid refresh token.",
"message": "Token refresh failed: Token refresh failed: status=403 error=invalid_grant error_description=Unknown or invalid refresh token.",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=267ce0983d1ace3747730a30ed6a62d9&at=1794909889&msg=Token+refresh+failed%3A+Token+refresh+failed%3A+status%3D403+error%3Dinvalid_grant+error_description%3DUnknown+or+invalid+refresh+token.",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "Token refresh failed: Token refresh failed: status=403 error=invalid_grant error_description=Unknown or invalid refresh token.",
"trace": "267ce0983d1ace3747730a30ed6a62d9",
"log": "stderr",
"ts": "2026-11-17T10:04:49.800098019Z"
},
"log_key": "99d8242980bbfad7afcc475d5e933d703e6c20dbd35e88331bc63da82188920a",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794909922,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52114,
"message_url": "https://xray.tilda.pizza/alerts/messages/52114"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52111 2026-11-17 09:28:37 UTC |
channel test | warning Error in schedsvc (tilda) on master
show the payload{
"title": "Error in schedsvc (tilda) on master",
"severity": "warning",
"headline": "Error in schedsvc (tilda) on master",
"text": "Error in schedsvc (tilda) on master\nerror parsing time periods",
"message": "error parsing time periods",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=schedsvc&namespace=tilda&trace=9b10438ba7044a6f077d276e761e23f6&at=1794907713&msg=error+parsing+time+periods",
"log": {
"severity": "ERROR",
"service": "schedsvc",
"cluster": "master",
"namespace": "tilda",
"message": "error parsing time periods",
"trace": "9b10438ba7044a6f077d276e761e23f6",
"log": "stdout",
"ts": "2026-11-17T09:28:33.370485649Z"
},
"log_key": "383df22466f28d9307b23fa6ffc5e8048ba0976f608f8d339e427d4b7b05b389",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794907717,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52111,
"message_url": "https://xray.tilda.pizza/alerts/messages/52111"
} |
#dev-notifications capped#in-app capped |
opened | over this alert's limit of 20 messages an hour — counted in its hourly roll-up over this alert's limit of 20 messages an hour — counted in its hourly roll-up |
| #52109 2026-11-17 09:24:12 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794907447",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T09:24:07.894959Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794907452,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52109,
"message_url": "https://xray.tilda.pizza/alerts/messages/52109"
} |
#dev-notifications capped#in-app capped |
opened | over this alert's limit of 20 messages an hour — counted in its hourly roll-up over this alert's limit of 20 messages an hour — counted in its hourly roll-up |
| #52105 2026-11-17 09:19:08 UTC |
channel test | warning Any error → notify me (master): 1 more message held by the rate limit in the last hour
show the payload{
"title": "Any error \u2192 notify me (master): 1 more message held by the rate limit in the last hour",
"text": "Any error \u2192 notify me (master): 1 more message held by the rate limit in the last hour\nThe alert allows 20 an hour. Most recent held: Error in execsvc (tilda) on master",
"body": "The alert allows 20 an hour. Most recent held: Error in execsvc (tilda) on master",
"url": "https://xray.tilda.pizza/alerts/8",
"severity": "warning",
"_scope": {},
"_job": {
"finding": null,
"force": true,
"retry_at": 1794907178,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52105,
"message_url": "https://xray.tilda.pizza/alerts/messages/52105"
} |
#dev-notifications sent#in-app delivered |
rollup | ok in-app: 1 inbox row(s) for [email protected] |
| #52103 2026-11-17 09:18:33 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nevalResult is not a bool",
"message": "evalResult is not a bool",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f07c10b3ed56b2fdace28225487175e6&at=1794907110&msg=evalResult+is+not+a+bool",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "evalResult is not a bool",
"trace": "f07c10b3ed56b2fdace28225487175e6",
"log": "stdout",
"ts": "2026-11-17T09:18:30.579916018Z"
},
"log_key": "5fe49d3829c318f25ee6cc600a4c5170e367760f7a03b10d87afef9e46d17198",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794907113,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52103,
"message_url": "https://xray.tilda.pizza/alerts/messages/52103"
} |
#dev-notifications capped#in-app capped |
opened | over this alert's limit of 20 messages an hour — counted in its hourly roll-up over this alert's limit of 20 messages an hour — counted in its hourly roll-up |
| #52101 2026-11-17 09:18:33 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nauto query re-evaluation failed",
"message": "auto query re-evaluation failed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f07c10b3ed56b2fdace28225487175e6&at=1794907108&msg=auto+query+re-evaluation+failed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "auto query re-evaluation failed",
"trace": "f07c10b3ed56b2fdace28225487175e6",
"log": "stdout",
"ts": "2026-11-17T09:18:28.670937555Z"
},
"log_key": "ab591fe230c52e574850a2b26a9eb3dcbbec2251f9d65aa2c21a908c4a887519",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794907143,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": [],
"reserved_at": 1794907113
},
"message_id": 52101,
"message_url": "/alerts/messages/52101"
} |
#in-app delivered#dev-notifications sent |
opened | in-app: 1 inbox row(s) for [email protected] ok |
| #52099 2026-11-17 09:18:32 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nJob completed",
"message": "Job completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f07c10b3ed56b2fdace28225487175e6&at=1794907108&msg=Job+completed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Job completed",
"trace": "f07c10b3ed56b2fdace28225487175e6",
"log": "stdout",
"ts": "2026-11-17T09:18:28.728542847Z"
},
"log_key": "c74deab9b0ebaca0dfd6164704293554f866fed15541770acc539e74f689ff56",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794907208,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52099,
"message_url": "https://xray.tilda.pizza/alerts/messages/52099"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52097 2026-11-17 09:18:32 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nfailed to evaluate and raise auto query for participant question",
"message": "failed to evaluate and raise auto query for participant question",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f07c10b3ed56b2fdace28225487175e6&at=1794907111&msg=failed+to+evaluate+and+raise+auto+query+for+participant+question",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to evaluate and raise auto query for participant question",
"trace": "f07c10b3ed56b2fdace28225487175e6",
"log": "stdout",
"ts": "2026-11-17T09:18:31.635961449Z"
},
"log_key": "6bd3587402488be63611acc5599dc01571e825e5157878dd4c0a52de2538572f",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794907143,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52097,
"message_url": "https://xray.tilda.pizza/alerts/messages/52097"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52095 2026-11-17 09:18:32 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nno result from branch evaluation",
"message": "no result from branch evaluation",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f07c10b3ed56b2fdace28225487175e6&at=1794907111&msg=no+result+from+branch+evaluation",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "no result from branch evaluation",
"trace": "f07c10b3ed56b2fdace28225487175e6",
"log": "stdout",
"ts": "2026-11-17T09:18:31.635906533Z"
},
"log_key": "99389d22a81127e2d8a7b9897343d56e8b9ff07cd2a5c2eaedff03af61df8f87",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794907142,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52095,
"message_url": "https://xray.tilda.pizza/alerts/messages/52095"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52093 2026-11-17 09:08:00 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794906474",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T09:07:54.708553Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794906510,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52093,
"message_url": "https://xray.tilda.pizza/alerts/messages/52093"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52087 2026-11-17 08:36:35 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\nmessage was never published because transaction was never committed",
"message": "message was never published because transaction was never committed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=6cdb5d0c42dc9ac77a44da4bde8a901c&at=1794904591&msg=message+was+never+published+because+transaction+was+never+committed",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "message was never published because transaction was never committed",
"trace": "6cdb5d0c42dc9ac77a44da4bde8a901c",
"log": "stdout",
"ts": "2026-11-17T08:36:31.66744662Z"
},
"log_key": "2f0e80c680892bdde82cb2c9e495d0250d68634179bec2f228ad86045550169f",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904625,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52087,
"message_url": "https://xray.tilda.pizza/alerts/messages/52087"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52085 2026-11-17 08:34:55 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\ncould not update document stateTag: Either object does not exist OR object has been updated since last read",
"message": "could not update document stateTag: Either object does not exist OR object has been updated since last read",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=6cdb5d0c42dc9ac77a44da4bde8a901c&at=1794904489&msg=could+not+update+document+stateTag%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not update document stateTag: Either object does not exist OR object has been updated since last read",
"trace": "6cdb5d0c42dc9ac77a44da4bde8a901c",
"log": "stdout",
"ts": "2026-11-17T08:34:49.319801944Z"
},
"log_key": "65148ebad9b32837f5019dc4dfbf5b7e345dfefcf618c31c9d05b3ce447d2951",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904526,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52085,
"message_url": "https://xray.tilda.pizza/alerts/messages/52085"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52083 2026-11-17 08:34:54 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\ncould not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update the document version: cannot use this transaction, because it has been committed or rolled back either because of an error or explicit termination",
"message": "could not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update the document version: cannot use this transaction, because it has been committed or rolled back either because of an error or explicit termination",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=bec6e867c738f443c04b6700e23abced&at=1794904486&msg=could+not+update+document+metadata+for+doc+id+8d77d110-eb2a-49d7-be95-bbe2dbabecb6%3A+could+not+update+the+document+version%3A+cannot+use+this+transaction%2C+because+it+has+been+committed+or+rolled+back+either+because+of+an+error+or+explicit+termination",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update the document version: cannot use this transaction, because it has been committed or rolled back either because of an error or explicit termination",
"trace": "bec6e867c738f443c04b6700e23abced",
"log": "stdout",
"ts": "2026-11-17T08:34:46.989314983Z"
},
"log_key": "fe66dcd80edbbbed2a532d7a5dca22acedcd7bb1f922975948ab6692ce004464",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904585,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52083,
"message_url": "https://xray.tilda.pizza/alerts/messages/52083"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52081 2026-11-17 08:34:54 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\nfailed to run cypher in transaction",
"message": "failed to run cypher in transaction",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=bec6e867c738f443c04b6700e23abced&at=1794904486&msg=failed+to+run+cypher+in+transaction",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to run cypher in transaction",
"trace": "bec6e867c738f443c04b6700e23abced",
"log": "stdout",
"ts": "2026-11-17T08:34:46.988957864Z"
},
"log_key": "666d7cdf6bb8bdc0a4979d6039dfc336ef34dcfb6e2f4b80bcc9a4658c479d10",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904525,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52081,
"message_url": "https://xray.tilda.pizza/alerts/messages/52081"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52079 2026-11-17 08:34:53 UTC |
channel test | warning Error in docproc (tilda) on master
show the payload{
"title": "Error in docproc (tilda) on master",
"severity": "warning",
"headline": "Error in docproc (tilda) on master",
"text": "Error in docproc (tilda) on master\ndocsvc PATCH failed for c7b214fe-b333-4a54-ae72-1e768f742ffd: 500 {\"message\":\"could not update document metadata for doc id c7b214fe-b333-4a54-ae72-1e768f742ffd: could not update document for categorization: Either object does not exist OR object has been updated since last read\"}",
"message": "docsvc PATCH failed for c7b214fe-b333-4a54-ae72-1e768f742ffd: 500 {\"message\":\"could not update document metadata for doc id c7b214fe-b333-4a54-ae72-1e768f742ffd: could not update document for categorization: Either object does not exist OR object has been updated since last read\"}",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docproc&namespace=tilda&trace=c064764e0dc90c4b062551c5af6018e9&at=1794904486&msg=docsvc+PATCH+failed+for+c7b214fe-b333-4a54-ae72-1e768f742ffd%3A+500+%7B%22message%22%3A%22could+not+update+document+metadata+for+doc+id+c7b214fe-b333-4a54-ae72-1e768f742ffd%3A+could+not+update+document+for+categorization%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read%22%7D",
"log": {
"severity": "ERROR",
"service": "docproc",
"cluster": "master",
"namespace": "tilda",
"message": "docsvc PATCH failed for c7b214fe-b333-4a54-ae72-1e768f742ffd: 500 {\"message\":\"could not update document metadata for doc id c7b214fe-b333-4a54-ae72-1e768f742ffd: could not update document for categorization: Either object does not exist OR object has been updated since last read\"}",
"trace": "c064764e0dc90c4b062551c5af6018e9",
"log": "stdout",
"ts": "2026-11-17T08:34:46.304992402Z"
},
"log_key": "91f919a87fb0b5e3bce7ba445a56cb71b5a8a8f1c12f621aa79a417e53fe7721",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904523,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52079,
"message_url": "https://xray.tilda.pizza/alerts/messages/52079"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52077 2026-11-17 08:34:53 UTC |
channel test | warning Error in docproc (tilda) on master
show the payload{
"title": "Error in docproc (tilda) on master",
"severity": "warning",
"headline": "Error in docproc (tilda) on master",
"text": "Error in docproc (tilda) on master\nCBR promote failed for 1d50555b-f57a-4a0a-afe2-a3c8487d5808: HTTP 500: {\"message\":\"could not update document stateTag: Either object does not exist OR object has been updated since last read\"}",
"message": "CBR promote failed for 1d50555b-f57a-4a0a-afe2-a3c8487d5808: HTTP 500: {\"message\":\"could not update document stateTag: Either object does not exist OR object has been updated since last read\"}",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docproc&namespace=tilda&trace=6cdb5d0c42dc9ac77a44da4bde8a901c&at=1794904489&msg=CBR+promote+failed+for+1d50555b-f57a-4a0a-afe2-a3c8487d5808%3A+HTTP+500%3A+%7B%22message%22%3A%22could+not+update+document+stateTag%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read%22%7D",
"log": {
"severity": "ERROR",
"service": "docproc",
"cluster": "master",
"namespace": "tilda",
"message": "CBR promote failed for 1d50555b-f57a-4a0a-afe2-a3c8487d5808: HTTP 500: {\"message\":\"could not update document stateTag: Either object does not exist OR object has been updated since last read\"}",
"trace": "6cdb5d0c42dc9ac77a44da4bde8a901c",
"log": "stderr",
"ts": "2026-11-17T08:34:49.324828058Z"
},
"log_key": "583360d93020a3242f3d1779a04060a7f8c2cb469c3d12e13029edd9fe5955c7",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904586,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52077,
"message_url": "https://xray.tilda.pizza/alerts/messages/52077"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52075 2026-11-17 08:34:53 UTC |
channel test | warning Error in docproc (tilda) on master
show the payload{
"title": "Error in docproc (tilda) on master",
"severity": "warning",
"headline": "Error in docproc (tilda) on master",
"text": "Error in docproc (tilda) on master\ndocsvc PATCH failed for 7db44208-f1f6-467f-b83c-dbcbb23dace7: 500 {\"message\":\"could not update document metadata for doc id 7db44208-f1f6-467f-b83c-dbcbb23dace7: could not update the document: Either object does not exist OR object has been updated since last read\"}",
"message": "docsvc PATCH failed for 7db44208-f1f6-467f-b83c-dbcbb23dace7: 500 {\"message\":\"could not update document metadata for doc id 7db44208-f1f6-467f-b83c-dbcbb23dace7: could not update the document: Either object does not exist OR object has been updated since last read\"}",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docproc&namespace=tilda&trace=b7b68a7116fb8eab82300256cdcf5798&at=1794904487&msg=docsvc+PATCH+failed+for+7db44208-f1f6-467f-b83c-dbcbb23dace7%3A+500+%7B%22message%22%3A%22could+not+update+document+metadata+for+doc+id+7db44208-f1f6-467f-b83c-dbcbb23dace7%3A+could+not+update+the+document%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read%22%7D",
"log": {
"severity": "ERROR",
"service": "docproc",
"cluster": "master",
"namespace": "tilda",
"message": "docsvc PATCH failed for 7db44208-f1f6-467f-b83c-dbcbb23dace7: 500 {\"message\":\"could not update document metadata for doc id 7db44208-f1f6-467f-b83c-dbcbb23dace7: could not update the document: Either object does not exist OR object has been updated since last read\"}",
"trace": "b7b68a7116fb8eab82300256cdcf5798",
"log": "stdout",
"ts": "2026-11-17T08:34:47.938613056Z"
},
"log_key": "3fc355714dff25d1afa5c817956c78ce060ff4f268207a99ab838805955c0a48",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904523,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52075,
"message_url": "https://xray.tilda.pizza/alerts/messages/52075"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52073 2026-11-17 08:34:52 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\ncould not update document metadata for doc id 7db44208-f1f6-467f-b83c-dbcbb23dace7: could not update the document: Either object does not exist OR object has been updated since last read",
"message": "could not update document metadata for doc id 7db44208-f1f6-467f-b83c-dbcbb23dace7: could not update the document: Either object does not exist OR object has been updated since last read",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=b7b68a7116fb8eab82300256cdcf5798&at=1794904487&msg=could+not+update+document+metadata+for+doc+id+7db44208-f1f6-467f-b83c-dbcbb23dace7%3A+could+not+update+the+document%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not update document metadata for doc id 7db44208-f1f6-467f-b83c-dbcbb23dace7: could not update the document: Either object does not exist OR object has been updated since last read",
"trace": "b7b68a7116fb8eab82300256cdcf5798",
"log": "stdout",
"ts": "2026-11-17T08:34:47.934324153Z"
},
"log_key": "880f828c1add82344341fccf86906d149de7822f7be05c44cde89278e3d11b4e",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904522,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52073,
"message_url": "https://xray.tilda.pizza/alerts/messages/52073"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52071 2026-11-17 08:34:49 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\ncould not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update document for categorization: Either object does not exist OR object has been updated since last read",
"message": "could not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update document for categorization: Either object does not exist OR object has been updated since last read",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=5fc5213f3dffb574d30253956b8cb23e&at=1794904486&msg=could+not+update+document+metadata+for+doc+id+8d77d110-eb2a-49d7-be95-bbe2dbabecb6%3A+could+not+update+document+for+categorization%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update document for categorization: Either object does not exist OR object has been updated since last read",
"trace": "5fc5213f3dffb574d30253956b8cb23e",
"log": "stdout",
"ts": "2026-11-17T08:34:46.168601732Z"
},
"log_key": "d9ed7ccbf189c2a651947a348909dd0865379215eb592d2a8da973a5afd5904c",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904519,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52071,
"message_url": "https://xray.tilda.pizza/alerts/messages/52071"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52069 2026-11-17 08:34:49 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\ncould not update document metadata for doc id c7b214fe-b333-4a54-ae72-1e768f742ffd: could not update document for categorization: Either object does not exist OR object has been updated since last read",
"message": "could not update document metadata for doc id c7b214fe-b333-4a54-ae72-1e768f742ffd: could not update document for categorization: Either object does not exist OR object has been updated since last read",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=c064764e0dc90c4b062551c5af6018e9&at=1794904486&msg=could+not+update+document+metadata+for+doc+id+c7b214fe-b333-4a54-ae72-1e768f742ffd%3A+could+not+update+document+for+categorization%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not update document metadata for doc id c7b214fe-b333-4a54-ae72-1e768f742ffd: could not update document for categorization: Either object does not exist OR object has been updated since last read",
"trace": "c064764e0dc90c4b062551c5af6018e9",
"log": "stdout",
"ts": "2026-11-17T08:34:46.297301087Z"
},
"log_key": "c762240c0cfa5115e58ec9bcf85e2e1b67f878c6d5314feac8d66641aca61ad3",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904520,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52069,
"message_url": "https://xray.tilda.pizza/alerts/messages/52069"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52067 2026-11-17 08:34:49 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\nRequest completed",
"message": "Request completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=5fc5213f3dffb574d30253956b8cb23e&at=1794904486&msg=Request+completed",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Request completed",
"trace": "5fc5213f3dffb574d30253956b8cb23e",
"log": "stdout",
"ts": "2026-11-17T08:34:46.168815602Z"
},
"log_key": "77db170144f3de14233f801b01d071ae079a710620d377007912f895aa3c5c3b",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904519,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52067,
"message_url": "https://xray.tilda.pizza/alerts/messages/52067"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52065 2026-11-17 08:34:48 UTC |
channel test | warning Error in docproc (tilda) on master
show the payload{
"title": "Error in docproc (tilda) on master",
"severity": "warning",
"headline": "Error in docproc (tilda) on master",
"text": "Error in docproc (tilda) on master\ndocsvc PATCH failed for 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: 500 {\"message\":\"could not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update document for categorization: Either object does not exist OR object has been updated since last read\"}",
"message": "docsvc PATCH failed for 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: 500 {\"message\":\"could not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update document for categorization: Either object does not exist OR object has been updated since last read\"}",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docproc&namespace=tilda&trace=5fc5213f3dffb574d30253956b8cb23e&at=1794904486&msg=docsvc+PATCH+failed+for+8d77d110-eb2a-49d7-be95-bbe2dbabecb6%3A+500+%7B%22message%22%3A%22could+not+update+document+metadata+for+doc+id+8d77d110-eb2a-49d7-be95-bbe2dbabecb6%3A+could+not+update+document+for+categorization%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read%22%7D",
"log": {
"severity": "ERROR",
"service": "docproc",
"cluster": "master",
"namespace": "tilda",
"message": "docsvc PATCH failed for 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: 500 {\"message\":\"could not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update document for categorization: Either object does not exist OR object has been updated since last read\"}",
"trace": "5fc5213f3dffb574d30253956b8cb23e",
"log": "stdout",
"ts": "2026-11-17T08:34:46.178951376Z"
},
"log_key": "07690fd09d96fac52a6ce4743a74feab480d7581d10f395fe567fd8c0420a442",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904518,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52065,
"message_url": "https://xray.tilda.pizza/alerts/messages/52065"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52063 2026-11-17 08:34:47 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794904484",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T08:34:44.779977Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904517,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52063,
"message_url": "https://xray.tilda.pizza/alerts/messages/52063"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52061 2026-11-17 08:30:14 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nNo cached node found for activation chain ref node",
"message": "No cached node found for activation chain ref node",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=b1881d3cd7fbff839e742d6f0d372723&at=1794904210&msg=No+cached+node+found+for+activation+chain+ref+node",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "No cached node found for activation chain ref node",
"trace": "b1881d3cd7fbff839e742d6f0d372723",
"log": "stdout",
"ts": "2026-11-17T08:30:10.939546298Z"
},
"log_key": "bf042eb1b187488d26cc24b28cd653f2aa4c55bb2320a2758b030eccc34a3247",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904244,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52061,
"message_url": "https://xray.tilda.pizza/alerts/messages/52061"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52059 2026-11-17 08:22:38 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nNo cached node found for activation chain ref node",
"message": "No cached node found for activation chain ref node",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=eb09f7ddcb816d48d074f92cc60fb3e9&at=1794903754&msg=No+cached+node+found+for+activation+chain+ref+node",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "No cached node found for activation chain ref node",
"trace": "eb09f7ddcb816d48d074f92cc60fb3e9",
"log": "stdout",
"ts": "2026-11-17T08:22:34.308527471Z"
},
"log_key": "bf042eb1b187488d26cc24b28cd653f2aa4c55bb2320a2758b030eccc34a3247",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794903788,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52059,
"message_url": "https://xray.tilda.pizza/alerts/messages/52059"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52057 2026-11-17 08:12:47 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=75313c24-f239-4f5a-9a30-693ecab5c0a2 greeting still wrong after 2 attempts (The greeting must reproduce the reviewer's own line exactly: 'Estimados {{principal_investigators}}:'. The draft opens 'Dear Pacific Clinical Studies Team,'.) \u2014 sending the best draft anyway",
"message": "[reminder] wo=75313c24-f239-4f5a-9a30-693ecab5c0a2 greeting still wrong after 2 attempts (The greeting must reproduce the reviewer's own line exactly: 'Estimados {{principal_investigators}}:'. The draft opens 'Dear Pacific Clinical Studies Team,'.) \u2014 sending the best draft anyway",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=a6681eb4bb478e7cef99be407d43fd1b&at=1794903163&msg=%5Breminder%5D+wo%3D75313c24-f239-4f5a-9a30-693ecab5c0a2+greeting+still+wrong+after+2+attempts+%28The+greeting+must+reproduce+the+reviewer%27s+own+line+exactly%3A+%27Estimados+%7B%7Bprincipal_investigators%7D%7D%3A%27.+The+draft+opens+%27Dear+Pacific+Clinical+Studies+Team%2C%27.%29+%E2%80%94+sending+the+best+draft+anyway",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=75313c24-f239-4f5a-9a30-693ecab5c0a2 greeting still wrong after 2 attempts (The greeting must reproduce the reviewer's own line exactly: 'Estimados {{principal_investigators}}:'. The draft opens 'Dear Pacific Clinical Studies Team,'.) \u2014 sending the best draft anyway",
"trace": "a6681eb4bb478e7cef99be407d43fd1b",
"log": "stderr",
"ts": "2026-11-17T08:12:43.036994764Z"
},
"log_key": "c4dfd34c090bb70c9801e796fcf0334c14491a4f4e4c11c9c9ca959c0c3807d6",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794903197,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52057,
"message_url": "https://xray.tilda.pizza/alerts/messages/52057"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52055 2026-11-17 08:11:29 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=a6681eb4bb478e7cef99be407d43fd1b&at=1794903083&msg=%5Breminder%5D+wo%3D97bcca47-6e42-49be-bcf8-715ee3a73d53+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ko%27-script+text+%28%27%ED%8C%80%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "a6681eb4bb478e7cef99be407d43fd1b",
"log": "stderr",
"ts": "2026-11-17T08:11:23.533375065Z"
},
"log_key": "a21b1a376c74f5195c8af08090b36f6360bb500c051ade1192a1ebd1eec933bd",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794903119,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52055,
"message_url": "https://xray.tilda.pizza/alerts/messages/52055"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52053 2026-11-17 08:10:53 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=be54167d5123d43e4ab2ca89a1a3ed67&at=1794903049&msg=%5Breminder%5D+wo%3D48275f5e-57af-4670-bd15-2b08e179b69b+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ja%27-script+text+%28%27%E6%A7%98%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "be54167d5123d43e4ab2ca89a1a3ed67",
"log": "stderr",
"ts": "2026-11-17T08:10:49.708853636Z"
},
"log_key": "db8c99fb601ac7cbb971b15f6bd7a430dcf42331e2810535259637448bfb7406",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794903083,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52053,
"message_url": "https://xray.tilda.pizza/alerts/messages/52053"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52051 2026-11-17 08:10:38 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=be54167d5123d43e4ab2ca89a1a3ed67&at=1794903036&msg=%5Breminder%5D+wo%3D850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27zh-tw%27-script+text+%28%27%E6%82%A8%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "be54167d5123d43e4ab2ca89a1a3ed67",
"log": "stderr",
"ts": "2026-11-17T08:10:36.001561484Z"
},
"log_key": "df998a0f4c524270d6d64977e8c1d486629224b4953daef9c122c60f8cfc74ad",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794903068,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52051,
"message_url": "https://xray.tilda.pizza/alerts/messages/52051"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52043 2026-11-17 07:14:35 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794899663",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T07:14:23.510106Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794899705,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52043,
"message_url": "https://xray.tilda.pizza/alerts/messages/52043"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52041 2026-11-17 07:05:57 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794899151",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T07:05:51.551406Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794899187,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52041,
"message_url": "https://xray.tilda.pizza/alerts/messages/52041"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52039 2026-11-17 06:56:40 UTC |
channel test | warning Error in studiosvc (tilda) on master
show the payload{
"title": "Error in studiosvc (tilda) on master",
"severity": "warning",
"headline": "Error in studiosvc (tilda) on master",
"text": "Error in studiosvc (tilda) on master\nhttp error response returned to client",
"message": "http error response returned to client",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=studiosvc&namespace=tilda&at=1794898598&msg=http+error+response+returned+to+client",
"log": {
"severity": "ERROR",
"service": "studiosvc",
"cluster": "master",
"namespace": "tilda",
"message": "http error response returned to client",
"trace": null,
"log": "stdout",
"ts": "2026-11-17T06:56:38.728063147Z"
},
"log_key": "ab310e879110b225580d45a31d16bebbb9b120817df3a3e7b1642a87afc82555",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794898630,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": [],
"reserved_at": 1794898600
},
"message_id": 52039,
"message_url": "/alerts/messages/52039"
} |
#in-app delivered#dev-notifications sent |
opened | in-app: 1 inbox row(s) for [email protected] ok |
| #52037 2026-11-17 06:56:39 UTC |
channel test | warning Error in studiosvc (tilda) on master
show the payload{
"title": "Error in studiosvc (tilda) on master",
"severity": "warning",
"headline": "Error in studiosvc (tilda) on master",
"text": "Error in studiosvc (tilda) on master\nresponse set creation failed",
"message": "response set creation failed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=studiosvc&namespace=tilda&trace=4c7fc825c23f5048ecfa6aee48c56797&at=1794898598&msg=response+set+creation+failed",
"log": {
"severity": "ERROR",
"service": "studiosvc",
"cluster": "master",
"namespace": "tilda",
"message": "response set creation failed",
"trace": "4c7fc825c23f5048ecfa6aee48c56797",
"log": "stdout",
"ts": "2026-11-17T06:56:38.72301539Z"
},
"log_key": "026cc253e3528321226f9cf754dcba90363259e82486a4cae61246851106fbcf",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794898690,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52037,
"message_url": "https://xray.tilda.pizza/alerts/messages/52037"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52035 2026-11-17 06:56:39 UTC |
channel test | warning Error in studiosvc (tilda) on master
show the payload{
"title": "Error in studiosvc (tilda) on master",
"severity": "warning",
"headline": "Error in studiosvc (tilda) on master",
"text": "Error in studiosvc (tilda) on master\nRequest completed",
"message": "Request completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=studiosvc&namespace=tilda&trace=4c7fc825c23f5048ecfa6aee48c56797&at=1794898598&msg=Request+completed",
"log": {
"severity": "ERROR",
"service": "studiosvc",
"cluster": "master",
"namespace": "tilda",
"message": "Request completed",
"trace": "4c7fc825c23f5048ecfa6aee48c56797",
"log": "stdout",
"ts": "2026-11-17T06:56:38.728542426Z"
},
"log_key": "12706001fcfc74398af6ce45a2871ca57e27612f99b2566b34c34dcc62dbf44d",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794898629,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52035,
"message_url": "https://xray.tilda.pizza/alerts/messages/52035"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52032 2026-11-17 06:38:22 UTC |
channel test | warning Error in chat-agent (tilda) on master
show the payload{
"title": "Error in chat-agent (tilda) on master",
"severity": "warning",
"headline": "Error in chat-agent (tilda) on master",
"text": "Error in chat-agent (tilda) on master\nException in ASGI application\n",
"message": "Exception in ASGI application\n",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=chat-agent&namespace=tilda&at=1794897500&msg=Exception+in+ASGI+application%0A",
"log": {
"severity": "ERROR",
"service": "chat-agent",
"cluster": "master",
"namespace": "tilda",
"message": "Exception in ASGI application\n",
"trace": null,
"log": "stderr",
"ts": "2026-11-17T06:38:20.071681218Z"
},
"log_key": "6da008b24934c57f7b3d8a2b1786e970fd9107e256c2508345279522456a2593",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794897533,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52032,
"message_url": "https://xray.tilda.pizza/alerts/messages/52032"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52030 2026-11-17 06:38:22 UTC |
channel test | warning Error in chat-agent (tilda) on master
show the payload{
"title": "Error in chat-agent (tilda) on master",
"severity": "warning",
"headline": "Error in chat-agent (tilda) on master",
"text": "Error in chat-agent (tilda) on master\nUnhandled TimeoutError on POST /api/v1/chat/44b4f1ec-4085-4783-980e-42f1896b3f81/runs/stream",
"message": "Unhandled TimeoutError on POST /api/v1/chat/44b4f1ec-4085-4783-980e-42f1896b3f81/runs/stream",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=chat-agent&namespace=tilda&trace=4a123c049b28dd7d6deb25be8306ecea&at=1794897500&msg=Unhandled+TimeoutError+on+POST+%2Fapi%2Fv1%2Fchat%2F44b4f1ec-4085-4783-980e-42f1896b3f81%2Fruns%2Fstream",
"log": {
"severity": "ERROR",
"service": "chat-agent",
"cluster": "master",
"namespace": "tilda",
"message": "Unhandled TimeoutError on POST /api/v1/chat/44b4f1ec-4085-4783-980e-42f1896b3f81/runs/stream",
"trace": "4a123c049b28dd7d6deb25be8306ecea",
"log": "stderr",
"ts": "2026-11-17T06:38:20.046775832Z"
},
"log_key": "faf4bb714e9b98fb3971917c05665db0808c8db5f734087325c1d8fba30561fe",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794897532,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52030,
"message_url": "https://xray.tilda.pizza/alerts/messages/52030"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52028 2026-11-17 06:11:23 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=e90bf6d10ed2f1b27ff1be1d00f36ac6&at=1794895878&msg=%5Breminder%5D+wo%3D97bcca47-6e42-49be-bcf8-715ee3a73d53+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ko%27-script+text+%28%27%ED%8C%80%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "e90bf6d10ed2f1b27ff1be1d00f36ac6",
"log": "stderr",
"ts": "2026-11-17T06:11:18.495984787Z"
},
"log_key": "a21b1a376c74f5195c8af08090b36f6360bb500c051ade1192a1ebd1eec933bd",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794895913,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52028,
"message_url": "https://xray.tilda.pizza/alerts/messages/52028"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52026 2026-11-17 06:11:07 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=5e506b9abbab0e53b3b7f5129308130a&at=1794895865&msg=%5Breminder%5D+wo%3D48275f5e-57af-4670-bd15-2b08e179b69b+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ja%27-script+text+%28%27%E6%A7%98%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "5e506b9abbab0e53b3b7f5129308130a",
"log": "stderr",
"ts": "2026-11-17T06:11:05.007340589Z"
},
"log_key": "db8c99fb601ac7cbb971b15f6bd7a430dcf42331e2810535259637448bfb7406",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794895897,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52026,
"message_url": "https://xray.tilda.pizza/alerts/messages/52026"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52024 2026-11-17 06:10:37 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=5e506b9abbab0e53b3b7f5129308130a&at=1794895836&msg=%5Breminder%5D+wo%3D850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27zh-tw%27-script+text+%28%27%E6%82%A8%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "5e506b9abbab0e53b3b7f5129308130a",
"log": "stderr",
"ts": "2026-11-17T06:10:36.07703947Z"
},
"log_key": "df998a0f4c524270d6d64977e8c1d486629224b4953daef9c122c60f8cfc74ad",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794895867,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52024,
"message_url": "https://xray.tilda.pizza/alerts/messages/52024"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52022 2026-11-17 05:51:02 UTC |
channel test | warning Error in schedsvc (tilda) on master
show the payload{
"title": "Error in schedsvc (tilda) on master",
"severity": "warning",
"headline": "Error in schedsvc (tilda) on master",
"text": "Error in schedsvc (tilda) on master\nerror parsing time periods",
"message": "error parsing time periods",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=schedsvc&namespace=tilda&trace=ace7099be9aeb9adceb57a9a300458db&at=1794894662&msg=error+parsing+time+periods",
"log": {
"severity": "ERROR",
"service": "schedsvc",
"cluster": "master",
"namespace": "tilda",
"message": "error parsing time periods",
"trace": "ace7099be9aeb9adceb57a9a300458db",
"log": "stdout",
"ts": "2026-11-17T05:51:02.034780594Z"
},
"log_key": "383df22466f28d9307b23fa6ffc5e8048ba0976f608f8d339e427d4b7b05b389",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794894692,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52022,
"message_url": "https://xray.tilda.pizza/alerts/messages/52022"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52020 2026-11-17 05:35:00 UTC |
channel test | info Tilda namespace findings (master): all clear — 1 resolved
show the card🔵 Tilda namespace findings (master): all clear — 1 resolved alert: Tilda namespace findings (master) · changed 1 resolved ✅ execsvc server-error rate 89.19% vs a normal 0.00% ± 1.00% for Tue 04:00 UTC (3 of the last 4 checks) — mostly POST /api/trialexec/v2/study/:id/activity/status (787× 5xx) message 2 for this group · last one 48m ago · X-ray message #52020 |
#dev-notifications sent#in-app delivered |
group | ok in-app: 52 inbox row(s) for 52 people on X-ray |
| #52014 2026-11-17 05:28:06 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794893278",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T05:27:58.598886Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794893316,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52014,
"message_url": "https://xray.tilda.pizza/alerts/messages/52014"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52012 2026-11-17 05:20:17 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794892809",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T05:20:09.007504Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794892847,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52012,
"message_url": "https://xray.tilda.pizza/alerts/messages/52012"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52009 2026-11-17 05:01:57 UTC |
channel test | warning Error in dashrsvc (tilda) on master
show the payload{
"title": "Error in dashrsvc (tilda) on master",
"severity": "warning",
"headline": "Error in dashrsvc (tilda) on master",
"text": "Error in dashrsvc (tilda) on master\nFailed to resolve document_id from ai_run_id",
"message": "Failed to resolve document_id from ai_run_id",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=dashrsvc&namespace=tilda&trace=cc6c195e6bb4691b947c00a66eabf24e&at=1794891712&msg=Failed+to+resolve+document_id+from+ai_run_id",
"log": {
"severity": "ERROR",
"service": "dashrsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Failed to resolve document_id from ai_run_id",
"trace": "cc6c195e6bb4691b947c00a66eabf24e",
"log": "stdout",
"ts": "2026-11-17T05:01:52.642482707Z"
},
"log_key": "6020cbf0f8aa1e940471331827fa9fb81f077ddd5b118c4b905177169da8ae06",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794891747,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52009,
"message_url": "https://xray.tilda.pizza/alerts/messages/52009"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52007 2026-11-17 05:01:31 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794891689",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T05:01:29.004453Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794891721,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52007,
"message_url": "https://xray.tilda.pizza/alerts/messages/52007"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52005 2026-11-17 04:46:53 UTC |
channel test | critical execsvc server-error rate 89.19% vs a normal 0.00% ± 1.00% for Tue 04:00 UTC (3 of the last 4 checks) — mostly POST /api/trialexec/v2/study/:id/activity/status (787× 5xx)
show the card🟠 HIGH · execsvc server-error rate 89.19% vs a normal 0.00% ± 1.00% for Tue 04:00 UTC (3 of the last 4 checks) — mostly POST /api/trialexec/v2/study/:id/activity/status (787× 5xx) master · Error-rate anomaly · high execsvc shows an 89.2% error rate on master, significantly above the normal 0.0% ± 1.0% baseline for this weekly time slot. The anomaly spans 3 of the last 4 five-minute buckets (z = 89.2), based on 4 weeks of historical data. The failing endpoint is POST /api/trialexec/v2/study/:id/activity/status, which experienced 787 errors out of 787 total calls. User impact depends on endpoint traffic distribution. The facts measure: err5_pct unit: % value: 89.19 expected_median: 0.00 expected_mad: 1.00 z: 89.19 rollup · observed 56s ago · 1× so far · X-ray message #52005 |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 52 inbox row(s) for 52 people on X-ray |
| #52003 2026-11-17 04:34:27 UTC |
channel test | warning Error in regsvc (tilda) on master
show the payload{
"title": "Error in regsvc (tilda) on master",
"severity": "warning",
"headline": "Error in regsvc (tilda) on master",
"text": "Error in regsvc (tilda) on master\noauth token refresh rejected by provider",
"message": "oauth token refresh rejected by provider",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=regsvc&namespace=tilda&at=1794890065&msg=oauth+token+refresh+rejected+by+provider",
"log": {
"severity": "ERROR",
"service": "regsvc",
"cluster": "master",
"namespace": "tilda",
"message": "oauth token refresh rejected by provider",
"trace": null,
"log": "stdout",
"ts": "2026-11-17T04:34:25.32672437Z"
},
"log_key": "6ca27a7d0989ad0479f52d6ff96c685d6de5df4d24fb66d7063f2174ae33cea0",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794890099,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52003,
"message_url": "https://xray.tilda.pizza/alerts/messages/52003"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52001 2026-11-17 04:34:27 UTC |
channel test | warning Error in regsvc (tilda) on master
show the payload{
"title": "Error in regsvc (tilda) on master",
"severity": "warning",
"headline": "Error in regsvc (tilda) on master",
"text": "Error in regsvc (tilda) on master\nfailed to refresh token; session refresh loop stopping",
"message": "failed to refresh token; session refresh loop stopping",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=regsvc&namespace=tilda&at=1794890065&msg=failed+to+refresh+token%3B+session+refresh+loop+stopping",
"log": {
"severity": "ERROR",
"service": "regsvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to refresh token; session refresh loop stopping",
"trace": null,
"log": "stdout",
"ts": "2026-11-17T04:34:25.32677349Z"
},
"log_key": "3e851c0107918c12cccdc5c5232f60c9aff54bcdca4164121b97e06e20deb4e8",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794890097,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52001,
"message_url": "https://xray.tilda.pizza/alerts/messages/52001"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51998 2026-11-17 04:16:17 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\ncould not match participant questionnaire/question/table/doc-upload to activity graph",
"message": "could not match participant questionnaire/question/table/doc-upload to activity graph",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=155a3bef7d375e6fd6a4cef1db49f6e3&at=1794888974&msg=could+not+match+participant+questionnaire%2Fquestion%2Ftable%2Fdoc-upload+to+activity+graph",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not match participant questionnaire/question/table/doc-upload to activity graph",
"trace": "155a3bef7d375e6fd6a4cef1db49f6e3",
"log": "stdout",
"ts": "2026-11-17T04:16:14.500150165Z"
},
"log_key": "d6dc1706a7113bb3b7c2d69ab60ed30804cb9bf28f56d4f2280bed1aa61b32a0",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794889069,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51998,
"message_url": "https://xray.tilda.pizza/alerts/messages/51998"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51996 2026-11-17 04:16:17 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nFailed to fetch activity status",
"message": "Failed to fetch activity status",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=7ecc0ff1e56ae17c05c88a1b04780a82&at=1794888974&msg=Failed+to+fetch+activity+status",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Failed to fetch activity status",
"trace": "7ecc0ff1e56ae17c05c88a1b04780a82",
"log": "stdout",
"ts": "2026-11-17T04:16:14.586932118Z"
},
"log_key": "826d3d9347bec580c70fbebf5014a82dd6d897ebccddd610c2fc8ba98cdfe3ae",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794889008,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51996,
"message_url": "https://xray.tilda.pizza/alerts/messages/51996"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51994 2026-11-17 04:16:17 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nRequest completed",
"message": "Request completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f1d365830b94cf97bf5c96d5e36d30d5&at=1794888972&msg=Request+completed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Request completed",
"trace": "f1d365830b94cf97bf5c96d5e36d30d5",
"log": "stdout",
"ts": "2026-11-17T04:16:12.502809085Z"
},
"log_key": "520c14762a99a5be7aefae37d3d55b28e8805375174fc9968943ebb288bd249c",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794889007,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51994,
"message_url": "https://xray.tilda.pizza/alerts/messages/51994"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51992 2026-11-17 04:11:33 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=018a75f866afc8521606b9f807ebcc9b&at=1794888682&msg=%5Breminder%5D+wo%3D97bcca47-6e42-49be-bcf8-715ee3a73d53+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ko%27-script+text+%28%27%ED%8C%80%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "018a75f866afc8521606b9f807ebcc9b",
"log": "stderr",
"ts": "2026-11-17T04:11:22.915624207Z"
},
"log_key": "a21b1a376c74f5195c8af08090b36f6360bb500c051ade1192a1ebd1eec933bd",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794888723,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51992,
"message_url": "https://xray.tilda.pizza/alerts/messages/51992"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51990 2026-11-17 04:11:12 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=690fc5aec2b80f9deacfd4742f575fb8&at=1794888668&msg=%5Breminder%5D+wo%3D48275f5e-57af-4670-bd15-2b08e179b69b+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ja%27-script+text+%28%27%E6%A7%98%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "690fc5aec2b80f9deacfd4742f575fb8",
"log": "stderr",
"ts": "2026-11-17T04:11:08.793961152Z"
},
"log_key": "db8c99fb601ac7cbb971b15f6bd7a430dcf42331e2810535259637448bfb7406",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794888702,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51990,
"message_url": "https://xray.tilda.pizza/alerts/messages/51990"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51988 2026-11-17 04:11:02 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=690fc5aec2b80f9deacfd4742f575fb8&at=1794888657&msg=%5Breminder%5D+wo%3D850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27zh-tw%27-script+text+%28%27%E6%82%A8%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "690fc5aec2b80f9deacfd4742f575fb8",
"log": "stderr",
"ts": "2026-11-17T04:10:57.801069754Z"
},
"log_key": "df998a0f4c524270d6d64977e8c1d486629224b4953daef9c122c60f8cfc74ad",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794888692,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51988,
"message_url": "https://xray.tilda.pizza/alerts/messages/51988"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51986 2026-11-17 04:10:29 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\ncould not match participant questionnaire/question/table/doc-upload to activity graph",
"message": "could not match participant questionnaire/question/table/doc-upload to activity graph",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=53245cec7eb9fed7222dc5d85462b620&at=1794888620&msg=could+not+match+participant+questionnaire%2Fquestion%2Ftable%2Fdoc-upload+to+activity+graph",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not match participant questionnaire/question/table/doc-upload to activity graph",
"trace": "53245cec7eb9fed7222dc5d85462b620",
"log": "stdout",
"ts": "2026-11-17T04:10:20.257053543Z"
},
"log_key": "d6dc1706a7113bb3b7c2d69ab60ed30804cb9bf28f56d4f2280bed1aa61b32a0",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794888659,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51986,
"message_url": "https://xray.tilda.pizza/alerts/messages/51986"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51984 2026-11-17 04:10:23 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nFailed to fetch activity status",
"message": "Failed to fetch activity status",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f09f75be21bec7599e192eb15f9913f4&at=1794888619&msg=Failed+to+fetch+activity+status",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Failed to fetch activity status",
"trace": "f09f75be21bec7599e192eb15f9913f4",
"log": "stdout",
"ts": "2026-11-17T04:10:19.411731586Z"
},
"log_key": "826d3d9347bec580c70fbebf5014a82dd6d897ebccddd610c2fc8ba98cdfe3ae",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794888654,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51984,
"message_url": "https://xray.tilda.pizza/alerts/messages/51984"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51982 2026-11-17 04:10:23 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nRequest completed",
"message": "Request completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f09f75be21bec7599e192eb15f9913f4&at=1794888619&msg=Request+completed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Request completed",
"trace": "f09f75be21bec7599e192eb15f9913f4",
"log": "stdout",
"ts": "2026-11-17T04:10:19.411868326Z"
},
"log_key": "520c14762a99a5be7aefae37d3d55b28e8805375174fc9968943ebb288bd249c",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794888653,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51982,
"message_url": "https://xray.tilda.pizza/alerts/messages/51982"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |