Interactive Controls
Set Interactive Time Range
All interactive sessions use a set time range to cache metrics data and perform quick tests.
Query Time Start:
Query Time Duration:
Query Time Scrubber:
All interactive sessions use a set time range to cache metrics data and perform quick tests.
Query Time Start:
Query Time Duration:
Query Time Scrubber:
Shows the JSON configs for this server configuration
{
"web_http_port": 3000,
"web_path": "./web",
"site_config_path": "config/site.json",
"curve_path_format": "config/curves/%s.json",
"server_loop_delay": "0.8s",
"query_lock_timeout": "60s",
"query_fast_interval": "2s",
"query_fast_duration": "30s",
"interactive_duration_minutes_default": 15,
"prometheus_export_port": 8611,
"enable_demo": true,
"demo_api_port": 8709,
"reload_templates_always": true,
"log_template_parsing": false
}
{
"name": "Sireus Prototype",
"info": "A demonstration of how to use: Edge -> App -> Database",
"bot_group_paths": [
"config/bot_groups/demo_edge.json",
"config/bot_groups/demo_app.json",
"config/bot_groups/demo_database.json",
"config/bot_groups/demo_sireus.json"
],
"query_servers": [
{
"server_type": 0,
"name": "prometheus_primary",
"info": "Primary Prometheus cluster",
"host": "localhost",
"port": 9090,
"auth_user": "",
"auth_secret": "",
"default_data_duration": 60,
"default_step": "15s",
"web_url_format": "http://localhost:9090/graph?g0.expr={{query}}"
}
]
}
{
"name": "Edge",
"info": "Example Edge, simulating traffic coming through into a web app from the Internet",
"states": [
{
"name": "Operation",
"info": "Basic operational states, to group Actions",
"labels": ["Default", "Problem", "Evaluate", "Escalate", "EscalateWait"]
},
{
"name": "Traffic",
"info": "What does our traffic situation look like?",
"labels": ["Default", "High", "Low", "None"]
},
{
"name": "Attack Risk",
"info": "What is the likelihood we are under attack now?",
"labels": ["Default", "Low", "High", "Critical"]
}
],
"lock_timers": [
{
"type": 1,
"name": "Single Bot Lock",
"info": "Per bot lock, so that each bot can operate independently. Use for Node or Agent level control."
},
{
"type": 0,
"name": "Full Bot Group Lock",
"info": "Lock for controlling the entire Bot Group. Use for Service or Platform level control."
}
],
"bot_timeout_stale": 60,
"bot_timeout_remove": 120,
"bot_remove_store_duration": "24h",
"refuse_bot_resumption": false,
"action_threshold": 0.5,
"journal_rollup_states": ["Operation.Problem", "Operation.Evaluate", "Operation.Escalate", "Operation.EscalateWait"],
"journal_rollup_duration": "30m",
"bot_extractor": {
"query_name": "Edge Octets In",
"key": "circuit"
},
"queries": [
{
"query_server": "prometheus_primary",
"query_type": 0,
"name": "Demo App Requests",
"info": "Demo-only information, simulation of requests from the Internet, outside the edge",
"query": "irate(demo_requests[30s])",
"interval": "5s"
},
{
"query_server": "prometheus_primary",
"query_type": 0,
"name": "Edge Octets In",
"info": "Rate of octet ingress",
"query": "irate(demo_if_in_octets{bot_group=\"Edge\"}[30s])",
"interval": "5s"
},
{
"query_server": "prometheus_primary",
"query_type": 0,
"name": "Edge Octets Out",
"info": "Rate of octet egress",
"query": "irate(demo_if_out_octets{bot_group=\"Edge\"}[30s])",
"interval": "5s"
},
{
"query_server": "prometheus_primary",
"query_type": 0,
"name": "Circuit Circuit State",
"info": "Is the circuit up or not?",
"query": "demo_if_link_state{bot_group=\"Edge\"}",
"interval": "5s"
}
],
"variables": [
{
"name": "demo_app_requests",
"format": 10,
"bot_key": "",
"query_name": "Demo App Requests",
"query_key": "",
"query_key_value": "",
"evaluate": "",
"bool_range_start": 1,
"bool_range_end": 1,
"bool_invert": false,
"export": false
},
{
"name": "octets_in",
"format": 3,
"bot_key": "circuit",
"query_name": "Edge Octets In",
"query_key": "",
"query_key_value": "",
"evaluate": "",
"bool_range_start": 1,
"bool_range_end": 1,
"bool_invert": false,
"export": false
},
{
"name": "octets_out",
"format": 3,
"bot_key": "circuit",
"query_name": "Edge Octets Out",
"query_key": "",
"query_key_value": "",
"evaluate": "",
"bool_range_start": 1,
"bool_range_end": 1,
"bool_invert": false,
"export": false
},
{
"name": "link_state",
"format": 1,
"bot_key": "circuit",
"query_name": "Circuit Circuit State",
"query_key": "",
"query_key_value": "",
"evaluate": "",
"bool_range_start": 1,
"bool_range_end": 1,
"bool_invert": false,
"export": false
}
],
"actions": [
{
"is_launched": true,
"name": "Circuit Down",
"info": "Circuit shows Circuit Status down",
"is_disabled": false,
"weight": 2.0,
"weight_min": 0.5,
"weight_threshold": 0.5,
"execute_repeat_delay": "5s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Operation.Default"
],
"considerations": [
{
"name": "Circuit Down",
"weight": 1.0,
"curve": "dec_boolean",
"range_start": 0,
"range_end": 1,
"evaluate": "link_state"
}
],
"command": {
"name": "Circuit is down. Set state to Problem",
"log_format": "{{conditionCommand.Name}}. Set forward states: {{format_array_string_csv conditionCommand.SetBotStates}}",
"type": 0,
"content": "",
"success_status": 0,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
"Operation.Problem"
],
"reset_bot_states": []
}
},
{
"is_launched": true,
"name": "Circuit could be repaired via API",
"info": "This is the fastest way to fix the link state, but it only works in some cases. Best to try this first.",
"is_disabled": false,
"weight": 5.0,
"weight_min": 0.5,
"weight_threshold": 0.5,
"execute_repeat_delay": "25s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Operation.Problem"
],
"considerations": [
{
"name": "Circuit Down",
"weight": 1.0,
"curve": "dec_boolean",
"range_start": 0,
"range_end": 1,
"evaluate": "link_state"
}
],
"command": {
"name": "Attempt to Automatic Circuit Repair",
"log_format": "{{conditionCommand.Name}}. Set forward states: {{format_array_string_csv conditionCommand.SetBotStates}}",
"type": 2,
"content": "http://localhost:{{appConfig.DemoApiPort}}/fix/circuit?name={{bot.Name}}",
"success_status": 201,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
"Operation.Evaluate"
],
"reset_bot_states": []
}
},
{
"is_launched": true,
"name": "Circuit has not been fixed",
"info": "We waited to see if anything could automatically fix the circuit, but nothing could.",
"is_disabled": false,
"weight": 4.0,
"weight_min": 0.5,
"weight_threshold": 0.5,
"execute_repeat_delay": "25s",
"required_available": "15s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Operation.Evaluate"
],
"considerations": [
{
"name": "Circuit Down",
"weight": 1.0,
"curve": "dec_boolean",
"range_start": 0,
"range_end": 1,
"evaluate": "link_state"
}
],
"command": {
"name": "Escalate to on-call",
"log_format": "{{conditionCommand.Name}}. Set forward states: {{format_array_string_csv conditionCommand.SetBotStates}}",
"type": 0,
"content": "",
"success_status": 201,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
"Operation.Escalate"
],
"reset_bot_states": []
}
},
{
"is_launched": true,
"name": "Escalation has not fixed the circuit",
"info": "We waited for on-call to fix the problem, but they have not yet.",
"is_disabled": false,
"weight": 4.0,
"weight_min": 0.5,
"weight_threshold": 0.5,
"execute_repeat_delay": "25s",
"required_available": "15s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Operation.Escalate"
],
"considerations": [
{
"name": "Circuit Down",
"weight": 1.0,
"curve": "dec_boolean",
"range_start": 0,
"range_end": 1,
"evaluate": "link_state"
}
],
"command": {
"name": "Reset to Default",
"log_format": "{{conditionCommand.Name}}. Reset forward states: {{format_array_string_csv conditionCommand.ResetBotStates}}",
"type": 0,
"content": "",
"success_status": 201,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
],
"reset_bot_states": ["Operation"]
}
},
{
"is_launched": true,
"name": "Circuit Is Repaired",
"info": "Is this link repaired? Then reset",
"is_disabled": false,
"weight": 5.0,
"weight_min": 0.5,
"weight_threshold": 0.5,
"execute_repeat_delay": "5s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Operation.Evaluate"
],
"considerations": [
{
"name": "Circuit Down",
"weight": 1.0,
"curve": "inc_boolean",
"range_start": 0,
"range_end": 1,
"evaluate": "link_state"
}
],
"command": {
"name": "Reset Problem, it was fixed",
"log_format": "{{conditionCommand.Name}}. Reset States: {{format_array_string_csv conditionCommand.ResetBotStates}}",
"type": 0,
"content": "",
"success_status": 0,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
],
"reset_bot_states": ["Operation"]
}
},
{
"is_launched": true,
"name": "Circuit Is Repaired Without Command",
"info": "Is this link repaired without waiting for a command? Then reset",
"is_disabled": false,
"weight": 5.0,
"weight_min": 0.5,
"weight_threshold": 0.5,
"execute_repeat_delay": "5s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Operation.Problem"
],
"considerations": [
{
"name": "Circuit Down",
"weight": 1.0,
"curve": "inc_boolean",
"range_start": 0,
"range_end": 1,
"evaluate": "link_state"
}
],
"command": {
"name": "Problem fixed itself. Reset Operation",
"log_format": "{{conditionCommand.Name}}. Reset States: {{format_array_string_csv conditionCommand.ResetBotStates}}",
"type": 0,
"content": "",
"success_status": 0,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
],
"reset_bot_states": ["Operation"]
}
},
{
"is_launched": true,
"name": "Traffic is High",
"info": "High amount of Traffic",
"is_disabled": false,
"weight": 5.0,
"weight_min": 0.5,
"weight_threshold": 3.5,
"execute_repeat_delay": "5s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Traffic.Default"
],
"considerations": [
{
"name": "High Traffic",
"weight": 1.0,
"curve": "inc_smooth",
"range_start": 500,
"range_end": 1500,
"evaluate": "demo_app_requests"
}
],
"command": {
"name": "Set High Traffic State",
"log_format": "{{conditionCommand.Name}}. Set States: {{format_array_string_csv conditionCommand.SetBotStates}}",
"type": 0,
"content": "",
"success_status": 0,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
"Traffic.High"
],
"reset_bot_states": []
}
},
{
"is_launched": true,
"name": "No longer High Traffic",
"info": "Used to have High Traffic, but do not now",
"is_disabled": false,
"weight": 6.0,
"weight_min": 1.3,
"weight_threshold": 3.5,
"execute_repeat_delay": "5s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Traffic.High"
],
"considerations": [
{
"name": "No Longer High Traffic",
"weight": 1.0,
"curve": "inc_smooth",
"range_start": 1500,
"range_end": 500,
"evaluate": "demo_app_requests"
}
],
"command": {
"name": "Set High Traffic State back to Default",
"log_format": "{{conditionCommand.Name}}. Reset States: {{format_array_string_csv conditionCommand.ResetBotStates}}",
"type": 0,
"content": "",
"success_status": 0,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
],
"reset_bot_states": ["Traffic"]
}
},
{
"is_launched": true,
"name": "No Traffic",
"info": "No traffic is coming in",
"is_disabled": false,
"weight": 5.0,
"weight_min": 0.5,
"weight_threshold": 3.5,
"execute_repeat_delay": "5s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Traffic.Default"
],
"considerations": [
{
"name": "No Traffic",
"weight": 1.0,
"curve": "dec_boolean",
"range_start": 0,
"range_end": 1,
"evaluate": "demo_app_requests"
}
],
"command": {
"name": "Set No Traffic State",
"log_format": "{{conditionCommand.Name}}. Set States: {{format_array_string_csv conditionCommand.SetBotStates}}",
"type": 0,
"content": "",
"success_status": 0,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
"Traffic.None"
],
"reset_bot_states": []
}
},
{
"is_launched": true,
"name": "Traffic has resumed from stopped",
"info": "Used to have No Traffic, but now there is traffic again",
"is_disabled": false,
"weight": 6.0,
"weight_min": 1.3,
"weight_threshold": 3.5,
"execute_repeat_delay": "5s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Traffic.None"
],
"considerations": [
{
"name": "Has Traffic",
"weight": 1.0,
"curve": "inc_boolean",
"range_start": 0,
"range_end": 1,
"evaluate": "demo_app_requests"
}
],
"command": {
"name": "Set No Traffic State back to Default",
"log_format": "{{conditionCommand.Name}}. Reset States: {{format_array_string_csv conditionCommand.ResetBotStates}}",
"type": 0,
"content": "",
"success_status": 0,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
],
"reset_bot_states": ["Traffic"]
}
}
]
}
{
"name": "App",
"info": "Example Application, to simulate a web app that can process or timeout requests",
"states": [
{
"name": "Operation",
"info": "Basic operational states, to group Actions",
"labels": ["Default", "Problem", "Evaluate", "Escalate", "EscalateWait"]
},
{
"name": "Traffic",
"info": "What does our traffic situation look like?",
"labels": ["Default", "High", "Low", "None"]
},
{
"name": "Attack Risk",
"info": "What is the likelihood we are under attack now?",
"labels": ["Default", "Low", "High", "Critical"]
}
],
"lock_timers": [
{
"type": 1,
"name": "Single Bot Lock",
"info": "Per bot lock, so that each bot can operate independently. Use for Node or Agent level control."
},
{
"type": 0,
"name": "Full Bot Group Lock",
"info": "Lock for controlling the entire Bot Group. Use for Service or Platform level control."
}
],
"bot_timeout_stale": 60,
"bot_timeout_remove": 120,
"bot_remove_store_duration": "24h",
"refuse_bot_resumption": false,
"action_threshold": 0.5,
"journal_rollup_states": ["Operation.Problem", "Operation.Evaluate", "Operation.Escalate", "Operation.EscalateWait"],
"journal_rollup_duration": "30m",
"bot_extractor": {
"query_name": "App Wait Queue",
"key": "job"
},
"queries": [
{
"query_server": "prometheus_primary",
"query_type": 0,
"name": "App Wait Queue",
"info": "Current waiting requests",
"query": "demo_req_queue_wait{bot_group=\"App\"}",
"interval": "5s"
},
{
"query_server": "prometheus_primary",
"query_type": 0,
"name": "App Timeouts",
"info": "Rate of timeouts over time",
"query": "irate(demo_req_timeout{bot_group=\"App\"}[30s])",
"interval": "5s"
},
{
"query_server": "prometheus_primary",
"query_type": 0,
"name": "App Processed",
"info": "Rate of successful requests over time",
"query": "irate(demo_req_success{bot_group=\"App\"}[30s])",
"interval": "5s"
}
],
"variables": [
{
"name": "wait_queue",
"format": 11,
"bot_key": "job",
"query_name": "App Wait Queue",
"query_key": "",
"query_key_value": "",
"evaluate": "",
"bool_range_start": 1,
"bool_range_end": 1,
"bool_invert": false,
"export": false
},
{
"name": "timeout_rate",
"format": 10,
"bot_key": "job",
"query_name": "App Timeouts",
"query_key": "",
"query_key_value": "",
"evaluate": "",
"bool_range_start": 1,
"bool_range_end": 1,
"bool_invert": false,
"export": false
},
{
"name": "processed",
"format": 10,
"bot_key": "job",
"query_name": "App Processed",
"query_key": "",
"query_key_value": "",
"evaluate": "",
"bool_range_start": 1,
"bool_range_end": 1,
"bool_invert": false,
"export": false
},
{
"name": "request_problem",
"format": 0,
"bot_key": "job",
"query_name": "",
"query_key": "",
"query_key_value": "",
"evaluate": "timeout_rate + wait_queue",
"bool_range_start": 1,
"bool_range_end": 1,
"bool_invert": false,
"export": true
}
],
"actions": [
{
"is_launched": true,
"name": "Long Wait Queue",
"info": "The wait queue is long and indicates a problem",
"is_disabled": false,
"weight": 2.0,
"weight_min": 0.5,
"weight_threshold": 0.5,
"execute_repeat_delay": "5s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Operation.Default"
],
"considerations": [
{
"name": "Wait Queue over 100",
"weight": 1.0,
"curve": "inc_boolean",
"range_start": 0,
"range_end": 200,
"evaluate": "wait_queue"
}
],
"command": {
"name": "Long wait queue. Set state to Problem",
"log_format": "{{condition.Name}}: {{wait_queue}} Set forward states: {{format_array_string_csv conditionCommand.SetBotStates}}",
"type": 0,
"content": "",
"success_status": 0,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
"Operation.Problem"
],
"reset_bot_states": []
}
},
{
"is_launched": true,
"name": "Many Timeouts",
"info": "Many requests are timing out",
"is_disabled": false,
"weight": 5.0,
"weight_min": 0.5,
"weight_threshold": 0.5,
"execute_repeat_delay": "5s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Operation.Default"
],
"considerations": [
{
"name": "Timeout Rate over 100",
"weight": 1.0,
"curve": "inc_boolean",
"range_start": 0,
"range_end": 200,
"evaluate": "timeout_rate"
}
],
"command": {
"name": "Many requesting timing out",
"log_format": "{{conditionCommand.Name}}: {{timeout_rate}} Set forward states: {{format_array_string_csv conditionCommand.SetBotStates}}",
"type": 0,
"content": "",
"success_status": 0,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
"Operation.Problem"
],
"reset_bot_states": []
}
},
{
"is_launched": true,
"name": "Normal Operation",
"info": "Things are back to normal",
"is_disabled": false,
"weight": 5.0,
"weight_min": 0.5,
"weight_threshold": 0.5,
"execute_repeat_delay": "5s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Operation.Problem"
],
"considerations": [
{
"name": "Timeout Rate over 100",
"weight": 1.0,
"curve": "inc_boolean",
"range_start": 200,
"range_end": 0,
"evaluate": "timeout_rate + wait_queue"
}
],
"command": {
"name": "Reverted to normal, no action",
"log_format": "{{conditionCommand.Name}}: Timeouts: {{timeout_rate}} Waiting: {{wait_queue}} Reset states: {{format_array_string_csv conditionCommand.ResetBotStates}}",
"type": 0,
"content": "",
"success_status": 0,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
],
"reset_bot_states": ["Operation"]
}
}
]
}
{
"name": "Database",
"info": "Example Database, to simulate requests that can back up and timeout if in a degraded state or too much traffic",
"states": [
{
"name": "Operation",
"info": "Basic operational states, to group Actions",
"labels": ["Default", "Problem", "Evaluate", "Escalate", "EscalateWait"]
},
{
"name": "Traffic",
"info": "What does our traffic situation look like?",
"labels": ["Default", "High", "Low", "None"]
},
{
"name": "Attack Risk",
"info": "What is the likelihood we are under attack now?",
"labels": ["Default", "Low", "High", "Critical"]
}
],
"lock_timers": [
{
"type": 1,
"name": "Single Bot Lock",
"info": "Per bot lock, so that each bot can operate independently. Use for Node or Agent level control."
},
{
"type": 0,
"name": "Full Bot Group Lock",
"info": "Lock for controlling the entire Bot Group. Use for Service or Platform level control."
}
],
"bot_timeout_stale": 60,
"bot_timeout_remove": 120,
"bot_remove_store_duration": "24h",
"refuse_bot_resumption": false,
"action_threshold": 0.5,
"journal_rollup_states": ["Operation.Problem", "Operation.Evaluate", "Operation.Escalate", "Operation.EscalateWait"],
"journal_rollup_duration": "30m",
"bot_extractor": {
"query_name": "Database Wait Queue",
"key": "job"
},
"queries": [
{
"query_server": "prometheus_primary",
"query_type": 0,
"name": "Database Wait Queue",
"info": "Current waiting requests",
"query": "demo_req_queue_wait{bot_group=\"Database\"}",
"interval": "5s"
},
{
"query_server": "prometheus_primary",
"query_type": 0,
"name": "Database Timeouts",
"info": "Rate of timeouts over time",
"query": "irate(demo_req_timeout{bot_group=\"Database\"}[30s])",
"interval": "5s"
},
{
"query_server": "prometheus_primary",
"query_type": 0,
"name": "Database Processed",
"info": "Rate of successful requests over time",
"query": "irate(demo_req_success{bot_group=\"Database\"}[30s])",
"interval": "5s"
}
],
"variables": [
{
"name": "wait_queue",
"format": 11,
"bot_key": "job",
"query_name": "Database Wait Queue",
"query_key": "",
"query_key_value": "",
"evaluate": "",
"bool_range_start": 1,
"bool_range_end": 1,
"bool_invert": false,
"export": false
},
{
"name": "timeout_rate",
"format": 10,
"bot_key": "job",
"query_name": "Database Timeouts",
"query_key": "",
"query_key_value": "",
"evaluate": "",
"bool_range_start": 1,
"bool_range_end": 1,
"bool_invert": false,
"export": false
},
{
"name": "processed",
"format": 10,
"bot_key": "job",
"query_name": "Database Processed",
"query_key": "",
"query_key_value": "",
"evaluate": "",
"bool_range_start": 1,
"bool_range_end": 1,
"bool_invert": false,
"export": false
},
{
"name": "request_problem",
"format": 0,
"bot_key": "job",
"query_name": "",
"query_key": "",
"query_key_value": "",
"evaluate": "timeout_rate + wait_queue",
"bool_range_start": 1,
"bool_range_end": 1,
"bool_invert": false,
"export": true
}
],
"actions": [
{
"is_launched": true,
"name": "Long Wait Queue",
"info": "The wait queue is long and indicates a problem",
"is_disabled": false,
"weight": 2.0,
"weight_min": 0.5,
"weight_threshold": 0.5,
"execute_repeat_delay": "5s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Operation.Default"
],
"considerations": [
{
"name": "Long Wait Queue",
"weight": 1.0,
"curve": "inc_smooth",
"range_start": 0,
"range_end": 500,
"evaluate": "wait_queue"
}
],
"command": {
"name": "Long wait queue. Set state to Problem.",
"log_format": "{{conditionCommand.Name}}. Set forward states: {{format_array_string_csv conditionCommand.SetBotStates}}",
"type": 0,
"content": "",
"success_status": 0,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
"Operation.Problem"
],
"reset_bot_states": []
}
},
{
"is_launched": true,
"name": "Normal Operation",
"info": "Things are back to normal",
"is_disabled": false,
"weight": 5.0,
"weight_min": 0.5,
"weight_threshold": 0.5,
"execute_repeat_delay": "5s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Operation.Problem"
],
"considerations": [
{
"name": "Timeout Rate over 100",
"weight": 1.0,
"curve": "inc_boolean",
"range_start": 200,
"range_end": 0,
"evaluate": "timeout_rate + wait_queue"
}
],
"command": {
"name": "Reverted to normal, no action.",
"log_format": "{{conditionCommand.Name}}: Timeouts: {{timeout_rate}} Waiting: {{wait_queue}} Reset states: {{format_array_string_csv conditionCommand.ResetBotStates}}",
"type": 0,
"content": "",
"success_status": 0,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
],
"reset_bot_states": ["Operation"]
}
},
{
"is_launched": true,
"name": "Is database storage degraded?",
"info": "There is a problem, maybe the storage is degraded",
"is_disabled": false,
"weight": 2.0,
"weight_min": 0.5,
"weight_threshold": 0.5,
"execute_repeat_delay": "5s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Operation.Problem"
],
"considerations": [
{
"name": "Long Wait Queue",
"weight": 1.0,
"curve": "inc_smooth",
"range_start": 0,
"range_end": 500,
"evaluate": "wait_queue"
}
],
"command": {
"name": "Attempt to fix degraded storage",
"log_format": "{{conditionCommand.Name}}. Set forward states: {{format_array_string_csv conditionCommand.SetBotStates}}",
"type": 2,
"content": "http://localhost:{{appConfig.DemoApiPort}}/fix/database_storage_degraded?name={{bot.Name}}",
"success_status": 200,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
"Operation.Evaluate"
],
"reset_bot_states": []
}
},
{
"is_launched": true,
"name": "Processing Fixed",
"info": "The database is processing normally again",
"is_disabled": false,
"weight": 2.0,
"weight_min": 0.5,
"weight_threshold": 0.5,
"execute_repeat_delay": "5s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Operation.Evaluate"
],
"considerations": [
{
"name": "Wait Queue over 2000",
"weight": 1.0,
"curve": "inc_smooth",
"range_start": 500,
"range_end": 0,
"evaluate": "wait_queue"
}
],
"command": {
"name": "Reset Operation to Default State",
"log_format": "{{conditionCommand.Name}}. Reset forward states: {{format_array_string_csv conditionCommand.ResetBotStates}}",
"type": 0,
"content": "",
"success_status": 200,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
],
"reset_bot_states": ["Operation"]
}
},
{
"is_launched": true,
"name": "Processing Bad After Fix Attempt",
"info": "We tried to fix it, but it didn't improve",
"is_disabled": false,
"weight": 2.0,
"weight_min": 0.5,
"weight_threshold": 0.5,
"execute_repeat_delay": "5s",
"required_available": "4s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Operation.Evaluate"
],
"considerations": [
{
"name": "Always True after 10s",
"weight": 1.0,
"curve": "inc_boolean",
"range_start": 0,
"range_end": 1,
"evaluate": "1"
}
],
"command": {
"name": "Escalate to On-Call",
"log_format": "{{conditionCommand.Name}}. Set forward states: {{format_array_string_csv conditionCommand.SetBotStates}}",
"type": 0,
"content": "",
"success_status": 0,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
"Operation.Escalate"
],
"reset_bot_states": []
}
},
{
"is_launched": true,
"name": "Waiting for On-Call",
"info": "Escalated and waiting for on-call to fix the problem",
"is_disabled": false,
"weight": 2.0,
"weight_min": 0.5,
"weight_threshold": 0.5,
"execute_repeat_delay": "5s",
"required_available": "10s",
"required_lock_timers": [
"Single Bot Lock"
],
"required_states": [
"Operation.Escalate"
],
"considerations": [
{
"name": "Always True after 10s",
"weight": 1.0,
"curve": "inc_boolean",
"range_start": 0,
"range_end": 1,
"evaluate": "1"
}
],
"command": {
"name": "Waited 10s for On-Call, Reset State and Try Again",
"log_format": "{{conditionCommand.Name}}. Reset forward states: {{format_array_string_csv conditionCommand.ResetBotStates}}",
"type": 0,
"content": "",
"success_status": 0,
"success_content": "",
"lock_timer_duration": "1s",
"host_exec_key": "",
"set_bot_states": [
],
"reset_bot_states": ["Operation"]
}
}
]
}
{
"name": "Sireus in Sireus",
"info": "Import data from Prometheus that we exported, so we can monitor and execute commands on ourself",
"states": [
{
"name": "In Use",
"info": "Is there at least 1 user using the demo?",
"labels": ["Default", "In Use", "Abandoned"]
}
],
"lock_timers": [
{
"type": 1,
"name": "Single Bot Lock",
"info": "Per bot lock, so that each bot can operate independently. Use for Node or Agent level control."
},
{
"type": 0,
"name": "Full Bot Group Lock",
"info": "Lock for controlling the entire Bot Group. Use for Service or Platform level control."
}
],
"bot_timeout_stale": 60,
"bot_timeout_remove": 120,
"bot_remove_store_duration": "24h",
"refuse_bot_resumption": false,
"action_threshold": 0.5,
"journal_rollup_states": [],
"journal_rollup_duration": "30m",
"bot_extractor": {
"query_name": "Sireus Bot Group Counts",
"key": "bot_group"
},
"queries": [
{
"query_server": "prometheus_primary",
"query_type": 0,
"name": "Sireus Bot Group Counts",
"info": "How many Bots are in each Bot Group in Sireus?",
"query": "sum(sireus_bot_exists) by (bot_group)",
"interval": "5s"
}
],
"variables": [
{
"name": "bot_count",
"format": 11,
"bot_key": "bot_group",
"query_name": "Sireus Bot Group Counts",
"query_key": "",
"query_key_value": "",
"evaluate": "",
"bool_range_start": 1,
"bool_range_end": 1,
"bool_invert": false,
"export": false
}
],
"actions": [
]
}