[{"data":1,"prerenderedAt":5137},["ShallowReactive",2],{"navigation":3,"\u002Freference\u002Fmetrics-reference":189,"\u002Freference\u002Fmetrics-reference-surround":5132},[4,34,60,116,137,153],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":33},"Getting Started","i-lucide-rocket","\u002Fgetting-started","1.getting-started",[10,15,18,23,28],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-lucide-house",{"title":5,"path":16,"stem":17,"icon":6},"\u002Fgetting-started\u002Fgetting-started","1.getting-started\u002F2.getting-started",{"title":19,"path":20,"stem":21,"icon":22},"Choosing a Preset","\u002Fgetting-started\u002Fchoosing-a-preset","1.getting-started\u002F3.choosing-a-preset","i-lucide-git-branch",{"title":24,"path":25,"stem":26,"icon":27},"Deployment","\u002Fgetting-started\u002Fdeployment","1.getting-started\u002F4.deployment","i-lucide-cloud",{"title":29,"path":30,"stem":31,"icon":32},"Background Maintenance","\u002Fgetting-started\u002Fbackground-maintenance","1.getting-started\u002F5.background-maintenance","i-lucide-timer-reset",false,{"title":35,"icon":36,"path":37,"stem":38,"children":39,"page":33},"Build","i-lucide-wrench","\u002Fbuild","2.build",[40,45,50,55],{"title":41,"path":42,"stem":43,"icon":44},"Routers & Prefixes","\u002Fbuild\u002Frouters-and-prefixes","2.build\u002F1.routers-and-prefixes","i-lucide-route",{"title":46,"path":47,"stem":48,"icon":49},"Configuration","\u002Fbuild\u002Fconfiguration","2.build\u002F2.configuration","i-lucide-settings",{"title":51,"path":52,"stem":53,"icon":54},"Authorization Dependencies","\u002Fbuild\u002Fauthorization-dependencies","2.build\u002F3.authorization-dependencies","i-lucide-shield-check",{"title":56,"path":57,"stem":58,"icon":59},"Command Line","\u002Fbuild\u002Fcli","2.build\u002F4.cli","i-lucide-terminal",{"title":61,"icon":62,"path":63,"stem":64,"children":65,"page":33},"Auth","i-lucide-lock","\u002Fauth","3.auth",[66,71,76,81,86,91,96,101,106,111],{"title":67,"path":68,"stem":69,"icon":70},"OAuth & Social Login","\u002Fauth\u002Foauth-and-social-login","3.auth\u002F1.oauth-and-social-login","i-lucide-log-in",{"title":72,"path":73,"stem":74,"icon":75},"Multi-Frontend Support","\u002Fauth\u002Fmulti-frontend","3.auth\u002F10.multi-frontend","i-lucide-layout-grid",{"title":77,"path":78,"stem":79,"icon":80},"Sessions & Audit","\u002Fauth\u002Fsessions-and-audit","3.auth\u002F2.sessions-and-audit","i-lucide-monitor-smartphone",{"title":82,"path":83,"stem":84,"icon":85},"Passwordless & Messaging","\u002Fauth\u002Fpasswordless-and-messaging","3.auth\u002F3.passwordless-and-messaging","i-lucide-mail",{"title":87,"path":88,"stem":89,"icon":90},"JWT Tokens","\u002Fauth\u002Fjwt-tokens","3.auth\u002F4.jwt-tokens","i-lucide-key-round",{"title":92,"path":93,"stem":94,"icon":95},"User Management API","\u002Fauth\u002Fuser-management-api","3.auth\u002F5.user-management-api","i-lucide-users",{"title":97,"path":98,"stem":99,"icon":100},"User Invitations","\u002Fauth\u002Fuser-invitations","3.auth\u002F6.user-invitations","i-lucide-send",{"title":102,"path":103,"stem":104,"icon":105},"Roles & Permissions","\u002Fauth\u002Froles-and-permissions","3.auth\u002F7.roles-and-permissions","i-lucide-shield",{"title":107,"path":108,"stem":109,"icon":110},"ABAC","\u002Fauth\u002Fabac","3.auth\u002F8.abac","i-lucide-filter",{"title":112,"path":113,"stem":114,"icon":115},"User Status","\u002Fauth\u002Fuser-status","3.auth\u002F9.user-status","i-lucide-user-cog",{"title":117,"icon":118,"path":119,"stem":120,"children":121,"page":33},"Enterprise","i-lucide-building-2","\u002Fenterprise","4.enterprise",[122,127,132],{"title":123,"path":124,"stem":125,"icon":126},"Core Authorization Concepts","\u002Fenterprise\u002Fcore-authorization-concepts","4.enterprise\u002F1.core-authorization-concepts","i-lucide-network",{"title":128,"path":129,"stem":130,"icon":131},"Entities","\u002Fenterprise\u002Fentities","4.enterprise\u002F2.entities","i-lucide-folder-tree",{"title":133,"path":134,"stem":135,"icon":136},"Entity Memberships","\u002Fenterprise\u002Fentity-memberships","4.enterprise\u002F3.entity-memberships","i-lucide-user-plus",{"title":138,"icon":139,"path":140,"stem":141,"children":142,"page":33},"Integrations","i-lucide-plug","\u002Fintegrations","5.integrations",[143,148],{"title":144,"path":145,"stem":146,"icon":147},"API Keys","\u002Fintegrations\u002Fapi-keys","5.integrations\u002F1.api-keys","i-lucide-key",{"title":149,"path":150,"stem":151,"icon":152},"OutlabsAuth UI","\u002Fintegrations\u002Foutlabsauth-ui","5.integrations\u002F2.outlabsauth-ui","i-lucide-layout-dashboard",{"title":154,"icon":155,"path":156,"stem":157,"children":158,"page":33},"Reference","i-lucide-book-marked","\u002Freference","6.reference",[159,164,169,174,179,184],{"title":160,"path":161,"stem":162,"icon":163},"Data Models","\u002Freference\u002Fdata-models","6.reference\u002F1.data-models","i-lucide-database",{"title":165,"path":166,"stem":167,"icon":168},"Activity Tracking","\u002Freference\u002Factivity-tracking","6.reference\u002F2.activity-tracking","i-lucide-activity",{"title":170,"path":171,"stem":172,"icon":173},"Testing","\u002Freference\u002Ftesting","6.reference\u002F3.testing","i-lucide-flask-conical",{"title":175,"path":176,"stem":177,"icon":178},"Observability","\u002Freference\u002Fobservability","6.reference\u002F4.observability","i-lucide-eye",{"title":180,"path":181,"stem":182,"icon":183},"Metrics Reference","\u002Freference\u002Fmetrics-reference","6.reference\u002F5.metrics-reference","i-lucide-chart-bar",{"title":185,"path":186,"stem":187,"icon":188},"Log Events Reference","\u002Freference\u002Flog-events-reference","6.reference\u002F6.log-events-reference","i-lucide-scroll-text",{"id":190,"title":180,"body":191,"description":5125,"extension":5126,"links":5127,"meta":5128,"navigation":5129,"path":181,"seo":5130,"stem":182,"__hash__":5131},"docs\u002F6.reference\u002F5.metrics-reference.md",{"type":192,"value":193,"toc":5064},"minimark",[194,205,211,216,236,242,262,265,269,273,283,288,354,359,447,452,653,655,658,666,670,677,686,690,770,774,858,860,863,870,874,901,905,934,936,939,946,950,969,973,1002,1006,1090,1092,1095,1102,1107,1111,1140,1144,1228,1230,1234,1237,1244,1248,1284,1288,1354,1358,1442,1444,1447,1454,1458,1466,1470,1522,1526,1610,1612,1615,1622,1626,1633,1637,1675,1677,1681,1684,1692,1696,1700,1743,1747,1831,1833,1836,1843,1847,1855,1859,1897,1899,1903,1906,1913,1917,1956,1960,2012,2016,2100,2102,2105,2112,2116,2127,2131,2174,2176,2179,2186,2190,2197,2201,2230,2234,2318,2320,2324,2327,2334,2338,2372,2376,2405,2409,2494,2496,2500,2503,2510,2514,2540,2544,2615,2619,2786,2788,2791,2798,2802,2828,2838,2870,2874,2945,2949,3117,3119,3122,3129,3133,3162,3171,3199,3203,3260,3264,3348,3350,3353,3360,3364,3392,3396,3467,3471,3638,3640,3644,3647,3654,3658,3689,3693,3731,3733,3736,3743,3747,3754,3758,3794,3798,3895,3897,3900,3907,3918,3922,3952,3959,3963,4006,4010,4094,4096,4100,4103,4110,4114,4145,4149,4192,4194,4197,4204,4208,4215,4222,4226,4255,4257,4261,4264,4271,4275,4307,4311,4354,4356,4359,4366,4370,4377,4384,4388,4403,4405,4409,4412,4419,4423,4451,4455,4484,4486,4489,4496,4500,4507,4511,4526,4528,4531,4538,4542,4550,4554,4569,4571,4575,4578,4585,4589,4632,4636,4679,4681,4684,4691,4695,4708,4712,4741,4745,4829,4831,4835,4839,4894,4898,4952,4956,5013,5015,5019,5045,5047,5060],[195,196,197],"note",{},[198,199,200,201,204],"p",{},"— Prometheus metric catalog. Setup: ",[202,203,175],"a",{"href":176},".",[198,206,207,208,204],{},"Lookup page — jump to the metric name you need. For how to wire registries and\nembedded vs standalone modes, start with ",[202,209,210],{"href":176},"97",[212,213,215],"h2",{"id":214},"overview","Overview",[198,217,218,219,223,224,228,229,232,233,235],{},"OutlabsAuth registers auth-domain Prometheus metrics. In ",[220,221,222],"strong",{},"embedded"," hosts they\nappear on the host’s ",[225,226,227],"code",{},"\u002Fmetrics"," scrape; in ",[220,230,231],{},"standalone"," demos the library may\nmount ",[225,234,227],{}," itself.",[198,237,238,241],{},[220,239,240],{},"Metric types:"," Counter (monotonic), Histogram (distributions), Gauge (up\u002Fdown).",[198,243,244,247,248,251,252,251,255,251,258,261],{},[220,245,246],{},"Common labels:"," ",[225,249,250],{},"status",", ",[225,253,254],{},"method",[225,256,257],{},"permission",[225,259,260],{},"result"," (exact sets vary\nper metric below).",[263,264],"hr",{},[212,266,268],{"id":267},"authentication-metrics","Authentication Metrics",[270,271,272],"h3",{"id":272},"outlabs_auth_login_attempts_total",[198,274,275,278,279,282],{},[220,276,277],{},"Type:"," Counter\n",[220,280,281],{},"Description:"," Total number of login attempts (successful and failed)",[198,284,285],{},[220,286,287],{},"Labels:",[289,290,291,311],"ul",{},[292,293,294,296,297],"li",{},[225,295,250],{}," - Login result\n",[289,298,299,305],{},[292,300,301,304],{},[225,302,303],{},"success"," - Login succeeded",[292,306,307,310],{},[225,308,309],{},"failed"," - Login failed (invalid credentials, locked account, etc.)",[292,312,313,315,316],{},[225,314,254],{}," - Authentication method\n",[289,317,318,324,330,336,342,348],{},[292,319,320,323],{},[225,321,322],{},"password"," - Email\u002Fpassword login",[292,325,326,329],{},[225,327,328],{},"google"," - Google OAuth login",[292,331,332,335],{},[225,333,334],{},"facebook"," - Facebook OAuth login",[292,337,338,341],{},[225,339,340],{},"apple"," - Apple Sign In",[292,343,344,347],{},[225,345,346],{},"github"," - GitHub OAuth login",[292,349,350,353],{},[225,351,352],{},"api_key"," - API key authentication",[198,355,356],{},[220,357,358],{},"Example Queries:",[360,361,366],"pre",{"className":362,"code":363,"language":364,"meta":365,"style":365},"language-promql shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","# Login success rate (last 5 minutes)\nrate(outlabs_auth_login_attempts_total{status=\"success\"}[5m])\n\n# Login failure rate (detect attacks)\nrate(outlabs_auth_login_attempts_total{status=\"failed\"}[5m])\n\n# Success ratio (should be high)\nsum(rate(outlabs_auth_login_attempts_total{status=\"success\"}[5m]))\n\u002F\nsum(rate(outlabs_auth_login_attempts_total[5m]))\n\n# Logins by method\nsum by (method) (rate(outlabs_auth_login_attempts_total{status=\"success\"}[1h]))\n","promql","",[225,367,368,376,382,389,395,401,406,412,418,424,430,435,441],{"__ignoreMap":365},[369,370,373],"span",{"class":371,"line":372},"line",1,[369,374,375],{},"# Login success rate (last 5 minutes)\n",[369,377,379],{"class":371,"line":378},2,[369,380,381],{},"rate(outlabs_auth_login_attempts_total{status=\"success\"}[5m])\n",[369,383,385],{"class":371,"line":384},3,[369,386,388],{"emptyLinePlaceholder":387},true,"\n",[369,390,392],{"class":371,"line":391},4,[369,393,394],{},"# Login failure rate (detect attacks)\n",[369,396,398],{"class":371,"line":397},5,[369,399,400],{},"rate(outlabs_auth_login_attempts_total{status=\"failed\"}[5m])\n",[369,402,404],{"class":371,"line":403},6,[369,405,388],{"emptyLinePlaceholder":387},[369,407,409],{"class":371,"line":408},7,[369,410,411],{},"# Success ratio (should be high)\n",[369,413,415],{"class":371,"line":414},8,[369,416,417],{},"sum(rate(outlabs_auth_login_attempts_total{status=\"success\"}[5m]))\n",[369,419,421],{"class":371,"line":420},9,[369,422,423],{},"\u002F\n",[369,425,427],{"class":371,"line":426},10,[369,428,429],{},"sum(rate(outlabs_auth_login_attempts_total[5m]))\n",[369,431,433],{"class":371,"line":432},11,[369,434,388],{"emptyLinePlaceholder":387},[369,436,438],{"class":371,"line":437},12,[369,439,440],{},"# Logins by method\n",[369,442,444],{"class":371,"line":443},13,[369,445,446],{},"sum by (method) (rate(outlabs_auth_login_attempts_total{status=\"success\"}[1h]))\n",[198,448,449],{},[220,450,451],{},"Recommended Alerts:",[360,453,457],{"className":454,"code":455,"language":456,"meta":365,"style":365},"language-yaml shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","# High failure rate (possible attack)\n- alert: HighLoginFailureRate\n  expr: rate(outlabs_auth_login_attempts_total{status=\"failed\"}[5m]) > 10\n  for: 5m\n  labels:\n    severity: warning\n  annotations:\n    summary: \"High login failure rate detected\"\n    description: \"{{ $value }} failed logins per second\"\n\n# Login success rate drop\n- alert: LoginSuccessRateDrop\n  expr: |\n    sum(rate(outlabs_auth_login_attempts_total{status=\"success\"}[5m]))\n    \u002F\n    sum(rate(outlabs_auth_login_attempts_total[5m])) \u003C 0.8\n  for: 10m\n  labels:\n    severity: critical\n  annotations:\n    summary: \"Login success rate below 80%\"\n","yaml",[225,458,459,465,482,492,502,510,520,527,543,557,561,566,577,587,593,599,605,615,622,632,639],{"__ignoreMap":365},[369,460,461],{"class":371,"line":372},[369,462,464],{"class":463},"sHwdD","# High failure rate (possible attack)\n",[369,466,467,471,475,478],{"class":371,"line":378},[369,468,470],{"class":469},"sMK4o","-",[369,472,474],{"class":473},"swJcz"," alert",[369,476,477],{"class":469},":",[369,479,481],{"class":480},"sfazB"," HighLoginFailureRate\n",[369,483,484,487,489],{"class":371,"line":384},[369,485,486],{"class":473},"  expr",[369,488,477],{"class":469},[369,490,491],{"class":480}," rate(outlabs_auth_login_attempts_total{status=\"failed\"}[5m]) > 10\n",[369,493,494,497,499],{"class":371,"line":391},[369,495,496],{"class":473},"  for",[369,498,477],{"class":469},[369,500,501],{"class":480}," 5m\n",[369,503,504,507],{"class":371,"line":397},[369,505,506],{"class":473},"  labels",[369,508,509],{"class":469},":\n",[369,511,512,515,517],{"class":371,"line":403},[369,513,514],{"class":473},"    severity",[369,516,477],{"class":469},[369,518,519],{"class":480}," warning\n",[369,521,522,525],{"class":371,"line":408},[369,523,524],{"class":473},"  annotations",[369,526,509],{"class":469},[369,528,529,532,534,537,540],{"class":371,"line":414},[369,530,531],{"class":473},"    summary",[369,533,477],{"class":469},[369,535,536],{"class":469}," \"",[369,538,539],{"class":480},"High login failure rate detected",[369,541,542],{"class":469},"\"\n",[369,544,545,548,550,552,555],{"class":371,"line":420},[369,546,547],{"class":473},"    description",[369,549,477],{"class":469},[369,551,536],{"class":469},[369,553,554],{"class":480},"{{ $value }} failed logins per second",[369,556,542],{"class":469},[369,558,559],{"class":371,"line":426},[369,560,388],{"emptyLinePlaceholder":387},[369,562,563],{"class":371,"line":432},[369,564,565],{"class":463},"# Login success rate drop\n",[369,567,568,570,572,574],{"class":371,"line":437},[369,569,470],{"class":469},[369,571,474],{"class":473},[369,573,477],{"class":469},[369,575,576],{"class":480}," LoginSuccessRateDrop\n",[369,578,579,581,583],{"class":371,"line":443},[369,580,486],{"class":473},[369,582,477],{"class":469},[369,584,586],{"class":585},"s7zQu"," |\n",[369,588,590],{"class":371,"line":589},14,[369,591,592],{"class":480},"    sum(rate(outlabs_auth_login_attempts_total{status=\"success\"}[5m]))\n",[369,594,596],{"class":371,"line":595},15,[369,597,598],{"class":480},"    \u002F\n",[369,600,602],{"class":371,"line":601},16,[369,603,604],{"class":480},"    sum(rate(outlabs_auth_login_attempts_total[5m])) \u003C 0.8\n",[369,606,608,610,612],{"class":371,"line":607},17,[369,609,496],{"class":473},[369,611,477],{"class":469},[369,613,614],{"class":480}," 10m\n",[369,616,618,620],{"class":371,"line":617},18,[369,619,506],{"class":473},[369,621,509],{"class":469},[369,623,625,627,629],{"class":371,"line":624},19,[369,626,514],{"class":473},[369,628,477],{"class":469},[369,630,631],{"class":480}," critical\n",[369,633,635,637],{"class":371,"line":634},20,[369,636,524],{"class":473},[369,638,509],{"class":469},[369,640,642,644,646,648,651],{"class":371,"line":641},21,[369,643,531],{"class":473},[369,645,477],{"class":469},[369,647,536],{"class":469},[369,649,650],{"class":480},"Login success rate below 80%",[369,652,542],{"class":469},[263,654],{},[270,656,657],{"id":657},"outlabs_auth_login_duration_seconds",[198,659,660,662,663,665],{},[220,661,277],{}," Histogram\n",[220,664,281],{}," Time taken for login operations (in seconds)",[198,667,668],{},[220,669,287],{},[289,671,672],{},[292,673,674,676],{},[225,675,254],{}," - Authentication method (password, google, api_key, etc.)",[198,678,679,247,682,685],{},[220,680,681],{},"Buckets:",[225,683,684],{},"[0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0]"," (1ms to 1s)",[198,687,688],{},[220,689,358],{},[360,691,693],{"className":362,"code":692,"language":364,"meta":365,"style":365},"# P50 (median) login latency\nhistogram_quantile(0.5, rate(outlabs_auth_login_duration_seconds_bucket[5m]))\n\n# P95 login latency\nhistogram_quantile(0.95, rate(outlabs_auth_login_duration_seconds_bucket[5m]))\n\n# P99 login latency (slowest 1%)\nhistogram_quantile(0.99, rate(outlabs_auth_login_duration_seconds_bucket[5m]))\n\n# Average login duration\nrate(outlabs_auth_login_duration_seconds_sum[5m])\n\u002F\nrate(outlabs_auth_login_duration_seconds_count[5m])\n\n# P95 latency by method\nhistogram_quantile(0.95, sum by (method, le) (rate(outlabs_auth_login_duration_seconds_bucket[5m])))\n",[225,694,695,700,705,709,714,719,723,728,733,737,742,747,751,756,760,765],{"__ignoreMap":365},[369,696,697],{"class":371,"line":372},[369,698,699],{},"# P50 (median) login latency\n",[369,701,702],{"class":371,"line":378},[369,703,704],{},"histogram_quantile(0.5, rate(outlabs_auth_login_duration_seconds_bucket[5m]))\n",[369,706,707],{"class":371,"line":384},[369,708,388],{"emptyLinePlaceholder":387},[369,710,711],{"class":371,"line":391},[369,712,713],{},"# P95 login latency\n",[369,715,716],{"class":371,"line":397},[369,717,718],{},"histogram_quantile(0.95, rate(outlabs_auth_login_duration_seconds_bucket[5m]))\n",[369,720,721],{"class":371,"line":403},[369,722,388],{"emptyLinePlaceholder":387},[369,724,725],{"class":371,"line":408},[369,726,727],{},"# P99 login latency (slowest 1%)\n",[369,729,730],{"class":371,"line":414},[369,731,732],{},"histogram_quantile(0.99, rate(outlabs_auth_login_duration_seconds_bucket[5m]))\n",[369,734,735],{"class":371,"line":420},[369,736,388],{"emptyLinePlaceholder":387},[369,738,739],{"class":371,"line":426},[369,740,741],{},"# Average login duration\n",[369,743,744],{"class":371,"line":432},[369,745,746],{},"rate(outlabs_auth_login_duration_seconds_sum[5m])\n",[369,748,749],{"class":371,"line":437},[369,750,423],{},[369,752,753],{"class":371,"line":443},[369,754,755],{},"rate(outlabs_auth_login_duration_seconds_count[5m])\n",[369,757,758],{"class":371,"line":589},[369,759,388],{"emptyLinePlaceholder":387},[369,761,762],{"class":371,"line":595},[369,763,764],{},"# P95 latency by method\n",[369,766,767],{"class":371,"line":601},[369,768,769],{},"histogram_quantile(0.95, sum by (method, le) (rate(outlabs_auth_login_duration_seconds_bucket[5m])))\n",[198,771,772],{},[220,773,451],{},[360,775,777],{"className":454,"code":776,"language":456,"meta":365,"style":365},"# Slow logins\n- alert: SlowLoginLatency\n  expr: histogram_quantile(0.95, rate(outlabs_auth_login_duration_seconds_bucket[5m])) > 0.5\n  for: 10m\n  labels:\n    severity: warning\n  annotations:\n    summary: \"P95 login latency above 500ms\"\n    description: \"Login operations are taking {{ $value }}s at P95\"\n",[225,778,779,784,795,804,812,818,826,832,845],{"__ignoreMap":365},[369,780,781],{"class":371,"line":372},[369,782,783],{"class":463},"# Slow logins\n",[369,785,786,788,790,792],{"class":371,"line":378},[369,787,470],{"class":469},[369,789,474],{"class":473},[369,791,477],{"class":469},[369,793,794],{"class":480}," SlowLoginLatency\n",[369,796,797,799,801],{"class":371,"line":384},[369,798,486],{"class":473},[369,800,477],{"class":469},[369,802,803],{"class":480}," histogram_quantile(0.95, rate(outlabs_auth_login_duration_seconds_bucket[5m])) > 0.5\n",[369,805,806,808,810],{"class":371,"line":391},[369,807,496],{"class":473},[369,809,477],{"class":469},[369,811,614],{"class":480},[369,813,814,816],{"class":371,"line":397},[369,815,506],{"class":473},[369,817,509],{"class":469},[369,819,820,822,824],{"class":371,"line":403},[369,821,514],{"class":473},[369,823,477],{"class":469},[369,825,519],{"class":480},[369,827,828,830],{"class":371,"line":408},[369,829,524],{"class":473},[369,831,509],{"class":469},[369,833,834,836,838,840,843],{"class":371,"line":414},[369,835,531],{"class":473},[369,837,477],{"class":469},[369,839,536],{"class":469},[369,841,842],{"class":480},"P95 login latency above 500ms",[369,844,542],{"class":469},[369,846,847,849,851,853,856],{"class":371,"line":420},[369,848,547],{"class":473},[369,850,477],{"class":469},[369,852,536],{"class":469},[369,854,855],{"class":480},"Login operations are taking {{ $value }}s at P95",[369,857,542],{"class":469},[263,859],{},[270,861,862],{"id":862},"outlabs_auth_logout_total",[198,864,865,278,867,869],{},[220,866,277],{},[220,868,281],{}," Total number of logout operations",[198,871,872],{},[220,873,287],{},[289,875,876],{},[292,877,878,880,881],{},[225,879,254],{}," - How logout was triggered\n",[289,882,883,889,895],{},[292,884,885,888],{},[225,886,887],{},"explicit"," - User clicked logout",[292,890,891,894],{},[225,892,893],{},"token_expiry"," - Token expired",[292,896,897,900],{},[225,898,899],{},"admin_revoke"," - Admin revoked session",[198,902,903],{},[220,904,358],{},[360,906,908],{"className":362,"code":907,"language":364,"meta":365,"style":365},"# Logout rate\nrate(outlabs_auth_logout_total[5m])\n\n# Logouts by method\nsum by (method) (rate(outlabs_auth_logout_total[1h]))\n",[225,909,910,915,920,924,929],{"__ignoreMap":365},[369,911,912],{"class":371,"line":372},[369,913,914],{},"# Logout rate\n",[369,916,917],{"class":371,"line":378},[369,918,919],{},"rate(outlabs_auth_logout_total[5m])\n",[369,921,922],{"class":371,"line":384},[369,923,388],{"emptyLinePlaceholder":387},[369,925,926],{"class":371,"line":391},[369,927,928],{},"# Logouts by method\n",[369,930,931],{"class":371,"line":397},[369,932,933],{},"sum by (method) (rate(outlabs_auth_logout_total[1h]))\n",[263,935],{},[270,937,938],{"id":938},"outlabs_auth_token_refresh_total",[198,940,941,278,943,945],{},[220,942,277],{},[220,944,281],{}," Total number of access token refresh operations",[198,947,948],{},[220,949,287],{},[289,951,952],{},[292,953,954,956,957],{},[225,955,250],{}," - Refresh result\n",[289,958,959,964],{},[292,960,961,963],{},[225,962,303],{}," - Token refreshed successfully",[292,965,966,968],{},[225,967,309],{}," - Refresh failed (invalid\u002Fexpired refresh token)",[198,970,971],{},[220,972,358],{},[360,974,976],{"className":362,"code":975,"language":364,"meta":365,"style":365},"# Token refresh rate\nrate(outlabs_auth_token_refresh_total{status=\"success\"}[5m])\n\n# Failed refresh rate (users need to re-login)\nrate(outlabs_auth_token_refresh_total{status=\"failed\"}[5m])\n",[225,977,978,983,988,992,997],{"__ignoreMap":365},[369,979,980],{"class":371,"line":372},[369,981,982],{},"# Token refresh rate\n",[369,984,985],{"class":371,"line":378},[369,986,987],{},"rate(outlabs_auth_token_refresh_total{status=\"success\"}[5m])\n",[369,989,990],{"class":371,"line":384},[369,991,388],{"emptyLinePlaceholder":387},[369,993,994],{"class":371,"line":391},[369,995,996],{},"# Failed refresh rate (users need to re-login)\n",[369,998,999],{"class":371,"line":397},[369,1000,1001],{},"rate(outlabs_auth_token_refresh_total{status=\"failed\"}[5m])\n",[198,1003,1004],{},[220,1005,451],{},[360,1007,1009],{"className":454,"code":1008,"language":456,"meta":365,"style":365},"# High refresh failure rate\n- alert: HighTokenRefreshFailureRate\n  expr: rate(outlabs_auth_token_refresh_total{status=\"failed\"}[5m]) > 5\n  for: 5m\n  labels:\n    severity: warning\n  annotations:\n    summary: \"High token refresh failure rate\"\n    description: \"Many users unable to refresh tokens (forced re-login)\"\n",[225,1010,1011,1016,1027,1036,1044,1050,1058,1064,1077],{"__ignoreMap":365},[369,1012,1013],{"class":371,"line":372},[369,1014,1015],{"class":463},"# High refresh failure rate\n",[369,1017,1018,1020,1022,1024],{"class":371,"line":378},[369,1019,470],{"class":469},[369,1021,474],{"class":473},[369,1023,477],{"class":469},[369,1025,1026],{"class":480}," HighTokenRefreshFailureRate\n",[369,1028,1029,1031,1033],{"class":371,"line":384},[369,1030,486],{"class":473},[369,1032,477],{"class":469},[369,1034,1035],{"class":480}," rate(outlabs_auth_token_refresh_total{status=\"failed\"}[5m]) > 5\n",[369,1037,1038,1040,1042],{"class":371,"line":391},[369,1039,496],{"class":473},[369,1041,477],{"class":469},[369,1043,501],{"class":480},[369,1045,1046,1048],{"class":371,"line":397},[369,1047,506],{"class":473},[369,1049,509],{"class":469},[369,1051,1052,1054,1056],{"class":371,"line":403},[369,1053,514],{"class":473},[369,1055,477],{"class":469},[369,1057,519],{"class":480},[369,1059,1060,1062],{"class":371,"line":408},[369,1061,524],{"class":473},[369,1063,509],{"class":469},[369,1065,1066,1068,1070,1072,1075],{"class":371,"line":414},[369,1067,531],{"class":473},[369,1069,477],{"class":469},[369,1071,536],{"class":469},[369,1073,1074],{"class":480},"High token refresh failure rate",[369,1076,542],{"class":469},[369,1078,1079,1081,1083,1085,1088],{"class":371,"line":420},[369,1080,547],{"class":473},[369,1082,477],{"class":469},[369,1084,536],{"class":469},[369,1086,1087],{"class":480},"Many users unable to refresh tokens (forced re-login)",[369,1089,542],{"class":469},[263,1091],{},[270,1093,1094],{"id":1094},"outlabs_auth_account_locked_total",[198,1096,1097,278,1099,1101],{},[220,1098,277],{},[220,1100,281],{}," Total number of accounts locked due to failed login attempts",[198,1103,1104,1106],{},[220,1105,287],{}," None",[198,1108,1109],{},[220,1110,358],{},[360,1112,1114],{"className":362,"code":1113,"language":364,"meta":365,"style":365},"# Account lockout rate\nrate(outlabs_auth_account_locked_total[1h])\n\n# Total locked accounts today\nincrease(outlabs_auth_account_locked_total[24h])\n",[225,1115,1116,1121,1126,1130,1135],{"__ignoreMap":365},[369,1117,1118],{"class":371,"line":372},[369,1119,1120],{},"# Account lockout rate\n",[369,1122,1123],{"class":371,"line":378},[369,1124,1125],{},"rate(outlabs_auth_account_locked_total[1h])\n",[369,1127,1128],{"class":371,"line":384},[369,1129,388],{"emptyLinePlaceholder":387},[369,1131,1132],{"class":371,"line":391},[369,1133,1134],{},"# Total locked accounts today\n",[369,1136,1137],{"class":371,"line":397},[369,1138,1139],{},"increase(outlabs_auth_account_locked_total[24h])\n",[198,1141,1142],{},[220,1143,451],{},[360,1145,1147],{"className":454,"code":1146,"language":456,"meta":365,"style":365},"# Spike in account lockouts (possible attack)\n- alert: HighAccountLockoutRate\n  expr: rate(outlabs_auth_account_locked_total[5m]) > 2\n  for: 5m\n  labels:\n    severity: critical\n  annotations:\n    summary: \"Unusual number of account lockouts\"\n    description: \"Possible brute force attack\"\n",[225,1148,1149,1154,1165,1174,1182,1188,1196,1202,1215],{"__ignoreMap":365},[369,1150,1151],{"class":371,"line":372},[369,1152,1153],{"class":463},"# Spike in account lockouts (possible attack)\n",[369,1155,1156,1158,1160,1162],{"class":371,"line":378},[369,1157,470],{"class":469},[369,1159,474],{"class":473},[369,1161,477],{"class":469},[369,1163,1164],{"class":480}," HighAccountLockoutRate\n",[369,1166,1167,1169,1171],{"class":371,"line":384},[369,1168,486],{"class":473},[369,1170,477],{"class":469},[369,1172,1173],{"class":480}," rate(outlabs_auth_account_locked_total[5m]) > 2\n",[369,1175,1176,1178,1180],{"class":371,"line":391},[369,1177,496],{"class":473},[369,1179,477],{"class":469},[369,1181,501],{"class":480},[369,1183,1184,1186],{"class":371,"line":397},[369,1185,506],{"class":473},[369,1187,509],{"class":469},[369,1189,1190,1192,1194],{"class":371,"line":403},[369,1191,514],{"class":473},[369,1193,477],{"class":469},[369,1195,631],{"class":480},[369,1197,1198,1200],{"class":371,"line":408},[369,1199,524],{"class":473},[369,1201,509],{"class":469},[369,1203,1204,1206,1208,1210,1213],{"class":371,"line":414},[369,1205,531],{"class":473},[369,1207,477],{"class":469},[369,1209,536],{"class":469},[369,1211,1212],{"class":480},"Unusual number of account lockouts",[369,1214,542],{"class":469},[369,1216,1217,1219,1221,1223,1226],{"class":371,"line":420},[369,1218,547],{"class":473},[369,1220,477],{"class":469},[369,1222,536],{"class":469},[369,1224,1225],{"class":480},"Possible brute force attack",[369,1227,542],{"class":469},[263,1229],{},[212,1231,1233],{"id":1232},"authorization-metrics","Authorization Metrics",[270,1235,1236],{"id":1236},"outlabs_auth_permission_checks_total",[198,1238,1239,278,1241,1243],{},[220,1240,277],{},[220,1242,281],{}," Total number of permission checks performed",[198,1245,1246],{},[220,1247,287],{},[289,1249,1250,1269],{},[292,1251,1252,1254,1255],{},[225,1253,260],{}," - Permission check result\n",[289,1256,1257,1263],{},[292,1258,1259,1262],{},[225,1260,1261],{},"granted"," - Permission allowed",[292,1264,1265,1268],{},[225,1266,1267],{},"denied"," - Permission denied",[292,1270,1271,1273,1274,251,1277,251,1280,1283],{},[225,1272,257],{}," - Permission being checked (e.g., ",[225,1275,1276],{},"user:read",[225,1278,1279],{},"post:create",[225,1281,1282],{},"*:*",")",[198,1285,1286],{},[220,1287,358],{},[360,1289,1291],{"className":362,"code":1290,"language":364,"meta":365,"style":365},"# Permission check rate\nrate(outlabs_auth_permission_checks_total[5m])\n\n# Denied permission rate\nrate(outlabs_auth_permission_checks_total{result=\"denied\"}[5m])\n\n# Top denied permissions\ntopk(10, sum by (permission) (rate(outlabs_auth_permission_checks_total{result=\"denied\"}[1h])))\n\n# Permission grant ratio (should be high)\nsum(rate(outlabs_auth_permission_checks_total{result=\"granted\"}[5m]))\n\u002F\nsum(rate(outlabs_auth_permission_checks_total[5m]))\n",[225,1292,1293,1298,1303,1307,1312,1317,1321,1326,1331,1335,1340,1345,1349],{"__ignoreMap":365},[369,1294,1295],{"class":371,"line":372},[369,1296,1297],{},"# Permission check rate\n",[369,1299,1300],{"class":371,"line":378},[369,1301,1302],{},"rate(outlabs_auth_permission_checks_total[5m])\n",[369,1304,1305],{"class":371,"line":384},[369,1306,388],{"emptyLinePlaceholder":387},[369,1308,1309],{"class":371,"line":391},[369,1310,1311],{},"# Denied permission rate\n",[369,1313,1314],{"class":371,"line":397},[369,1315,1316],{},"rate(outlabs_auth_permission_checks_total{result=\"denied\"}[5m])\n",[369,1318,1319],{"class":371,"line":403},[369,1320,388],{"emptyLinePlaceholder":387},[369,1322,1323],{"class":371,"line":408},[369,1324,1325],{},"# Top denied permissions\n",[369,1327,1328],{"class":371,"line":414},[369,1329,1330],{},"topk(10, sum by (permission) (rate(outlabs_auth_permission_checks_total{result=\"denied\"}[1h])))\n",[369,1332,1333],{"class":371,"line":420},[369,1334,388],{"emptyLinePlaceholder":387},[369,1336,1337],{"class":371,"line":426},[369,1338,1339],{},"# Permission grant ratio (should be high)\n",[369,1341,1342],{"class":371,"line":432},[369,1343,1344],{},"sum(rate(outlabs_auth_permission_checks_total{result=\"granted\"}[5m]))\n",[369,1346,1347],{"class":371,"line":437},[369,1348,423],{},[369,1350,1351],{"class":371,"line":443},[369,1352,1353],{},"sum(rate(outlabs_auth_permission_checks_total[5m]))\n",[198,1355,1356],{},[220,1357,451],{},[360,1359,1361],{"className":454,"code":1360,"language":456,"meta":365,"style":365},"# Sudden spike in denied permissions\n- alert: HighPermissionDenialRate\n  expr: rate(outlabs_auth_permission_checks_total{result=\"denied\"}[5m]) > 50\n  for: 5m\n  labels:\n    severity: warning\n  annotations:\n    summary: \"Unusual number of permission denials\"\n    description: \"Possible misconfigured roles or unauthorized access attempts\"\n",[225,1362,1363,1368,1379,1388,1396,1402,1410,1416,1429],{"__ignoreMap":365},[369,1364,1365],{"class":371,"line":372},[369,1366,1367],{"class":463},"# Sudden spike in denied permissions\n",[369,1369,1370,1372,1374,1376],{"class":371,"line":378},[369,1371,470],{"class":469},[369,1373,474],{"class":473},[369,1375,477],{"class":469},[369,1377,1378],{"class":480}," HighPermissionDenialRate\n",[369,1380,1381,1383,1385],{"class":371,"line":384},[369,1382,486],{"class":473},[369,1384,477],{"class":469},[369,1386,1387],{"class":480}," rate(outlabs_auth_permission_checks_total{result=\"denied\"}[5m]) > 50\n",[369,1389,1390,1392,1394],{"class":371,"line":391},[369,1391,496],{"class":473},[369,1393,477],{"class":469},[369,1395,501],{"class":480},[369,1397,1398,1400],{"class":371,"line":397},[369,1399,506],{"class":473},[369,1401,509],{"class":469},[369,1403,1404,1406,1408],{"class":371,"line":403},[369,1405,514],{"class":473},[369,1407,477],{"class":469},[369,1409,519],{"class":480},[369,1411,1412,1414],{"class":371,"line":408},[369,1413,524],{"class":473},[369,1415,509],{"class":469},[369,1417,1418,1420,1422,1424,1427],{"class":371,"line":414},[369,1419,531],{"class":473},[369,1421,477],{"class":469},[369,1423,536],{"class":469},[369,1425,1426],{"class":480},"Unusual number of permission denials",[369,1428,542],{"class":469},[369,1430,1431,1433,1435,1437,1440],{"class":371,"line":420},[369,1432,547],{"class":473},[369,1434,477],{"class":469},[369,1436,536],{"class":469},[369,1438,1439],{"class":480},"Possible misconfigured roles or unauthorized access attempts",[369,1441,542],{"class":469},[263,1443],{},[270,1445,1446],{"id":1446},"outlabs_auth_permission_check_duration_seconds",[198,1448,1449,662,1451,1453],{},[220,1450,277],{},[220,1452,281],{}," Time taken for permission checks (in seconds)",[198,1455,1456,1106],{},[220,1457,287],{},[198,1459,1460,247,1462,1465],{},[220,1461,681],{},[225,1463,1464],{},"[0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1]"," (0.1ms to 100ms)",[198,1467,1468],{},[220,1469,358],{},[360,1471,1473],{"className":362,"code":1472,"language":364,"meta":365,"style":365},"# P95 permission check latency\nhistogram_quantile(0.95, rate(outlabs_auth_permission_check_duration_seconds_bucket[5m]))\n\n# P99 permission check latency\nhistogram_quantile(0.99, rate(outlabs_auth_permission_check_duration_seconds_bucket[5m]))\n\n# Average permission check duration\nrate(outlabs_auth_permission_check_duration_seconds_sum[5m])\n\u002F\nrate(outlabs_auth_permission_check_duration_seconds_count[5m])\n",[225,1474,1475,1480,1485,1489,1494,1499,1503,1508,1513,1517],{"__ignoreMap":365},[369,1476,1477],{"class":371,"line":372},[369,1478,1479],{},"# P95 permission check latency\n",[369,1481,1482],{"class":371,"line":378},[369,1483,1484],{},"histogram_quantile(0.95, rate(outlabs_auth_permission_check_duration_seconds_bucket[5m]))\n",[369,1486,1487],{"class":371,"line":384},[369,1488,388],{"emptyLinePlaceholder":387},[369,1490,1491],{"class":371,"line":391},[369,1492,1493],{},"# P99 permission check latency\n",[369,1495,1496],{"class":371,"line":397},[369,1497,1498],{},"histogram_quantile(0.99, rate(outlabs_auth_permission_check_duration_seconds_bucket[5m]))\n",[369,1500,1501],{"class":371,"line":403},[369,1502,388],{"emptyLinePlaceholder":387},[369,1504,1505],{"class":371,"line":408},[369,1506,1507],{},"# Average permission check duration\n",[369,1509,1510],{"class":371,"line":414},[369,1511,1512],{},"rate(outlabs_auth_permission_check_duration_seconds_sum[5m])\n",[369,1514,1515],{"class":371,"line":420},[369,1516,423],{},[369,1518,1519],{"class":371,"line":426},[369,1520,1521],{},"rate(outlabs_auth_permission_check_duration_seconds_count[5m])\n",[198,1523,1524],{},[220,1525,451],{},[360,1527,1529],{"className":454,"code":1528,"language":456,"meta":365,"style":365},"# Slow permission checks\n- alert: SlowPermissionChecks\n  expr: histogram_quantile(0.95, rate(outlabs_auth_permission_check_duration_seconds_bucket[5m])) > 0.1\n  for: 10m\n  labels:\n    severity: warning\n  annotations:\n    summary: \"P95 permission check latency above 100ms\"\n    description: \"Permission checks are slow (possible complex hierarchy or ABAC policies)\"\n",[225,1530,1531,1536,1547,1556,1564,1570,1578,1584,1597],{"__ignoreMap":365},[369,1532,1533],{"class":371,"line":372},[369,1534,1535],{"class":463},"# Slow permission checks\n",[369,1537,1538,1540,1542,1544],{"class":371,"line":378},[369,1539,470],{"class":469},[369,1541,474],{"class":473},[369,1543,477],{"class":469},[369,1545,1546],{"class":480}," SlowPermissionChecks\n",[369,1548,1549,1551,1553],{"class":371,"line":384},[369,1550,486],{"class":473},[369,1552,477],{"class":469},[369,1554,1555],{"class":480}," histogram_quantile(0.95, rate(outlabs_auth_permission_check_duration_seconds_bucket[5m])) > 0.1\n",[369,1557,1558,1560,1562],{"class":371,"line":391},[369,1559,496],{"class":473},[369,1561,477],{"class":469},[369,1563,614],{"class":480},[369,1565,1566,1568],{"class":371,"line":397},[369,1567,506],{"class":473},[369,1569,509],{"class":469},[369,1571,1572,1574,1576],{"class":371,"line":403},[369,1573,514],{"class":473},[369,1575,477],{"class":469},[369,1577,519],{"class":480},[369,1579,1580,1582],{"class":371,"line":408},[369,1581,524],{"class":473},[369,1583,509],{"class":469},[369,1585,1586,1588,1590,1592,1595],{"class":371,"line":414},[369,1587,531],{"class":473},[369,1589,477],{"class":469},[369,1591,536],{"class":469},[369,1593,1594],{"class":480},"P95 permission check latency above 100ms",[369,1596,542],{"class":469},[369,1598,1599,1601,1603,1605,1608],{"class":371,"line":420},[369,1600,547],{"class":473},[369,1602,477],{"class":469},[369,1604,536],{"class":469},[369,1606,1607],{"class":480},"Permission checks are slow (possible complex hierarchy or ABAC policies)",[369,1609,542],{"class":469},[263,1611],{},[270,1613,1614],{"id":1614},"outlabs_auth_tree_permission_depth",[198,1616,1617,662,1619,1621],{},[220,1618,277],{},[220,1620,281],{}," Depth of entity hierarchy traversed during tree permission checks",[198,1623,1624,1106],{},[220,1625,287],{},[198,1627,1628,247,1630],{},[220,1629,681],{},[225,1631,1632],{},"[1, 2, 3, 5, 10, 20, 50]",[198,1634,1635],{},[220,1636,358],{},[360,1638,1640],{"className":362,"code":1639,"language":364,"meta":365,"style":365},"# Average hierarchy depth\nrate(outlabs_auth_tree_permission_depth_sum[5m])\n\u002F\nrate(outlabs_auth_tree_permission_depth_count[5m])\n\n# P95 hierarchy depth\nhistogram_quantile(0.95, rate(outlabs_auth_tree_permission_depth_bucket[5m]))\n",[225,1641,1642,1647,1652,1656,1661,1665,1670],{"__ignoreMap":365},[369,1643,1644],{"class":371,"line":372},[369,1645,1646],{},"# Average hierarchy depth\n",[369,1648,1649],{"class":371,"line":378},[369,1650,1651],{},"rate(outlabs_auth_tree_permission_depth_sum[5m])\n",[369,1653,1654],{"class":371,"line":384},[369,1655,423],{},[369,1657,1658],{"class":371,"line":391},[369,1659,1660],{},"rate(outlabs_auth_tree_permission_depth_count[5m])\n",[369,1662,1663],{"class":371,"line":397},[369,1664,388],{"emptyLinePlaceholder":387},[369,1666,1667],{"class":371,"line":403},[369,1668,1669],{},"# P95 hierarchy depth\n",[369,1671,1672],{"class":371,"line":408},[369,1673,1674],{},"histogram_quantile(0.95, rate(outlabs_auth_tree_permission_depth_bucket[5m]))\n",[263,1676],{},[212,1678,1680],{"id":1679},"session-metrics","Session Metrics",[270,1682,1683],{"id":1683},"outlabs_auth_active_sessions",[198,1685,1686,1688,1689,1691],{},[220,1687,277],{}," Gauge\n",[220,1690,281],{}," Current number of active user sessions",[198,1693,1694,1106],{},[220,1695,287],{},[198,1697,1698],{},[220,1699,358],{},[360,1701,1703],{"className":362,"code":1702,"language":364,"meta":365,"style":365},"# Current active sessions\noutlabs_auth_active_sessions\n\n# Peak sessions in last 24 hours\nmax_over_time(outlabs_auth_active_sessions[24h])\n\n# Average sessions in last hour\navg_over_time(outlabs_auth_active_sessions[1h])\n",[225,1704,1705,1710,1715,1719,1724,1729,1733,1738],{"__ignoreMap":365},[369,1706,1707],{"class":371,"line":372},[369,1708,1709],{},"# Current active sessions\n",[369,1711,1712],{"class":371,"line":378},[369,1713,1714],{},"outlabs_auth_active_sessions\n",[369,1716,1717],{"class":371,"line":384},[369,1718,388],{"emptyLinePlaceholder":387},[369,1720,1721],{"class":371,"line":391},[369,1722,1723],{},"# Peak sessions in last 24 hours\n",[369,1725,1726],{"class":371,"line":397},[369,1727,1728],{},"max_over_time(outlabs_auth_active_sessions[24h])\n",[369,1730,1731],{"class":371,"line":403},[369,1732,388],{"emptyLinePlaceholder":387},[369,1734,1735],{"class":371,"line":408},[369,1736,1737],{},"# Average sessions in last hour\n",[369,1739,1740],{"class":371,"line":414},[369,1741,1742],{},"avg_over_time(outlabs_auth_active_sessions[1h])\n",[198,1744,1745],{},[220,1746,451],{},[360,1748,1750],{"className":454,"code":1749,"language":456,"meta":365,"style":365},"# Unusual session spike\n- alert: UnusualSessionSpike\n  expr: outlabs_auth_active_sessions > 10000\n  for: 5m\n  labels:\n    severity: warning\n  annotations:\n    summary: \"Unusual number of active sessions\"\n    description: \"{{ $value }} active sessions (possible attack or viral growth)\"\n",[225,1751,1752,1757,1768,1777,1785,1791,1799,1805,1818],{"__ignoreMap":365},[369,1753,1754],{"class":371,"line":372},[369,1755,1756],{"class":463},"# Unusual session spike\n",[369,1758,1759,1761,1763,1765],{"class":371,"line":378},[369,1760,470],{"class":469},[369,1762,474],{"class":473},[369,1764,477],{"class":469},[369,1766,1767],{"class":480}," UnusualSessionSpike\n",[369,1769,1770,1772,1774],{"class":371,"line":384},[369,1771,486],{"class":473},[369,1773,477],{"class":469},[369,1775,1776],{"class":480}," outlabs_auth_active_sessions > 10000\n",[369,1778,1779,1781,1783],{"class":371,"line":391},[369,1780,496],{"class":473},[369,1782,477],{"class":469},[369,1784,501],{"class":480},[369,1786,1787,1789],{"class":371,"line":397},[369,1788,506],{"class":473},[369,1790,509],{"class":469},[369,1792,1793,1795,1797],{"class":371,"line":403},[369,1794,514],{"class":473},[369,1796,477],{"class":469},[369,1798,519],{"class":480},[369,1800,1801,1803],{"class":371,"line":408},[369,1802,524],{"class":473},[369,1804,509],{"class":469},[369,1806,1807,1809,1811,1813,1816],{"class":371,"line":414},[369,1808,531],{"class":473},[369,1810,477],{"class":469},[369,1812,536],{"class":469},[369,1814,1815],{"class":480},"Unusual number of active sessions",[369,1817,542],{"class":469},[369,1819,1820,1822,1824,1826,1829],{"class":371,"line":420},[369,1821,547],{"class":473},[369,1823,477],{"class":469},[369,1825,536],{"class":469},[369,1827,1828],{"class":480},"{{ $value }} active sessions (possible attack or viral growth)",[369,1830,542],{"class":469},[263,1832],{},[270,1834,1835],{"id":1835},"outlabs_auth_session_duration_seconds",[198,1837,1838,662,1840,1842],{},[220,1839,277],{},[220,1841,281],{}," Session duration from login to logout (in seconds)",[198,1844,1845,1106],{},[220,1846,287],{},[198,1848,1849,247,1851,1854],{},[220,1850,681],{},[225,1852,1853],{},"[60, 300, 900, 1800, 3600, 7200, 14400, 28800]"," (1min to 8hrs)",[198,1856,1857],{},[220,1858,358],{},[360,1860,1862],{"className":362,"code":1861,"language":364,"meta":365,"style":365},"# Median session duration\nhistogram_quantile(0.5, rate(outlabs_auth_session_duration_seconds_bucket[1h]))\n\n# Average session duration\nrate(outlabs_auth_session_duration_seconds_sum[1h])\n\u002F\nrate(outlabs_auth_session_duration_seconds_count[1h])\n",[225,1863,1864,1869,1874,1878,1883,1888,1892],{"__ignoreMap":365},[369,1865,1866],{"class":371,"line":372},[369,1867,1868],{},"# Median session duration\n",[369,1870,1871],{"class":371,"line":378},[369,1872,1873],{},"histogram_quantile(0.5, rate(outlabs_auth_session_duration_seconds_bucket[1h]))\n",[369,1875,1876],{"class":371,"line":384},[369,1877,388],{"emptyLinePlaceholder":387},[369,1879,1880],{"class":371,"line":391},[369,1881,1882],{},"# Average session duration\n",[369,1884,1885],{"class":371,"line":397},[369,1886,1887],{},"rate(outlabs_auth_session_duration_seconds_sum[1h])\n",[369,1889,1890],{"class":371,"line":403},[369,1891,423],{},[369,1893,1894],{"class":371,"line":408},[369,1895,1896],{},"rate(outlabs_auth_session_duration_seconds_count[1h])\n",[263,1898],{},[212,1900,1902],{"id":1901},"api-key-metrics","API Key Metrics",[270,1904,1905],{"id":1905},"outlabs_auth_api_key_validations_total",[198,1907,1908,278,1910,1912],{},[220,1909,277],{},[220,1911,281],{}," Total number of API key validation attempts",[198,1914,1915],{},[220,1916,287],{},[289,1918,1919],{},[292,1920,1921,1923,1924],{},[225,1922,250],{}," - Validation result\n",[289,1925,1926,1932,1938,1944,1950],{},[292,1927,1928,1931],{},[225,1929,1930],{},"valid"," - API key is valid",[292,1933,1934,1937],{},[225,1935,1936],{},"invalid"," - API key doesn't exist or is malformed",[292,1939,1940,1943],{},[225,1941,1942],{},"expired"," - API key has expired",[292,1945,1946,1949],{},[225,1947,1948],{},"rate_limited"," - API key hit rate limit",[292,1951,1952,1955],{},[225,1953,1954],{},"disabled"," - API key has been disabled",[198,1957,1958],{},[220,1959,358],{},[360,1961,1963],{"className":362,"code":1962,"language":364,"meta":365,"style":365},"# API key validation rate\nrate(outlabs_auth_api_key_validations_total{status=\"valid\"}[5m])\n\n# Invalid API key rate (possible attacks)\nrate(outlabs_auth_api_key_validations_total{status=\"invalid\"}[5m])\n\n# Success ratio\nsum(rate(outlabs_auth_api_key_validations_total{status=\"valid\"}[5m]))\n\u002F\nsum(rate(outlabs_auth_api_key_validations_total[5m]))\n",[225,1964,1965,1970,1975,1979,1984,1989,1993,1998,2003,2007],{"__ignoreMap":365},[369,1966,1967],{"class":371,"line":372},[369,1968,1969],{},"# API key validation rate\n",[369,1971,1972],{"class":371,"line":378},[369,1973,1974],{},"rate(outlabs_auth_api_key_validations_total{status=\"valid\"}[5m])\n",[369,1976,1977],{"class":371,"line":384},[369,1978,388],{"emptyLinePlaceholder":387},[369,1980,1981],{"class":371,"line":391},[369,1982,1983],{},"# Invalid API key rate (possible attacks)\n",[369,1985,1986],{"class":371,"line":397},[369,1987,1988],{},"rate(outlabs_auth_api_key_validations_total{status=\"invalid\"}[5m])\n",[369,1990,1991],{"class":371,"line":403},[369,1992,388],{"emptyLinePlaceholder":387},[369,1994,1995],{"class":371,"line":408},[369,1996,1997],{},"# Success ratio\n",[369,1999,2000],{"class":371,"line":414},[369,2001,2002],{},"sum(rate(outlabs_auth_api_key_validations_total{status=\"valid\"}[5m]))\n",[369,2004,2005],{"class":371,"line":420},[369,2006,423],{},[369,2008,2009],{"class":371,"line":426},[369,2010,2011],{},"sum(rate(outlabs_auth_api_key_validations_total[5m]))\n",[198,2013,2014],{},[220,2015,451],{},[360,2017,2019],{"className":454,"code":2018,"language":456,"meta":365,"style":365},"# High invalid API key rate\n- alert: HighInvalidApiKeyRate\n  expr: rate(outlabs_auth_api_key_validations_total{status=\"invalid\"}[5m]) > 10\n  for: 5m\n  labels:\n    severity: warning\n  annotations:\n    summary: \"High rate of invalid API keys\"\n    description: \"Possible API key scanning attack\"\n",[225,2020,2021,2026,2037,2046,2054,2060,2068,2074,2087],{"__ignoreMap":365},[369,2022,2023],{"class":371,"line":372},[369,2024,2025],{"class":463},"# High invalid API key rate\n",[369,2027,2028,2030,2032,2034],{"class":371,"line":378},[369,2029,470],{"class":469},[369,2031,474],{"class":473},[369,2033,477],{"class":469},[369,2035,2036],{"class":480}," HighInvalidApiKeyRate\n",[369,2038,2039,2041,2043],{"class":371,"line":384},[369,2040,486],{"class":473},[369,2042,477],{"class":469},[369,2044,2045],{"class":480}," rate(outlabs_auth_api_key_validations_total{status=\"invalid\"}[5m]) > 10\n",[369,2047,2048,2050,2052],{"class":371,"line":391},[369,2049,496],{"class":473},[369,2051,477],{"class":469},[369,2053,501],{"class":480},[369,2055,2056,2058],{"class":371,"line":397},[369,2057,506],{"class":473},[369,2059,509],{"class":469},[369,2061,2062,2064,2066],{"class":371,"line":403},[369,2063,514],{"class":473},[369,2065,477],{"class":469},[369,2067,519],{"class":480},[369,2069,2070,2072],{"class":371,"line":408},[369,2071,524],{"class":473},[369,2073,509],{"class":469},[369,2075,2076,2078,2080,2082,2085],{"class":371,"line":414},[369,2077,531],{"class":473},[369,2079,477],{"class":469},[369,2081,536],{"class":469},[369,2083,2084],{"class":480},"High rate of invalid API keys",[369,2086,542],{"class":469},[369,2088,2089,2091,2093,2095,2098],{"class":371,"line":420},[369,2090,547],{"class":473},[369,2092,477],{"class":469},[369,2094,536],{"class":469},[369,2096,2097],{"class":480},"Possible API key scanning attack",[369,2099,542],{"class":469},[263,2101],{},[270,2103,2104],{"id":2104},"outlabs_auth_api_key_usage_total",[198,2106,2107,278,2109,2111],{},[220,2108,277],{},[220,2110,281],{}," Total number of requests authenticated with API keys (by key prefix)",[198,2113,2114],{},[220,2115,287],{},[289,2117,2118],{},[292,2119,2120,2123,2124,1283],{},[225,2121,2122],{},"prefix"," - API key prefix (first 12 characters, e.g., ",[225,2125,2126],{},"sk_live_abc1",[198,2128,2129],{},[220,2130,358],{},[360,2132,2134],{"className":362,"code":2133,"language":364,"meta":365,"style":365},"# API key usage rate\nrate(outlabs_auth_api_key_usage_total[5m])\n\n# Top API keys by usage\ntopk(10, sum by (prefix) (rate(outlabs_auth_api_key_usage_total[1h])))\n\n# Usage by specific key\nrate(outlabs_auth_api_key_usage_total{prefix=\"sk_live_abc1\"}[5m])\n",[225,2135,2136,2141,2146,2150,2155,2160,2164,2169],{"__ignoreMap":365},[369,2137,2138],{"class":371,"line":372},[369,2139,2140],{},"# API key usage rate\n",[369,2142,2143],{"class":371,"line":378},[369,2144,2145],{},"rate(outlabs_auth_api_key_usage_total[5m])\n",[369,2147,2148],{"class":371,"line":384},[369,2149,388],{"emptyLinePlaceholder":387},[369,2151,2152],{"class":371,"line":391},[369,2153,2154],{},"# Top API keys by usage\n",[369,2156,2157],{"class":371,"line":397},[369,2158,2159],{},"topk(10, sum by (prefix) (rate(outlabs_auth_api_key_usage_total[1h])))\n",[369,2161,2162],{"class":371,"line":403},[369,2163,388],{"emptyLinePlaceholder":387},[369,2165,2166],{"class":371,"line":408},[369,2167,2168],{},"# Usage by specific key\n",[369,2170,2171],{"class":371,"line":414},[369,2172,2173],{},"rate(outlabs_auth_api_key_usage_total{prefix=\"sk_live_abc1\"}[5m])\n",[263,2175],{},[270,2177,2178],{"id":2178},"outlabs_auth_api_key_rate_limit_hits_total",[198,2180,2181,278,2183,2185],{},[220,2182,277],{},[220,2184,281],{}," Total number of API key rate limit hits",[198,2187,2188],{},[220,2189,287],{},[289,2191,2192],{},[292,2193,2194,2196],{},[225,2195,2122],{}," - API key prefix",[198,2198,2199],{},[220,2200,358],{},[360,2202,2204],{"className":362,"code":2203,"language":364,"meta":365,"style":365},"# Rate limit hit rate\nrate(outlabs_auth_api_key_rate_limit_hits_total[5m])\n\n# Keys hitting rate limits\nsum by (prefix) (rate(outlabs_auth_api_key_rate_limit_hits_total[1h]))\n",[225,2205,2206,2211,2216,2220,2225],{"__ignoreMap":365},[369,2207,2208],{"class":371,"line":372},[369,2209,2210],{},"# Rate limit hit rate\n",[369,2212,2213],{"class":371,"line":378},[369,2214,2215],{},"rate(outlabs_auth_api_key_rate_limit_hits_total[5m])\n",[369,2217,2218],{"class":371,"line":384},[369,2219,388],{"emptyLinePlaceholder":387},[369,2221,2222],{"class":371,"line":391},[369,2223,2224],{},"# Keys hitting rate limits\n",[369,2226,2227],{"class":371,"line":397},[369,2228,2229],{},"sum by (prefix) (rate(outlabs_auth_api_key_rate_limit_hits_total[1h]))\n",[198,2231,2232],{},[220,2233,451],{},[360,2235,2237],{"className":454,"code":2236,"language":456,"meta":365,"style":365},"# API key hitting rate limits frequently\n- alert: ApiKeyRateLimitExceeded\n  expr: rate(outlabs_auth_api_key_rate_limit_hits_total[5m]) > 1\n  for: 10m\n  labels:\n    severity: warning\n  annotations:\n    summary: \"API key {{ $labels.prefix }} hitting rate limits\"\n    description: \"Consider increasing limits or investigating usage patterns\"\n",[225,2238,2239,2244,2255,2264,2272,2278,2286,2292,2305],{"__ignoreMap":365},[369,2240,2241],{"class":371,"line":372},[369,2242,2243],{"class":463},"# API key hitting rate limits frequently\n",[369,2245,2246,2248,2250,2252],{"class":371,"line":378},[369,2247,470],{"class":469},[369,2249,474],{"class":473},[369,2251,477],{"class":469},[369,2253,2254],{"class":480}," ApiKeyRateLimitExceeded\n",[369,2256,2257,2259,2261],{"class":371,"line":384},[369,2258,486],{"class":473},[369,2260,477],{"class":469},[369,2262,2263],{"class":480}," rate(outlabs_auth_api_key_rate_limit_hits_total[5m]) > 1\n",[369,2265,2266,2268,2270],{"class":371,"line":391},[369,2267,496],{"class":473},[369,2269,477],{"class":469},[369,2271,614],{"class":480},[369,2273,2274,2276],{"class":371,"line":397},[369,2275,506],{"class":473},[369,2277,509],{"class":469},[369,2279,2280,2282,2284],{"class":371,"line":403},[369,2281,514],{"class":473},[369,2283,477],{"class":469},[369,2285,519],{"class":480},[369,2287,2288,2290],{"class":371,"line":408},[369,2289,524],{"class":473},[369,2291,509],{"class":469},[369,2293,2294,2296,2298,2300,2303],{"class":371,"line":414},[369,2295,531],{"class":473},[369,2297,477],{"class":469},[369,2299,536],{"class":469},[369,2301,2302],{"class":480},"API key {{ $labels.prefix }} hitting rate limits",[369,2304,542],{"class":469},[369,2306,2307,2309,2311,2313,2316],{"class":371,"line":420},[369,2308,547],{"class":473},[369,2310,477],{"class":469},[369,2312,536],{"class":469},[369,2314,2315],{"class":480},"Consider increasing limits or investigating usage patterns",[369,2317,542],{"class":469},[263,2319],{},[212,2321,2323],{"id":2322},"security-metrics","Security Metrics",[270,2325,2326],{"id":2326},"outlabs_auth_suspicious_activity_total",[198,2328,2329,278,2331,2333],{},[220,2330,277],{},[220,2332,281],{}," Total number of suspicious activity detections",[198,2335,2336],{},[220,2337,287],{},[289,2339,2340],{},[292,2341,2342,2345,2346],{},[225,2343,2344],{},"type"," - Type of suspicious activity\n",[289,2347,2348,2354,2360,2366],{},[292,2349,2350,2353],{},[225,2351,2352],{},"brute_force"," - Multiple failed login attempts",[292,2355,2356,2359],{},[225,2357,2358],{},"session_hijack"," - IP or user agent changed mid-session",[292,2361,2362,2365],{},[225,2363,2364],{},"rate_limit_abuse"," - Excessive requests",[292,2367,2368,2371],{},[225,2369,2370],{},"invalid_token_flood"," - Many invalid token attempts",[198,2373,2374],{},[220,2375,358],{},[360,2377,2379],{"className":362,"code":2378,"language":364,"meta":365,"style":365},"# Suspicious activity rate\nrate(outlabs_auth_suspicious_activity_total[5m])\n\n# Activity by type\nsum by (type) (rate(outlabs_auth_suspicious_activity_total[1h]))\n",[225,2380,2381,2386,2391,2395,2400],{"__ignoreMap":365},[369,2382,2383],{"class":371,"line":372},[369,2384,2385],{},"# Suspicious activity rate\n",[369,2387,2388],{"class":371,"line":378},[369,2389,2390],{},"rate(outlabs_auth_suspicious_activity_total[5m])\n",[369,2392,2393],{"class":371,"line":384},[369,2394,388],{"emptyLinePlaceholder":387},[369,2396,2397],{"class":371,"line":391},[369,2398,2399],{},"# Activity by type\n",[369,2401,2402],{"class":371,"line":397},[369,2403,2404],{},"sum by (type) (rate(outlabs_auth_suspicious_activity_total[1h]))\n",[198,2406,2407],{},[220,2408,451],{},[360,2410,2412],{"className":454,"code":2411,"language":456,"meta":365,"style":365},"# Brute force attack detected\n- alert: BruteForceAttackDetected\n  expr: rate(outlabs_auth_suspicious_activity_total{type=\"brute_force\"}[5m]) > 5\n  for: 2m\n  labels:\n    severity: critical\n  annotations:\n    summary: \"Brute force attack detected\"\n    description: \"Multiple users experiencing failed login attempts\"\n",[225,2413,2414,2419,2430,2439,2448,2454,2462,2468,2481],{"__ignoreMap":365},[369,2415,2416],{"class":371,"line":372},[369,2417,2418],{"class":463},"# Brute force attack detected\n",[369,2420,2421,2423,2425,2427],{"class":371,"line":378},[369,2422,470],{"class":469},[369,2424,474],{"class":473},[369,2426,477],{"class":469},[369,2428,2429],{"class":480}," BruteForceAttackDetected\n",[369,2431,2432,2434,2436],{"class":371,"line":384},[369,2433,486],{"class":473},[369,2435,477],{"class":469},[369,2437,2438],{"class":480}," rate(outlabs_auth_suspicious_activity_total{type=\"brute_force\"}[5m]) > 5\n",[369,2440,2441,2443,2445],{"class":371,"line":391},[369,2442,496],{"class":473},[369,2444,477],{"class":469},[369,2446,2447],{"class":480}," 2m\n",[369,2449,2450,2452],{"class":371,"line":397},[369,2451,506],{"class":473},[369,2453,509],{"class":469},[369,2455,2456,2458,2460],{"class":371,"line":403},[369,2457,514],{"class":473},[369,2459,477],{"class":469},[369,2461,631],{"class":480},[369,2463,2464,2466],{"class":371,"line":408},[369,2465,524],{"class":473},[369,2467,509],{"class":469},[369,2469,2470,2472,2474,2476,2479],{"class":371,"line":414},[369,2471,531],{"class":473},[369,2473,477],{"class":469},[369,2475,536],{"class":469},[369,2477,2478],{"class":480},"Brute force attack detected",[369,2480,542],{"class":469},[369,2482,2483,2485,2487,2489,2492],{"class":371,"line":420},[369,2484,547],{"class":473},[369,2486,477],{"class":469},[369,2488,536],{"class":469},[369,2490,2491],{"class":480},"Multiple users experiencing failed login attempts",[369,2493,542],{"class":469},[263,2495],{},[212,2497,2499],{"id":2498},"error-metrics","Error Metrics",[270,2501,2502],{"id":2502},"outlabs_auth_errors_total",[198,2504,2505,278,2507,2509],{},[220,2506,277],{},[220,2508,281],{}," Total errors by type and location",[198,2511,2512],{},[220,2513,287],{},[289,2515,2516,2528],{},[292,2517,2518,2521,2522,251,2525,1283],{},[225,2519,2520],{},"error_type"," - Exception class name (e.g., ",[225,2523,2524],{},"DatabaseError",[225,2526,2527],{},"ValueError",[292,2529,2530,2533,2534,251,2537,1283],{},[225,2531,2532],{},"location"," - Where error occurred (e.g., ",[225,2535,2536],{},"users_router",[225,2538,2539],{},"auth_service",[198,2541,2542],{},[220,2543,358],{},[360,2545,2547],{"className":362,"code":2546,"language":364,"meta":365,"style":365},"# Total error rate\nsum(rate(outlabs_auth_errors_total[5m]))\n\n# Errors by type\nsum by (error_type) (rate(outlabs_auth_errors_total[5m]))\n\n# Top 10 error types\ntopk(10, sum by (error_type) (increase(outlabs_auth_errors_total[1h])))\n\n# Database errors\nsum(rate(outlabs_auth_errors_total{error_type=~\".*Database.*\"}[5m]))\n\n# Errors by location\nsum by (location) (rate(outlabs_auth_errors_total[5m]))\n",[225,2548,2549,2554,2559,2563,2568,2573,2577,2582,2587,2591,2596,2601,2605,2610],{"__ignoreMap":365},[369,2550,2551],{"class":371,"line":372},[369,2552,2553],{},"# Total error rate\n",[369,2555,2556],{"class":371,"line":378},[369,2557,2558],{},"sum(rate(outlabs_auth_errors_total[5m]))\n",[369,2560,2561],{"class":371,"line":384},[369,2562,388],{"emptyLinePlaceholder":387},[369,2564,2565],{"class":371,"line":391},[369,2566,2567],{},"# Errors by type\n",[369,2569,2570],{"class":371,"line":397},[369,2571,2572],{},"sum by (error_type) (rate(outlabs_auth_errors_total[5m]))\n",[369,2574,2575],{"class":371,"line":403},[369,2576,388],{"emptyLinePlaceholder":387},[369,2578,2579],{"class":371,"line":408},[369,2580,2581],{},"# Top 10 error types\n",[369,2583,2584],{"class":371,"line":414},[369,2585,2586],{},"topk(10, sum by (error_type) (increase(outlabs_auth_errors_total[1h])))\n",[369,2588,2589],{"class":371,"line":420},[369,2590,388],{"emptyLinePlaceholder":387},[369,2592,2593],{"class":371,"line":426},[369,2594,2595],{},"# Database errors\n",[369,2597,2598],{"class":371,"line":432},[369,2599,2600],{},"sum(rate(outlabs_auth_errors_total{error_type=~\".*Database.*\"}[5m]))\n",[369,2602,2603],{"class":371,"line":437},[369,2604,388],{"emptyLinePlaceholder":387},[369,2606,2607],{"class":371,"line":443},[369,2608,2609],{},"# Errors by location\n",[369,2611,2612],{"class":371,"line":589},[369,2613,2614],{},"sum by (location) (rate(outlabs_auth_errors_total[5m]))\n",[198,2616,2617],{},[220,2618,451],{},[360,2620,2622],{"className":454,"code":2621,"language":456,"meta":365,"style":365},"# High error rate\n- alert: HighErrorRate\n  expr: sum(rate(outlabs_auth_errors_total[5m])) > 1\n  for: 5m\n  labels:\n    severity: warning\n  annotations:\n    summary: \"High error rate detected\"\n    description: \"{{ $value }} errors per second\"\n\n# Specific error type spike\n- alert: DatabaseErrorSpike\n  expr: sum(rate(outlabs_auth_errors_total{error_type=~\".*Database.*\"}[5m])) > 0.5\n  for: 2m\n  labels:\n    severity: critical\n  annotations:\n    summary: \"Database errors increasing\"\n    description: \"Database errors: {{ $value }}\u002Fsec\"\n",[225,2623,2624,2629,2640,2649,2657,2663,2671,2677,2690,2703,2707,2712,2723,2732,2740,2746,2754,2760,2773],{"__ignoreMap":365},[369,2625,2626],{"class":371,"line":372},[369,2627,2628],{"class":463},"# High error rate\n",[369,2630,2631,2633,2635,2637],{"class":371,"line":378},[369,2632,470],{"class":469},[369,2634,474],{"class":473},[369,2636,477],{"class":469},[369,2638,2639],{"class":480}," HighErrorRate\n",[369,2641,2642,2644,2646],{"class":371,"line":384},[369,2643,486],{"class":473},[369,2645,477],{"class":469},[369,2647,2648],{"class":480}," sum(rate(outlabs_auth_errors_total[5m])) > 1\n",[369,2650,2651,2653,2655],{"class":371,"line":391},[369,2652,496],{"class":473},[369,2654,477],{"class":469},[369,2656,501],{"class":480},[369,2658,2659,2661],{"class":371,"line":397},[369,2660,506],{"class":473},[369,2662,509],{"class":469},[369,2664,2665,2667,2669],{"class":371,"line":403},[369,2666,514],{"class":473},[369,2668,477],{"class":469},[369,2670,519],{"class":480},[369,2672,2673,2675],{"class":371,"line":408},[369,2674,524],{"class":473},[369,2676,509],{"class":469},[369,2678,2679,2681,2683,2685,2688],{"class":371,"line":414},[369,2680,531],{"class":473},[369,2682,477],{"class":469},[369,2684,536],{"class":469},[369,2686,2687],{"class":480},"High error rate detected",[369,2689,542],{"class":469},[369,2691,2692,2694,2696,2698,2701],{"class":371,"line":420},[369,2693,547],{"class":473},[369,2695,477],{"class":469},[369,2697,536],{"class":469},[369,2699,2700],{"class":480},"{{ $value }} errors per second",[369,2702,542],{"class":469},[369,2704,2705],{"class":371,"line":426},[369,2706,388],{"emptyLinePlaceholder":387},[369,2708,2709],{"class":371,"line":432},[369,2710,2711],{"class":463},"# Specific error type spike\n",[369,2713,2714,2716,2718,2720],{"class":371,"line":437},[369,2715,470],{"class":469},[369,2717,474],{"class":473},[369,2719,477],{"class":469},[369,2721,2722],{"class":480}," DatabaseErrorSpike\n",[369,2724,2725,2727,2729],{"class":371,"line":443},[369,2726,486],{"class":473},[369,2728,477],{"class":469},[369,2730,2731],{"class":480}," sum(rate(outlabs_auth_errors_total{error_type=~\".*Database.*\"}[5m])) > 0.5\n",[369,2733,2734,2736,2738],{"class":371,"line":589},[369,2735,496],{"class":473},[369,2737,477],{"class":469},[369,2739,2447],{"class":480},[369,2741,2742,2744],{"class":371,"line":595},[369,2743,506],{"class":473},[369,2745,509],{"class":469},[369,2747,2748,2750,2752],{"class":371,"line":601},[369,2749,514],{"class":473},[369,2751,477],{"class":469},[369,2753,631],{"class":480},[369,2755,2756,2758],{"class":371,"line":607},[369,2757,524],{"class":473},[369,2759,509],{"class":469},[369,2761,2762,2764,2766,2768,2771],{"class":371,"line":617},[369,2763,531],{"class":473},[369,2765,477],{"class":469},[369,2767,536],{"class":469},[369,2769,2770],{"class":480},"Database errors increasing",[369,2772,542],{"class":469},[369,2774,2775,2777,2779,2781,2784],{"class":371,"line":624},[369,2776,547],{"class":473},[369,2778,477],{"class":469},[369,2780,536],{"class":469},[369,2782,2783],{"class":480},"Database errors: {{ $value }}\u002Fsec",[369,2785,542],{"class":469},[263,2787],{},[270,2789,2790],{"id":2790},"outlabs_auth_500_errors_total",[198,2792,2793,278,2795,2797],{},[220,2794,277],{},[220,2796,281],{}," Total HTTP 500 Internal Server Errors",[198,2799,2800],{},[220,2801,287],{},[289,2803,2804,2816],{},[292,2805,2806,2809,2810,251,2813,1283],{},[225,2807,2808],{},"endpoint"," - API endpoint (e.g., ",[225,2811,2812],{},"\u002Fv1\u002Fusers\u002F",[225,2814,2815],{},"\u002Fv1\u002Fauth\u002Flogin",[292,2817,2818,2821,2822,251,2825,1283],{},[225,2819,2820],{},"error_class"," - Exception class (e.g., ",[225,2823,2824],{},"UserAlreadyExistsError",[225,2826,2827],{},"AttributeError",[198,2829,2830,2833,2834,2837],{},[220,2831,2832],{},"How it's incremented:","\nAutomatically incremented when using ",[225,2835,2836],{},"ObservabilityContext.log_500_error()"," in your routes:",[360,2839,2843],{"className":2840,"code":2841,"language":2842,"meta":365,"style":365},"language-python shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","try:\n    user = await auth.user_service.create_user(data)\nexcept Exception as e:\n    obs.log_500_error(e)  # Increments this metric\n    raise HTTPException(500, detail=\"Failed to create user\")\n","python",[225,2844,2845,2850,2855,2860,2865],{"__ignoreMap":365},[369,2846,2847],{"class":371,"line":372},[369,2848,2849],{},"try:\n",[369,2851,2852],{"class":371,"line":378},[369,2853,2854],{},"    user = await auth.user_service.create_user(data)\n",[369,2856,2857],{"class":371,"line":384},[369,2858,2859],{},"except Exception as e:\n",[369,2861,2862],{"class":371,"line":391},[369,2863,2864],{},"    obs.log_500_error(e)  # Increments this metric\n",[369,2866,2867],{"class":371,"line":397},[369,2868,2869],{},"    raise HTTPException(500, detail=\"Failed to create user\")\n",[198,2871,2872],{},[220,2873,358],{},[360,2875,2877],{"className":362,"code":2876,"language":364,"meta":365,"style":365},"# 500 error rate\nsum(rate(outlabs_auth_500_errors_total[5m]))\n\n# 500 errors by endpoint\nsum by (endpoint) (rate(outlabs_auth_500_errors_total[5m]))\n\n# Top endpoints with 500s\ntopk(5, sum by (endpoint) (increase(outlabs_auth_500_errors_total[1h])))\n\n# 500 errors by error type\nsum by (error_class) (rate(outlabs_auth_500_errors_total[5m]))\n\n# Specific endpoint error rate\nrate(outlabs_auth_500_errors_total{endpoint=\"\u002Fv1\u002Fusers\"}[5m])\n",[225,2878,2879,2884,2889,2893,2898,2903,2907,2912,2917,2921,2926,2931,2935,2940],{"__ignoreMap":365},[369,2880,2881],{"class":371,"line":372},[369,2882,2883],{},"# 500 error rate\n",[369,2885,2886],{"class":371,"line":378},[369,2887,2888],{},"sum(rate(outlabs_auth_500_errors_total[5m]))\n",[369,2890,2891],{"class":371,"line":384},[369,2892,388],{"emptyLinePlaceholder":387},[369,2894,2895],{"class":371,"line":391},[369,2896,2897],{},"# 500 errors by endpoint\n",[369,2899,2900],{"class":371,"line":397},[369,2901,2902],{},"sum by (endpoint) (rate(outlabs_auth_500_errors_total[5m]))\n",[369,2904,2905],{"class":371,"line":403},[369,2906,388],{"emptyLinePlaceholder":387},[369,2908,2909],{"class":371,"line":408},[369,2910,2911],{},"# Top endpoints with 500s\n",[369,2913,2914],{"class":371,"line":414},[369,2915,2916],{},"topk(5, sum by (endpoint) (increase(outlabs_auth_500_errors_total[1h])))\n",[369,2918,2919],{"class":371,"line":420},[369,2920,388],{"emptyLinePlaceholder":387},[369,2922,2923],{"class":371,"line":426},[369,2924,2925],{},"# 500 errors by error type\n",[369,2927,2928],{"class":371,"line":432},[369,2929,2930],{},"sum by (error_class) (rate(outlabs_auth_500_errors_total[5m]))\n",[369,2932,2933],{"class":371,"line":437},[369,2934,388],{"emptyLinePlaceholder":387},[369,2936,2937],{"class":371,"line":443},[369,2938,2939],{},"# Specific endpoint error rate\n",[369,2941,2942],{"class":371,"line":589},[369,2943,2944],{},"rate(outlabs_auth_500_errors_total{endpoint=\"\u002Fv1\u002Fusers\"}[5m])\n",[198,2946,2947],{},[220,2948,451],{},[360,2950,2952],{"className":454,"code":2951,"language":456,"meta":365,"style":365},"# Any 500 errors\n- alert: InternalServerErrors\n  expr: sum(rate(outlabs_auth_500_errors_total[5m])) > 0\n  for: 1m\n  labels:\n    severity: critical\n  annotations:\n    summary: \"500 errors detected\"\n    description: \"{{ $value }} 500 errors per second\"\n\n# Specific endpoint failing\n- alert: EndpointFailing\n  expr: rate(outlabs_auth_500_errors_total{endpoint=\"\u002Fv1\u002Fauth\u002Flogin\"}[5m]) > 0.1\n  for: 2m\n  labels:\n    severity: critical\n  annotations:\n    summary: \"Login endpoint failing\"\n    description: \"Login returning 500s: {{ $value }}\u002Fsec\"\n",[225,2953,2954,2959,2970,2979,2988,2994,3002,3008,3021,3034,3038,3043,3054,3063,3071,3077,3085,3091,3104],{"__ignoreMap":365},[369,2955,2956],{"class":371,"line":372},[369,2957,2958],{"class":463},"# Any 500 errors\n",[369,2960,2961,2963,2965,2967],{"class":371,"line":378},[369,2962,470],{"class":469},[369,2964,474],{"class":473},[369,2966,477],{"class":469},[369,2968,2969],{"class":480}," InternalServerErrors\n",[369,2971,2972,2974,2976],{"class":371,"line":384},[369,2973,486],{"class":473},[369,2975,477],{"class":469},[369,2977,2978],{"class":480}," sum(rate(outlabs_auth_500_errors_total[5m])) > 0\n",[369,2980,2981,2983,2985],{"class":371,"line":391},[369,2982,496],{"class":473},[369,2984,477],{"class":469},[369,2986,2987],{"class":480}," 1m\n",[369,2989,2990,2992],{"class":371,"line":397},[369,2991,506],{"class":473},[369,2993,509],{"class":469},[369,2995,2996,2998,3000],{"class":371,"line":403},[369,2997,514],{"class":473},[369,2999,477],{"class":469},[369,3001,631],{"class":480},[369,3003,3004,3006],{"class":371,"line":408},[369,3005,524],{"class":473},[369,3007,509],{"class":469},[369,3009,3010,3012,3014,3016,3019],{"class":371,"line":414},[369,3011,531],{"class":473},[369,3013,477],{"class":469},[369,3015,536],{"class":469},[369,3017,3018],{"class":480},"500 errors detected",[369,3020,542],{"class":469},[369,3022,3023,3025,3027,3029,3032],{"class":371,"line":420},[369,3024,547],{"class":473},[369,3026,477],{"class":469},[369,3028,536],{"class":469},[369,3030,3031],{"class":480},"{{ $value }} 500 errors per second",[369,3033,542],{"class":469},[369,3035,3036],{"class":371,"line":426},[369,3037,388],{"emptyLinePlaceholder":387},[369,3039,3040],{"class":371,"line":432},[369,3041,3042],{"class":463},"# Specific endpoint failing\n",[369,3044,3045,3047,3049,3051],{"class":371,"line":437},[369,3046,470],{"class":469},[369,3048,474],{"class":473},[369,3050,477],{"class":469},[369,3052,3053],{"class":480}," EndpointFailing\n",[369,3055,3056,3058,3060],{"class":371,"line":443},[369,3057,486],{"class":473},[369,3059,477],{"class":469},[369,3061,3062],{"class":480}," rate(outlabs_auth_500_errors_total{endpoint=\"\u002Fv1\u002Fauth\u002Flogin\"}[5m]) > 0.1\n",[369,3064,3065,3067,3069],{"class":371,"line":589},[369,3066,496],{"class":473},[369,3068,477],{"class":469},[369,3070,2447],{"class":480},[369,3072,3073,3075],{"class":371,"line":595},[369,3074,506],{"class":473},[369,3076,509],{"class":469},[369,3078,3079,3081,3083],{"class":371,"line":601},[369,3080,514],{"class":473},[369,3082,477],{"class":469},[369,3084,631],{"class":480},[369,3086,3087,3089],{"class":371,"line":607},[369,3088,524],{"class":473},[369,3090,509],{"class":469},[369,3092,3093,3095,3097,3099,3102],{"class":371,"line":617},[369,3094,531],{"class":473},[369,3096,477],{"class":469},[369,3098,536],{"class":469},[369,3100,3101],{"class":480},"Login endpoint failing",[369,3103,542],{"class":469},[369,3105,3106,3108,3110,3112,3115],{"class":371,"line":624},[369,3107,547],{"class":473},[369,3109,477],{"class":469},[369,3111,536],{"class":469},[369,3113,3114],{"class":480},"Login returning 500s: {{ $value }}\u002Fsec",[369,3116,542],{"class":469},[263,3118],{},[270,3120,3121],{"id":3121},"outlabs_auth_router_errors_total",[198,3123,3124,278,3126,3128],{},[220,3125,277],{},[220,3127,281],{}," Total router-level errors (for tracking non-500 errors you still want to monitor)",[198,3130,3131],{},[220,3132,287],{},[289,3134,3135,3150],{},[292,3136,3137,3140,3141,251,3144,251,3147,1283],{},[225,3138,3139],{},"router"," - Router name (e.g., ",[225,3142,3143],{},"users",[225,3145,3146],{},"roles",[225,3148,3149],{},"auth",[292,3151,3152,3155,3156,251,3159,1283],{},[225,3153,3154],{},"operation"," - Operation name (e.g., ",[225,3157,3158],{},"create_user",[225,3160,3161],{},"update_role",[198,3163,3164,3166,3167,3170],{},[220,3165,2832],{},"\nUse ",[225,3168,3169],{},"ObservabilityContext.log_router_error()"," for errors you want to track but aren't 500s:",[360,3172,3174],{"className":2840,"code":3173,"language":2842,"meta":365,"style":365},"try:\n    user = await auth.user_service.create_user(data)\nexcept ValidationError as e:\n    obs.log_router_error(\"users\", \"create_user\", e)  # Increments this metric\n    raise HTTPException(400, detail=\"Invalid data\")\n",[225,3175,3176,3180,3184,3189,3194],{"__ignoreMap":365},[369,3177,3178],{"class":371,"line":372},[369,3179,2849],{},[369,3181,3182],{"class":371,"line":378},[369,3183,2854],{},[369,3185,3186],{"class":371,"line":384},[369,3187,3188],{},"except ValidationError as e:\n",[369,3190,3191],{"class":371,"line":391},[369,3192,3193],{},"    obs.log_router_error(\"users\", \"create_user\", e)  # Increments this metric\n",[369,3195,3196],{"class":371,"line":397},[369,3197,3198],{},"    raise HTTPException(400, detail=\"Invalid data\")\n",[198,3200,3201],{},[220,3202,358],{},[360,3204,3206],{"className":362,"code":3205,"language":364,"meta":365,"style":365},"# Errors by router\nsum by (router) (rate(outlabs_auth_router_errors_total[5m]))\n\n# Most error-prone router\ntopk(1, sum by (router) (increase(outlabs_auth_router_errors_total[1h])))\n\n# Users router error rate\nsum(rate(outlabs_auth_router_errors_total{router=\"users\"}[5m]))\n\n# Error rate heatmap (by router and endpoint)\nsum by (router, endpoint) (rate(outlabs_auth_router_errors_total[5m]))\n",[225,3207,3208,3213,3218,3222,3227,3232,3236,3241,3246,3250,3255],{"__ignoreMap":365},[369,3209,3210],{"class":371,"line":372},[369,3211,3212],{},"# Errors by router\n",[369,3214,3215],{"class":371,"line":378},[369,3216,3217],{},"sum by (router) (rate(outlabs_auth_router_errors_total[5m]))\n",[369,3219,3220],{"class":371,"line":384},[369,3221,388],{"emptyLinePlaceholder":387},[369,3223,3224],{"class":371,"line":391},[369,3225,3226],{},"# Most error-prone router\n",[369,3228,3229],{"class":371,"line":397},[369,3230,3231],{},"topk(1, sum by (router) (increase(outlabs_auth_router_errors_total[1h])))\n",[369,3233,3234],{"class":371,"line":403},[369,3235,388],{"emptyLinePlaceholder":387},[369,3237,3238],{"class":371,"line":408},[369,3239,3240],{},"# Users router error rate\n",[369,3242,3243],{"class":371,"line":414},[369,3244,3245],{},"sum(rate(outlabs_auth_router_errors_total{router=\"users\"}[5m]))\n",[369,3247,3248],{"class":371,"line":420},[369,3249,388],{"emptyLinePlaceholder":387},[369,3251,3252],{"class":371,"line":426},[369,3253,3254],{},"# Error rate heatmap (by router and endpoint)\n",[369,3256,3257],{"class":371,"line":432},[369,3258,3259],{},"sum by (router, endpoint) (rate(outlabs_auth_router_errors_total[5m]))\n",[198,3261,3262],{},[220,3263,451],{},[360,3265,3267],{"className":454,"code":3266,"language":456,"meta":365,"style":365},"# Router error spike\n- alert: RouterErrorSpike\n  expr: sum by (router) (rate(outlabs_auth_router_errors_total[5m])) > 0.5\n  for: 5m\n  labels:\n    severity: warning\n  annotations:\n    summary: \"{{ $labels.router }} router errors spiking\"\n    description: \"{{ $value }} errors\u002Fsec in {{ $labels.router }} router\"\n",[225,3268,3269,3274,3285,3294,3302,3308,3316,3322,3335],{"__ignoreMap":365},[369,3270,3271],{"class":371,"line":372},[369,3272,3273],{"class":463},"# Router error spike\n",[369,3275,3276,3278,3280,3282],{"class":371,"line":378},[369,3277,470],{"class":469},[369,3279,474],{"class":473},[369,3281,477],{"class":469},[369,3283,3284],{"class":480}," RouterErrorSpike\n",[369,3286,3287,3289,3291],{"class":371,"line":384},[369,3288,486],{"class":473},[369,3290,477],{"class":469},[369,3292,3293],{"class":480}," sum by (router) (rate(outlabs_auth_router_errors_total[5m])) > 0.5\n",[369,3295,3296,3298,3300],{"class":371,"line":391},[369,3297,496],{"class":473},[369,3299,477],{"class":469},[369,3301,501],{"class":480},[369,3303,3304,3306],{"class":371,"line":397},[369,3305,506],{"class":473},[369,3307,509],{"class":469},[369,3309,3310,3312,3314],{"class":371,"line":403},[369,3311,514],{"class":473},[369,3313,477],{"class":469},[369,3315,519],{"class":480},[369,3317,3318,3320],{"class":371,"line":408},[369,3319,524],{"class":473},[369,3321,509],{"class":469},[369,3323,3324,3326,3328,3330,3333],{"class":371,"line":414},[369,3325,531],{"class":473},[369,3327,477],{"class":469},[369,3329,536],{"class":469},[369,3331,3332],{"class":480},"{{ $labels.router }} router errors spiking",[369,3334,542],{"class":469},[369,3336,3337,3339,3341,3343,3346],{"class":371,"line":420},[369,3338,547],{"class":473},[369,3340,477],{"class":469},[369,3342,536],{"class":469},[369,3344,3345],{"class":480},"{{ $value }} errors\u002Fsec in {{ $labels.router }} router",[369,3347,542],{"class":469},[263,3349],{},[270,3351,3352],{"id":3352},"outlabs_auth_service_errors_total",[198,3354,3355,278,3357,3359],{},[220,3356,277],{},[220,3358,281],{}," Total service-level errors",[198,3361,3362],{},[220,3363,287],{},[289,3365,3366,3382],{},[292,3367,3368,3371,3372,251,3374,251,3377,251,3380,1283],{},[225,3369,3370],{},"service"," - Service name (e.g., ",[225,3373,3149],{},[225,3375,3376],{},"user",[225,3378,3379],{},"role",[225,3381,257],{},[292,3383,3384,3386,3387,251,3390,1283],{},[225,3385,3154],{}," - Operation being performed (e.g., ",[225,3388,3389],{},"login",[225,3391,3158],{},[198,3393,3394],{},[220,3395,358],{},[360,3397,3399],{"className":362,"code":3398,"language":364,"meta":365,"style":365},"# Errors by service\nsum by (service) (rate(outlabs_auth_service_errors_total[5m]))\n\n# Errors by operation\nsum by (operation) (rate(outlabs_auth_service_errors_total[5m]))\n\n# Auth service errors\nsum(rate(outlabs_auth_service_errors_total{service=\"auth\"}[5m]))\n\n# Login operation errors\nsum(rate(outlabs_auth_service_errors_total{operation=\"login\"}[5m]))\n\n# Most error-prone operations\ntopk(10, sum by (operation) (increase(outlabs_auth_service_errors_total[1h])))\n",[225,3400,3401,3406,3411,3415,3420,3425,3429,3434,3439,3443,3448,3453,3457,3462],{"__ignoreMap":365},[369,3402,3403],{"class":371,"line":372},[369,3404,3405],{},"# Errors by service\n",[369,3407,3408],{"class":371,"line":378},[369,3409,3410],{},"sum by (service) (rate(outlabs_auth_service_errors_total[5m]))\n",[369,3412,3413],{"class":371,"line":384},[369,3414,388],{"emptyLinePlaceholder":387},[369,3416,3417],{"class":371,"line":391},[369,3418,3419],{},"# Errors by operation\n",[369,3421,3422],{"class":371,"line":397},[369,3423,3424],{},"sum by (operation) (rate(outlabs_auth_service_errors_total[5m]))\n",[369,3426,3427],{"class":371,"line":403},[369,3428,388],{"emptyLinePlaceholder":387},[369,3430,3431],{"class":371,"line":408},[369,3432,3433],{},"# Auth service errors\n",[369,3435,3436],{"class":371,"line":414},[369,3437,3438],{},"sum(rate(outlabs_auth_service_errors_total{service=\"auth\"}[5m]))\n",[369,3440,3441],{"class":371,"line":420},[369,3442,388],{"emptyLinePlaceholder":387},[369,3444,3445],{"class":371,"line":426},[369,3446,3447],{},"# Login operation errors\n",[369,3449,3450],{"class":371,"line":432},[369,3451,3452],{},"sum(rate(outlabs_auth_service_errors_total{operation=\"login\"}[5m]))\n",[369,3454,3455],{"class":371,"line":437},[369,3456,388],{"emptyLinePlaceholder":387},[369,3458,3459],{"class":371,"line":443},[369,3460,3461],{},"# Most error-prone operations\n",[369,3463,3464],{"class":371,"line":589},[369,3465,3466],{},"topk(10, sum by (operation) (increase(outlabs_auth_service_errors_total[1h])))\n",[198,3468,3469],{},[220,3470,451],{},[360,3472,3474],{"className":454,"code":3473,"language":456,"meta":365,"style":365},"# Service error spike\n- alert: ServiceErrorSpike\n  expr: sum by (service) (rate(outlabs_auth_service_errors_total[5m])) > 0.5\n  for: 5m\n  labels:\n    severity: warning\n  annotations:\n    summary: \"{{ $labels.service }} service errors spiking\"\n    description: \"{{ $value }} errors\u002Fsec in {{ $labels.service }} service\"\n\n# Login failures\n- alert: LoginServiceFailing\n  expr: sum(rate(outlabs_auth_service_errors_total{operation=\"login\"}[5m])) > 0.1\n  for: 2m\n  labels:\n    severity: critical\n  annotations:\n    summary: \"Login service experiencing errors\"\n    description: \"{{ $value }} login errors per second\"\n",[225,3475,3476,3481,3492,3501,3509,3515,3523,3529,3542,3555,3559,3564,3575,3584,3592,3598,3606,3612,3625],{"__ignoreMap":365},[369,3477,3478],{"class":371,"line":372},[369,3479,3480],{"class":463},"# Service error spike\n",[369,3482,3483,3485,3487,3489],{"class":371,"line":378},[369,3484,470],{"class":469},[369,3486,474],{"class":473},[369,3488,477],{"class":469},[369,3490,3491],{"class":480}," ServiceErrorSpike\n",[369,3493,3494,3496,3498],{"class":371,"line":384},[369,3495,486],{"class":473},[369,3497,477],{"class":469},[369,3499,3500],{"class":480}," sum by (service) (rate(outlabs_auth_service_errors_total[5m])) > 0.5\n",[369,3502,3503,3505,3507],{"class":371,"line":391},[369,3504,496],{"class":473},[369,3506,477],{"class":469},[369,3508,501],{"class":480},[369,3510,3511,3513],{"class":371,"line":397},[369,3512,506],{"class":473},[369,3514,509],{"class":469},[369,3516,3517,3519,3521],{"class":371,"line":403},[369,3518,514],{"class":473},[369,3520,477],{"class":469},[369,3522,519],{"class":480},[369,3524,3525,3527],{"class":371,"line":408},[369,3526,524],{"class":473},[369,3528,509],{"class":469},[369,3530,3531,3533,3535,3537,3540],{"class":371,"line":414},[369,3532,531],{"class":473},[369,3534,477],{"class":469},[369,3536,536],{"class":469},[369,3538,3539],{"class":480},"{{ $labels.service }} service errors spiking",[369,3541,542],{"class":469},[369,3543,3544,3546,3548,3550,3553],{"class":371,"line":420},[369,3545,547],{"class":473},[369,3547,477],{"class":469},[369,3549,536],{"class":469},[369,3551,3552],{"class":480},"{{ $value }} errors\u002Fsec in {{ $labels.service }} service",[369,3554,542],{"class":469},[369,3556,3557],{"class":371,"line":426},[369,3558,388],{"emptyLinePlaceholder":387},[369,3560,3561],{"class":371,"line":432},[369,3562,3563],{"class":463},"# Login failures\n",[369,3565,3566,3568,3570,3572],{"class":371,"line":437},[369,3567,470],{"class":469},[369,3569,474],{"class":473},[369,3571,477],{"class":469},[369,3573,3574],{"class":480}," LoginServiceFailing\n",[369,3576,3577,3579,3581],{"class":371,"line":443},[369,3578,486],{"class":473},[369,3580,477],{"class":469},[369,3582,3583],{"class":480}," sum(rate(outlabs_auth_service_errors_total{operation=\"login\"}[5m])) > 0.1\n",[369,3585,3586,3588,3590],{"class":371,"line":589},[369,3587,496],{"class":473},[369,3589,477],{"class":469},[369,3591,2447],{"class":480},[369,3593,3594,3596],{"class":371,"line":595},[369,3595,506],{"class":473},[369,3597,509],{"class":469},[369,3599,3600,3602,3604],{"class":371,"line":601},[369,3601,514],{"class":473},[369,3603,477],{"class":469},[369,3605,631],{"class":480},[369,3607,3608,3610],{"class":371,"line":607},[369,3609,524],{"class":473},[369,3611,509],{"class":469},[369,3613,3614,3616,3618,3620,3623],{"class":371,"line":617},[369,3615,531],{"class":473},[369,3617,477],{"class":469},[369,3619,536],{"class":469},[369,3621,3622],{"class":480},"Login service experiencing errors",[369,3624,542],{"class":469},[369,3626,3627,3629,3631,3633,3636],{"class":371,"line":624},[369,3628,547],{"class":473},[369,3630,477],{"class":469},[369,3632,536],{"class":469},[369,3634,3635],{"class":480},"{{ $value }} login errors per second",[369,3637,542],{"class":469},[263,3639],{},[212,3641,3643],{"id":3642},"performance-metrics","Performance Metrics",[270,3645,3646],{"id":3646},"outlabs_auth_cache_hits_total",[198,3648,3649,278,3651,3653],{},[220,3650,277],{},[220,3652,281],{}," Total number of cache hits (Redis)",[198,3655,3656],{},[220,3657,287],{},[289,3659,3660],{},[292,3661,3662,3665,3666],{},[225,3663,3664],{},"cache_type"," - Type of cache\n",[289,3667,3668,3673,3678,3683],{},[292,3669,3670,3672],{},[225,3671,257],{}," - Permission check results",[292,3674,3675,3677],{},[225,3676,3379],{}," - Role lookups",[292,3679,3680,3682],{},[225,3681,3376],{}," - User data",[292,3684,3685,3688],{},[225,3686,3687],{},"entity"," - Entity data",[198,3690,3691],{},[220,3692,358],{},[360,3694,3696],{"className":362,"code":3695,"language":364,"meta":365,"style":365},"# Cache hit rate\nsum(rate(outlabs_auth_cache_hits_total[5m]))\n\u002F\n(sum(rate(outlabs_auth_cache_hits_total[5m])) + sum(rate(outlabs_auth_cache_misses_total[5m])))\n\n# Hits by cache type\nsum by (cache_type) (rate(outlabs_auth_cache_hits_total[5m]))\n",[225,3697,3698,3703,3708,3712,3717,3721,3726],{"__ignoreMap":365},[369,3699,3700],{"class":371,"line":372},[369,3701,3702],{},"# Cache hit rate\n",[369,3704,3705],{"class":371,"line":378},[369,3706,3707],{},"sum(rate(outlabs_auth_cache_hits_total[5m]))\n",[369,3709,3710],{"class":371,"line":384},[369,3711,423],{},[369,3713,3714],{"class":371,"line":391},[369,3715,3716],{},"(sum(rate(outlabs_auth_cache_hits_total[5m])) + sum(rate(outlabs_auth_cache_misses_total[5m])))\n",[369,3718,3719],{"class":371,"line":397},[369,3720,388],{"emptyLinePlaceholder":387},[369,3722,3723],{"class":371,"line":403},[369,3724,3725],{},"# Hits by cache type\n",[369,3727,3728],{"class":371,"line":408},[369,3729,3730],{},"sum by (cache_type) (rate(outlabs_auth_cache_hits_total[5m]))\n",[263,3732],{},[270,3734,3735],{"id":3735},"outlabs_auth_cache_misses_total",[198,3737,3738,278,3740,3742],{},[220,3739,277],{},[220,3741,281],{}," Total number of cache misses (Redis)",[198,3744,3745],{},[220,3746,287],{},[289,3748,3749],{},[292,3750,3751,3753],{},[225,3752,3664],{}," - Type of cache (permission, role, user, entity)",[198,3755,3756],{},[220,3757,358],{},[360,3759,3761],{"className":362,"code":3760,"language":364,"meta":365,"style":365},"# Cache miss rate\nrate(outlabs_auth_cache_misses_total[5m])\n\n# Cache efficiency (should be >90%)\nsum(rate(outlabs_auth_cache_hits_total[5m]))\n\u002F\n(sum(rate(outlabs_auth_cache_hits_total[5m])) + sum(rate(outlabs_auth_cache_misses_total[5m])))\n",[225,3762,3763,3768,3773,3777,3782,3786,3790],{"__ignoreMap":365},[369,3764,3765],{"class":371,"line":372},[369,3766,3767],{},"# Cache miss rate\n",[369,3769,3770],{"class":371,"line":378},[369,3771,3772],{},"rate(outlabs_auth_cache_misses_total[5m])\n",[369,3774,3775],{"class":371,"line":384},[369,3776,388],{"emptyLinePlaceholder":387},[369,3778,3779],{"class":371,"line":391},[369,3780,3781],{},"# Cache efficiency (should be >90%)\n",[369,3783,3784],{"class":371,"line":397},[369,3785,3707],{},[369,3787,3788],{"class":371,"line":403},[369,3789,423],{},[369,3791,3792],{"class":371,"line":408},[369,3793,3716],{},[198,3795,3796],{},[220,3797,451],{},[360,3799,3801],{"className":454,"code":3800,"language":456,"meta":365,"style":365},"# Low cache hit rate\n- alert: LowCacheHitRate\n  expr: |\n    sum(rate(outlabs_auth_cache_hits_total[5m]))\n    \u002F\n    (sum(rate(outlabs_auth_cache_hits_total[5m])) + sum(rate(outlabs_auth_cache_misses_total[5m]))) \u003C 0.7\n  for: 10m\n  labels:\n    severity: warning\n  annotations:\n    summary: \"Cache hit rate below 70%\"\n    description: \"Cache may need tuning or TTL adjustment\"\n",[225,3802,3803,3808,3819,3827,3832,3836,3841,3849,3855,3863,3869,3882],{"__ignoreMap":365},[369,3804,3805],{"class":371,"line":372},[369,3806,3807],{"class":463},"# Low cache hit rate\n",[369,3809,3810,3812,3814,3816],{"class":371,"line":378},[369,3811,470],{"class":469},[369,3813,474],{"class":473},[369,3815,477],{"class":469},[369,3817,3818],{"class":480}," LowCacheHitRate\n",[369,3820,3821,3823,3825],{"class":371,"line":384},[369,3822,486],{"class":473},[369,3824,477],{"class":469},[369,3826,586],{"class":585},[369,3828,3829],{"class":371,"line":391},[369,3830,3831],{"class":480},"    sum(rate(outlabs_auth_cache_hits_total[5m]))\n",[369,3833,3834],{"class":371,"line":397},[369,3835,598],{"class":480},[369,3837,3838],{"class":371,"line":403},[369,3839,3840],{"class":480},"    (sum(rate(outlabs_auth_cache_hits_total[5m])) + sum(rate(outlabs_auth_cache_misses_total[5m]))) \u003C 0.7\n",[369,3842,3843,3845,3847],{"class":371,"line":408},[369,3844,496],{"class":473},[369,3846,477],{"class":469},[369,3848,614],{"class":480},[369,3850,3851,3853],{"class":371,"line":414},[369,3852,506],{"class":473},[369,3854,509],{"class":469},[369,3856,3857,3859,3861],{"class":371,"line":420},[369,3858,514],{"class":473},[369,3860,477],{"class":469},[369,3862,519],{"class":480},[369,3864,3865,3867],{"class":371,"line":426},[369,3866,524],{"class":473},[369,3868,509],{"class":469},[369,3870,3871,3873,3875,3877,3880],{"class":371,"line":432},[369,3872,531],{"class":473},[369,3874,477],{"class":469},[369,3876,536],{"class":469},[369,3878,3879],{"class":480},"Cache hit rate below 70%",[369,3881,542],{"class":469},[369,3883,3884,3886,3888,3890,3893],{"class":371,"line":437},[369,3885,547],{"class":473},[369,3887,477],{"class":469},[369,3889,536],{"class":469},[369,3891,3892],{"class":480},"Cache may need tuning or TTL adjustment",[369,3894,542],{"class":469},[263,3896],{},[270,3898,3899],{"id":3899},"outlabs_auth_db_query_duration_seconds",[198,3901,3902,662,3904,3906],{},[220,3903,277],{},[220,3905,281],{}," PostgreSQL query duration (in seconds)",[198,3908,3909,3910,3913,3914,3917],{},"Emitted automatically for every statement executed on the SQLAlchemy async\nengine, via ",[225,3911,3912],{},"before_cursor_execute"," \u002F ",[225,3915,3916],{},"after_cursor_execute"," listeners.",[198,3919,3920],{},[220,3921,287],{},[289,3923,3924],{},[292,3925,3926,3928,3929,251,3932,251,3935,251,3938,251,3941,251,3944,3947,3948,3951],{},[225,3927,3154],{}," - SQL verb, taken as the first token of the statement, lowercased\n(",[225,3930,3931],{},"select",[225,3933,3934],{},"insert",[225,3936,3937],{},"update",[225,3939,3940],{},"delete",[225,3942,3943],{},"begin",[225,3945,3946],{},"commit",", ...; ",[225,3949,3950],{},"unknown"," if\nthe statement is empty)",[198,3953,3954,247,3956],{},[220,3955,681],{},[225,3957,3958],{},"[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5]",[198,3960,3961],{},[220,3962,358],{},[360,3964,3966],{"className":362,"code":3965,"language":364,"meta":365,"style":365},"# P95 database query latency\nhistogram_quantile(0.95, rate(outlabs_auth_db_query_duration_seconds_bucket[5m]))\n\n# P95 latency by SQL operation\nhistogram_quantile(0.95, sum by (operation, le) (rate(outlabs_auth_db_query_duration_seconds_bucket[5m])))\n\n# SELECT throughput\nsum(rate(outlabs_auth_db_query_duration_seconds_count{operation=\"select\"}[5m]))\n",[225,3967,3968,3973,3978,3982,3987,3992,3996,4001],{"__ignoreMap":365},[369,3969,3970],{"class":371,"line":372},[369,3971,3972],{},"# P95 database query latency\n",[369,3974,3975],{"class":371,"line":378},[369,3976,3977],{},"histogram_quantile(0.95, rate(outlabs_auth_db_query_duration_seconds_bucket[5m]))\n",[369,3979,3980],{"class":371,"line":384},[369,3981,388],{"emptyLinePlaceholder":387},[369,3983,3984],{"class":371,"line":391},[369,3985,3986],{},"# P95 latency by SQL operation\n",[369,3988,3989],{"class":371,"line":397},[369,3990,3991],{},"histogram_quantile(0.95, sum by (operation, le) (rate(outlabs_auth_db_query_duration_seconds_bucket[5m])))\n",[369,3993,3994],{"class":371,"line":403},[369,3995,388],{"emptyLinePlaceholder":387},[369,3997,3998],{"class":371,"line":408},[369,3999,4000],{},"# SELECT throughput\n",[369,4002,4003],{"class":371,"line":414},[369,4004,4005],{},"sum(rate(outlabs_auth_db_query_duration_seconds_count{operation=\"select\"}[5m]))\n",[198,4007,4008],{},[220,4009,451],{},[360,4011,4013],{"className":454,"code":4012,"language":456,"meta":365,"style":365},"# Slow database queries\n- alert: SlowDatabaseQueries\n  expr: histogram_quantile(0.95, rate(outlabs_auth_db_query_duration_seconds_bucket[5m])) > 0.1\n  for: 10m\n  labels:\n    severity: warning\n  annotations:\n    summary: \"P95 database query latency above 100ms\"\n    description: \"Database queries are slow (check indexes)\"\n",[225,4014,4015,4020,4031,4040,4048,4054,4062,4068,4081],{"__ignoreMap":365},[369,4016,4017],{"class":371,"line":372},[369,4018,4019],{"class":463},"# Slow database queries\n",[369,4021,4022,4024,4026,4028],{"class":371,"line":378},[369,4023,470],{"class":469},[369,4025,474],{"class":473},[369,4027,477],{"class":469},[369,4029,4030],{"class":480}," SlowDatabaseQueries\n",[369,4032,4033,4035,4037],{"class":371,"line":384},[369,4034,486],{"class":473},[369,4036,477],{"class":469},[369,4038,4039],{"class":480}," histogram_quantile(0.95, rate(outlabs_auth_db_query_duration_seconds_bucket[5m])) > 0.1\n",[369,4041,4042,4044,4046],{"class":371,"line":391},[369,4043,496],{"class":473},[369,4045,477],{"class":469},[369,4047,614],{"class":480},[369,4049,4050,4052],{"class":371,"line":397},[369,4051,506],{"class":473},[369,4053,509],{"class":469},[369,4055,4056,4058,4060],{"class":371,"line":403},[369,4057,514],{"class":473},[369,4059,477],{"class":469},[369,4061,519],{"class":480},[369,4063,4064,4066],{"class":371,"line":408},[369,4065,524],{"class":473},[369,4067,509],{"class":469},[369,4069,4070,4072,4074,4076,4079],{"class":371,"line":414},[369,4071,531],{"class":473},[369,4073,477],{"class":469},[369,4075,536],{"class":469},[369,4077,4078],{"class":480},"P95 database query latency above 100ms",[369,4080,542],{"class":469},[369,4082,4083,4085,4087,4089,4092],{"class":371,"line":420},[369,4084,547],{"class":473},[369,4086,477],{"class":469},[369,4088,536],{"class":469},[369,4090,4091],{"class":480},"Database queries are slow (check indexes)",[369,4093,542],{"class":469},[263,4095],{},[212,4097,4099],{"id":4098},"entity-operations-metrics","Entity Operations Metrics",[270,4101,4102],{"id":4102},"outlabs_auth_entity_operations_total",[198,4104,4105,278,4107,4109],{},[220,4106,277],{},[220,4108,281],{}," Total number of entity operations (create, update, move, delete)",[198,4111,4112],{},[220,4113,287],{},[289,4115,4116],{},[292,4117,4118,4120,4121],{},[225,4119,3154],{}," - Operation type\n",[289,4122,4123,4129,4134,4140],{},[292,4124,4125,4128],{},[225,4126,4127],{},"create"," - Entity created",[292,4130,4131,4133],{},[225,4132,3937],{}," - Entity updated",[292,4135,4136,4139],{},[225,4137,4138],{},"move"," - Entity moved in hierarchy",[292,4141,4142,4144],{},[225,4143,3940],{}," - Entity deleted",[198,4146,4147],{},[220,4148,358],{},[360,4150,4152],{"className":362,"code":4151,"language":364,"meta":365,"style":365},"# Entity operation rate\nrate(outlabs_auth_entity_operations_total[5m])\n\n# Operations by type\nsum by (operation) (rate(outlabs_auth_entity_operations_total[5m]))\n\n# Entity creations per hour\nsum(increase(outlabs_auth_entity_operations_total{operation=\"create\"}[1h]))\n",[225,4153,4154,4159,4164,4168,4173,4178,4182,4187],{"__ignoreMap":365},[369,4155,4156],{"class":371,"line":372},[369,4157,4158],{},"# Entity operation rate\n",[369,4160,4161],{"class":371,"line":378},[369,4162,4163],{},"rate(outlabs_auth_entity_operations_total[5m])\n",[369,4165,4166],{"class":371,"line":384},[369,4167,388],{"emptyLinePlaceholder":387},[369,4169,4170],{"class":371,"line":391},[369,4171,4172],{},"# Operations by type\n",[369,4174,4175],{"class":371,"line":397},[369,4176,4177],{},"sum by (operation) (rate(outlabs_auth_entity_operations_total[5m]))\n",[369,4179,4180],{"class":371,"line":403},[369,4181,388],{"emptyLinePlaceholder":387},[369,4183,4184],{"class":371,"line":408},[369,4185,4186],{},"# Entity creations per hour\n",[369,4188,4189],{"class":371,"line":414},[369,4190,4191],{},"sum(increase(outlabs_auth_entity_operations_total{operation=\"create\"}[1h]))\n",[263,4193],{},[270,4195,4196],{"id":4196},"outlabs_auth_entity_operation_duration_seconds",[198,4198,4199,662,4201,4203],{},[220,4200,277],{},[220,4202,281],{}," Time taken for entity operations (in seconds)",[198,4205,4206],{},[220,4207,287],{},[289,4209,4210],{},[292,4211,4212,4214],{},[225,4213,3154],{}," - Operation type (create, update, move, delete)",[198,4216,4217,247,4219],{},[220,4218,681],{},[225,4220,4221],{},"[0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0]",[198,4223,4224],{},[220,4225,358],{},[360,4227,4229],{"className":362,"code":4228,"language":364,"meta":365,"style":365},"# P95 entity operation latency\nhistogram_quantile(0.95, rate(outlabs_auth_entity_operation_duration_seconds_bucket[5m]))\n\n# P95 move operation latency (hierarchy operations are slower)\nhistogram_quantile(0.95, rate(outlabs_auth_entity_operation_duration_seconds_bucket{operation=\"move\"}[5m]))\n",[225,4230,4231,4236,4241,4245,4250],{"__ignoreMap":365},[369,4232,4233],{"class":371,"line":372},[369,4234,4235],{},"# P95 entity operation latency\n",[369,4237,4238],{"class":371,"line":378},[369,4239,4240],{},"histogram_quantile(0.95, rate(outlabs_auth_entity_operation_duration_seconds_bucket[5m]))\n",[369,4242,4243],{"class":371,"line":384},[369,4244,388],{"emptyLinePlaceholder":387},[369,4246,4247],{"class":371,"line":391},[369,4248,4249],{},"# P95 move operation latency (hierarchy operations are slower)\n",[369,4251,4252],{"class":371,"line":397},[369,4253,4254],{},"histogram_quantile(0.95, rate(outlabs_auth_entity_operation_duration_seconds_bucket{operation=\"move\"}[5m]))\n",[263,4256],{},[212,4258,4260],{"id":4259},"membership-operations-metrics","Membership Operations Metrics",[270,4262,4263],{"id":4263},"outlabs_auth_membership_operations_total",[198,4265,4266,278,4268,4270],{},[220,4267,277],{},[220,4269,281],{}," Total number of membership operations",[198,4272,4273],{},[220,4274,287],{},[289,4276,4277],{},[292,4278,4279,4120,4281],{},[225,4280,3154],{},[289,4282,4283,4289,4295,4301],{},[292,4284,4285,4288],{},[225,4286,4287],{},"add"," - Member added to entity",[292,4290,4291,4294],{},[225,4292,4293],{},"remove"," - Member removed from entity",[292,4296,4297,4300],{},[225,4298,4299],{},"suspend"," - Membership suspended",[292,4302,4303,4306],{},[225,4304,4305],{},"reactivate"," - Membership reactivated",[198,4308,4309],{},[220,4310,358],{},[360,4312,4314],{"className":362,"code":4313,"language":364,"meta":365,"style":365},"# Membership changes rate\nrate(outlabs_auth_membership_operations_total[5m])\n\n# Membership additions per hour\nsum(increase(outlabs_auth_membership_operations_total{operation=\"add\"}[1h]))\n\n# Suspensions (potential security events)\nrate(outlabs_auth_membership_operations_total{operation=\"suspend\"}[5m])\n",[225,4315,4316,4321,4326,4330,4335,4340,4344,4349],{"__ignoreMap":365},[369,4317,4318],{"class":371,"line":372},[369,4319,4320],{},"# Membership changes rate\n",[369,4322,4323],{"class":371,"line":378},[369,4324,4325],{},"rate(outlabs_auth_membership_operations_total[5m])\n",[369,4327,4328],{"class":371,"line":384},[369,4329,388],{"emptyLinePlaceholder":387},[369,4331,4332],{"class":371,"line":391},[369,4333,4334],{},"# Membership additions per hour\n",[369,4336,4337],{"class":371,"line":397},[369,4338,4339],{},"sum(increase(outlabs_auth_membership_operations_total{operation=\"add\"}[1h]))\n",[369,4341,4342],{"class":371,"line":403},[369,4343,388],{"emptyLinePlaceholder":387},[369,4345,4346],{"class":371,"line":408},[369,4347,4348],{},"# Suspensions (potential security events)\n",[369,4350,4351],{"class":371,"line":414},[369,4352,4353],{},"rate(outlabs_auth_membership_operations_total{operation=\"suspend\"}[5m])\n",[263,4355],{},[270,4357,4358],{"id":4358},"outlabs_auth_membership_operation_duration_seconds",[198,4360,4361,662,4363,4365],{},[220,4362,277],{},[220,4364,281],{}," Time taken for membership operations (in seconds)",[198,4367,4368],{},[220,4369,287],{},[289,4371,4372],{},[292,4373,4374,4376],{},[225,4375,3154],{}," - Operation type (add, remove, suspend, reactivate)",[198,4378,4379,247,4381],{},[220,4380,681],{},[225,4382,4383],{},"[0.01, 0.05, 0.1, 0.25, 0.5, 1.0]",[198,4385,4386],{},[220,4387,358],{},[360,4389,4391],{"className":362,"code":4390,"language":364,"meta":365,"style":365},"# P95 membership operation latency\nhistogram_quantile(0.95, rate(outlabs_auth_membership_operation_duration_seconds_bucket[5m]))\n",[225,4392,4393,4398],{"__ignoreMap":365},[369,4394,4395],{"class":371,"line":372},[369,4396,4397],{},"# P95 membership operation latency\n",[369,4399,4400],{"class":371,"line":378},[369,4401,4402],{},"histogram_quantile(0.95, rate(outlabs_auth_membership_operation_duration_seconds_bucket[5m]))\n",[263,4404],{},[212,4406,4408],{"id":4407},"activity-tracking-metrics","Activity Tracking Metrics",[270,4410,4411],{"id":4411},"outlabs_auth_activity_track_total",[198,4413,4414,278,4416,4418],{},[220,4415,277],{},[220,4417,281],{}," Activity tracking operations by period type",[198,4420,4421],{},[220,4422,287],{},[289,4424,4425],{},[292,4426,4427,4430,4431],{},[225,4428,4429],{},"period"," - Period type\n",[289,4432,4433,4439,4445],{},[292,4434,4435,4438],{},[225,4436,4437],{},"daily"," - Daily active user tracking (DAU)",[292,4440,4441,4444],{},[225,4442,4443],{},"monthly"," - Monthly active user tracking (MAU)",[292,4446,4447,4450],{},[225,4448,4449],{},"quarterly"," - Quarterly active user tracking (QAU)",[198,4452,4453],{},[220,4454,358],{},[360,4456,4458],{"className":362,"code":4457,"language":364,"meta":365,"style":365},"# Activity tracking rate\nrate(outlabs_auth_activity_track_total[5m])\n\n# Daily activity tracking events\nrate(outlabs_auth_activity_track_total{period=\"daily\"}[5m])\n",[225,4459,4460,4465,4470,4474,4479],{"__ignoreMap":365},[369,4461,4462],{"class":371,"line":372},[369,4463,4464],{},"# Activity tracking rate\n",[369,4466,4467],{"class":371,"line":378},[369,4468,4469],{},"rate(outlabs_auth_activity_track_total[5m])\n",[369,4471,4472],{"class":371,"line":384},[369,4473,388],{"emptyLinePlaceholder":387},[369,4475,4476],{"class":371,"line":391},[369,4477,4478],{},"# Daily activity tracking events\n",[369,4480,4481],{"class":371,"line":397},[369,4482,4483],{},"rate(outlabs_auth_activity_track_total{period=\"daily\"}[5m])\n",[263,4485],{},[270,4487,4488],{"id":4488},"outlabs_auth_activity_sync_duration_seconds",[198,4490,4491,662,4493,4495],{},[220,4492,277],{},[220,4494,281],{}," Duration of activity sync operations to database (in seconds)",[198,4497,4498,1106],{},[220,4499,287],{},[198,4501,4502,247,4504],{},[220,4503,681],{},[225,4505,4506],{},"[0.1, 0.5, 1.0, 2.5, 5.0, 10.0]",[198,4508,4509],{},[220,4510,358],{},[360,4512,4514],{"className":362,"code":4513,"language":364,"meta":365,"style":365},"# P95 activity sync duration\nhistogram_quantile(0.95, rate(outlabs_auth_activity_sync_duration_seconds_bucket[5m]))\n",[225,4515,4516,4521],{"__ignoreMap":365},[369,4517,4518],{"class":371,"line":372},[369,4519,4520],{},"# P95 activity sync duration\n",[369,4522,4523],{"class":371,"line":378},[369,4524,4525],{},"histogram_quantile(0.95, rate(outlabs_auth_activity_sync_duration_seconds_bucket[5m]))\n",[263,4527],{},[270,4529,4530],{"id":4530},"outlabs_auth_activity_sync_records_total",[198,4532,4533,278,4535,4537],{},[220,4534,277],{},[220,4536,281],{}," Number of activity records synced to database",[198,4539,4540],{},[220,4541,287],{},[289,4543,4544],{},[292,4545,4546,4549],{},[225,4547,4548],{},"metric_type"," - Type of metric synced (daily, monthly, quarterly)",[198,4551,4552],{},[220,4553,358],{},[360,4555,4557],{"className":362,"code":4556,"language":364,"meta":365,"style":365},"# Records synced per sync operation\nrate(outlabs_auth_activity_sync_records_total[5m])\n",[225,4558,4559,4564],{"__ignoreMap":365},[369,4560,4561],{"class":371,"line":372},[369,4562,4563],{},"# Records synced per sync operation\n",[369,4565,4566],{"class":371,"line":378},[369,4567,4568],{},"rate(outlabs_auth_activity_sync_records_total[5m])\n",[263,4570],{},[212,4572,4574],{"id":4573},"notification-metrics","Notification Metrics",[270,4576,4577],{"id":4577},"outlabs_auth_notification_events_total",[198,4579,4580,278,4582,4584],{},[220,4581,277],{},[220,4583,281],{}," Total notification events emitted",[198,4586,4587],{},[220,4588,287],{},[289,4590,4591],{},[292,4592,4593,4596,4597],{},[225,4594,4595],{},"event_type"," - Type of notification event\n",[289,4598,4599,4605,4611,4617,4623,4629],{},[292,4600,4601,4604],{},[225,4602,4603],{},"user_registered"," - User registration notification",[292,4606,4607,4610],{},[225,4608,4609],{},"user_verified"," - Email verification notification",[292,4612,4613,4616],{},[225,4614,4615],{},"password_reset"," - Password reset notification",[292,4618,4619,4622],{},[225,4620,4621],{},"login_success"," - Login notification",[292,4624,4625,4628],{},[225,4626,4627],{},"permission_denied"," - Permission denial notification",[292,4630,4631],{},"etc.",[198,4633,4634],{},[220,4635,358],{},[360,4637,4639],{"className":362,"code":4638,"language":364,"meta":365,"style":365},"# Notification rate\nrate(outlabs_auth_notification_events_total[5m])\n\n# Notifications by type\nsum by (event_type) (rate(outlabs_auth_notification_events_total[5m]))\n\n# User registrations per hour\nsum(increase(outlabs_auth_notification_events_total{event_type=\"user_registered\"}[1h]))\n",[225,4640,4641,4646,4651,4655,4660,4665,4669,4674],{"__ignoreMap":365},[369,4642,4643],{"class":371,"line":372},[369,4644,4645],{},"# Notification rate\n",[369,4647,4648],{"class":371,"line":378},[369,4649,4650],{},"rate(outlabs_auth_notification_events_total[5m])\n",[369,4652,4653],{"class":371,"line":384},[369,4654,388],{"emptyLinePlaceholder":387},[369,4656,4657],{"class":371,"line":391},[369,4658,4659],{},"# Notifications by type\n",[369,4661,4662],{"class":371,"line":397},[369,4663,4664],{},"sum by (event_type) (rate(outlabs_auth_notification_events_total[5m]))\n",[369,4666,4667],{"class":371,"line":403},[369,4668,388],{"emptyLinePlaceholder":387},[369,4670,4671],{"class":371,"line":408},[369,4672,4673],{},"# User registrations per hour\n",[369,4675,4676],{"class":371,"line":414},[369,4677,4678],{},"sum(increase(outlabs_auth_notification_events_total{event_type=\"user_registered\"}[1h]))\n",[263,4680],{},[270,4682,4683],{"id":4683},"outlabs_auth_notification_delivery_failures_total",[198,4685,4686,278,4688,4690],{},[220,4687,277],{},[220,4689,281],{}," Total notification delivery failures",[198,4692,4693],{},[220,4694,287],{},[289,4696,4697,4703],{},[292,4698,4699,4702],{},[225,4700,4701],{},"channel"," - Delivery channel (email, webhook, pubsub)",[292,4704,4705,4707],{},[225,4706,4595],{}," - Type of notification that failed",[198,4709,4710],{},[220,4711,358],{},[360,4713,4715],{"className":362,"code":4714,"language":364,"meta":365,"style":365},"# Delivery failure rate\nrate(outlabs_auth_notification_delivery_failures_total[5m])\n\n# Failures by channel\nsum by (channel) (rate(outlabs_auth_notification_delivery_failures_total[5m]))\n",[225,4716,4717,4722,4727,4731,4736],{"__ignoreMap":365},[369,4718,4719],{"class":371,"line":372},[369,4720,4721],{},"# Delivery failure rate\n",[369,4723,4724],{"class":371,"line":378},[369,4725,4726],{},"rate(outlabs_auth_notification_delivery_failures_total[5m])\n",[369,4728,4729],{"class":371,"line":384},[369,4730,388],{"emptyLinePlaceholder":387},[369,4732,4733],{"class":371,"line":391},[369,4734,4735],{},"# Failures by channel\n",[369,4737,4738],{"class":371,"line":397},[369,4739,4740],{},"sum by (channel) (rate(outlabs_auth_notification_delivery_failures_total[5m]))\n",[198,4742,4743],{},[220,4744,451],{},[360,4746,4748],{"className":454,"code":4747,"language":456,"meta":365,"style":365},"# High notification failure rate\n- alert: NotificationDeliveryFailures\n  expr: rate(outlabs_auth_notification_delivery_failures_total[5m]) > 0.5\n  for: 5m\n  labels:\n    severity: warning\n  annotations:\n    summary: \"Notification delivery failures\"\n    description: \"{{ $value }} notification failures per second on {{ $labels.channel }}\"\n",[225,4749,4750,4755,4766,4775,4783,4789,4797,4803,4816],{"__ignoreMap":365},[369,4751,4752],{"class":371,"line":372},[369,4753,4754],{"class":463},"# High notification failure rate\n",[369,4756,4757,4759,4761,4763],{"class":371,"line":378},[369,4758,470],{"class":469},[369,4760,474],{"class":473},[369,4762,477],{"class":469},[369,4764,4765],{"class":480}," NotificationDeliveryFailures\n",[369,4767,4768,4770,4772],{"class":371,"line":384},[369,4769,486],{"class":473},[369,4771,477],{"class":469},[369,4773,4774],{"class":480}," rate(outlabs_auth_notification_delivery_failures_total[5m]) > 0.5\n",[369,4776,4777,4779,4781],{"class":371,"line":391},[369,4778,496],{"class":473},[369,4780,477],{"class":469},[369,4782,501],{"class":480},[369,4784,4785,4787],{"class":371,"line":397},[369,4786,506],{"class":473},[369,4788,509],{"class":469},[369,4790,4791,4793,4795],{"class":371,"line":403},[369,4792,514],{"class":473},[369,4794,477],{"class":469},[369,4796,519],{"class":480},[369,4798,4799,4801],{"class":371,"line":408},[369,4800,524],{"class":473},[369,4802,509],{"class":469},[369,4804,4805,4807,4809,4811,4814],{"class":371,"line":414},[369,4806,531],{"class":473},[369,4808,477],{"class":469},[369,4810,536],{"class":469},[369,4812,4813],{"class":480},"Notification delivery failures",[369,4815,542],{"class":469},[369,4817,4818,4820,4822,4824,4827],{"class":371,"line":420},[369,4819,547],{"class":473},[369,4821,477],{"class":469},[369,4823,536],{"class":469},[369,4825,4826],{"class":480},"{{ $value }} notification failures per second on {{ $labels.channel }}",[369,4828,542],{"class":469},[263,4830],{},[212,4832,4834],{"id":4833},"example-grafana-dashboard-queries","Example Grafana Dashboard Queries",[270,4836,4838],{"id":4837},"authentication-dashboard","Authentication Dashboard",[360,4840,4842],{"className":362,"code":4841,"language":364,"meta":365,"style":365},"# Login Success Rate (last 24h)\nsum(increase(outlabs_auth_login_attempts_total{status=\"success\"}[24h]))\n\n# Failed Logins (last 24h)\nsum(increase(outlabs_auth_login_attempts_total{status=\"failed\"}[24h]))\n\n# P95 Login Latency (5m)\nhistogram_quantile(0.95, rate(outlabs_auth_login_duration_seconds_bucket[5m]))\n\n# Active Sessions\noutlabs_auth_active_sessions\n",[225,4843,4844,4849,4854,4858,4863,4868,4872,4877,4881,4885,4890],{"__ignoreMap":365},[369,4845,4846],{"class":371,"line":372},[369,4847,4848],{},"# Login Success Rate (last 24h)\n",[369,4850,4851],{"class":371,"line":378},[369,4852,4853],{},"sum(increase(outlabs_auth_login_attempts_total{status=\"success\"}[24h]))\n",[369,4855,4856],{"class":371,"line":384},[369,4857,388],{"emptyLinePlaceholder":387},[369,4859,4860],{"class":371,"line":391},[369,4861,4862],{},"# Failed Logins (last 24h)\n",[369,4864,4865],{"class":371,"line":397},[369,4866,4867],{},"sum(increase(outlabs_auth_login_attempts_total{status=\"failed\"}[24h]))\n",[369,4869,4870],{"class":371,"line":403},[369,4871,388],{"emptyLinePlaceholder":387},[369,4873,4874],{"class":371,"line":408},[369,4875,4876],{},"# P95 Login Latency (5m)\n",[369,4878,4879],{"class":371,"line":414},[369,4880,718],{},[369,4882,4883],{"class":371,"line":420},[369,4884,388],{"emptyLinePlaceholder":387},[369,4886,4887],{"class":371,"line":426},[369,4888,4889],{},"# Active Sessions\n",[369,4891,4892],{"class":371,"line":432},[369,4893,1714],{},[270,4895,4897],{"id":4896},"authorization-dashboard","Authorization Dashboard",[360,4899,4901],{"className":362,"code":4900,"language":364,"meta":365,"style":365},"# Permission Checks\u002Fsec\nrate(outlabs_auth_permission_checks_total[5m])\n\n# Permission Denial Rate\nrate(outlabs_auth_permission_checks_total{result=\"denied\"}[5m])\n\n# P95 Permission Check Latency\nhistogram_quantile(0.95, rate(outlabs_auth_permission_check_duration_seconds_bucket[5m]))\n\n# Top Denied Permissions\ntopk(10, sum by (permission) (increase(outlabs_auth_permission_checks_total{result=\"denied\"}[1h])))\n",[225,4902,4903,4908,4912,4916,4921,4925,4929,4934,4938,4942,4947],{"__ignoreMap":365},[369,4904,4905],{"class":371,"line":372},[369,4906,4907],{},"# Permission Checks\u002Fsec\n",[369,4909,4910],{"class":371,"line":378},[369,4911,1302],{},[369,4913,4914],{"class":371,"line":384},[369,4915,388],{"emptyLinePlaceholder":387},[369,4917,4918],{"class":371,"line":391},[369,4919,4920],{},"# Permission Denial Rate\n",[369,4922,4923],{"class":371,"line":397},[369,4924,1316],{},[369,4926,4927],{"class":371,"line":403},[369,4928,388],{"emptyLinePlaceholder":387},[369,4930,4931],{"class":371,"line":408},[369,4932,4933],{},"# P95 Permission Check Latency\n",[369,4935,4936],{"class":371,"line":414},[369,4937,1484],{},[369,4939,4940],{"class":371,"line":420},[369,4941,388],{"emptyLinePlaceholder":387},[369,4943,4944],{"class":371,"line":426},[369,4945,4946],{},"# Top Denied Permissions\n",[369,4948,4949],{"class":371,"line":432},[369,4950,4951],{},"topk(10, sum by (permission) (increase(outlabs_auth_permission_checks_total{result=\"denied\"}[1h])))\n",[270,4953,4955],{"id":4954},"security-dashboard","Security Dashboard",[360,4957,4959],{"className":362,"code":4958,"language":364,"meta":365,"style":365},"# Failed Login Attempts (last hour)\nsum(increase(outlabs_auth_login_attempts_total{status=\"failed\"}[1h]))\n\n# Account Lockouts (last hour)\nincrease(outlabs_auth_account_locked_total[1h])\n\n# Suspicious Activity (last 24h)\nsum by (type) (increase(outlabs_auth_suspicious_activity_total[24h]))\n\n# Invalid API Keys (last hour)\nsum(increase(outlabs_auth_api_key_validations_total{status=\"invalid\"}[1h]))\n",[225,4960,4961,4966,4971,4975,4980,4985,4989,4994,4999,5003,5008],{"__ignoreMap":365},[369,4962,4963],{"class":371,"line":372},[369,4964,4965],{},"# Failed Login Attempts (last hour)\n",[369,4967,4968],{"class":371,"line":378},[369,4969,4970],{},"sum(increase(outlabs_auth_login_attempts_total{status=\"failed\"}[1h]))\n",[369,4972,4973],{"class":371,"line":384},[369,4974,388],{"emptyLinePlaceholder":387},[369,4976,4977],{"class":371,"line":391},[369,4978,4979],{},"# Account Lockouts (last hour)\n",[369,4981,4982],{"class":371,"line":397},[369,4983,4984],{},"increase(outlabs_auth_account_locked_total[1h])\n",[369,4986,4987],{"class":371,"line":403},[369,4988,388],{"emptyLinePlaceholder":387},[369,4990,4991],{"class":371,"line":408},[369,4992,4993],{},"# Suspicious Activity (last 24h)\n",[369,4995,4996],{"class":371,"line":414},[369,4997,4998],{},"sum by (type) (increase(outlabs_auth_suspicious_activity_total[24h]))\n",[369,5000,5001],{"class":371,"line":420},[369,5002,388],{"emptyLinePlaceholder":387},[369,5004,5005],{"class":371,"line":426},[369,5006,5007],{},"# Invalid API Keys (last hour)\n",[369,5009,5010],{"class":371,"line":432},[369,5011,5012],{},"sum(increase(outlabs_auth_api_key_validations_total{status=\"invalid\"}[1h]))\n",[263,5014],{},[212,5016,5018],{"id":5017},"next-steps","Next Steps",[289,5020,5021,5029,5037],{},[292,5022,5023,5028],{},[220,5024,5025],{},[202,5026,5027],{"href":176},"97-Observability.md"," - Main observability guide",[292,5030,5031,5036],{},[220,5032,5033],{},[202,5034,5035],{"href":186},"99-Log-Events-Reference.md"," - Complete log events catalog",[292,5038,5039,5044],{},[220,5040,5041],{},[202,5042,5043],{"href":12},"Grafana Dashboard"," - Pre-built dashboard using these metrics",[263,5046],{},[198,5048,5049,5052,5053,247,5056,251,5058],{},[220,5050,5051],{},"Last Updated:"," 2025-01-20\n",[220,5054,5055],{},"Related:",[202,5057,5027],{"href":176},[202,5059,5035],{"href":186},[5061,5062,5063],"style",{},"html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sHwdD, html code.shiki .sHwdD{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#546E7A;--shiki-default-font-style:italic;--shiki-dark:#676E95;--shiki-dark-font-style:italic}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .swJcz, html code.shiki .swJcz{--shiki-light:#E53935;--shiki-default:#F07178;--shiki-dark:#F07178}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html pre.shiki code .s7zQu, html code.shiki .s7zQu{--shiki-light:#39ADB5;--shiki-light-font-style:italic;--shiki-default:#89DDFF;--shiki-default-font-style:italic;--shiki-dark:#89DDFF;--shiki-dark-font-style:italic}",{"title":365,"searchDepth":372,"depth":378,"links":5065},[5066,5067,5074,5079,5083,5088,5091,5097,5102,5106,5110,5115,5119,5124],{"id":214,"depth":378,"text":215},{"id":267,"depth":378,"text":268,"children":5068},[5069,5070,5071,5072,5073],{"id":272,"depth":384,"text":272},{"id":657,"depth":384,"text":657},{"id":862,"depth":384,"text":862},{"id":938,"depth":384,"text":938},{"id":1094,"depth":384,"text":1094},{"id":1232,"depth":378,"text":1233,"children":5075},[5076,5077,5078],{"id":1236,"depth":384,"text":1236},{"id":1446,"depth":384,"text":1446},{"id":1614,"depth":384,"text":1614},{"id":1679,"depth":378,"text":1680,"children":5080},[5081,5082],{"id":1683,"depth":384,"text":1683},{"id":1835,"depth":384,"text":1835},{"id":1901,"depth":378,"text":1902,"children":5084},[5085,5086,5087],{"id":1905,"depth":384,"text":1905},{"id":2104,"depth":384,"text":2104},{"id":2178,"depth":384,"text":2178},{"id":2322,"depth":378,"text":2323,"children":5089},[5090],{"id":2326,"depth":384,"text":2326},{"id":2498,"depth":378,"text":2499,"children":5092},[5093,5094,5095,5096],{"id":2502,"depth":384,"text":2502},{"id":2790,"depth":384,"text":2790},{"id":3121,"depth":384,"text":3121},{"id":3352,"depth":384,"text":3352},{"id":3642,"depth":378,"text":3643,"children":5098},[5099,5100,5101],{"id":3646,"depth":384,"text":3646},{"id":3735,"depth":384,"text":3735},{"id":3899,"depth":384,"text":3899},{"id":4098,"depth":378,"text":4099,"children":5103},[5104,5105],{"id":4102,"depth":384,"text":4102},{"id":4196,"depth":384,"text":4196},{"id":4259,"depth":378,"text":4260,"children":5107},[5108,5109],{"id":4263,"depth":384,"text":4263},{"id":4358,"depth":384,"text":4358},{"id":4407,"depth":378,"text":4408,"children":5111},[5112,5113,5114],{"id":4411,"depth":384,"text":4411},{"id":4488,"depth":384,"text":4488},{"id":4530,"depth":384,"text":4530},{"id":4573,"depth":378,"text":4574,"children":5116},[5117,5118],{"id":4577,"depth":384,"text":4577},{"id":4683,"depth":384,"text":4683},{"id":4833,"depth":378,"text":4834,"children":5120},[5121,5122,5123],{"id":4837,"depth":384,"text":4838},{"id":4896,"depth":384,"text":4897},{"id":4954,"depth":384,"text":4955},{"id":5017,"depth":378,"text":5018},"Prometheus metric catalog.","md",null,{},{"icon":183},{"title":180,"description":5125},"742A0aSlGgA8tikziwZtC6jv3qMWflO9V7BH9bt5slg",[5133,5135],{"title":175,"path":176,"stem":177,"description":5134,"icon":178,"children":-1},"Metrics and logs without taking over your FastAPI app.",{"title":185,"path":186,"stem":187,"description":5136,"icon":188,"children":-1},"Structured log event catalog.",1787472745731]