Merge pull request #87 from sebadob/audit-part-7

Audit part 7
sebadob · Oct 23, 2023 · a9af494 · a9af494
2 parents 5f0c9c9 + 0cd9a1e
commit a9af494
Show file tree

Hide file tree

Showing 12 changed files with 249 additions and 62 deletions.
diff --git a/frontend/src/components/admin/events/Event.svelte b/frontend/src/components/admin/events/Event.svelte
@@ -54,6 +54,7 @@
 </script>
 
 <div
+        role="contentinfo"
         class={cls}
         class:showCollapsed
         style:border-left={`${borderWidth} solid ${eventColor(event.level)}`}

diff --git a/frontend/src/components/admin/events/Events.svelte b/frontend/src/components/admin/events/Events.svelte
@@ -2,20 +2,56 @@
     import {onMount} from "svelte";
     import Event from "./Event.svelte";
     import EventsLegend from "./EventsLegend.svelte";
+    import Button from "$lib/Button.svelte";
+    import {postTestEvent} from "../../../utils/dataFetching.js";
+    import OptionSelect from "$lib/OptionSelect.svelte";
+    import {EVENT_LEVELS} from "../../../utils/constants.js";
 
     export let collapsed = true;
     export let wide;
 
     let latest = 50;
     let es;
     let events = [];
+    let eventsFiltered = [];
+    let eventLevel;
     let isHover = false;
 
     $: widthDefault = !collapsed && !wide || isHover;
     $: widthCollapsed = collapsed && !wide && !isHover;
     $: widthWide = !collapsed && wide;
 
-    onMount(() => {
+    $: if (events) {
+        switch (eventLevel) {
+            case 'Info':
+                eventsFiltered = events;
+                break;
+            case 'Notice':
+                eventsFiltered = events.filter(
+                    evt => evt.typ === 'Test'
+                        || evt.level === 'notice'
+                        || evt.level === 'warning'
+                        || evt.level === 'critical'
+                );
+                break;
+            case 'Warning':
+                eventsFiltered = events.filter(
+                    evt => evt.typ === 'Test' || evt.level === 'warning' || evt.level === 'critical'
+                );
+                break;
+            case 'Critical':
+                eventsFiltered = events.filter(evt => evt.typ === 'Test' || evt.level === 'critical');
+                break;
+        }
+    }
+
+    $: if (eventLevel) {
+        saveEventLevel(eventLevel);
+    }
+
+    onMount(async () => {
+        eventLevel = await readSavedEventLevel();
+
         if (!es || es.closed) {
             es = new EventSource(`/auth/v1/events?latest=${latest}`);
 
@@ -53,9 +89,22 @@
         }
     }
 
+    async function readSavedEventLevel() {
+        return localStorage.getItem('eventLevel') || 'Info';
+    }
+
+    async function saveEventLevel(level) {
+        localStorage.setItem('eventLevel', level);
+    }
+
+    async function sendTestEvent() {
+        await postTestEvent();
+    }
+
 </script>
 
 <div
+        role="contentinfo"
         id="events"
         class:widthDefault
         class:widthCollapsed
@@ -64,19 +113,46 @@
         on:mouseleave={() => isHover = false}
 >
     <div class="upper">
-        {#if !widthCollapsed}
-            <b>Events</b><br/><br/>
+        <div class="header">
+            {#if !widthCollapsed}
+                {#if widthWide}
+                    <div class="headerWideInner">
+                        <b>Events</b>
+                        <OptionSelect
+                                bind:value={eventLevel}
+                                options={EVENT_LEVELS}
+                        />
+                    </div>
+                {:else}
+                    <b>Events</b>
+                {/if}
+
+                <Button
+                        on:click={sendTestEvent}
+                        level=3
+                >
+                    TEST
+                </Button>
+            {/if}
+        </div>
+        {#if !widthWide}
+            <div class="opts">
+                <OptionSelect
+                        bind:value={eventLevel}
+                        options={EVENT_LEVELS}
+                />
+            </div>
         {/if}
 
         <div class={widthWide ? 'dataWide' : widthCollapsed ? 'dataCollapsed' : 'data'}>
-            {#each events as event (event.id)}
-                <Event bind:event eventColor={eventColor} collapsed={collapsed && !isHover} bind:wide />
+            {#each eventsFiltered as event (event.id)}
+                <Event bind:event eventColor={eventColor} collapsed={collapsed && !isHover} bind:wide/>
             {/each}
         </div>
     </div>
 
     {#if !collapsed || isHover}
-        <EventsLegend eventColor={eventColor} bind:wide />
+        <EventsLegend eventColor={eventColor} bind:wide/>
     {/if}
 </div>
 
@@ -109,7 +185,22 @@
         max-height: calc(100dvh - 5.1rem);
     }
 
-    .upper {
+    .header, .opts {
+        display: flex;
+        justify-content: space-between;
+        align-items: center;
+        padding: 0 .5rem 0 1.25rem;
+    }
+
+    .headerWideInner {
+        display: flex;
+        align-items: center;
+        gap: 2rem;
+    }
+
+    .opts {
+        padding: 0 1.25rem;
+        width: 100%;
     }
 
     .widthDefault {

diff --git a/frontend/src/utils/constants.js b/frontend/src/utils/constants.js
@@ -26,6 +26,12 @@ export const REGEX_URI = /^[a-zA-Z0-9,.:/_\-&?=~#!$'()*+%]+$/gm;
 export const REGEX_CSS_COLOR = /(#(?:[0-9a-f]{2}){2,4}$|(#[0-9a-f]{3}$)|(rgb|hsl)a?\((-?\d+%?[,\s]+){2,3}\s*[\d\.]+%?\)$|black$|silver$|gray$|whitesmoke$|maroon$|red$|purple$|fuchsia$|green$|lime$|olivedrab$|yellow$|navy$|blue$|teal$|aquamarine$|orange$|aliceblue$|antiquewhite$|aqua$|azure$|beige$|bisque$|blanchedalmond$|blueviolet$|brown$|burlywood$|cadetblue$|chartreuse$|chocolate$|coral$|cornflowerblue$|cornsilk$|crimson$|currentcolor$|darkblue$|darkcyan$|darkgoldenrod$|darkgray$|darkgreen$|darkgrey$|darkkhaki$|darkmagenta$|darkolivegreen$|darkorange$|darkorchid$|darkred$|darksalmon$|darkseagreen$|darkslateblue$|darkslategray$|darkslategrey$|darkturquoise$|darkviolet$|deeppink$|deepskyblue$|dimgray$|dimgrey$|dodgerblue$|firebrick$|floralwhite$|forestgreen$|gainsboro$|ghostwhite$|goldenrod$|gold$|greenyellow$|grey$|honeydew$|hotpink$|indianred$|indigo$|ivory$|khaki$|lavenderblush$|lavender$|lawngreen$|lemonchiffon$|lightblue$|lightcoral$|lightcyan$|lightgoldenrodyellow$|lightgray$|lightgreen$|lightgrey$|lightpink$|lightsalmon$|lightseagreen$|lightskyblue$|lightslategray$|lightslategrey$|lightsteelblue$|lightyellow$|limegreen$|linen$|mediumaquamarine$|mediumblue$|mediumorchid$|mediumpurple$|mediumseagreen$|mediumslateblue$|mediumspringgreen$|mediumturquoise$|mediumvioletred$|midnightblue$|mintcream$|mistyrose$|moccasin$|navajowhite$|oldlace$|olive$|orangered$|orchid$|palegoldenrod$|palegreen$|paleturquoise$|palevioletred$|papayawhip$|peachpuff$|peru$|pink$|plum$|powderblue$|rosybrown$|royalblue$|saddlebrown$|salmon$|sandybrown$|seagreen$|seashell$|sienna$|skyblue$|slateblue$|slategray$|slategrey$|snow$|springgreen$|steelblue$|tan$|thistle$|tomato$|transparent$|turquoise$|violet$|wheat$|white$|yellowgreen$|rebeccapurple$)/i;
 
 
+export const EVENT_LEVELS = [
+	'Info',
+	'Notice',
+	'Warning',
+	'Critical'
+]
 export const LANGUAGES = ['DE', 'EN'];
 export const TOKEN_ALGS = [
 	'RS256',

diff --git a/frontend/src/utils/dataFetching.js b/frontend/src/utils/dataFetching.js
@@ -125,6 +125,13 @@ export async function postPasswordResetRequest(data) {
 	});
 }
 
+export async function postTestEvent() {
+	return await fetch('/auth/v1/events/test', {
+		method: 'POST',
+		headers: getCsrfHeaders(),
+	});
+}
+
 export async function registerUser(data) {
 	return await fetch('/auth/v1/users/register', {
 		method: 'POST',

diff --git a/rauthy-book/src/config/config.md b/rauthy-book/src/config/config.md
@@ -226,6 +226,19 @@ MAX_HASH_THREADS=1
 # default: 'notice'
 #EVENT_NOTIFY_LEVEL=notice
 
+# Define the level from which on events should be persisted inside the database.
+# Possible values:
+# - info
+# - notice
+# - warning
+# - critical
+# default: 'notice'
+#EVENT_PERSIST_LEVEL=notice
+
+# Define the number of days when events should be cleaned up from the database.
+# default: 31
+#EVENT_CLEANUP_DAYS=31
+
 #####################################
 ####### LIFETIMES / TIMEOUTS ########
 #####################################

diff --git a/rauthy-main/src/main.rs b/rauthy-main/src/main.rs
@@ -14,23 +14,11 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-use std::error::Error;
-use std::net::Ipv4Addr;
-use std::str::FromStr;
-use std::time::Duration;
-use std::{env, thread};
-
 use actix_web::rt::System;
 use actix_web::{middleware, web, App, HttpServer};
 use actix_web_grants::GrantsMiddleware;
 use actix_web_prom::PrometheusMetricsBuilder;
 use prometheus::Registry;
-use sqlx::{query, query_as};
-use tokio::sync::mpsc;
-use tokio::time;
-use tracing::{debug, error, info};
-use utoipa_swagger_ui::SwaggerUi;
-
 use rauthy_common::constants::{
     CACHE_NAME_12HR, CACHE_NAME_AUTH_CODES, CACHE_NAME_LOGIN_DELAY, CACHE_NAME_POW,
     CACHE_NAME_SESSIONS, CACHE_NAME_WEBAUTHN, CACHE_NAME_WEBAUTHN_DATA, POW_EXP, RAUTHY_VERSION,
@@ -44,10 +32,20 @@ use rauthy_handlers::openapi::ApiDoc;
 use rauthy_handlers::{clients, events, generic, groups, oidc, roles, scopes, sessions, users};
 use rauthy_models::app_state::{AppState, Caches, DbPool};
 use rauthy_models::email::EMail;
+use rauthy_models::events::init_event_vars;
 use rauthy_models::events::listener::EventListener;
-use rauthy_models::events::notifier::EventNotifier;
 use rauthy_models::{email, ListenScheme};
 use rauthy_service::auth;
+use sqlx::{query, query_as};
+use std::error::Error;
+use std::net::Ipv4Addr;
+use std::str::FromStr;
+use std::time::Duration;
+use std::{env, thread};
+use tokio::sync::mpsc;
+use tokio::time;
+use tracing::{debug, error, info};
+use utoipa_swagger_ui::SwaggerUi;
 
 use crate::cache_notify::handle_notify;
 use crate::logging::setup_logging;
@@ -180,7 +178,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
         .expect("Passkey UV migration to not fail");
 
     // events listener
-    EventNotifier::init().unwrap();
+    init_event_vars().unwrap();
     tokio::spawn(EventListener::listen(
         tx_email,
         tx_events_router,

diff --git a/rauthy-main/src/schedulers.rs b/rauthy-main/src/schedulers.rs
@@ -1,5 +1,6 @@
 use ::time::OffsetDateTime;
 use actix_web::web;
+use chrono::Utc;
 use rauthy_common::constants::{CACHE_NAME_12HR, DB_TYPE, IDX_JWK_KID, OFFLINE_TOKEN_LT};
 use rauthy_common::DbType;
 use rauthy_models::app_state::{AppState, DbPool};
@@ -25,6 +26,7 @@ pub async fn scheduler_main(data: web::Data<AppState>) {
     let rx_health = data.caches.ha_cache_config.rx_health_state.clone();
 
     tokio::spawn(db_backup(data.db.clone()));
+    tokio::spawn(events_cleanup(data.db.clone(), rx_health.clone()));
     tokio::spawn(magic_link_cleanup(data.db.clone(), rx_health.clone()));
     tokio::spawn(refresh_tokens_cleanup(data.db.clone(), rx_health.clone()));
     tokio::spawn(sessions_cleanup(data.db.clone(), rx_health.clone()));
@@ -33,7 +35,6 @@ pub async fn scheduler_main(data: web::Data<AppState>) {
     tokio::spawn(user_expiry_checker(data, rx_health));
 }
 
-// TODO -> adapt to RDBMS
 // Creates a backup of the data store
 pub async fn db_backup(db: DbPool) {
     if *DB_TYPE == DbType::Postgres {
@@ -67,6 +68,45 @@ pub async fn db_backup(db: DbPool) {
     }
 }
 
+// Cleans up all Events that exceed the configured EVENT_CLEANUP_DAYS
+pub async fn events_cleanup(db: DbPool, rx_health: Receiver<Option<QuorumHealthState>>) {
+    let mut interval = time::interval(Duration::from_secs(3600));
+
+    let cleanup_days = env::var("EVENT_CLEANUP_DAYS")
+        .unwrap_or_else(|_| "31".to_string())
+        .parse::<u32>()
+        .expect("Cannot parse EVENT_CLEANUP_DAYS to u32") as i64;
+
+    loop {
+        interval.tick().await;
+
+        // will return None in a non-HA deployment
+        if let Some(is_ha_leader) = is_ha_leader(&rx_health) {
+            if !is_ha_leader {
+                debug!(
+                    "Running HA mode without being the leader - skipping events_cleanup scheduler"
+                );
+                continue;
+            }
+        }
+
+        debug!("Running events_cleanup scheduler");
+
+        let threshold = Utc::now()
+            .sub(chrono::Duration::days(cleanup_days))
+            .timestamp_millis();
+        let res = sqlx::query!("DELETE FROM events WHERE timestamp < $1", threshold)
+            .execute(&db)
+            .await;
+        match res {
+            Ok(r) => {
+                debug!("Cleaned up {} expired events", r.rows_affected());
+            }
+            Err(err) => error!("Events cleanup error: {:?}", err),
+        }
+    }
+}
+
 // Cleans up old / expired magic links and deletes users, that have never used their
 // 'set first ever password' magic link to keep the database clean in case of an open user registration.
 // Runs every 6 hours.
@@ -84,8 +124,6 @@ pub async fn magic_link_cleanup(db: DbPool, rx_health: Receiver<Option<QuorumHea
             }
         }
 
-        // if this channel has an error, just go on and ignore it -> no HA mode
-        // if let Ok(cache_state) = rx_health.recv_async().await {}
         debug!("Running magic_link_cleanup scheduler");
 
         // allow 300 seconds of clock skew before cleaning up magic links

diff --git a/rauthy-models/src/events/listener.rs b/rauthy-models/src/events/listener.rs
@@ -2,6 +2,7 @@ use crate::app_state::DbPool;
 use crate::email::EMail;
 use crate::events::event::Event;
 use crate::events::notifier::EventNotifier;
+use crate::events::EVENT_PERSIST_LEVEL;
 use actix_web_lab::sse;
 use rauthy_common::constants::{DATABASE_URL, DB_TYPE, EVENTS_LATEST_LIMIT};
 use rauthy_common::error_response::ErrorResponse;
@@ -70,9 +71,11 @@ impl EventListener {
         tx: flume::Sender<EventRouterMsg>,
     ) {
         // insert into DB
-        while let Err(err) = event.insert(&db).await {
-            error!("Inserting Event into Database: {:?}", err);
-            time::sleep(Duration::from_secs(1)).await;
+        if &event.level.value() >= EVENT_PERSIST_LEVEL.get().unwrap() {
+            while let Err(err) = event.insert(&db).await {
+                error!("Inserting Event into Database: {:?}", err);
+                time::sleep(Duration::from_secs(1)).await;
+            }
         }
 
         // forward to event router
@@ -93,9 +96,11 @@ impl EventListener {
     #[tracing::instrument(level = "debug", skip_all)]
     async fn handle_event_ha(tx_email: mpsc::Sender<EMail>, event: Event, db: DbPool) {
         // insert into DB
-        while let Err(err) = event.insert(&db).await {
-            error!("Inserting Event into Database: {:?}", err);
-            time::sleep(Duration::from_secs(1)).await;
+        if &event.level.value() >= EVENT_PERSIST_LEVEL.get().unwrap() {
+            while let Err(err) = event.insert(&db).await {
+                error!("Inserting Event into Database: {:?}", err);
+                time::sleep(Duration::from_secs(1)).await;
+            }
         }
 
         // notify postgres listeners