Merge pull request #223 from sebadob/outsource-users-into-own-cache

Outsource users into own cache
sebadob · Dec 26, 2023 · 3137927 · 3137927
2 parents 8fd2c99 + c10deed
commit 3137927
Show file tree

Hide file tree

Showing 7 changed files with 86 additions and 51 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -56,9 +56,9 @@ but also even encryption algorithm encryptions really easy in the future.
 - Migrate to [spow](https://github.com/sebadob/spow)
 [ff579f6](https://github.com/sebadob/rauthy/commit/ff579f60414cb529d727ae27fd83e9506ad770d5)
 - Pre-Compute CSP's for all HTML content at build-time and get rid of the per-request nonce computation
-[]()
+[8fd2c99](https://github.com/sebadob/rauthy/commit/8fd2c99d25aea2f307e0197f6f91a585b4408dce)
 - `noindex, nofollow` globally via headers and meta tag -> Rauthy as an Auth provider should never be indexed
-[]()
+[38a2a52](https://github.com/sebadob/rauthy/commit/38a2a52fe6530cf4efdedfe96d2b3041959fcd3d)
 
 ### Bugfixes
 

diff --git a/dev_notes.md b/dev_notes.md
@@ -2,13 +2,8 @@
 
 ## CURRENT WORK
 
-- integrate `cryptr` and migrate all old ENC_KEYS in:
-    - [ ] db_migrate 
-    - [ ] api_keys 
-    - [ ] jwk 
-    - [ ] clients
-- adopt Webauthn cookies to cryptr and check compatibility
-- move auth::rotate_jwks into Jwk Entity and adopt encryption
+- migrate all exising user emails to lowercase in the DB and only save lowercase ever again
+- change the login and always convert given users emails to lowercase only to avoid conflicts
 
 ## Stage 1 - essentials
 

diff --git a/rauthy-book/src/config/config.md b/rauthy-book/src/config/config.md
@@ -156,6 +156,14 @@ extract these values, create Kubernetes Secrets and provide them as environment
 # operations (default: 128)
 #CACHE_BUF_CLIENT=128
 
+# The max cache size for users. If you can afford it memory-wise, make it possible to fit
+# all active users inside the cache.
+# default: 100
+CACHE_USERS_SIZE=100
+# The lifespan of the users cache in seconds. Cache eviction on updates will be handled automatically.
+# default: 28800
+CACHE_USERS_LIFESPAN=28800
+
 # Secret token, which is used to authenticate the cache members
 #CACHE_AUTH_TOKEN=SomeSuperSecretAndVerySafeToken1337
 

diff --git a/rauthy-common/src/constants.rs b/rauthy-common/src/constants.rs
@@ -38,6 +38,7 @@ pub const CACHE_NAME_EPHEMERAL_CLIENTS: &str = "ephemeral-clients";
 pub const CACHE_NAME_LOGIN_DELAY: &str = "login-dly";
 pub const CACHE_NAME_SESSIONS: &str = "sessions";
 pub const CACHE_NAME_POW: &str = "pow";
+pub const CACHE_NAME_USERS: &str = "users";
 pub const CACHE_NAME_WEBAUTHN: &str = "webauthn";
 pub const CACHE_NAME_WEBAUTHN_DATA: &str = "webauthn-data";
 

diff --git a/rauthy-main/src/main.rs b/rauthy-main/src/main.rs
@@ -10,9 +10,10 @@ use cryptr::{EncKeys, EncValue};
 use prometheus::Registry;
 use rauthy_common::constants::{
     CACHE_NAME_12HR, CACHE_NAME_AUTH_CODES, CACHE_NAME_DPOP_NONCES, CACHE_NAME_EPHEMERAL_CLIENTS,
-    CACHE_NAME_LOGIN_DELAY, CACHE_NAME_POW, CACHE_NAME_SESSIONS, CACHE_NAME_WEBAUTHN,
-    CACHE_NAME_WEBAUTHN_DATA, DPOP_NONCE_EXP, EPHEMERAL_CLIENTS_CACHE_LIFETIME, POW_EXP,
-    RAUTHY_VERSION, SWAGGER_UI_EXTERNAL, SWAGGER_UI_INTERNAL, WEBAUTHN_DATA_EXP, WEBAUTHN_REQ_EXP,
+    CACHE_NAME_LOGIN_DELAY, CACHE_NAME_POW, CACHE_NAME_SESSIONS, CACHE_NAME_USERS,
+    CACHE_NAME_WEBAUTHN, CACHE_NAME_WEBAUTHN_DATA, DPOP_NONCE_EXP,
+    EPHEMERAL_CLIENTS_CACHE_LIFETIME, POW_EXP, RAUTHY_VERSION, SWAGGER_UI_EXTERNAL,
+    SWAGGER_UI_INTERNAL, WEBAUTHN_DATA_EXP, WEBAUTHN_REQ_EXP,
 };
 use rauthy_common::error_response::ErrorResponse;
 use rauthy_common::password_hasher;
@@ -184,14 +185,15 @@ async fn main() -> Result<(), Box<dyn Error>> {
     );
 
     // sessions
-    let session_lifetime = env::var("SESSION_LIFETIME")
-        .unwrap_or_else(|_| String::from("14400"))
-        .trim()
-        .parse::<u32>()
-        .expect("SESSION_LIFETIME cannot be parsed to u32 - bad format");
     cache_config.spawn_cache(
         CACHE_NAME_SESSIONS.to_string(),
-        redhac::TimedCache::with_lifespan(session_lifetime as u64),
+        redhac::TimedCache::with_lifespan(
+            env::var("SESSION_LIFETIME")
+                .unwrap_or_else(|_| String::from("14400"))
+                .trim()
+                .parse::<u64>()
+                .expect("SESSION_LIFETIME cannot be parsed to u64 - bad format"),
+        ),
         Some(64),
     );
 
@@ -202,6 +204,24 @@ async fn main() -> Result<(), Box<dyn Error>> {
         Some(16),
     );
 
+    // Users
+    cache_config.spawn_cache(
+        CACHE_NAME_USERS.to_string(),
+        redhac::TimedCache::with_lifespan_and_capacity(
+            env::var("CACHE_USERS_LIFESPAN")
+                .unwrap_or_else(|_| String::from("28800"))
+                .trim()
+                .parse::<u64>()
+                .expect("CACHE_USERS_LIFESPAN cannot be parsed to u64 - bad format"),
+            env::var("CACHE_USERS_SIZE")
+                .unwrap_or_else(|_| String::from("100"))
+                .trim()
+                .parse::<usize>()
+                .expect("CACHE_USERS_SIZE cannot be parsed to usize - bad format"),
+        ),
+        Some(16),
+    );
+
     // webauthn requests
     cache_config.spawn_cache(
         CACHE_NAME_WEBAUTHN.to_string(),
@@ -723,5 +743,8 @@ async fn V20_migrate_to_cryptr(app_state: &Data<AppState>) -> Result<(), ErrorRe
 
     // TODO maybe encrypt webauthn data while we are doing it already anyway?
 
+    // TODO check the user's database and convert all existing emails to lowercase only to avoid
+    // duplicate issues if a user is added more than once just with an uppercase character
+
     Ok(())
 }
diff --git a/rauthy-models/src/entity/users.rs b/rauthy-models/src/entity/users.rs
@@ -19,7 +19,7 @@ use crate::templates::UserEmailChangeConfirmHtml;
 use actix_web::{web, HttpRequest};
 use argon2::PasswordHash;
 use rauthy_common::constants::{
-    CACHE_NAME_12HR, IDX_USERS, RAUTHY_ADMIN_ROLE, WEBAUTHN_NO_PASSWORD_EXPIRY,
+    CACHE_NAME_USERS, IDX_USERS, RAUTHY_ADMIN_ROLE, WEBAUTHN_NO_PASSWORD_EXPIRY,
 };
 use rauthy_common::error_response::{ErrorResponse, ErrorResponseType};
 use rauthy_common::password_hasher::{ComparePasswords, HashPassword};
@@ -99,7 +99,7 @@ impl User {
         let mut users = User::find_all(data).await?;
         users.push(new_user.clone());
         cache_insert(
-            CACHE_NAME_12HR.to_string(),
+            CACHE_NAME_USERS.to_string(),
             IDX_USERS.to_string(),
             &data.caches.ha_cache_config,
             &users,
@@ -150,7 +150,7 @@ impl User {
             .collect::<Vec<Self>>();
 
         cache_insert(
-            CACHE_NAME_12HR.to_string(),
+            CACHE_NAME_USERS.to_string(),
             IDX_USERS.to_string(),
             &data.caches.ha_cache_config,
             &users,
@@ -160,7 +160,7 @@ impl User {
 
         let idx = format!("{}_{}", IDX_USERS, &self.id);
         cache_remove(
-            CACHE_NAME_12HR.to_string(),
+            CACHE_NAME_USERS.to_string(),
             idx,
             &data.caches.ha_cache_config,
             AckLevel::Quorum,
@@ -169,7 +169,7 @@ impl User {
 
         let idx = format!("{}_{}", IDX_USERS, &self.email);
         cache_remove(
-            CACHE_NAME_12HR.to_string(),
+            CACHE_NAME_USERS.to_string(),
             idx,
             &data.caches.ha_cache_config,
             AckLevel::Quorum,
@@ -184,7 +184,7 @@ impl User {
         let idx = format!("{}_{}", IDX_USERS, id);
         let user_opt = cache_get!(
             User,
-            CACHE_NAME_12HR.to_string(),
+            CACHE_NAME_USERS.to_string(),
             idx.to_string(),
             &data.caches.ha_cache_config,
             false
@@ -206,7 +206,7 @@ impl User {
         let idx = format!("{}_{}", IDX_USERS, id);
         let user_opt = cache_get!(
             User,
-            CACHE_NAME_12HR.to_string(),
+            CACHE_NAME_USERS.to_string(),
             idx.to_string(),
             &data.caches.ha_cache_config,
             false
@@ -223,7 +223,7 @@ impl User {
             .await?;
 
         cache_insert(
-            CACHE_NAME_12HR.to_string(),
+            CACHE_NAME_USERS.to_string(),
             idx,
             &data.caches.ha_cache_config,
             &user,
@@ -241,7 +241,7 @@ impl User {
         let idx = format!("{}_{}", IDX_USERS, email);
         let user_opt = cache_get!(
             User,
-            CACHE_NAME_12HR.to_string(),
+            CACHE_NAME_USERS.to_string(),
             idx.clone(),
             &data.caches.ha_cache_config,
             false
@@ -257,7 +257,7 @@ impl User {
             .await?;
 
         cache_insert(
-            CACHE_NAME_12HR.to_string(),
+            CACHE_NAME_USERS.to_string(),
             idx,
             &data.caches.ha_cache_config,
             &user,
@@ -269,30 +269,30 @@ impl User {
 
     // Returns all existing users
     pub async fn find_all(data: &web::Data<AppState>) -> Result<Vec<Self>, ErrorResponse> {
-        let users = cache_get!(
-            Vec<User>,
-            CACHE_NAME_12HR.to_string(),
-            IDX_USERS.to_string(),
-            &data.caches.ha_cache_config,
-            false
-        )
-        .await?;
-        if let Some(users) = users {
-            return Ok(users);
-        }
+        // let users = cache_get!(
+        //     Vec<User>,
+        //     CACHE_NAME_USERS.to_string(),
+        //     IDX_USERS.to_string(),
+        //     &data.caches.ha_cache_config,
+        //     false
+        // )
+        // .await?;
+        // if let Some(users) = users {
+        //     return Ok(users);
+        // }
 
         let res = sqlx::query_as::<_, Self>("select * from users")
             .fetch_all(&data.db)
             .await?;
 
-        cache_insert(
-            CACHE_NAME_12HR.to_string(),
-            IDX_USERS.to_string(),
-            &data.caches.ha_cache_config,
-            &res,
-            AckLevel::Quorum,
-        )
-        .await?;
+        // cache_insert(
+        //     CACHE_NAME_12HR.to_string(),
+        //     IDX_USERS.to_string(),
+        //     &data.caches.ha_cache_config,
+        //     &res,
+        //     AckLevel::Quorum,
+        // )
+        // .await?;
         Ok(res)
     }
 
@@ -373,15 +373,15 @@ impl User {
         if let Some(email) = old_email {
             let idx = format!("{}_{}", IDX_USERS, email);
             cache_del(
-                CACHE_NAME_12HR.to_string(),
+                CACHE_NAME_USERS.to_string(),
                 idx,
                 &data.caches.ha_cache_config,
             )
             .await?;
         }
 
         cache_insert(
-            CACHE_NAME_12HR.to_string(),
+            CACHE_NAME_USERS.to_string(),
             IDX_USERS.to_string(),
             &data.caches.ha_cache_config,
             &users,
@@ -391,7 +391,7 @@ impl User {
 
         let idx = format!("{}_{}", IDX_USERS, &self.id);
         cache_insert(
-            CACHE_NAME_12HR.to_string(),
+            CACHE_NAME_USERS.to_string(),
             idx,
             &data.caches.ha_cache_config,
             &self,
@@ -401,7 +401,7 @@ impl User {
 
         let idx = format!("{}_{}", IDX_USERS, &self.email);
         cache_insert(
-            CACHE_NAME_12HR.to_string(),
+            CACHE_NAME_USERS.to_string(),
             idx,
             &data.caches.ha_cache_config,
             &self,

diff --git a/rauthy.cfg b/rauthy.cfg
@@ -135,6 +135,14 @@ CACHE_BUF_SERVER=128
 # Buffer for client requests to remote servers for all cache operations (default: 128)
 CACHE_BUF_CLIENT=128
 
+# The max cache size for users. If you can afford it memory-wise, make it possible to fit
+# all active users inside the cache.
+# default: 100
+CACHE_USERS_SIZE=100
+# The lifespan of the users cache in seconds. Cache eviction on updates will be handled automatically.
+# default: 28800
+CACHE_USERS_LIFESPAN=28800
+
 # Secret token, which is used to authenticate the cache members
 #CACHE_AUTH_TOKEN=