From ecfbf2b474fb527af24d45382e3260b264ed20ba Mon Sep 17 00:00:00 2001 From: hamidreza kalbasi Date: Fri, 4 Jun 2021 08:06:12 +0430 Subject: [PATCH 1/6] put media in filesystem --- src/client_server/media.rs | 14 +++++---- src/database/globals.rs | 21 ++++++++++---- src/database/media.rs | 58 ++++++++++++++++++++++++++++---------- src/error.rs | 5 ++++ 4 files changed, 73 insertions(+), 25 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index f9350e0f..f59c0a8a 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -38,10 +38,11 @@ pub async fn create_content_route( ); db.media.create( mxc.clone(), + &db.globals, &body.filename.as_deref(), &body.content_type.as_deref(), &body.file, - )?; + ).await?; db.flush().await?; @@ -67,7 +68,7 @@ pub async fn get_content_route( filename, content_type, file, - }) = db.media.get(&mxc)? + }) = db.media.get(&db.globals, &mxc).await? { Ok(get_content::Response { file, @@ -91,10 +92,11 @@ pub async fn get_content_route( db.media.create( mxc, + &db.globals, &get_content_response.content_disposition.as_deref(), &get_content_response.content_type.as_deref(), &get_content_response.file, - )?; + ).await?; Ok(get_content_response.into()) } else { @@ -117,13 +119,14 @@ pub async fn get_content_thumbnail_route( content_type, file, .. }) = db.media.get_thumbnail( mxc.clone(), + &db.globals, body.width .try_into() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, body.height .try_into() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, - )? { + ).await? { Ok(get_content_thumbnail::Response { file, content_type }.into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { let get_thumbnail_response = db @@ -144,12 +147,13 @@ pub async fn get_content_thumbnail_route( db.media.upload_thumbnail( mxc, + &db.globals, &None, &get_thumbnail_response.content_type, body.width.try_into().expect("all UInts are valid u32s"), body.height.try_into().expect("all UInts are valid u32s"), &get_thumbnail_response.file, - )?; + ).await?; Ok(get_thumbnail_response.into()) } else { diff --git a/src/database/globals.rs b/src/database/globals.rs index 5d91d374..311ff256 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -5,11 +5,7 @@ use ruma::{ EventId, MilliSecondsSinceUnixEpoch, ServerName, ServerSigningKeyId, }; use rustls::{ServerCertVerifier, WebPKIVerifier}; -use std::{ - collections::{BTreeMap, HashMap}, - sync::{Arc, RwLock}, - time::{Duration, Instant}, -}; +use std::{collections::{BTreeMap, HashMap}, path::{PathBuf}, sync::{Arc, RwLock}, time::{Duration, Instant}}; use tokio::sync::Semaphore; use trust_dns_resolver::TokioAsyncResolver; @@ -264,4 +260,19 @@ impl Globals { self.globals.insert("version", &new_version.to_be_bytes())?; Ok(()) } + + pub fn get_media_folder(&self) -> PathBuf { + let mut r = PathBuf::new(); + r.push(self.config.database_path.clone()); + r.push("media"); + r + } + + pub fn get_media_file(&self, key: &Vec) -> PathBuf { + let mut r = PathBuf::new(); + r.push(self.config.database_path.clone()); + r.push("media"); + r.push(base64::encode_config(key, base64::URL_SAFE_NO_PAD)); + r + } } diff --git a/src/database/media.rs b/src/database/media.rs index 37fcb741..6a86c047 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -1,7 +1,9 @@ use image::{imageops::FilterType, GenericImageView}; +use crate::database::globals::Globals; use crate::{utils, Error, Result}; -use std::mem; +use std::{mem}; +use tokio::{fs::{self, File}, io::AsyncWriteExt, io::AsyncReadExt}; pub struct FileMeta { pub filename: Option, @@ -15,10 +17,11 @@ pub struct Media { } impl Media { - /// Uploads or replaces a file. - pub fn create( + /// Uploads a file. + pub async fn create( &self, mxc: String, + globals: &Globals, filename: &Option<&str>, content_type: &Option<&str>, file: &[u8], @@ -37,15 +40,20 @@ impl Media { .unwrap_or_default(), ); - self.mediaid_file.insert(key, file)?; + let path = globals.get_media_file(&key); + fs::create_dir_all(path.parent().unwrap()).await?; + let mut f = File::create(path).await?; + f.write_all(file).await?; + self.mediaid_file.insert(key, vec![])?; Ok(()) } /// Uploads or replaces a file thumbnail. - pub fn upload_thumbnail( + pub async fn upload_thumbnail( &self, mxc: String, + globals: &Globals, filename: &Option, content_type: &Option, width: u32, @@ -66,13 +74,18 @@ impl Media { .unwrap_or_default(), ); - self.mediaid_file.insert(key, file)?; + let path = globals.get_media_file(&key); + fs::create_dir_all(path.parent().unwrap()).await?; + let mut f = File::create(path).await?; + f.write_all(file).await?; + + self.mediaid_file.insert(key, vec![])?; Ok(()) } /// Downloads a file. - pub fn get(&self, mxc: &str) -> Result> { + pub async fn get(&self, globals: &Globals, mxc: &str) -> Result> { let mut prefix = mxc.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail @@ -80,7 +93,10 @@ impl Media { prefix.push(0xff); if let Some(r) = self.mediaid_file.scan_prefix(&prefix).next() { - let (key, file) = r?; + let (key, _file) = r?; + let path = globals.get_media_file(&key.to_vec()); + let mut file = vec![]; + File::open(path).await?.read_to_end(&mut file).await?; let mut parts = key.rsplit(|&b| b == 0xff); let content_type = parts @@ -107,7 +123,7 @@ impl Media { Ok(Some(FileMeta { filename, content_type, - file: file.to_vec(), + file, })) } else { Ok(None) @@ -137,7 +153,7 @@ impl Media { /// - Server creates the thumbnail and sends it to the user /// /// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards. - pub fn get_thumbnail(&self, mxc: String, width: u32, height: u32) -> Result> { + pub async fn get_thumbnail(&self, mxc: String, globals: &Globals, width: u32, height: u32) -> Result> { let (width, height, crop) = self .thumbnail_properties(width, height) .unwrap_or((0, 0, false)); // 0, 0 because that's the original file @@ -157,7 +173,10 @@ impl Media { if let Some(r) = self.mediaid_file.scan_prefix(&thumbnail_prefix).next() { // Using saved thumbnail - let (key, file) = r?; + let (key, _file) = r?; + let path = globals.get_media_file(&key.to_vec()); + let mut file = vec![]; + File::open(path).await?.read_to_end(&mut file).await?; let mut parts = key.rsplit(|&b| b == 0xff); let content_type = parts @@ -190,7 +209,11 @@ impl Media { } else if let Some(r) = self.mediaid_file.scan_prefix(&original_prefix).next() { // Generate a thumbnail - let (key, file) = r?; + let (key, _file) = r?; + let path = globals.get_media_file(&key.to_vec()); + let mut file = vec![]; + File::open(path).await?.read_to_end(&mut file).await?; + let mut parts = key.rsplit(|&b| b == 0xff); let content_type = parts @@ -283,19 +306,24 @@ impl Media { widthheight, ); - self.mediaid_file.insert(thumbnail_key, &*thumbnail_bytes)?; + let path = globals.get_media_file(&thumbnail_key); + fs::create_dir_all(path.parent().unwrap()).await?; + let mut f = File::create(path).await?; + f.write_all(&thumbnail_bytes).await?; + + self.mediaid_file.insert(thumbnail_key, vec![])?; Ok(Some(FileMeta { filename, content_type, - file: thumbnail_bytes.to_vec(), + file: thumbnail_bytes.to_vec() })) } else { // Couldn't parse file to generate thumbnail, send original Ok(Some(FileMeta { filename, content_type, - file: file.to_vec(), + file: file.to_vec() })) } } else { diff --git a/src/error.rs b/src/error.rs index e2664e21..e36f863f 100644 --- a/src/error.rs +++ b/src/error.rs @@ -40,6 +40,11 @@ pub enum Error { }, #[error("{0}")] FederationError(Box, RumaError), + #[error("Could not do this io: {source}")] + IoError { + #[from] + source: std::io::Error, + }, #[error("{0}")] BadServerResponse(&'static str), #[error("{0}")] -- GitLab From 13c6c5f2e6d982ceed0102fe25ac14a4c3db74cb Mon Sep 17 00:00:00 2001 From: hamidreza kalbasi Date: Sun, 6 Jun 2021 16:58:32 +0430 Subject: [PATCH 2/6] fix fmt and clippy warnings --- src/client_server/media.rs | 76 +++++++++++++++++++++----------------- src/database/globals.rs | 9 ++++- src/database/media.rs | 26 +++++++++---- 3 files changed, 68 insertions(+), 43 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index f59c0a8a..beefe116 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -36,13 +36,15 @@ pub async fn create_content_route( db.globals.server_name(), utils::random_string(MXC_LENGTH) ); - db.media.create( - mxc.clone(), - &db.globals, - &body.filename.as_deref(), - &body.content_type.as_deref(), - &body.file, - ).await?; + db.media + .create( + mxc.clone(), + &db.globals, + &body.filename.as_deref(), + &body.content_type.as_deref(), + &body.file, + ) + .await?; db.flush().await?; @@ -90,13 +92,15 @@ pub async fn get_content_route( ) .await?; - db.media.create( - mxc, - &db.globals, - &get_content_response.content_disposition.as_deref(), - &get_content_response.content_type.as_deref(), - &get_content_response.file, - ).await?; + db.media + .create( + mxc, + &db.globals, + &get_content_response.content_disposition.as_deref(), + &get_content_response.content_type.as_deref(), + &get_content_response.file, + ) + .await?; Ok(get_content_response.into()) } else { @@ -117,16 +121,20 @@ pub async fn get_content_thumbnail_route( if let Some(FileMeta { content_type, file, .. - }) = db.media.get_thumbnail( - mxc.clone(), - &db.globals, - body.width - .try_into() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, - body.height - .try_into() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, - ).await? { + }) = db + .media + .get_thumbnail( + mxc.clone(), + &db.globals, + body.width + .try_into() + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, + body.height + .try_into() + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, + ) + .await? + { Ok(get_content_thumbnail::Response { file, content_type }.into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { let get_thumbnail_response = db @@ -145,15 +153,17 @@ pub async fn get_content_thumbnail_route( ) .await?; - db.media.upload_thumbnail( - mxc, - &db.globals, - &None, - &get_thumbnail_response.content_type, - body.width.try_into().expect("all UInts are valid u32s"), - body.height.try_into().expect("all UInts are valid u32s"), - &get_thumbnail_response.file, - ).await?; + db.media + .upload_thumbnail( + mxc, + &db.globals, + &None, + &get_thumbnail_response.content_type, + body.width.try_into().expect("all UInts are valid u32s"), + body.height.try_into().expect("all UInts are valid u32s"), + &get_thumbnail_response.file, + ) + .await?; Ok(get_thumbnail_response.into()) } else { diff --git a/src/database/globals.rs b/src/database/globals.rs index 311ff256..e382f0c4 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -5,7 +5,12 @@ use ruma::{ EventId, MilliSecondsSinceUnixEpoch, ServerName, ServerSigningKeyId, }; use rustls::{ServerCertVerifier, WebPKIVerifier}; -use std::{collections::{BTreeMap, HashMap}, path::{PathBuf}, sync::{Arc, RwLock}, time::{Duration, Instant}}; +use std::{ + collections::{BTreeMap, HashMap}, + path::PathBuf, + sync::{Arc, RwLock}, + time::{Duration, Instant}, +}; use tokio::sync::Semaphore; use trust_dns_resolver::TokioAsyncResolver; @@ -268,7 +273,7 @@ impl Globals { r } - pub fn get_media_file(&self, key: &Vec) -> PathBuf { + pub fn get_media_file(&self, key: &[u8]) -> PathBuf { let mut r = PathBuf::new(); r.push(self.config.database_path.clone()); r.push("media"); diff --git a/src/database/media.rs b/src/database/media.rs index 6a86c047..d35be10e 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -1,9 +1,13 @@ -use image::{imageops::FilterType, GenericImageView}; use crate::database::globals::Globals; +use image::{imageops::FilterType, GenericImageView}; use crate::{utils, Error, Result}; -use std::{mem}; -use tokio::{fs::{self, File}, io::AsyncWriteExt, io::AsyncReadExt}; +use std::mem; +use tokio::{ + fs::{self, File}, + io::AsyncReadExt, + io::AsyncWriteExt, +}; pub struct FileMeta { pub filename: Option, @@ -153,7 +157,13 @@ impl Media { /// - Server creates the thumbnail and sends it to the user /// /// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards. - pub async fn get_thumbnail(&self, mxc: String, globals: &Globals, width: u32, height: u32) -> Result> { + pub async fn get_thumbnail( + &self, + mxc: String, + globals: &Globals, + width: u32, + height: u32, + ) -> Result> { let (width, height, crop) = self .thumbnail_properties(width, height) .unwrap_or((0, 0, false)); // 0, 0 because that's the original file @@ -213,7 +223,7 @@ impl Media { let path = globals.get_media_file(&key.to_vec()); let mut file = vec![]; File::open(path).await?.read_to_end(&mut file).await?; - + let mut parts = key.rsplit(|&b| b == 0xff); let content_type = parts @@ -310,20 +320,20 @@ impl Media { fs::create_dir_all(path.parent().unwrap()).await?; let mut f = File::create(path).await?; f.write_all(&thumbnail_bytes).await?; - + self.mediaid_file.insert(thumbnail_key, vec![])?; Ok(Some(FileMeta { filename, content_type, - file: thumbnail_bytes.to_vec() + file: thumbnail_bytes.to_vec(), })) } else { // Couldn't parse file to generate thumbnail, send original Ok(Some(FileMeta { filename, content_type, - file: file.to_vec() + file: file.to_vec(), })) } } else { -- GitLab From f038422d69bab0ae4813991bdd47ce08814d9a91 Mon Sep 17 00:00:00 2001 From: hamidreza kalbasi Date: Tue, 8 Jun 2021 17:05:13 +0430 Subject: [PATCH 3/6] use .keys() and remove unneccery .to_vec() --- src/database/media.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/database/media.rs b/src/database/media.rs index a92164f6..f2caa118 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -106,9 +106,9 @@ impl Media { prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail prefix.push(0xff); - if let Some(r) = self.mediaid_file.scan_prefix(&prefix).next() { - let (key, _file) = r?; - let path = globals.get_media_file(&key.to_vec()); + if let Some(r) = self.mediaid_file.scan_prefix(&prefix).keys().next() { + let key = r?; + let path = globals.get_media_file(&key); let mut file = vec![]; File::open(path).await?.read_to_end(&mut file).await?; let mut parts = key.rsplit(|&b| b == 0xff); @@ -195,10 +195,10 @@ impl Media { original_prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail original_prefix.push(0xff); - if let Some(r) = self.mediaid_file.scan_prefix(&thumbnail_prefix).next() { + if let Some(r) = self.mediaid_file.scan_prefix(&thumbnail_prefix).keys().next() { // Using saved thumbnail - let (key, _file) = r?; - let path = globals.get_media_file(&key.to_vec()); + let key = r?; + let path = globals.get_media_file(&key); let mut file = vec![]; File::open(path).await?.read_to_end(&mut file).await?; let mut parts = key.rsplit(|&b| b == 0xff); @@ -231,11 +231,11 @@ impl Media { content_type, file: file.to_vec(), })) - } else if let Some(r) = self.mediaid_file.scan_prefix(&original_prefix).next() { + } else if let Some(r) = self.mediaid_file.scan_prefix(&original_prefix).keys().next() { // Generate a thumbnail - let (key, _file) = r?; - let path = globals.get_media_file(&key.to_vec()); + let key = r?; + let path = globals.get_media_file(&key); let mut file = vec![]; File::open(path).await?.read_to_end(&mut file).await?; -- GitLab From 93d8205f315e8b24826edfd17c60e0a37486465d Mon Sep 17 00:00:00 2001 From: hamidreza kalbasi Date: Tue, 8 Jun 2021 17:50:06 +0430 Subject: [PATCH 4/6] create media folder in init --- src/database/globals.rs | 9 +++++++-- src/database/media.rs | 23 +++++++++++++---------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/database/globals.rs b/src/database/globals.rs index e382f0c4..e17e563a 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -7,6 +7,7 @@ use ruma::{ use rustls::{ServerCertVerifier, WebPKIVerifier}; use std::{ collections::{BTreeMap, HashMap}, + fs, path::PathBuf, sync::{Arc, RwLock}, time::{Duration, Instant}, @@ -131,7 +132,7 @@ impl Globals { .as_ref() .map(|secret| jsonwebtoken::DecodingKey::from_secret(secret.as_bytes()).into_static()); - Ok(Self { + let s = Self { globals, config, keypair: Arc::new(keypair), @@ -146,7 +147,11 @@ impl Globals { bad_event_ratelimiter: Arc::new(RwLock::new(BTreeMap::new())), bad_signature_ratelimiter: Arc::new(RwLock::new(BTreeMap::new())), servername_ratelimiter: Arc::new(RwLock::new(BTreeMap::new())), - }) + }; + + fs::create_dir_all(s.get_media_folder())?; + + Ok(s) } /// Returns this server's keypair. diff --git a/src/database/media.rs b/src/database/media.rs index f2caa118..34b1fab7 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -3,11 +3,7 @@ use image::{imageops::FilterType, GenericImageView}; use crate::{utils, Error, Result}; use std::mem; -use tokio::{ - fs::{self, File}, - io::AsyncReadExt, - io::AsyncWriteExt, -}; +use tokio::{fs::File, io::AsyncReadExt, io::AsyncWriteExt}; pub struct FileMeta { pub content_disposition: Option, @@ -50,7 +46,6 @@ impl Media { ); let path = globals.get_media_file(&key); - fs::create_dir_all(path.parent().unwrap()).await?; let mut f = File::create(path).await?; f.write_all(file).await?; @@ -89,7 +84,6 @@ impl Media { ); let path = globals.get_media_file(&key); - fs::create_dir_all(path.parent().unwrap()).await?; let mut f = File::create(path).await?; f.write_all(file).await?; @@ -195,7 +189,12 @@ impl Media { original_prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail original_prefix.push(0xff); - if let Some(r) = self.mediaid_file.scan_prefix(&thumbnail_prefix).keys().next() { + if let Some(r) = self + .mediaid_file + .scan_prefix(&thumbnail_prefix) + .keys() + .next() + { // Using saved thumbnail let key = r?; let path = globals.get_media_file(&key); @@ -231,7 +230,12 @@ impl Media { content_type, file: file.to_vec(), })) - } else if let Some(r) = self.mediaid_file.scan_prefix(&original_prefix).keys().next() { + } else if let Some(r) = self + .mediaid_file + .scan_prefix(&original_prefix) + .keys() + .next() + { // Generate a thumbnail let key = r?; @@ -336,7 +340,6 @@ impl Media { ); let path = globals.get_media_file(&thumbnail_key); - fs::create_dir_all(path.parent().unwrap()).await?; let mut f = File::create(path).await?; f.write_all(&thumbnail_bytes).await?; -- GitLab From 9b11ec3dab48ace2d0e0e1527cfb145a46e11387 Mon Sep 17 00:00:00 2001 From: hamidreza kalbasi Date: Tue, 8 Jun 2021 20:53:24 +0430 Subject: [PATCH 5/6] add migrations --- src/database.rs | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/database.rs b/src/database.rs index 7a55b030..553512e1 100644 --- a/src/database.rs +++ b/src/database.rs @@ -18,11 +18,7 @@ use log::{error, info}; use rocket::futures::{self, channel::mpsc}; use ruma::{DeviceId, ServerName, UserId}; use serde::Deserialize; -use std::{ - collections::HashMap, - fs::remove_dir_all, - sync::{Arc, RwLock}, -}; +use std::{collections::HashMap, fs::{self, remove_dir_all}, io::Write, sync::{Arc, RwLock}}; use tokio::sync::Semaphore; #[derive(Clone, Debug, Deserialize)] @@ -253,9 +249,11 @@ impl Database { let password = utils::string_from_bytes(&password); - if password.map_or(false, |password| { + let password_not_exists = password.map_or(false, |password| { argon2::verify_encoded(&password, b"").unwrap_or(false) - }) { + }); + + if password_not_exists { db.users.userid_password.insert(userid, b"")?; } } @@ -265,6 +263,22 @@ impl Database { info!("Migration: 1 -> 2 finished"); } + if db.globals.database_version()? < 3 { + + // Move media to filesystem + for r in db.media.mediaid_file.iter() { + let (key, content) = r?; + let path = db.globals.get_media_file(&key); + let mut file = fs::File::create(path)?; + file.write_all(&content)?; + db.media.mediaid_file.remove(&key)?; + db.media.mediaid_file.insert(&key, vec![])?; + } + + db.globals.bump_database_version(3)?; + + info!("Migration: 2 -> 3 finished"); + } // This data is probably outdated db.rooms.edus.presenceid_presence.clear()?; -- GitLab From 6947dce02c664e1d46346d2b0cdf53bc549f0e5f Mon Sep 17 00:00:00 2001 From: hamidreza kalbasi Date: Tue, 8 Jun 2021 20:54:36 +0430 Subject: [PATCH 6/6] fix fmt problems --- src/database.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/database.rs b/src/database.rs index 553512e1..ae1122c1 100644 --- a/src/database.rs +++ b/src/database.rs @@ -18,7 +18,12 @@ use log::{error, info}; use rocket::futures::{self, channel::mpsc}; use ruma::{DeviceId, ServerName, UserId}; use serde::Deserialize; -use std::{collections::HashMap, fs::{self, remove_dir_all}, io::Write, sync::{Arc, RwLock}}; +use std::{ + collections::HashMap, + fs::{self, remove_dir_all}, + io::Write, + sync::{Arc, RwLock}, +}; use tokio::sync::Semaphore; #[derive(Clone, Debug, Deserialize)] @@ -264,7 +269,6 @@ impl Database { } if db.globals.database_version()? < 3 { - // Move media to filesystem for r in db.media.mediaid_file.iter() { let (key, content) = r?; -- GitLab