Split code into cargo workspaces (#67)

More fixes

- fixed docker builds
- fixed mentions regex test
- fixed DATABASE_URL stuff
- change schema path in diesel.toml

Address review comments

- add jsonb column back into activity table
- remove authors field from cargo.toml
- adjust LEMMY_DATABASE_URL env var usage
- rename all occurences of LEMMY_DATABASE_URL to DATABASE_URL

Decouple utils and db

Split code into cargo workspaces

Co-authored-by: Felix Ableitner <me@nutomic.com>
Reviewed-on: https://yerbamate.dev/LemmyNet/lemmy/pulls/67

63 files changed, 1071 insertions, 921 deletions

diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile
index 82a03f3c..4445e4fe 100644
--- a/docker/dev/Dockerfile
+++ b/docker/dev/Dockerfile
@@ -17,13 +17,20 @@ WORKDIR /app
 RUN sudo chown -R rust:rust .
 RUN USER=root cargo new server
 WORKDIR /app/server
+RUN mkdir -p lemmy_db/src/ lemmy_utils/src/
 COPY server/Cargo.toml server/Cargo.lock ./
+COPY server/lemmy_db/Cargo.toml ./lemmy_db/
+COPY server/lemmy_utils/Cargo.toml ./lemmy_utils/
 RUN sudo chown -R rust:rust .
 RUN mkdir -p ./src/bin \
-  && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs 
+   && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs \
+   && cp ./src/bin/main.rs ./lemmy_db/src/main.rs \
+   && cp ./src/bin/main.rs ./lemmy_utils/src/main.rs
 RUN cargo build
 RUN rm -f ./target/x86_64-unknown-linux-musl/release/deps/lemmy_server*
 COPY server/src ./src/
+COPY server/lemmy_db ./lemmy_db/
+COPY server/lemmy_utils ./lemmy_utils/
 COPY server/migrations ./migrations/
 
 # Build for debug
diff --git a/docker/prod/Dockerfile b/docker/prod/Dockerfile
index 54485a37..9000ca3a 100644
--- a/docker/prod/Dockerfile
+++ b/docker/prod/Dockerfile
@@ -10,13 +10,19 @@ WORKDIR /app
 RUN sudo chown -R rust:rust .
 RUN USER=root cargo new server
 WORKDIR /app/server
+RUN mkdir -p lemmy_db/src/ lemmy_utils/src/
 COPY --chown=rust:rust server/Cargo.toml server/Cargo.lock ./
-#RUN sudo chown -R rust:rust .
+COPY --chown=rust:rust server/lemmy_db/Cargo.toml ./lemmy_db/
+COPY --chown=rust:rust server/lemmy_utils/Cargo.toml ./lemmy_utils/
 RUN mkdir -p ./src/bin \
-   && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs
+   && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs \
+   && cp ./src/bin/main.rs ./lemmy_db/src/main.rs \
+   && cp ./src/bin/main.rs ./lemmy_utils/src/main.rs
 RUN cargo build --release
 RUN rm -f ./target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR/deps/lemmy_server*
 COPY --chown=rust:rust server/src ./src/
+COPY --chown=rust:rust server/lemmy_db ./lemmy_db/
+COPY --chown=rust:rust server/lemmy_utils ./lemmy_utils/
 COPY --chown=rust:rust server/migrations ./migrations/
 
 # build for release
diff --git a/docs/src/contributing_tests.md b/docs/src/contributing_tests.md
index 13e5d122..ccce6725 100644
--- a/docs/src/contributing_tests.md
+++ b/docs/src/contributing_tests.md
@@ -9,7 +9,7 @@ following commands in the `server` subfolder:
 psql -U lemmy -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"
 export DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy
 diesel migration run
-RUST_TEST_THREADS=1 cargo test
+RUST_TEST_THREADS=1 cargo test --workspace
 ```
 
 ### Federation
diff --git a/install.sh b/install.sh
index fb42b26d..19b847b1 100755
--- a/install.sh
+++ b/install.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 set -e
 
 # Set the database variable to the default first.
diff --git a/server/Cargo.lock b/server/Cargo.lock
index 6d6364d4..62438593 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -1400,12 +1400,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "htmlescape"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
-
-[[package]]
 name = "http"
 version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1573,6 +1567,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
+name = "lemmy_db"
+version = "0.1.0"
+dependencies = [
+ "bcrypt",
+ "chrono",
+ "diesel",
+ "log",
+ "serde 1.0.114",
+ "serde_json",
+ "sha2",
+ "strum",
+ "strum_macros",
+]
+
+[[package]]
 name = "lemmy_server"
 version = "0.0.1"
 dependencies = [
@@ -1589,27 +1598,23 @@ dependencies = [
  "base64 0.12.3",
  "bcrypt",
  "chrono",
- "comrak",
- "config",
  "diesel",
  "diesel_migrations",
  "dotenv",
  "env_logger",
  "failure",
  "futures",
- "htmlescape",
  "http",
  "http-signature-normalization-actix",
  "itertools",
  "jsonwebtoken",
  "lazy_static",
- "lettre",
- "lettre_email",
+ "lemmy_db",
+ "lemmy_utils",
  "log",
  "openssl",
  "percent-encoding",
  "rand 0.7.3",
- "regex",
  "rss",
  "serde 1.0.114",
  "serde_json",
@@ -1622,6 +1627,26 @@ dependencies = [
 ]
 
 [[package]]
+name = "lemmy_utils"
+version = "0.1.0"
+dependencies = [
+ "chrono",
+ "comrak",
+ "config",
+ "itertools",
+ "lazy_static",
+ "lettre",
+ "lettre_email",
+ "log",
+ "openssl",
+ "rand 0.7.3",
+ "regex",
+ "serde 1.0.114",
+ "serde_json",
+ "url",
+]
+
+[[package]]
 name = "lettre"
 version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 8daf72c4..a5e5a583 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -1,14 +1,21 @@
 [package]
 name = "lemmy_server"
 version = "0.0.1"
-authors = ["Dessalines <tyhou13@gmx.com>"]
 edition = "2018"
 
 [profile.release]
 lto = true
 
+[workspace]
+members = [
+    "lemmy_utils",
+    "lemmy_db"
+]
+
 [dependencies]
-diesel = { version = "1.4.4", features = ["postgres","chrono","r2d2","64-column-tables","serde_json"] }
+lemmy_utils = { path = "./lemmy_utils" }
+lemmy_db = { path = "./lemmy_db" }
+diesel = "1.4.4"
 diesel_migrations = "1.4.0"
 dotenv = "0.15.0"
 activitystreams = "0.6.2"
@@ -31,16 +38,10 @@ rand = "0.7.3"
 strum = "0.18.0"
 strum_macros = "0.18.0"
 jsonwebtoken = "7.0.1"
-regex = "1.3.5"
 lazy_static = "1.3.0"
-lettre = "0.9.3"
-lettre_email = "0.9.4"
 rss = "1.9.0"
-htmlescape = "0.3.1"
 url = { version = "2.1.1", features = ["serde"] }
-config = {version = "0.10.1", default-features = false, features = ["hjson"] }
 percent-encoding = "2.1.0"
-comrak = "0.7"
 openssl = "0.10"
 http = "0.2.1"
 http-signature-normalization-actix = { version = "0.4.0-alpha.0", default-features = false, features = ["sha-2"] }
diff --git a/server/db-init.sh b/server/db-init.sh
index a2ad77b5..ccecb7de 100755
--- a/server/db-init.sh
+++ b/server/db-init.sh
@@ -1,4 +1,5 @@
-#!/bin/sh
+#!/bin/bash
+set -e
 
 # Default configurations
 username=lemmy
diff --git a/server/diesel.toml b/server/diesel.toml
index 92267c82..1644558f 100644
--- a/server/diesel.toml
+++ b/server/diesel.toml
@@ -2,4 +2,4 @@
 # see diesel.rs/guides/configuring-diesel-cli
 
 [print_schema]
-file = "src/schema.rs"
+file = "lemmy_db/src/schema.rs"
diff --git a/server/lemmy_db/Cargo.toml b/server/lemmy_db/Cargo.toml
new file mode 100644
index 00000000..d94cf5fc
--- /dev/null
+++ b/server/lemmy_db/Cargo.toml
@@ -0,0 +1,15 @@
+[package]
+name = "lemmy_db"
+version = "0.1.0"
+edition = "2018"
+
+[dependencies]
+diesel = { version = "1.4.4", features = ["postgres","chrono","r2d2","64-column-tables","serde_json"] }
+chrono = { version = "0.4.7", features = ["serde"] }
+serde = { version = "1.0.105", features = ["derive"] }
+serde_json = { version = "1.0.52", features = ["preserve_order"]}
+strum = "0.18.0"
+strum_macros = "0.18.0"
+log = "0.4.0"
+sha2 = "0.9"
+bcrypt = "0.8.0"
\ No newline at end of file
diff --git a/server/src/db/activity.rs b/server/lemmy_db/src/activity.rs
index 8c2b0c74..83f85ca1 100644
--- a/server/src/db/activity.rs
+++ b/server/lemmy_db/src/activity.rs
@@ -1,9 +1,12 @@
-use crate::{blocking, db::Crud, schema::activity, DbPool, LemmyError};
+use crate::{schema::activity, Crud};
 use diesel::{dsl::*, result::Error, *};
 use log::debug;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use std::fmt::Debug;
+use std::{
+  fmt::Debug,
+  io::{Error as IoError, ErrorKind},
+};
 
 #[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, Deserialize)]
 #[table_name = "activity"]
@@ -55,46 +58,43 @@ impl Crud<ActivityForm> for Activity {
   }
 }
 
-pub async fn insert_activity<T>(
-  user_id: i32,
-  data: T,
-  local: bool,
-  pool: &DbPool,
-) -> Result<(), LemmyError>
-where
-  T: Serialize + Debug + Send + 'static,
-{
-  blocking(pool, move |conn| {
-    do_insert_activity(conn, user_id, &data, local)
-  })
-  .await??;
-  Ok(())
-}
-
-fn do_insert_activity<T>(
+pub fn do_insert_activity<T>(
   conn: &PgConnection,
   user_id: i32,
   data: &T,
   local: bool,
-) -> Result<(), LemmyError>
+) -> Result<Activity, IoError>
 where
   T: Serialize + Debug,
 {
+  debug!("inserting activity for user {}, data {:?}", user_id, &data);
   let activity_form = ActivityForm {
     user_id,
     data: serde_json::to_value(&data)?,
     local,
     updated: None,
   };
-  debug!("inserting activity for user {}, data {:?}", user_id, data);
-  Activity::create(&conn, &activity_form)?;
-  Ok(())
+  let result = Activity::create(&conn, &activity_form);
+  match result {
+    Ok(s) => Ok(s),
+    Err(e) => Err(IoError::new(
+      ErrorKind::Other,
+      format!("Failed to insert activity into database: {}", e),
+    )),
+  }
 }
 
 #[cfg(test)]
 mod tests {
-  use super::{super::user::*, *};
-  use crate::db::{establish_unpooled_connection, Crud, ListingType, SortType};
+  use crate::{
+    activity::{Activity, ActivityForm},
+    tests::establish_unpooled_connection,
+    user::{UserForm, User_},
+    Crud,
+    ListingType,
+    SortType,
+  };
+  use serde_json::Value;
 
   #[test]
   fn test_crud() {
diff --git a/server/src/db/category.rs b/server/lemmy_db/src/category.rs
index ff49bbbe..ec2efc7b 100644
--- a/server/src/db/category.rs
+++ b/server/lemmy_db/src/category.rs
@@ -1,6 +1,6 @@
 use crate::{
-  db::Crud,
   schema::{category, category::dsl::*},
+  Crud,
 };
 use diesel::{dsl::*, result::Error, *};
 use serde::{Deserialize, Serialize};
@@ -52,8 +52,7 @@ impl Category {
 
 #[cfg(test)]
 mod tests {
-  use super::*;
-  use crate::db::establish_unpooled_connection;
+  use crate::{category::Category, tests::establish_unpooled_connection};
 
   #[test]
   fn test_crud() {
diff --git a/server/src/db/comment.rs b/server/lemmy_db/src/comment.rs
index 7e76770f..602070d5 100644
--- a/server/src/db/comment.rs
+++ b/server/lemmy_db/src/comment.rs
@@ -1,9 +1,5 @@
 use super::{post::Post, *};
-use crate::{
-  apub::{make_apub_endpoint, EndpointType},
-  naive_now,
-  schema::{comment, comment_like, comment_saved},
-};
+use crate::schema::{comment, comment_like, comment_saved};
 
 // WITH RECURSIVE MyTree AS (
 //     SELECT * FROM comment WHERE parent_id IS NULL
@@ -77,12 +73,15 @@ impl Crud<CommentForm> for Comment {
 }
 
 impl Comment {
-  pub fn update_ap_id(conn: &PgConnection, comment_id: i32) -> Result<Self, Error> {
+  pub fn update_ap_id(
+    conn: &PgConnection,
+    comment_id: i32,
+    apub_id: String,
+  ) -> Result<Self, Error> {
     use crate::schema::comment::dsl::*;
 
-    let apid = make_apub_endpoint(EndpointType::Comment, &comment_id.to_string()).to_string();
     diesel::update(comment.find(comment_id))
-      .set(ap_id.eq(apid))
+      .set(ap_id.eq(apub_id))
       .get_result::<Self>(conn)
   }
 
@@ -204,10 +203,8 @@ impl Saveable<CommentSavedForm> for CommentSaved {
 
 #[cfg(test)]
 mod tests {
-  use super::{
-    super::{community::*, post::*, user::*},
-    *,
-  };
+  use crate::{comment::*, community::*, post::*, tests::establish_unpooled_connection, user::*};
+
   #[test]
   fn test_crud() {
     let conn = establish_unpooled_connection();
diff --git a/server/src/db/comment_view.rs b/server/lemmy_db/src/comment_view.rs
index 7c853a81..914e568c 100644
--- a/server/src/db/comment_view.rs
+++ b/server/lemmy_db/src/comment_view.rs
@@ -1,5 +1,5 @@
 // TODO, remove the cross join here, just join to user directly
-use crate::db::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType};
+use crate::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType};
 use diesel::{dsl::*, pg::Pg, result::Error, *};
 use serde::{Deserialize, Serialize};
 
@@ -460,11 +460,17 @@ impl<'a> ReplyQueryBuilder<'a> {
 
 #[cfg(test)]
 mod tests {
-  use super::{
-    super::{comment::*, community::*, post::*, user::*},
+  use crate::{
+    comment::*,
+    comment_view::*,
+    community::*,
+    post::*,
+    tests::establish_unpooled_connection,
+    user::*,
+    Crud,
+    Likeable,
     *,
   };
-  use crate::db::{establish_unpooled_connection, Crud, Likeable};
 
   #[test]
   fn test_crud() {
diff --git a/server/src/db/community.rs b/server/lemmy_db/src/community.rs
index 461ba473..60752080 100644
--- a/server/src/db/community.rs
+++ b/server/lemmy_db/src/community.rs
@@ -1,6 +1,9 @@
 use crate::{
-  db::{Bannable, Crud, Followable, Joinable},
   schema::{community, community_follower, community_moderator, community_user_ban},
+  Bannable,
+  Crud,
+  Followable,
+  Joinable,
 };
 use diesel::{dsl::*, result::Error, *};
 use serde::{Deserialize, Serialize};
@@ -232,8 +235,7 @@ impl Followable<CommunityFollowerForm> for CommunityFollower {
 
 #[cfg(test)]
 mod tests {
-  use super::{super::user::*, *};
-  use crate::db::{establish_unpooled_connection, ListingType, SortType};
+  use crate::{community::*, tests::establish_unpooled_connection, user::*, ListingType, SortType};
 
   #[test]
   fn test_crud() {
diff --git a/server/src/db/community_view.rs b/server/lemmy_db/src/community_view.rs
index 4ec839ac..b465ddab 100644
--- a/server/src/db/community_view.rs
+++ b/server/lemmy_db/src/community_view.rs
@@ -1,5 +1,5 @@
 use super::community_view::community_fast_view::BoxedQuery;
-use crate::db::{fuzzy_search, limit_and_offset, MaybeOptional, SortType};
+use crate::{fuzzy_search, limit_and_offset, MaybeOptional, SortType};
 use diesel::{pg::Pg, result::Error, *};
 use serde::{Deserialize, Serialize};
 
diff --git a/server/src/db/mod.rs b/server/lemmy_db/src/lib.rs
index da69f8dc..b34919cd 100644
--- a/server/src/db/mod.rs
+++ b/server/lemmy_db/src/lib.rs
@@ -1,10 +1,22 @@
-use crate::settings::Settings;
+#[macro_use]
+pub extern crate diesel;
+#[macro_use]
+pub extern crate strum_macros;
+pub extern crate bcrypt;
+pub extern crate chrono;
+pub extern crate log;
+pub extern crate serde;
+pub extern crate serde_json;
+pub extern crate sha2;
+pub extern crate strum;
+
+use chrono::NaiveDateTime;
 use diesel::{dsl::*, result::Error, *};
 use serde::{Deserialize, Serialize};
+use std::{env, env::VarError};
 
 pub mod activity;
 pub mod category;
-pub mod code_migrations;
 pub mod comment;
 pub mod comment_view;
 pub mod community;
@@ -16,6 +28,7 @@ pub mod post;
 pub mod post_view;
 pub mod private_message;
 pub mod private_message_view;
+pub mod schema;
 pub mod site;
 pub mod site_view;
 pub mod user;
@@ -111,9 +124,8 @@ impl<T> MaybeOptional<T> for Option<T> {
   }
 }
 
-pub fn establish_unpooled_connection() -> PgConnection {
-  let db_url = Settings::get().get_database_url();
-  PgConnection::establish(&db_url).unwrap_or_else(|_| panic!("Error connecting to {}", db_url))
+pub fn get_database_url_from_env() -> Result<String, VarError> {
+  env::var("LEMMY_DATABASE_URL")
 }
 
 #[derive(EnumString, ToString, Debug, Serialize, Deserialize)]
@@ -155,9 +167,25 @@ pub fn limit_and_offset(page: Option<i64>, limit: Option<i64>) -> (i64, i64) {
   let offset = limit * (page - 1);
   (limit, offset)
 }
+
+pub fn naive_now() -> NaiveDateTime {
+  chrono::prelude::Utc::now().naive_utc()
+}
+
 #[cfg(test)]
 mod tests {
   use super::fuzzy_search;
+  use crate::get_database_url_from_env;
+  use diesel::{Connection, PgConnection};
+
+  pub fn establish_unpooled_connection() -> PgConnection {
+    let db_url = match get_database_url_from_env() {
+      Ok(url) => url,
+      Err(_) => panic!("Failed to read database URL from env var LEMMY_DATABASE_URL"),
+    };
+    PgConnection::establish(&db_url).unwrap_or_else(|_| panic!("Error connecting to {}", db_url))
+  }
+
   #[test]
   fn test_fuzzy_search() {
     let test = "This is a fuzzy search";
diff --git a/server/src/db/moderator.rs b/server/lemmy_db/src/moderator.rs
index 44b04ec6..f5d33d96 100644
--- a/server/src/db/moderator.rs
+++ b/server/lemmy_db/src/moderator.rs
@@ -1,5 +1,4 @@
 use crate::{
-  db::Crud,
   schema::{
     mod_add,
     mod_add_community,
@@ -11,6 +10,7 @@ use crate::{
     mod_remove_post,
     mod_sticky_post,
   },
+  Crud,
 };
 use diesel::{dsl::*, result::Error, *};
 use serde::{Deserialize, Serialize};
@@ -437,11 +437,16 @@ impl Crud<ModAddForm> for ModAdd {
 
 #[cfg(test)]
 mod tests {
-  use super::{
-    super::{comment::*, community::*, post::*, user::*},
-    *,
+  use crate::{
+    comment::*,
+    community::*,
+    moderator::*,
+    post::*,
+    tests::establish_unpooled_connection,
+    user::*,
+    ListingType,
+    SortType,
   };
-  use crate::db::{establish_unpooled_connection, ListingType, SortType};
 
   // use Crud;
   #[test]
diff --git a/server/src/db/moderator_views.rs b/server/lemmy_db/src/moderator_views.rs
index f5b109fe..024907c3 100644
--- a/server/src/db/moderator_views