Iframely and pictshare backend mostly done.

12 files changed, 213 insertions, 40 deletions

diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs
index 155c706a..e4fdfee6 100644
--- a/server/src/api/mod.rs
+++ b/server/src/api/mod.rs
@@ -18,7 +18,8 @@ use crate::db::user_mention_view::*;
 use crate::db::user_view::*;
 use crate::db::*;
 use crate::{
-  extract_usernames, naive_from_unix, naive_now, remove_slurs, slur_check, slurs_vec_to_str,
+  extract_usernames, fetch_iframely_and_pictshare_data, naive_from_unix, naive_now, remove_slurs,
+  slur_check, slurs_vec_to_str,
 };
 use diesel::PgConnection;
 use failure::Error;
diff --git a/server/src/api/post.rs b/server/src/api/post.rs
index 00bf8e11..fb022589 100644
--- a/server/src/api/post.rs
+++ b/server/src/api/post.rs
@@ -110,6 +110,10 @@ impl Perform<PostResponse> for Oper<CreatePost> {
       return Err(APIError::err("site_ban").into());
     }
 
+    // Fetch Iframely and Pictshare cached image
+    let (iframely_title, iframely_description, iframely_html, pictshare_thumbnail) =
+      fetch_iframely_and_pictshare_data(data.url.to_owned());
+
     let post_form = PostForm {
       name: data.name.to_owned(),
       url: data.url.to_owned(),
@@ -122,6 +126,10 @@ impl Perform<PostResponse> for Oper<CreatePost> {
       locked: None,
       stickied: None,
       updated: None,
+      embed_title: iframely_title,
+      embed_description: iframely_description,
+      embed_html: iframely_html,
+      thumbnail_url: pictshare_thumbnail,
     };
 
     let inserted_post = match Post::create(&conn, &post_form) {
@@ -353,6 +361,10 @@ impl Perform<PostResponse> for Oper<EditPost> {
       return Err(APIError::err("site_ban").into());
     }
 
+    // Fetch Iframely and Pictshare cached image
+    let (iframely_title, iframely_description, iframely_html, pictshare_thumbnail) =
+      fetch_iframely_and_pictshare_data(data.url.to_owned());
+
     let post_form = PostForm {
       name: data.name.to_owned(),
       url: data.url.to_owned(),
@@ -365,6 +377,10 @@ impl Perform<PostResponse> for Oper<EditPost> {
       locked: data.locked.to_owned(),
       stickied: data.stickied.to_owned(),
       updated: Some(naive_now()),
+      embed_title: iframely_title,
+      embed_description: iframely_description,
+      embed_html: iframely_html,
+      thumbnail_url: pictshare_thumbnail,
     };
 
     let _updated_post = match Post::update(&conn, data.edit_id, &post_form) {
diff --git a/server/src/api/user.rs b/server/src/api/user.rs
index 1d332b90..f7313895 100644
--- a/server/src/api/user.rs
+++ b/server/src/api/user.rs
@@ -883,6 +883,10 @@ impl Perform<LoginResponse> for Oper<DeleteAccount> {
         locked: None,
         stickied: None,
         updated: Some(naive_now()),
+        embed_title: None,
+        embed_description: None,
+        embed_html: None,
+        thumbnail_url: None,
       };
 
       let _updated_post = match Post::update(&conn, post.id, &post_form) {
diff --git a/server/src/apub/mod.rs b/server/src/apub/mod.rs
index c5a0b2f0..e224e259 100644
--- a/server/src/apub/mod.rs
+++ b/server/src/apub/mod.rs
@@ -83,6 +83,10 @@ mod tests {
       nsfw: false,
       deleted: false,
       updated: None,
+      embed_title: None,
+      embed_description: None,
+      embed_html: None,
+      thumbnail_url: None,
     };
 
     let page = post.as_page();
diff --git a/server/src/db/comment.rs b/server/src/db/comment.rs
index efba07a5..c9bfbac6 100644
--- a/server/src/db/comment.rs
+++ b/server/src/db/comment.rs
@@ -216,6 +216,10 @@ mod tests {
       stickied: None,
       updated: None,
       nsfw: false,
+      embed_title: None,
+      embed_description: None,
+      embed_html: None,
+      thumbnail_url: None,
     };
 
     let inserted_post = Post::create(&conn, &new_post).unwrap();
diff --git a/server/src/db/comment_view.rs b/server/src/db/comment_view.rs
index ff915d5e..85b41d82 100644
--- a/server/src/db/comment_view.rs
+++ b/server/src/db/comment_view.rs
@@ -480,6 +480,10 @@ mod tests {
       stickied: None,
       updated: None,
       nsfw: false,
+      embed_title: None,
+      embed_description: None,
+      embed_html: None,
+      thumbnail_url: None,
     };
 
     let inserted_post = Post::create(&conn, &new_post).unwrap();
diff --git a/server/src/db/moderator.rs b/server/src/db/moderator.rs
index 4fd532af..a8c3df4f 100644
--- a/server/src/db/moderator.rs
+++ b/server/src/db/moderator.rs
@@ -506,6 +506,10 @@ mod tests {
       stickied: None,
       updated: None,
       nsfw: false,
+      embed_title: None,
+      embed_description: None,
+      embed_html: None,
+      thumbnail_url: None,
     };
 
     let inserted_post = Post::create(&conn, &new_post).unwrap();
diff --git a/server/src/db/post.rs b/server/src/db/post.rs
index 9e7a4341..ffde14d3 100644
--- a/server/src/db/post.rs
+++ b/server/src/db/post.rs
@@ -17,6 +17,10 @@ pub struct Post {
   pub deleted: bool,
   pub nsfw: bool,
   pub stickied: bool,
+  pub embed_title: Option<String>,
+  pub embed_description: Option<String>,
+  pub embed_html: Option<String>,
+  pub thumbnail_url: Option<String>,
 }
 
 #[derive(Insertable, AsChangeset, Clone)]
@@ -33,6 +37,10 @@ pub struct PostForm {
   pub deleted: Option<bool>,
   pub nsfw: bool,
   pub stickied: Option<bool>,
+  pub embed_title: Option<String>,
+  pub embed_description: Option<String>,
+  pub embed_html: Option<String>,
+  pub thumbnail_url: Option<String>,
 }
 
 impl Crud<PostForm> for Post {
@@ -229,6 +237,10 @@ mod tests {
       stickied: None,
       nsfw: false,
       updated: None,
+      embed_title: None,
+      embed_description: None,
+      embed_html: None,
+      thumbnail_url: None,
     };
 
     let inserted_post = Post::create(&conn, &new_post).unwrap();
@@ -247,6 +259,10 @@ mod tests {
       nsfw: false,
       deleted: false,
       updated: None,
+      embed_title: None,
+      embed_description: None,
+      embed_html: None,
+      thumbnail_url: None,
     };
 
     // Post Like
diff --git a/server/src/db/post_view.rs b/server/src/db/post_view.rs
index 3f385077..f48f4f68 100644
--- a/server/src/db/post_view.rs
+++ b/server/src/db/post_view.rs
@@ -17,9 +17,54 @@ table! {
     updated -> Nullable<Timestamp>,
     deleted -> Bool,
     nsfw -> Bool,
+    stickied -> Bool,
+    embed_title -> Nullable<Text>,
+    embed_description -> Nullable<Text>,
+    embed_html -> Nullable<Text>,
+    thumbnail_url -> Nullable<Text>,
     banned -> Bool,
     banned_from_community -> Bool,
+    creator_name -> Varchar,
+    creator_avatar -> Nullable<Text>,
+    community_name -> Varchar,
+    community_removed -> Bool,
+    community_deleted -> Bool,
+    community_nsfw -> Bool,
+    number_of_comments -> BigInt,
+    score -> BigInt,
+    upvotes -> BigInt,
+    downvotes -> BigInt,
+    hot_rank -> Int4,
+    newest_activity_time -> Timestamp,
+    user_id -> Nullable<Int4>,
+    my_vote -> Nullable<Int4>,
+    subscribed -> Nullable<Bool>,
+    read -> Nullable<Bool>,
+    saved -> Nullable<Bool>,
+  }
+}
+
+table! {
+  post_mview (id) {
+    id -> Int4,
+    name -> Varchar,
+    url -> Nullable<Text>,
+    body -> Nullable<Text>,
+    creator_id -> Int4,
+    community_id -> Int4,
+    removed -> Bool,
+    locked -> Bool,
+    published -> Timestamp,
+    updated -> Nullable<Timestamp>,
+    deleted -> Bool,
+    nsfw -> Bool,
     stickied -> Bool,
+    embed_title -> Nullable<Text>,
+    embed_description -> Nullable<Text>,
+    embed_html -> Nullable<Text>,
+    thumbnail_url -> Nullable<Text>,
+    banned -> Bool,
+    banned_from_community -> Bool,
     creator_name -> Varchar,
     creator_avatar -> Nullable<Text>,
     community_name -> Varchar,
@@ -57,9 +102,13 @@ pub struct PostView {
   pub updated: Option<chrono::NaiveDateTime>,
   pub deleted: bool,
   pub nsfw: bool,
+  pub stickied: bool,
+  pub embed_title: Option<String>,
+  pub embed_description: Option<String>,
+  pub embed_html: Option<String>,
+  pub thumbnail_url: Option<String>,
   pub banned: bool,
   pub banned_from_community: bool,
-  pub stickied: bool,
   pub creator_name: String,
   pub creator_avatar: Option<String>,
   pub community_name: String,
@@ -79,44 +128,6 @@ pub struct PostView {
   pub saved: Option<bool>,
 }
 
-// The faked schema since diesel doesn't do views
-table! {
-  post_mview (id) {
-    id -> Int4,
-    name -> Varchar,
-    url -> Nullable<Text>,
-    body -> Nullable<Text>,
-    creator_id -> Int4,
-    community_id -> Int4,
-    removed -> Bool,
-    locked -> Bool,
-    published -> Timestamp,
-    updated -> Nullable<Timestamp>,
-    deleted -> Bool,
-    nsfw -> Bool,
-    banned -> Bool,
-    banned_from_community -> Bool,
-    stickied -> Bool,
-    creator_name -> Varchar,
-    creator_avatar -> Nullable<Text>,
-    community_name -> Varchar,
-    community_removed -> Bool,
-    community_deleted -> Bool,
-    community_nsfw -> Bool,
-    number_of_comments -> BigInt,
-    score -> BigInt,
-    upvotes -> BigInt,
-    downvotes -> BigInt,
-    hot_rank -> Int4,
-    newest_activity_time -> Timestamp,
-    user_id -> Nullable<Int4>,
-    my_vote -> Nullable<Int4>,
-    subscribed -> Nullable<Bool>,
-    read -> Nullable<Bool>,
-    saved -> Nullable<Bool>,
-  }
-}
-
 pub struct PostQueryBuilder<'a> {
   conn: &'a PgConnection,
   query: BoxedQuery<'a, Pg>,
@@ -394,6 +405,10 @@ mod tests {
       stickied: None,
       updated: None,
       nsfw: false,
+      embed_title: None,
+      embed_description: None,
+      embed_html: None,
+      thumbnail_url: None,
     };
 
     let inserted_post = Post::create(&conn, &new_post).unwrap();
@@ -454,6 +469,10 @@ mod tests {
       read: None,
       saved: None,
       nsfw: false,
+      embed_title: None,
+      embed_description: None,
+      embed_html: None,
+      thumbnail_url: None,
     };
 
     let expected_post_listing_with_user = PostView {
@@ -489,6 +508,10 @@ mod tests {
       read: None,
       saved: None,
       nsfw: false,
+      embed_title: None,
+      embed_description: None,
+      embed_html: None,
+      thumbnail_url: None,
     };
 
     let read_post_listings_with_user = PostQueryBuilder::create(&conn)
diff --git a/server/src/db/user_mention.rs b/server/src/db/user_mention.rs
index 3b10fd0f..0cf25795 100644
--- a/server/src/db/user_mention.rs
+++ b/server/src/db/user_mention.rs
@@ -132,6 +132,10 @@ mod tests {
       stickied: None,
       updated: None,
       nsfw: false,
+      embed_title: None,
+      embed_description: None,
+      embed_html: None,
+      thumbnail_url: None,
     };
 
     let inserted_post = Post::create(&conn, &new_post).unwrap();
diff --git a/server/src/lib.rs b/server/src/lib.rs
index 3e22585d..60addd27 100644
--- a/server/src/lib.rs
+++ b/server/src/lib.rs
@@ -33,14 +33,17 @@ pub mod websocket;
 
 use crate::settings::Settings;
 use chrono::{DateTime, NaiveDateTime, Utc};
+use chttp::prelude::*;
 use lettre::smtp::authentication::{Credentials, Mechanism};
 use lettre::smtp::extension::ClientId;
 use lettre::smtp::ConnectionReuseParameters;
 use lettre::{ClientSecurity, SmtpClient, Transport};
 use lettre_email::Email;
+use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use rand::distributions::Alphanumeric;
 use rand::{thread_rng, Rng};
 use regex::{Regex, RegexBuilder};
+use serde::Deserialize;
 
 pub fn to_datetime_utc(ndt: NaiveDateTime) -> DateTime<Utc> {
   DateTime::<Utc>::from_utc(ndt, Utc)
@@ -143,6 +146,77 @@ pub fn send_email(
   }
 }
 
+#[derive(Deserialize, Debug)]
+pub struct IframelyResponse {
+  title: Option<String>,
+  description: Option<String>,
+  thumbnail_url: Option<String>,
+  html: Option<String>,
+}
+
+pub fn fetch_iframely(url: &str) -> Result<IframelyResponse, failure::Error> {
+  let fetch_url = format!("http://lemmy_iframely:8061/oembed?url={}", url);
+  let text = chttp::get(&fetch_url)?.text()?;
+  let res: IframelyResponse = serde_json::from_str(&text)?;
+  Ok(res)
+}
+
+#[derive(Deserialize, Debug)]
+pub struct PictshareResponse {
+  status: String,
+  url: String,
+}
+
+pub fn fetch_pictshare(image_url: &str) -> Result<PictshareResponse, failure::Error> {
+  let fetch_url = format!(
+    "http://lemmy_pictshare/api/geturl.php?url={}",
+    utf8_percent_encode(image_url, NON_ALPHANUMERIC)
+  );
+  let text = chttp::get(&fetch_url)?.text()?;
+  let res: PictshareResponse = serde_json::from_str(&text)?;
+  Ok(res)
+}
+
+fn fetch_iframely_and_pictshare_data(
+  url: Option<String>,
+) -> (
+  Option<String>,
+  Option<String>,
+  Option<String>,
+  Option<String>,
+) {
+  // Fetch iframely data
+  let (iframely_title, iframely_description, iframely_thumbnail_url, iframely_html) = match url {
+    Some(url) => match fetch_iframely(&url) {
+      Ok(res) => (res.title, res.description, res.thumbnail_url, res.html),
+      Err(e) => {
+        eprintln!("iframely err: {}", e);
+        (None, None, None, None)
+      }
+    },
+    None => (None, None, None, None),
+  };
+
+  // Fetch pictshare thumbnail
+  let pictshare_thumbnail = match iframely_thumbnail_url {
+    Some(iframely_thumbnail_url) => match fetch_pictshare(&iframely_thumbnail_url) {
+      Ok(res) => Some(res.url),
+      Err(e) => {
+        eprintln!("pictshare err: {}", e);
+        None
+      }
+    },
+    None => None,
+  };
+
+  (
+    iframely_title,
+    iframely_description,
+    iframely_html,
+    pictshare_thumbnail,
+  )
+}
+
 #[cfg(test)]
 mod tests {
   use crate::{extract_usernames, is_email_regex, remove_slurs, slur_check, slurs_vec_to_str};
@@ -188,6 +262,21 @@ mod tests {
     assert_eq!(usernames, expected);
   }
 
+  // These helped with testing
+  // #[test]
+  // fn test_iframely() {
+  //   let res = fetch_iframely("https://www.redspark.nu/?p=15341");
+  //   assert!(res.is_ok());
+  // }
+
+  // #[test]
+  // fn test_pictshare() {
+  //   let res = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpg");
+  //   assert!(res.is_ok());
+  //   let res_other = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpgaoeu");
+  //   assert!(res_other.is_err());
+  // }
+
   // #[test]
   // fn test_send_email() {
   //  let result =  send_email("not a subject", "test_email@gmail.com", "ur user", "<h1>HI there</h1>");
diff --git a/server/src/schema.rs b/server/src/schema.rs
index 5330ed07..d9449fef 100644
--- a/server/src/schema.rs
+++ b/server/src/schema.rs
@@ -207,6 +207,10 @@ table! {
         deleted -> Bool,
         nsfw -> Bool,
         stickied -> Bool,
+        embed_title -> Nullable<Text>,
+        embed_description -> Nullable<Text>,
+        embed_html -> Nullable<Text>,
+        thumbnail_url -> Nullable<Text>,
     }
 }