Parse out in reply to field. Fixes #694 (#799)

- When a comment or post doesn't exist locally, yet we receive
  an activitypub with it (for example, a nested comment update,
  for a community we just subscribed to, also with a post we
  don't have...), then fetch it.

3 files changed, 68 insertions, 25 deletions

diff --git a/server/src/apub/comment.rs b/server/src/apub/comment.rs
index d18908d0..0a513f33 100644
--- a/server/src/apub/comment.rs
+++ b/server/src/apub/comment.rs
@@ -5,7 +5,11 @@ use crate::{
     create_apub_tombstone_response,
     create_tombstone,
     fetch_webfinger_url,
-    fetcher::get_or_fetch_and_upsert_remote_user,
+    fetcher::{
+      get_or_fetch_and_insert_remote_comment,
+      get_or_fetch_and_insert_remote_post,
+      get_or_fetch_and_upsert_remote_user,
+    },
     ActorType,
     ApubLikeableType,
     ApubObjectType,
@@ -115,22 +119,21 @@ impl FromApub for CommentForm {
     let mut in_reply_tos = oprops.get_many_in_reply_to_xsd_any_uris().unwrap();
     let post_ap_id = in_reply_tos.next().unwrap().to_string();
 
+    // This post, or the parent comment might not yet exist on this server yet, fetch them.
+    let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?;
+
     // The 2nd item, if it exists, is the parent comment apub_id
+    // For deeply nested comments, FromApub automatically gets called recursively
     let parent_id: Option<i32> = match in_reply_tos.next() {
       Some(parent_comment_uri) => {
-        let parent_comment_uri_str = &parent_comment_uri.to_string();
-        let parent_comment = Comment::read_from_apub_id(&conn, &parent_comment_uri_str)?;
+        let parent_comment_ap_id = &parent_comment_uri.to_string();
+        let parent_comment = get_or_fetch_and_insert_remote_comment(&parent_comment_ap_id, &conn)?;
 
         Some(parent_comment.id)
       }
       None => None,
     };
 
-    // TODO this failed because a mention on a post that wasn't on this server yet. Has to do with
-    // fetching replytos
-    dbg!(&post_ap_id);
-    let post = Post::read_from_apub_id(&conn, &post_ap_id)?;
-
     Ok(CommentForm {
       creator_id: creator.id,
       post_id: post.id,
diff --git a/server/src/apub/fetcher.rs b/server/src/apub/fetcher.rs
index c7467c5b..05b7492e 100644
--- a/server/src/apub/fetcher.rs
+++ b/server/src/apub/fetcher.rs
@@ -254,6 +254,22 @@ fn upsert_post(post_form: &PostForm, conn: &PgConnection) -> Result<Post, Error>
   }
 }
 
+pub fn get_or_fetch_and_insert_remote_post(
+  post_ap_id: &str,
+  conn: &PgConnection,
+) -> Result<Post, Error> {
+  match Post::read_from_apub_id(conn, post_ap_id) {
+    Ok(p) => Ok(p),
+    Err(NotFound {}) => {
+      debug!("Fetching and creating remote post: {}", post_ap_id);
+      let post = fetch_remote_object::<PageExt>(&Url::parse(post_ap_id)?)?;
+      let post_form = PostForm::from_apub(&post, conn)?;
+      Ok(Post::create(conn, &post_form)?)
+    }
+    Err(e) => Err(Error::from(e)),
+  }
+}
+
 fn upsert_comment(comment_form: &CommentForm, conn: &PgConnection) -> Result<Comment, Error> {
   let existing = Comment::read_from_apub_id(conn, &comment_form.ap_id);
   match existing {
@@ -263,6 +279,25 @@ fn upsert_comment(comment_form: &CommentForm, conn: &PgConnection) -> Result<Com
   }
 }
 
+pub fn get_or_fetch_and_insert_remote_comment(
+  comment_ap_id: &str,
+  conn: &PgConnection,
+) -> Result<Comment, Error> {
+  match Comment::read_from_apub_id(conn, comment_ap_id) {
+    Ok(p) => Ok(p),
+    Err(NotFound {}) => {
+      debug!(
+        "Fetching and creating remote comment and its parents: {}",
+        comment_ap_id
+      );
+      let comment = fetch_remote_object::<Note>(&Url::parse(comment_ap_id)?)?;
+      let comment_form = CommentForm::from_apub(&comment, conn)?;
+      Ok(Comment::create(conn, &comment_form)?)
+    }
+    Err(e) => Err(Error::from(e)),
+  }
+}
+
 // TODO It should not be fetching data from a community outbox.
 // All posts, comments, comment likes, etc should be posts to our community_inbox
 // The only data we should be periodically fetching (if it hasn't been fetched in the last day
diff --git a/server/src/apub/shared_inbox.rs b/server/src/apub/shared_inbox.rs
index 3ecc873e..1ada6ad1 100644
--- a/server/src/apub/shared_inbox.rs
+++ b/server/src/apub/shared_inbox.rs
@@ -6,7 +6,12 @@ use crate::{
   },
   apub::{
     extensions::signatures::verify,
-    fetcher::{get_or_fetch_and_upsert_remote_community, get_or_fetch_and_upsert_remote_user},
+    fetcher::{
+      get_or_fetch_and_insert_remote_comment,
+      get_or_fetch_and_insert_remote_post,
+      get_or_fetch_and_upsert_remote_community,
+      get_or_fetch_and_upsert_remote_user,
+    },
     FromApub,
     GroupExt,
     PageExt,
@@ -427,7 +432,7 @@ fn receive_update_post(
   insert_activity(&conn, user.id, &update, false)?;
 
   let post = PostForm::from_apub(&page, conn)?;
-  let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id;
+  let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id;
   Post::update(conn, post_id, &post)?;
 
   // Refetch the view
@@ -465,7 +470,7 @@ fn receive_like_post(
   insert_activity(&conn, user.id, &like, false)?;
 
   let post = PostForm::from_apub(&page, conn)?;
-  let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id;
+  let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id;
 
   let like_form = PostLikeForm {
     post_id,
@@ -514,7 +519,7 @@ fn receive_dislike_post(
   insert_activity(&conn, user.id, &dislike, false)?;
 
   let post = PostForm::from_apub(&page, conn)?;
-  let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id;
+  let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id;
 
   let like_form = PostLikeForm {
     post_id,
@@ -563,7 +568,7 @@ fn receive_update_comment(
   insert_activity(&conn, user.id, &update, false)?;
 
   let comment = CommentForm::from_apub(&note, &conn)?;
-  let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id;
+  let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id;
   let updated_comment = Comment::update(conn, comment_id, &comment)?;
   let post = Post::read(&conn, updated_comment.post_id)?;
 
@@ -608,7 +613,7 @@ fn receive_like_comment(
   insert_activity(&conn, user.id, &like, false)?;
 
   let comment = CommentForm::from_apub(&note, &conn)?;
-  let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id;
+  let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id;
   let like_form = CommentLikeForm {
     comment_id,
     post_id: comment.post_id,
@@ -662,7 +667,7 @@ fn receive_dislike_comment(
   insert_activity(&conn, user.id, &dislike, false)?;
 
   let comment = CommentForm::from_apub(&note, &conn)?;
-  let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id;
+  let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id;
   let like_form = CommentLikeForm {
     comment_id,
     post_id: comment.post_id,
@@ -838,7 +843,7 @@ fn receive_delete_post(
   insert_activity(&conn, user.id, &delete, false)?;
 
   let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id;
-  let post = Post::read_from_apub_id(conn, &post_ap_id)?;
+  let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?;
 
   let post_form = PostForm {
     name: post.name.to_owned(),
@@ -901,7 +906,7 @@ fn receive_remove_post(
   insert_activity(&conn, mod_.id, &remove, false)?;
 
   let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id;
-  let post = Post::read_from_apub_id(conn, &post_ap_id)?;
+  let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?;
 
   let post_form = PostForm {
     name: post.name.to_owned(),
@@ -964,7 +969,7 @@ fn receive_delete_comment(
   insert_activity(&conn, user.id, &delete, false)?;
 
   let comment_ap_id = CommentForm::from_apub(&note, &conn)?.ap_id;
-  let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?;
+  let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?;
   let comment_form = CommentForm {
     content: comment.content.to_owned(),
     parent_id: comment.parent_id,
@@ -1024,7 +1029,7 @@ fn receive_remove_comment(
   insert_activity(&conn, mod_.id, &remove, false)?;
 
   let comment_ap_id = CommentForm::from_apub(&note, &conn)?.ap_id;
-  let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?;
+  let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?;
   let comment_form = CommentForm {
     content: comment.content.to_owned(),
     parent_id: comment.parent_id,
@@ -1144,7 +1149,7 @@ fn receive_undo_delete_comment(
   insert_activity(&conn, user.id, &delete, false)?;
 
   let comment_ap_id = CommentForm::from_apub(&note, &conn)?.ap_id;
-  let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?;
+  let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?;
   let comment_form = CommentForm {
     content: comment.content.to_owned(),
     parent_id: comment.parent_id,
@@ -1204,7 +1209,7 @@ fn receive_undo_remove_comment(
   insert_activity(&conn, mod_.id, &remove, false)?;
 
   let comment_ap_id = CommentForm::from_apub(&note, &conn)?.ap_id;
-  let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?;
+  let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?;
   let comment_form = CommentForm {
     content: comment.content.to_owned(),
     parent_id: comment.parent_id,
@@ -1264,7 +1269,7 @@ fn receive_undo_delete_post(
   insert_activity(&conn, user.id, &delete, false)?;
 
   let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id;
-  let post = Post::read_from_apub_id(conn, &post_ap_id)?;
+  let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?;
 
   let post_form = PostForm {
     name: post.name.to_owned(),
@@ -1327,7 +1332,7 @@ fn receive_undo_remove_post(
   insert_activity(&conn, mod_.id, &remove, false)?;
 
   let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id;
-  let post = Post::read_from_apub_id(conn, &post_ap_id)?;
+  let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?;
 
   let post_form = PostForm {
     name: post.name.to_owned(),
@@ -1537,7 +1542,7 @@ fn receive_undo_like_comment(
   insert_activity(&conn, user.id, &like, false)?;
 
   let comment = CommentForm::from_apub(&note, &conn)?;
-  let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id;
+  let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id;
   let like_form = CommentLikeForm {
     comment_id,
     post_id: comment.post_id,
@@ -1586,7 +1591,7 @@ fn receive_undo_like_post(
   insert_activity(&conn, user.id, &like, false)?;
 
   let post = PostForm::from_apub(&page, conn)?;
-  let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id;
+  let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id;
 
   let like_form = PostLikeForm {
     post_id,