Parse out in reply to field. Fixes #694 (#799)

- When a comment or post doesn't exist locally, yet we receive
  an activitypub with it (for example, a nested comment update,
  for a community we just subscribed to, also with a post we
  don't have...), then fetch it.

4 files changed, 237 insertions, 28 deletions

diff --git a/server/src/apub/comment.rs b/server/src/apub/comment.rs
index d18908d0..0a513f33 100644
--- a/server/src/apub/comment.rs
+++ b/server/src/apub/comment.rs
@@ -5,7 +5,11 @@ use crate::{
     create_apub_tombstone_response,
     create_tombstone,
     fetch_webfinger_url,
-    fetcher::get_or_fetch_and_upsert_remote_user,
+    fetcher::{
+      get_or_fetch_and_insert_remote_comment,
+      get_or_fetch_and_insert_remote_post,
+      get_or_fetch_and_upsert_remote_user,
+    },
     ActorType,
     ApubLikeableType,
     ApubObjectType,
@@ -115,22 +119,21 @@ impl FromApub for CommentForm {
     let mut in_reply_tos = oprops.get_many_in_reply_to_xsd_any_uris().unwrap();
     let post_ap_id = in_reply_tos.next().unwrap().to_string();
 
+    // This post, or the parent comment might not yet exist on this server yet, fetch them.
+    let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?;
+
     // The 2nd item, if it exists, is the parent comment apub_id
+    // For deeply nested comments, FromApub automatically gets called recursively
     let parent_id: Option<i32> = match in_reply_tos.next() {
       Some(parent_comment_uri) => {
-        let parent_comment_uri_str = &parent_comment_uri.to_string();
-        let parent_comment = Comment::read_from_apub_id(&conn, &parent_comment_uri_str)?;
+        let parent_comment_ap_id = &parent_comment_uri.to_string();
+        let parent_comment = get_or_fetch_and_insert_remote_comment(&parent_comment_ap_id, &conn)?;
 
         Some(parent_comment.id)
       }
       None => None,
     };
 
-    // TODO this failed because a mention on a post that wasn't on this server yet. Has to do with
-    // fetching replytos
-    dbg!(&post_ap_id);
-    let post = Post::read_from_apub_id(&conn, &post_ap_id)?;
-
     Ok(CommentForm {
       creator_id: creator.id,
       post_id: post.id,
diff --git a/server/src/apub/fetcher.rs b/server/src/apub/fetcher.rs
index c7467c5b..05b7492e 100644
--- a/server/src/apub/fetcher.rs
+++ b/server/src/apub/fetcher.rs
@@ -254,6 +254,22 @@ fn upsert_post(post_form: &PostForm, conn: &PgConnection) -> Result<Post, Error>
   }
 }
 
+pub fn get_or_fetch_and_insert_remote_post(
+  post_ap_id: &str,
+  conn: &PgConnection,
+) -> Result<Post, Error> {
+  match Post::read_from_apub_id(conn, post_ap_id) {
+    Ok(p) => Ok(p),
+    Err(NotFound {}) => {
+      debug!("Fetching and creating remote post: {}", post_ap_id);
+      let post = fetch_remote_object::<PageExt>(&Url::parse(post_ap_id)?)?;
+      let post_form = PostForm::from_apub(&post, conn)?;
+      Ok(Post::create(conn, &post_form)?)
+    }
+    Err(e) => Err(Error::from(e)),
+  }
+}
+
 fn upsert_comment(comment_form: &CommentForm, conn: &PgConnection) -> Result<Comment, Error> {
   let existing = Comment::read_from_apub_id(conn, &comment_form.ap_id);
   match existing {
@@ -263,6 +279,25 @@ fn upsert_comment(comment_form: &CommentForm, conn: &PgConnection) -> Result<Com
   }
 }
 
+pub fn get_or_fetch_and_insert_remote_comment(
+  comment_ap_id: &str,
+  conn: &PgConnection,
+) -> Result<Comment, Error> {
+  match Comment::read_from_apub_id(conn, comment_ap_id) {
+    Ok(p) => Ok(p),
+    Err(NotFound {}) => {
+      debug!(
+        "Fetching and creating remote comment and its parents: {}",
+        comment_ap_id
+      );
+      let comment = fetch_remote_object::<Note>(&Url::parse(comment_ap_id)?)?;
+      let comment_form = CommentForm::from_apub(&comment, conn)?;
+      Ok(Comment::create(conn, &comment_form)?)
+    }
+    Err(e) => Err(Error::from(e)),
+  }
+}
+
 // TODO It should not be fetching data from a community outbox.
 // All posts, comments, comment likes, etc should be posts to our community_inbox
 // The only data we should be periodically fetching (if it hasn't been fetched in the last day
diff --git a/server/src/apub/shared_inbox.rs b/server/src/apub/shared_inbox.rs
index 3ecc873e..1ada6ad1 100644
--- a/server/src/apub/shared_inbox.rs
+++ b/server/src/apub/shared_inbox.rs
@@ -6,7 +6,12 @@ use crate::{
   },
   apub::{
     extensions::signatures::verify,
-    fetcher::{get_or_fetch_and_upsert_remote_community, get_or_fetch_and_upsert_remote_user},
+    fetcher::{
+      get_or_fetch_and_insert_remote_comment,
+      get_or_fetch_and_insert_remote_post,
+      get_or_fetch_and_upsert_remote_community,
+      get_or_fetch_and_upsert_remote_user,
+    },
     FromApub,
     GroupExt,
     PageExt,
@@ -427,7 +432,7 @@ fn receive_update_post(
   insert_activity(&conn, user.id, &update, false)?;
 
   let post = PostForm::from_apub(&page, conn)?;
-  let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id;
+  let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id;
   Post::update(conn, post_id, &post)?;
 
   // Refetch the view
@@ -465,7 +470,7 @@ fn receive_like_post(
   insert_activity(&conn, user.id, &like, false)?;
 
   let post = PostForm::from_apub(&page, conn)?;
-  let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id;
+  let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id;
 
   let like_form = PostLikeForm {
     post_id,
@@ -514,7 +519,7 @@ fn receive_dislike_post(
   insert_activity(&conn, user.id, &dislike, false)?;
 
   let post = PostForm::from_apub(&page, conn)?;
-  let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id;
+  let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id;
 
   let like_form = PostLikeForm {
     post_id,
@@ -563,7 +568,7 @@ fn receive_update_comment(
   insert_activity(&conn, user.id, &update, false)?;
 
   let comment = CommentForm::from_apub(&note, &conn)?;
-  let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id;
+  let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id;
   let updated_comment = Comment::update(conn, comment_id, &comment)?;
   let post = Post::read(&conn, updated_comment.post_id)?;
 
@@ -608,7 +613,7 @@ fn receive_like_comment(
   insert_activity(&conn, user.id, &like, false)?;
 
   let comment = CommentForm::from_apub(&note, &conn)?;
-  let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id;
+  let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id;
   let like_form = CommentLikeForm {
     comment_id,
     post_id: comment.post_id,
@@ -662,7 +667,7 @@ fn receive_dislike_comment(
   insert_activity(&conn, user.id, &dislike, false)?;
 
   let comment = CommentForm::from_apub(&note, &conn)?;
-  let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id;
+  let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id;
   let like_form = CommentLikeForm {
     comment_id,
     post_id: comment.post_id,
@@ -838,7 +843,7 @@ fn receive_delete_post(
   insert_activity(&conn, user.id, &delete, false)?;
 
   let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id;
-  let post = Post::read_from_apub_id(conn, &post_ap_id)?;
+  let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?;
 
   let post_form = PostForm {
     name: post.name.to_owned(),
@@ -901,7 +906,7 @@ fn receive_remove_post(
   insert_activity(&conn, mod_.id, &remove, false)?;
 
   let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id;
-  let post = Post::read_from_apub_id(conn, &post_ap_id)?;
+  let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?;
 
   let post_form = PostForm {
     name: post.name.to_owned(),
@@ -964,7 +969,7 @@ fn receive_delete_comment(
   insert_activity(&conn, user.id, &delete, false)?;
 
   let comment_ap_id = CommentForm::from_apub(&note, &conn)?.ap_id;
-  let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?;
+  let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?;
   let comment_form = CommentForm {
     content: comment.content.to_owned(),
     parent_id: comment.parent_id,
@@ -1024,7 +1029,7 @@ fn receive_remove_comment(
   insert_activity(&conn, mod_.id, &remove, false)?;
 
   let comment_ap_id = CommentForm::from_apub(&note, &conn)?.ap_id;
-  let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?;
+  let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?;
   let comment_form = CommentForm {
     content: comment.content.to_owned(),
     parent_id: comment.parent_id,
@@ -1144,7 +1149,7 @@ fn receive_undo_delete_comment(
   insert_activity(&conn, user.id, &delete, false)?;
 
   let comment_ap_id = CommentForm::from_apub(&note, &conn)?.ap_id;
-  let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?;
+  let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?;
   let comment_form = CommentForm {
     content: comment.content.to_owned(),
     parent_id: comment.parent_id,
@@ -1204,7 +1209,7 @@ fn receive_undo_remove_comment(
   insert_activity(&conn, mod_.id, &remove, false)?;
 
   let comment_ap_id = CommentForm::from_apub(&note, &conn)?.ap_id;
-  let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?;
+  let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?;
   let comment_form = CommentForm {
     content: comment.content.to_owned(),
     parent_id: comment.parent_id,
@@ -1264,7 +1269,7 @@ fn receive_undo_delete_post(
   insert_activity(&conn, user.id, &delete, false)?;
 
   let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id;
-  let post = Post::read_from_apub_id(conn, &post_ap_id)?;
+  let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?;
 
   let post_form = PostForm {
     name: post.name.to_owned(),
@@ -1327,7 +1332,7 @@ fn receive_undo_remove_post(
   insert_activity(&conn, mod_.id, &remove, false)?;
 
   let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id;
-  let post = Post::read_from_apub_id(conn, &post_ap_id)?;
+  let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?;
 
   let post_form = PostForm {
     name: post.name.to_owned(),
@@ -1537,7 +1542,7 @@ fn receive_undo_like_comment(
   insert_activity(&conn, user.id, &like, false)?;
 
   let comment = CommentForm::from_apub(&note, &conn)?;
-  let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id;
+  let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id;
   let like_form = CommentLikeForm {
     comment_id,
     post_id: comment.post_id,
@@ -1586,7 +1591,7 @@ fn receive_undo_like_post(
   insert_activity(&conn, user.id, &like, false)?;
 
   let post = PostForm::from_apub(&page, conn)?;
-  let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id;
+  let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id;
 
   let like_form = PostLikeForm {
     post_id,
diff --git a/ui/src/api_tests/api.spec.ts b/ui/src/api_tests/api.spec.ts
index 4e38cfee..7337201c 100644
--- a/ui/src/api_tests/api.spec.ts
+++ b/ui/src/api_tests/api.spec.ts
@@ -101,7 +101,7 @@ describe('main', () => {
         nsfw: false,
       };
 
-      let createResponse: PostResponse = await fetch(
+      let createPostRes: PostResponse = await fetch(
         `${lemmyAlphaApiUrl}/post`,
         {
           method: 'POST',
@@ -111,9 +111,9 @@ describe('main', () => {
           body: wrapper(postForm),
         }
       ).then(d => d.json());
-      expect(createResponse.post.name).toBe(name);
+      expect(createPostRes.post.name).toBe(name);
 
-      let searchUrl = `${lemmyBetaApiUrl}/search?q=${createResponse.post.ap_id}&type_=All&sort=TopAll`;
+      let searchUrl = `${lemmyBetaApiUrl}/search?q=${createPostRes.post.ap_id}&type_=All&sort=TopAll`;
       let searchResponse: SearchResponse = await fetch(searchUrl, {
         method: 'GET',
       }).then(d => d.json());
@@ -1314,6 +1314,172 @@ describe('main', () => {
       expect(getPostRes.comments[0].score).toBe(1);
     });
   });
+
+  describe('fetch inreplytos', () => {
+    test('A is unsubbed from B, B makes a post, and some embedded comments, A subs to B, B updates the lowest level comment, A fetches both the post and all the inreplyto comments for that post.', async () => {
+      // Check that A is subscribed to B
+      let followedCommunitiesUrl = `${lemmyAlphaApiUrl}/user/followed_communities?&auth=${lemmyAlphaAuth}`;
+      let followedCommunitiesRes: GetFollowedCommunitiesResponse = await fetch(
+        followedCommunitiesUrl,
+        {
+          method: 'GET',
+        }
+      ).then(d => d.json());
+      expect(followedCommunitiesRes.communities[1].community_local).toBe(false);
+
+      // A unsubs from B (communities ids 3-5)
+      for (let i = 3; i <= 5; i++) {
+        let unfollowForm: FollowCommunityForm = {
+          community_id: i,
+          follow: false,
+          auth: lemmyAlphaAuth,
+        };
+
+        let unfollowRes: CommunityResponse = await fetch(
+          `${lemmyAlphaApiUrl}/community/follow`,
+          {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json',
+            },
+            body: wrapper(unfollowForm),
+          }
+        ).then(d => d.json());
+        expect(unfollowRes.community.local).toBe(false);
+      }
+
+      // Check that you are unsubscribed from all of them locally
+      let followedCommunitiesResAgain: GetFollowedCommunitiesResponse = await fetch(
+        followedCommunitiesUrl,
+        {
+          method: 'GET',
+        }
+      ).then(d => d.json());
+      expect(followedCommunitiesResAgain.communities.length).toBe(1);
+
+      // B creates a post, and two comments, should be invisible to A
+      let betaPostName = 'Test post on B, invisible to A at first';
+      let postForm: PostForm = {
+        name: betaPostName,
+        auth: lemmyBetaAuth,
+        community_id: 2,
+        creator_id: 2,
+        nsfw: false,
+      };
+
+      let createPostRes: PostResponse = await fetch(`${lemmyBetaApiUrl}/post`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: wrapper(postForm),
+      }).then(d => d.json());
+      expect(createPostRes.post.name).toBe(betaPostName);
+
+      // B creates a comment, then a child one of that.
+      let parentCommentContent = 'An invisible top level comment from beta';
+      let createParentCommentForm: CommentForm = {
+        content: parentCommentContent,
+        post_id: createPostRes.post.id,
+        auth: lemmyBetaAuth,
+      };
+
+      let createParentCommentRes: CommentResponse = await fetch(
+        `${lemmyBetaApiUrl}/comment`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+          body: wrapper(createParentCommentForm),
+        }
+      ).then(d => d.json());
+      expect(createParentCommentRes.comment.content).toBe(parentCommentContent);
+
+      let childCommentContent = 'An invisible child comment from beta';
+      let createChildCommentForm: CommentForm = {
+        content: childCommentContent,
+        parent_id: createParentCommentRes.comment.id,
+        post_id: createPostRes.post.id,
+        auth: lemmyBetaAuth,
+      };
+
+      let createChildCommentRes: CommentResponse = await fetch(
+        `${lemmyBetaApiUrl}/comment`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+          body: wrapper(createChildCommentForm),
+        }
+      ).then(d => d.json());
+      expect(createChildCommentRes.comment.content).toBe(childCommentContent);
+
+      // Follow again, for other tests
+      let searchUrl = `${lemmyAlphaApiUrl}/search?q=!main@lemmy_beta:8550&type_=All&sort=TopAll`;
+
+      let searchResponse: SearchResponse = await fetch(searchUrl, {
+        method: 'GET',
+      }).then(d => d.json());
+
+      expect(searchResponse.communities[0].name).toBe('main');
+
+      let followForm: FollowCommunityForm = {
+        community_id: searchResponse.communities[0].id,
+        follow: true,
+        auth: lemmyAlphaAuth,
+      };
+
+      let followResAgain: CommunityResponse = await fetch(
+        `${lemmyAlphaApiUrl}/community/follow`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+          body: wrapper(followForm),
+        }
+      ).then(d => d.json());
+
+      // Make sure the follow response went through
+      expect(followResAgain.community.local).toBe(false);
+      expect(followResAgain.community.name).toBe('main');
+
+      let updatedCommentContent = 'An update child comment from beta';
+      let updatedCommentForm: CommentForm = {
+        content: updatedCommentContent,
+        post_id: createPostRes.post.id,
+        edit_id: createChildCommentRes.comment.id,
+        auth: lemmyBetaAuth,
+        creator_id: 2,
+      };
+
+      let updateResponse: CommentResponse = await fetch(
+        `${lemmyBetaApiUrl}/comment`,
+        {
+          method: 'PUT',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+          body: wrapper(updatedCommentForm),
+        }
+      ).then(d => d.json());
+      expect(updateResponse.comment.content).toBe(updatedCommentContent);
+
+      // Make sure that A picked up the post, parent comment, and child comment
+      let getPostUrl = `${lemmyAlphaApiUrl}/post?id=6`;
+      let getPostRes: GetPostResponse = await fetch(getPostUrl, {
+        method: 'GET',
+      }).then(d => d.json());
+
+      expect(getPostRes.post.name).toBe(betaPostName);
+      expect(getPostRes.comments[1].content).toBe(parentCommentContent);
+      expect(getPostRes.comments[0].content).toBe(updatedCommentContent);
+      expect(getPostRes.post.community_local).toBe(false);
+      expect(getPostRes.post.creator_local).toBe(false);
+    });
+  });
 });
 
 function wrapper(form: any): string {