From 86dc50f9f0f6ea98273eb3845d5486d2fad8c1ff Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 26 Jun 2020 21:12:41 -0400 Subject: Some fixes to federation. - Advanced code migrations now disable then re-enable triggers. Brings run time down to < 15 seconds, no need to thread them. - Changing ap_ids and actor_ids in migrations to a fake url, so it doesn't break XsdAnyUri in activitystreams. --- docker/federation/docker-compose.yml | 11 ------ .../up.sql | 4 +-- .../up.sql | 4 +-- .../down.sql | 2 +- .../up.sql | 2 +- server/src/api/comment.rs | 2 +- server/src/api/post.rs | 2 +- server/src/api/user.rs | 2 +- server/src/db/activity.rs | 2 +- server/src/db/code_migrations.rs | 41 ++++++++++++++++------ server/src/db/comment.rs | 12 +++---- server/src/db/comment_view.rs | 12 +++---- server/src/db/community.rs | 6 ++-- server/src/db/moderator.rs | 10 +++--- server/src/db/password_reset_request.rs | 2 +- server/src/db/post.rs | 8 ++--- server/src/db/post_view.rs | 10 +++--- server/src/db/private_message.rs | 8 ++--- server/src/db/user.rs | 4 +-- server/src/db/user_mention.rs | 10 +++--- 20 files changed, 81 insertions(+), 73 deletions(-) diff --git a/docker/federation/docker-compose.yml b/docker/federation/docker-compose.yml index 585f2b4b..900f7c9c 100644 --- a/docker/federation/docker-compose.yml +++ b/docker/federation/docker-compose.yml @@ -18,7 +18,6 @@ services: - lemmy_gamma - pictrs_gamma - iframely - restart: "always" lemmy_alpha: image: lemmy-federation:latest @@ -36,7 +35,6 @@ services: - LEMMY_SETUP__SITE_NAME=lemmy_alpha - RUST_BACKTRACE=1 - RUST_LOG=debug - restart: always depends_on: - postgres_alpha postgres_alpha: @@ -47,13 +45,11 @@ services: - POSTGRES_DB=lemmy volumes: - ./volumes/postgres_alpha:/var/lib/postgresql/data - restart: always pictrs_alpha: image: asonix/pictrs:v0.1.13-r0 user: 991:991 volumes: - ./volumes/pictrs_alpha:/mnt - restart: always lemmy_beta: image: lemmy-federation:latest @@ -71,7 +67,6 @@ services: - LEMMY_SETUP__SITE_NAME=lemmy_beta - RUST_BACKTRACE=1 - RUST_LOG=debug - restart: always depends_on: - postgres_beta postgres_beta: @@ -82,13 +77,11 @@ services: - POSTGRES_DB=lemmy volumes: - ./volumes/postgres_beta:/var/lib/postgresql/data - restart: always pictrs_beta: image: asonix/pictrs:v0.1.13-r0 user: 991:991 volumes: - ./volumes/pictrs_beta:/mnt - restart: always lemmy_gamma: image: lemmy-federation:latest @@ -106,7 +99,6 @@ services: - LEMMY_SETUP__SITE_NAME=lemmy_gamma - RUST_BACKTRACE=1 - RUST_LOG=debug - restart: always depends_on: - postgres_gamma postgres_gamma: @@ -117,16 +109,13 @@ services: - POSTGRES_DB=lemmy volumes: - ./volumes/postgres_gamma:/var/lib/postgresql/data - restart: always pictrs_gamma: image: asonix/pictrs:v0.1.13-r0 user: 991:991 volumes: - ./volumes/pictrs_gamma:/mnt - restart: always iframely: image: dogbin/iframely:latest volumes: - ../iframely.config.local.js:/iframely/config.local.js:ro - restart: always diff --git a/server/migrations/2020-03-26-192410_add_activitypub_tables/up.sql b/server/migrations/2020-03-26-192410_add_activitypub_tables/up.sql index 8fe3b8ed..06db580e 100644 --- a/server/migrations/2020-03-26-192410_add_activitypub_tables/up.sql +++ b/server/migrations/2020-03-26-192410_add_activitypub_tables/up.sql @@ -15,7 +15,7 @@ create unique index idx_activity_unique_apid on activity ((data ->> 'id'::text)) -- Add federation columns to the two actor tables alter table user_ -- TODO uniqueness constraints should be added on these 3 columns later -add column actor_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local +add column actor_id character varying(255) not null default 'http://fake.com', -- This needs to be checked and updated in code, building from the site url if local add column bio text, -- not on community, already has description add column local boolean not null default true, add column private_key text, -- These need to be generated from code @@ -25,7 +25,7 @@ add column last_refreshed_at timestamp not null default now() -- Used to re-fetc -- Community alter table community -add column actor_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local +add column actor_id character varying(255) not null default 'http://fake.com', -- This needs to be checked and updated in code, building from the site url if local add column local boolean not null default true, add column private_key text, -- These need to be generated from code add column public_key text, diff --git a/server/migrations/2020-04-03-194936_add_activitypub_for_posts_and_comments/up.sql b/server/migrations/2020-04-03-194936_add_activitypub_for_posts_and_comments/up.sql index a3fb9562..6c930cd8 100644 --- a/server/migrations/2020-04-03-194936_add_activitypub_for_posts_and_comments/up.sql +++ b/server/migrations/2020-04-03-194936_add_activitypub_for_posts_and_comments/up.sql @@ -2,13 +2,13 @@ alter table post -- TODO uniqueness constraints should be added on these 3 columns later -add column ap_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local +add column ap_id character varying(255) not null default 'http://fake.com', -- This needs to be checked and updated in code, building from the site url if local add column local boolean not null default true ; alter table comment -- TODO uniqueness constraints should be added on these 3 columns later -add column ap_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local +add column ap_id character varying(255) not null default 'http://fake.com', -- This needs to be checked and updated in code, building from the site url if local add column local boolean not null default true ; diff --git a/server/migrations/2020-04-07-135912_add_user_community_apub_constraints/down.sql b/server/migrations/2020-04-07-135912_add_user_community_apub_constraints/down.sql index faf24fdc..46fc953a 100644 --- a/server/migrations/2020-04-07-135912_add_user_community_apub_constraints/down.sql +++ b/server/migrations/2020-04-07-135912_add_user_community_apub_constraints/down.sql @@ -2,7 +2,7 @@ drop view user_view cascade; alter table user_ -add column fedi_name varchar(40) not null default 'changeme'; +add column fedi_name varchar(40) not null default 'http://fake.com'; alter table user_ add constraint user__name_fedi_name_key unique (name, fedi_name); diff --git a/server/migrations/2020-05-05-210233_add_activitypub_for_private_messages/up.sql b/server/migrations/2020-05-05-210233_add_activitypub_for_private_messages/up.sql index 627be1f3..9f152ada 100644 --- a/server/migrations/2020-05-05-210233_add_activitypub_for_private_messages/up.sql +++ b/server/migrations/2020-05-05-210233_add_activitypub_for_private_messages/up.sql @@ -1,5 +1,5 @@ alter table private_message -add column ap_id character varying(255) not null default 'changeme', -- This needs to be checked and updated in code, building from the site url if local +add column ap_id character varying(255) not null default 'http://fake.com', -- This needs to be checked and updated in code, building from the site url if local add column local boolean not null default true ; diff --git a/server/src/api/comment.rs b/server/src/api/comment.rs index 369cba5c..56217458 100644 --- a/server/src/api/comment.rs +++ b/server/src/api/comment.rs @@ -140,7 +140,7 @@ impl Perform for Oper { read: None, published: None, updated: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, }; diff --git a/server/src/api/post.rs b/server/src/api/post.rs index 420bef1f..a3ac4915 100644 --- a/server/src/api/post.rs +++ b/server/src/api/post.rs @@ -172,7 +172,7 @@ impl Perform for Oper { embed_description: iframely_description, embed_html: iframely_html, thumbnail_url: pictrs_thumbnail, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, published: None, }; diff --git a/server/src/api/user.rs b/server/src/api/user.rs index f68a1a82..0b6458e7 100644 --- a/server/src/api/user.rs +++ b/server/src/api/user.rs @@ -1107,7 +1107,7 @@ impl Perform for Oper { deleted: None, read: None, updated: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, published: None, }; diff --git a/server/src/db/activity.rs b/server/src/db/activity.rs index 714c8202..ccd1e682 100644 --- a/server/src/db/activity.rs +++ b/server/src/db/activity.rs @@ -101,7 +101,7 @@ mod tests { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), bio: None, local: true, private_key: None, diff --git a/server/src/db/code_migrations.rs b/server/src/db/code_migrations.rs index 204bfe79..6f1b656f 100644 --- a/server/src/db/code_migrations.rs +++ b/server/src/db/code_migrations.rs @@ -16,12 +16,11 @@ use failure::Error; use log::info; pub fn run_advanced_migrations(conn: &PgConnection) -> Result<(), Error> { - user_updates_2020_04_02(conn)?; - community_updates_2020_04_02(conn)?; - post_updates_2020_04_03(conn)?; - comment_updates_2020_04_03(conn)?; - private_message_updates_2020_05_05(conn)?; - + user_updates_2020_04_02(&conn)?; + community_updates_2020_04_02(&conn)?; + post_updates_2020_04_03(&conn)?; + comment_updates_2020_04_03(&conn)?; + private_message_updates_2020_05_05(&conn)?; Ok(()) } @@ -32,10 +31,12 @@ fn user_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> { // Update the actor_id, private_key, and public_key, last_refreshed_at let incorrect_users = user_ - .filter(actor_id.eq("changeme")) + .filter(actor_id.eq("http://fake.com")) .filter(local.eq(true)) .load::(conn)?; + sql_query("alter table user_ disable trigger refresh_user").execute(conn)?; + for cuser in &incorrect_users { let keypair = generate_actor_keypair()?; @@ -67,6 +68,8 @@ fn user_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> { User_::update(&conn, cuser.id, &form)?; } + sql_query("alter table user_ enable trigger refresh_user").execute(conn)?; + info!("{} user rows updated.", incorrect_users.len()); Ok(()) @@ -79,10 +82,12 @@ fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> { // Update the actor_id, private_key, and public_key, last_refreshed_at let incorrect_communities = community - .filter(actor_id.eq("changeme")) + .filter(actor_id.eq("http://fake.com")) .filter(local.eq(true)) .load::(conn)?; + sql_query("alter table community disable trigger refresh_community").execute(conn)?; + for ccommunity in &incorrect_communities { let keypair = generate_actor_keypair()?; @@ -107,6 +112,8 @@ fn community_updates_2020_04_02(conn: &PgConnection) -> Result<(), Error> { Community::update(&conn, ccommunity.id, &form)?; } + sql_query("alter table community enable trigger refresh_community").execute(conn)?; + info!("{} community rows updated.", incorrect_communities.len()); Ok(()) @@ -119,16 +126,20 @@ fn post_updates_2020_04_03(conn: &PgConnection) -> Result<(), Error> { // Update the ap_id let incorrect_posts = post - .filter(ap_id.eq("changeme")) + .filter(ap_id.eq("http://fake.com")) .filter(local.eq(true)) .load::(conn)?; + sql_query("alter table post disable trigger refresh_post").execute(conn)?; + for cpost in &incorrect_posts { Post::update_ap_id(&conn, cpost.id)?; } info!("{} post rows updated.", incorrect_posts.len()); + sql_query("alter table post enable trigger refresh_post").execute(conn)?; + Ok(()) } @@ -139,14 +150,18 @@ fn comment_updates_2020_04_03(conn: &PgConnection) -> Result<(), Error> { // Update the ap_id let incorrect_comments = comment - .filter(ap_id.eq("changeme")) + .filter(ap_id.eq("http://fake.com")) .filter(local.eq(true)) .load::(conn)?; + sql_query("alter table comment disable trigger refresh_comment").execute(conn)?; + for ccomment in &incorrect_comments { Comment::update_ap_id(&conn, ccomment.id)?; } + sql_query("alter table comment enable trigger refresh_comment").execute(conn)?; + info!("{} comment rows updated.", incorrect_comments.len()); Ok(()) @@ -159,14 +174,18 @@ fn private_message_updates_2020_05_05(conn: &PgConnection) -> Result<(), Error> // Update the ap_id let incorrect_pms = private_message - .filter(ap_id.eq("changeme")) + .filter(ap_id.eq("http://fake.com")) .filter(local.eq(true)) .load::(conn)?; + sql_query("alter table private_message disable trigger refresh_private_message").execute(conn)?; + for cpm in &incorrect_pms { PrivateMessage::update_ap_id(&conn, cpm.id)?; } + sql_query("alter table private_message enable trigger refresh_private_message").execute(conn)?; + info!("{} private message rows updated.", incorrect_pms.len()); Ok(()) diff --git a/server/src/db/comment.rs b/server/src/db/comment.rs index ddf1feef..8a2aa889 100644 --- a/server/src/db/comment.rs +++ b/server/src/db/comment.rs @@ -229,7 +229,7 @@ mod tests { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), bio: None, local: true, private_key: None, @@ -249,7 +249,7 @@ mod tests { deleted: None, updated: None, nsfw: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), local: true, private_key: None, public_key: None, @@ -275,7 +275,7 @@ mod tests { embed_description: None, embed_html: None, thumbnail_url: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, published: None, }; @@ -292,7 +292,7 @@ mod tests { parent_id: None, published: None, updated: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, }; @@ -309,7 +309,7 @@ mod tests { parent_id: None, published: inserted_comment.published, updated: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, }; @@ -323,7 +323,7 @@ mod tests { read: None, published: None, updated: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, }; diff --git a/server/src/db/comment_view.rs b/server/src/db/comment_view.rs index 52f7d949..a37cdbcd 100644 --- a/server/src/db/comment_view.rs +++ b/server/src/db/comment_view.rs @@ -481,7 +481,7 @@ mod tests { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), bio: None, local: true, private_key: None, @@ -501,7 +501,7 @@ mod tests { deleted: None, updated: None, nsfw: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), local: true, private_key: None, public_key: None, @@ -527,7 +527,7 @@ mod tests { embed_description: None, embed_html: None, thumbnail_url: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, published: None, }; @@ -544,7 +544,7 @@ mod tests { read: None, published: None, updated: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, }; @@ -584,7 +584,7 @@ mod tests { my_vote: None, subscribed: None, saved: None, - ap_id: "changeme".to_string(), + ap_id: "http://fake.com".to_string(), local: true, community_actor_id: inserted_community.actor_id.to_owned(), community_local: true, @@ -617,7 +617,7 @@ mod tests { my_vote: Some(1), subscribed: None, saved: None, - ap_id: "changeme".to_string(), + ap_id: "http://fake.com".to_string(), local: true, community_actor_id: inserted_community.actor_id.to_owned(), community_local: true, diff --git a/server/src/db/community.rs b/server/src/db/community.rs index 885c9779..38ad07fc 100644 --- a/server/src/db/community.rs +++ b/server/src/db/community.rs @@ -256,7 +256,7 @@ mod tests { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), bio: None, local: true, private_key: None, @@ -276,7 +276,7 @@ mod tests { removed: None, deleted: None, updated: None, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), local: true, private_key: None, public_key: None, @@ -298,7 +298,7 @@ mod tests { deleted: false, published: inserted_community.published, updated: None, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), local: true, private_key: None, public_key: None, diff --git a/server/src/db/moderator.rs b/server/src/db/moderator.rs index d0f589e3..44b04ec6 100644 --- a/server/src/db/moderator.rs +++ b/server/src/db/moderator.rs @@ -465,7 +465,7 @@ mod tests { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), bio: None, local: true, private_key: None, @@ -492,7 +492,7 @@ mod tests { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), bio: None, local: true, private_key: None, @@ -512,7 +512,7 @@ mod tests { deleted: None, updated: None, nsfw: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), local: true, private_key: None, public_key: None, @@ -538,7 +538,7 @@ mod tests { embed_description: None, embed_html: None, thumbnail_url: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, published: None, }; @@ -555,7 +555,7 @@ mod tests { parent_id: None, published: None, updated: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, }; diff --git a/server/src/db/password_reset_request.rs b/server/src/db/password_reset_request.rs index 32d7f187..b92d70ed 100644 --- a/server/src/db/password_reset_request.rs +++ b/server/src/db/password_reset_request.rs @@ -105,7 +105,7 @@ mod tests { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), bio: None, local: true, private_key: None, diff --git a/server/src/db/post.rs b/server/src/db/post.rs index d12f98d8..91c1dcbf 100644 --- a/server/src/db/post.rs +++ b/server/src/db/post.rs @@ -268,7 +268,7 @@ mod tests { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), bio: None, local: true, private_key: None, @@ -288,7 +288,7 @@ mod tests { deleted: None, updated: None, nsfw: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), local: true, private_key: None, public_key: None, @@ -314,7 +314,7 @@ mod tests { embed_description: None, embed_html: None, thumbnail_url: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, published: None, }; @@ -339,7 +339,7 @@ mod tests { embed_description: None, embed_html: None, thumbnail_url: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, }; diff --git a/server/src/db/post_view.rs b/server/src/db/post_view.rs index 09488942..fbbf658d 100644 --- a/server/src/db/post_view.rs +++ b/server/src/db/post_view.rs @@ -395,7 +395,7 @@ mod tests { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), bio: None, local: true, private_key: None, @@ -415,7 +415,7 @@ mod tests { deleted: None, updated: None, nsfw: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), local: true, private_key: None, public_key: None, @@ -441,7 +441,7 @@ mod tests { embed_description: None, embed_html: None, thumbnail_url: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, published: None, }; @@ -508,7 +508,7 @@ mod tests { embed_description: None, embed_html: None, thumbnail_url: None, - ap_id: "changeme".to_string(), + ap_id: "http://fake.com".to_string(), local: true, creator_actor_id: inserted_user.actor_id.to_owned(), creator_local: true, @@ -553,7 +553,7 @@ mod tests { embed_description: None, embed_html: None, thumbnail_url: None, - ap_id: "changeme".to_string(), + ap_id: "http://fake.com".to_string(), local: true, creator_actor_id: inserted_user.actor_id.to_owned(), creator_local: true, diff --git a/server/src/db/private_message.rs b/server/src/db/private_message.rs index e8b28485..9d362bbf 100644 --- a/server/src/db/private_message.rs +++ b/server/src/db/private_message.rs @@ -113,7 +113,7 @@ mod tests { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), bio: None, local: true, private_key: None, @@ -140,7 +140,7 @@ mod tests { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), bio: None, local: true, private_key: None, @@ -158,7 +158,7 @@ mod tests { read: None, published: None, updated: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, }; @@ -173,7 +173,7 @@ mod tests { read: false, updated: None, published: inserted_private_message.published, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, }; diff --git a/server/src/db/user.rs b/server/src/db/user.rs index 8b2ff62a..eaf9d292 100644 --- a/server/src/db/user.rs +++ b/server/src/db/user.rs @@ -237,7 +237,7 @@ mod tests { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), bio: None, local: true, private_key: None, @@ -266,7 +266,7 @@ mod tests { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), bio: None, local: true, private_key: None, diff --git a/server/src/db/user_mention.rs b/server/src/db/user_mention.rs index b9129b01..1d54fa98 100644 --- a/server/src/db/user_mention.rs +++ b/server/src/db/user_mention.rs @@ -81,7 +81,7 @@ mod tests { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), bio: None, local: true, private_key: None, @@ -108,7 +108,7 @@ mod tests { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), bio: None, local: true, private_key: None, @@ -128,7 +128,7 @@ mod tests { deleted: None, updated: None, nsfw: false, - actor_id: "changeme".into(), + actor_id: "http://fake.com".into(), local: true, private_key: None, public_key: None, @@ -154,7 +154,7 @@ mod tests { embed_description: None, embed_html: None, thumbnail_url: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, published: None, }; @@ -171,7 +171,7 @@ mod tests { parent_id: None, published: None, updated: None, - ap_id: "changeme".into(), + ap_id: "http://fake.com".into(), local: true, }; -- cgit v1.2.3