Add basic support for cleaning up Activity Pub notes, rewriting mentions and converting HTML to Markdown

2 months ago · 288341f719
parent 8272c10704
commit 288341f719
5 changed files with 133 additions and 64 deletions
--- a/go.mod
+++ b/go.mod
@ -11,6 +11,7 @@ require (
 	git.mills.io/prologic/go-gopher v0.0.0-20220331140345-72e36e5710a1
 	git.mills.io/prologic/observe v0.0.0-20210712230028-fc31c7aa2bd1
 	git.mills.io/prologic/read-file-last-line v0.0.0-20210710073401-af293d63a6d0
+	github.com/JohannesKaufmann/html-to-markdown v1.3.7
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/PuerkitoBio/goquery v1.8.1
@ -80,10 +81,10 @@ require (
 	go.yarn.social/lextwt v0.0.0-20230305034352-b23f5582ef81
 	go.yarn.social/types v0.0.0-20230305013457-e4d91e351ac8
 	golang.org/x/crypto v0.6.0
-	golang.org/x/net v0.7.0
+	golang.org/x/net v0.8.0
 	golang.org/x/sync v0.1.0
-	golang.org/x/term v0.5.0
-	golang.org/x/text v0.7.0
+	golang.org/x/term v0.6.0
+	golang.org/x/text v0.8.0
 	gopkg.in/yaml.v2 v2.4.0
 	willnorris.com/go/microformats v1.2.0
 )
@ -166,7 +167,7 @@ require (
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
-	github.com/sergi/go-diff v1.0.0 // indirect
+	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/sour-is/go-passwd v0.1.0
 	github.com/spf13/afero v1.9.4 // indirect
--- a/go.sum
+++ b/go.sum
@ -52,6 +52,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU=
 github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/JohannesKaufmann/html-to-markdown v1.3.7 h1:06rF6ct6hDbB7ur380y9Vv26UowFdTFYljSv6f4VjdI=
+github.com/JohannesKaufmann/html-to-markdown v1.3.7/go.mod h1:BzWBqKEgKeVFX4EHEF98koY2ZnAfUM6ahWmXSWAAq9o=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=
@ -623,10 +625,13 @@ github.com/savsgio/gotils v0.0.0-20210921075833-21a6215cb0e4 h1:ocK/D6lCgLji37Z2
 github.com/schollz/progressbar/v3 v3.13.0 h1:9TeeWRcjW2qd05I8Kf9knPkW4vLM/hYoa6z9ABvxje8=
 github.com/schollz/progressbar/v3 v3.13.0/go.mod h1:ZBYnSuLAX2LU8P8UiKN/KgF2DY58AJC8yfVYLPC8Ly4=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y=
+github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
 github.com/securisec/go-keywords v0.0.0-20200619134240-769e7273f2ed h1:8ZFy/8C1JaByuTedmMDrLgK7dH/7KPKKXiuJDU0KJYg=
 github.com/securisec/go-keywords v0.0.0-20200619134240-769e7273f2ed/go.mod h1:ewJJMApUajQGvQOaQb/QyzTLoL619B5D02XOZlGnlNo=
-github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
+github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
@ -752,6 +757,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU=
+github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
@ -779,16 +786,9 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 go.yarn.social/client v0.0.0-20221026065557-81f92f9fecbc h1:2NyyP+dsr0Z3g2b585pKOBNkPPrJx6VTCTLSXdU3PAY=
 go.yarn.social/client v0.0.0-20221026065557-81f92f9fecbc/go.mod h1:yihDO7q8YbxV2oVnqpNjj6vFlmjnDfDed2MLdYe1Z6k=
-go.yarn.social/lextwt v0.0.0-20230304005743-703ab710c2bd h1:zmtvG8b+7yX2grc0I46kPMyzf9QmVCZ74WkrOJiA83Y=
-go.yarn.social/lextwt v0.0.0-20230304005743-703ab710c2bd/go.mod h1:3n4ul1qR3tYnV9og38s5YcQxnRjbz4w/XGywPFZpg4k=
-go.yarn.social/lextwt v0.0.0-20230305030618-194713276fa6 h1:I1wYV7zzDiQO44vxE+oBSrekBeXYt3CepfIsxjJsh3o=
-go.yarn.social/lextwt v0.0.0-20230305030618-194713276fa6/go.mod h1:VekSjgME7E/pEDo5dx1aohNr/9/vOTDAFoCdoI7mOKE=
-go.yarn.social/lextwt v0.0.0-20230305033205-d2ace007ed88 h1:nc7RAZ8a/jYjQcV2rKam7QG80lrv5xhud0Qf/ko+qlo=
-go.yarn.social/lextwt v0.0.0-20230305033205-d2ace007ed88/go.mod h1:VekSjgME7E/pEDo5dx1aohNr/9/vOTDAFoCdoI7mOKE=
 go.yarn.social/lextwt v0.0.0-20230305034352-b23f5582ef81 h1:8H4Kdzgarv2fgQdhKi57bgNl2jDUs+cRLjpfvw9Q8y0=
 go.yarn.social/lextwt v0.0.0-20230305034352-b23f5582ef81/go.mod h1:VekSjgME7E/pEDo5dx1aohNr/9/vOTDAFoCdoI7mOKE=
 go.yarn.social/types v0.0.0-20221026065125-bb0847d28781/go.mod h1:XN+G4HprNn/Gp7OF2zveqsCRSWFCHtOaIRh2GlcK+U4=
-go.yarn.social/types v0.0.0-20230129042829-96789c694b24/go.mod h1:+xnDkQ0T0S8emxWIsvxlCAoyF8gBaj0q81hr/VrKc0c=
 go.yarn.social/types v0.0.0-20230305013457-e4d91e351ac8 h1:zfnniiSO/WO65mSpdQzGYJ9pM0rYg/BKgrSm8h2mTyA=
 go.yarn.social/types v0.0.0-20230305013457-e4d91e351ac8/go.mod h1:+xnDkQ0T0S8emxWIsvxlCAoyF8gBaj0q81hr/VrKc0c=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@ -857,6 +857,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -905,8 +906,9 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
+golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -999,7 +1001,6 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@ -1007,8 +1008,9 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
-golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
+golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -1019,8 +1021,9 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
+golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@ -1080,6 +1083,7 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -1221,6 +1225,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
--- a/internal/activitypub_handlers.go
+++ b/internal/activitypub_handlers.go
@ -1,16 +1,86 @@
 package internal

 import (
+	"fmt"
+	"strings"
+
 	"encoding/json"
 	"net/http"

 	"git.mills.io/yarnsocial/yarn/internal/activitypub"
+	md "github.com/JohannesKaufmann/html-to-markdown"
+	"github.com/PuerkitoBio/goquery"
 	"github.com/julienschmidt/httprouter"
 	"github.com/rickb777/accept"
 	log "github.com/sirupsen/logrus"
 	"go.yarn.social/types"
 )

+// TwtFromNote ...
+type TwtFromNote func(note *activitypub.Note) (types.Twt, error)
+
+// CleanNote ...
+func CleanNote(s string) (string, error) {
+	p := strings.NewReader(s)
+
+	doc, err := goquery.NewDocumentFromReader(p)
+	if err != nil {
+		return "", err
+	}
+
+	doc.Find(".h-card").Each(func(i int, el *goquery.Selection) {
+		uri := el.Find(".u-url.mention").First().AttrOr("href", "")
+		user := strings.TrimPrefix(el.Text(), "@")
+		el.SetText(fmt.Sprintf(`@<%s %s>`, user, uri))
+	})
+
+	converter := md.NewConverter("", true, nil)
+	return converter.Convert(doc.Selection), nil
+}
+
+// TwtFromNoteFactory ...
+func TwtFromNoteFactory(conf *Config, cache *Cache, archive Archiver) TwtFromNote {
+	return func(note *activitypub.Note) (types.Twt, error) {
+		actor, err := ap.Lookup(note.AttributeTo)
+		if err != nil {
+			return types.NilTwt, err
+		}
+
+		twter := TwterFromActor(actor)
+		GetExternalAvatar(conf, *twter)
+
+		published := note.Published
+		content, err := CleanNote(note.Content)
+		if err != nil {
+			return types.NilTwt, err
+		}
+
+		if note.InReplyTo != "" {
+			note, err := ap.GetNote(note.InReplyTo)
+			if err != nil {
+				log.WithError(err).Warnf("error looking up note for inReplyTo=%s", note.InReplyTo)
+			} else {
+				if twt, err := TwtFromNoteFactory(conf, cache, archive)(note); err == nil {
+					hash := twt.Hash()
+					if _, ok := cache.Lookup(hash); ok || archive.Has(hash) {
+						content = fmt.Sprintf("(#%s) %s", hash, content)
+					}
+				} else {
+					log.WithError(err).Warnf("error converting note to twt")
+				}
+			}
+		}
+
+		twt := types.MakeTwt(*twter, published, content)
+
+		cache.SetTwter(actor.ID, twter)
+		cache.InjectFeed(actor.ID, twt)
+		cache.Refresh()
+
+		return twt, nil
+	}
+}
+
 // InboxHandler ...
 func (s *Server) InboxHandler() httprouter.Handle {
 	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
--- a/internal/activitypub_test.go
+++ b/internal/activitypub_test.go
@ -0,0 +1,41 @@
+package internal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCleanNote(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	tests := []struct {
+		Name          string
+		Content       string
+		ExpectedError error
+		ExpectedText  string
+	}{
+		{
+			Name:          "Handle mf2 mentions",
+			Content:       `<p>Hey <span class="h-card"><a href="https://yarn.mills.io/user/james/" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>james</span></a></span></p>`,
+			ExpectedError: nil,
+			ExpectedText:  `Hey @<james https://yarn.mills.io/user/james/>`,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			text, err := CleanNote(test.Content)
+			if test.ExpectedError == nil {
+				require.NoError(err)
+			} else {
+				require.Error(err)
+				assert.EqualError(err, test.ExpectedError.Error())
+			}
+
+			assert.Equal(test.ExpectedText, text)
+		})
+	}
+}
--- a/internal/server.go
+++ b/internal/server.go
@ -688,54 +688,6 @@ func (s *Server) setupWebSub() error {
 	return nil
 }

-// TwtFromNote ...
-type TwtFromNote func(note *activitypub.Note) (types.Twt, error)
-
-// CleanNote ...
-func CleanNote(content string) string {
-	return content
-}
-
-// TwtFromNoteFactory ...
-func TwtFromNoteFactory(conf *Config, cache *Cache, archive Archiver) TwtFromNote {
-	return func(note *activitypub.Note) (types.Twt, error) {
-		actor, err := ap.Lookup(note.AttributeTo)
-		if err != nil {
-			return types.NilTwt, err
-		}
-
-		twter := TwterFromActor(actor)
-		GetExternalAvatar(conf, *twter)
-
-		published := note.Published
-		content := CleanNote(note.Content)
-
-		if note.InReplyTo != "" {
-			note, err := ap.GetNote(note.InReplyTo)
-			if err != nil {
-				log.WithError(err).Warnf("error looking up note for inReplyTo=%s", note.InReplyTo)
-			} else {
-				if twt, err := TwtFromNoteFactory(conf, cache, archive)(note); err == nil {
-					hash := twt.Hash()
-					if _, ok := cache.Lookup(hash); ok || archive.Has(hash) {
-						content = fmt.Sprintf("(#%s) %s", hash, content)
-					}
-				} else {
-					log.WithError(err).Warnf("error converting note to twt")
-				}
-			}
-		}
-
-		twt := types.MakeTwt(*twter, published, content)
-
-		cache.SetTwter(actor.ID, twter)
-		cache.InjectFeed(actor.ID, twt)
-		cache.Refresh()
-
-		return twt, nil
-	}
-}
-
 func (s *Server) setupAP() error {
 	fn := filepath.Join(s.config.Data, "ap.json")