fix(og): broaden social-crawler coverage (Bluesky Cardyb, WeChat-scraper, fediverse, etc.)
From alexvaltchev's field UA list on #699. Adds CRAWLER-EXCLUSIVE tokens to both the nginx UA regex and SOCIAL_CRAWLER_PATTERNS (kept in sync): Cardyb (Bluesky's actual link-card fetcher), facebookcatalog, Signal, Misskey, Pleroma, Synapse, Nextcloud, Rocket.Chat, kakaotalk-scrap, Google-PageRenderer, OdklBot, ZoomBot. Deliberately NOT added: UAs shared with real human in-app browsers (WeChat MicroMessenger, LINE 'Line/', Zalo) and broad strings ('InAppBrowser', 'preview', 'unfurl', 'XING' → matches 'boxing'). Our OG response is meta-only with no redirect, so matching those would serve a human the bare stub. New negative test locks that exclusion in. Verified: nginx -t passes; live harness confirms the new tokens rewrite to /og while the in-app-browser UAs still get the SPA. Backend suite 15/15.
This commit is contained in:
@@ -343,14 +343,34 @@ describe('isSocialCrawler — extended bot coverage (#521)', () => {
|
||||
// 3rd-party preview services used by business-messaging stacks
|
||||
'LinkPreview/1.0',
|
||||
'Slack-ImgProxy/1.0',
|
||||
// Viber link-preview fetcher (#699 follow-up)
|
||||
// Viber + broader crawler set (#699 follow-up)
|
||||
'Mozilla/5.0 (compatible; Viber)',
|
||||
'Mozilla/5.0 (compatible; Bluesky Cardyb/1.1)',
|
||||
'facebookcatalog/1.0',
|
||||
'kakaotalk-scrap/1.0',
|
||||
'Mozilla/5.0 (compatible; Synapse/1.98)',
|
||||
'Rocket.Chat/6.0',
|
||||
];
|
||||
for (const ua of knownBots) {
|
||||
expect(isSocialCrawler(ua)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('does NOT match human in-app-browser UAs (our OG response is meta-only, no redirect)', () => {
|
||||
// These share a token with a preview bot but are also sent by real users
|
||||
// browsing inside the app's webview — matching them would serve a human
|
||||
// the bare OG stub. Deliberately excluded; guard against re-adding them.
|
||||
const inAppBrowsers = [
|
||||
'Mozilla/5.0 (iPhone) AppleWebKit MicroMessenger/8.0.0', // WeChat in-app
|
||||
'Mozilla/5.0 (iPhone) AppleWebKit Line/13.0.0', // LINE in-app
|
||||
'Mozilla/5.0 (Linux; Android) Zalo', // Zalo in-app
|
||||
'Mozilla/5.0 (Macintosh) Chrome/120.0 Safari/537.36 boxing', // "XING" substring trap
|
||||
];
|
||||
for (const ua of inAppBrowsers) {
|
||||
expect(isSocialCrawler(ua)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it('does not match a regular browser UA', () => {
|
||||
const browsers = [
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36',
|
||||
|
||||
@@ -38,7 +38,25 @@ const SOCIAL_CRAWLER_PATTERNS = [
|
||||
// Viber's link-preview fetcher — was never detected, so shared links
|
||||
// showed no rich preview in Viber (#699 follow-up). Keep in sync with the
|
||||
// UA list in frontend/nginx.conf.
|
||||
/Viber/i
|
||||
/Viber/i,
|
||||
// Broader crawler coverage (#699 follow-up, from alexvaltchev's field list).
|
||||
// IMPORTANT: only CRAWLER-EXCLUSIVE tokens are added here. Our OG response is
|
||||
// meta-only (no client redirect), so a UA shared with a real human in-app
|
||||
// browser would serve that human the bare stub. That rules out WeChat
|
||||
// (MicroMessenger), LINE (Line/), Zalo, and generic strings like
|
||||
// "InAppBrowser"/"preview"/"unfurl" — deliberately NOT added.
|
||||
/Cardyb/i, // Bluesky's link-card service (the actual fetcher UA)
|
||||
/facebookcatalog/i, // Facebook catalog crawler
|
||||
/Signal/i, // Signal link preview
|
||||
/Misskey/i, // fediverse (server-side preview fetch)
|
||||
/Pleroma/i, // fediverse
|
||||
/Synapse/i, // Matrix homeserver URL preview
|
||||
/Nextcloud/i, // Nextcloud Talk/News link crawler
|
||||
/Rocket\.Chat/i, // Rocket.Chat server preview
|
||||
/kakaotalk-scrap/i, // KakaoTalk's scraper (NOT the in-app browser UA)
|
||||
/Google-PageRenderer/i, // Google Chat previews (not Search)
|
||||
/OdklBot/i, // Odnoklassniki
|
||||
/ZoomBot/i // Zoom Team Chat
|
||||
];
|
||||
|
||||
function isSocialCrawler(userAgent) {
|
||||
|
||||
+5
-2
@@ -251,8 +251,11 @@ server {
|
||||
# backend/src/services/galleryOgService.js. WhatsAppBot / wa-bot
|
||||
# and LinkPreview / Slack-ImgProxy added in #521 to catch
|
||||
# business-API preview fetchers that aren't the main WhatsApp app.
|
||||
# Viber added in #699 follow-up (its preview fetcher was never detected).
|
||||
if ($http_user_agent ~* "(facebookexternalhit|facebot|Twitterbot|WhatsApp|WhatsAppBot|wa-bot|Slackbot|Slack-ImgProxy|TelegramBot|SkypeUriPreview|Discordbot|LinkedInBot|Pinterest|vkShare|redditbot|Embedly|iframely|Snapchat|Applebot|Mastodon|Bluesky|OpenGraph|LinkPreview|Viber)") {
|
||||
# Viber + the broader set below added in #699 follow-up. Only
|
||||
# CRAWLER-EXCLUSIVE tokens — the backend OG response is meta-only (no
|
||||
# redirect), so UAs shared with real human in-app browsers (WeChat's
|
||||
# MicroMessenger, LINE's "Line/", Zalo, "InAppBrowser") are NOT added.
|
||||
if ($http_user_agent ~* "(facebookexternalhit|facebookcatalog|facebot|Twitterbot|WhatsApp|WhatsAppBot|wa-bot|Slackbot|Slack-ImgProxy|TelegramBot|SkypeUriPreview|Discordbot|LinkedInBot|Pinterest|vkShare|redditbot|Embedly|iframely|Snapchat|Applebot|Mastodon|Bluesky|Cardyb|OpenGraph|LinkPreview|Viber|Signal|Misskey|Pleroma|Synapse|Nextcloud|Rocket\.Chat|kakaotalk-scrap|Google-PageRenderer|OdklBot|ZoomBot)") {
|
||||
rewrite ^ /og/gallery/$gallery_slug last;
|
||||
}
|
||||
try_files $uri $uri/ /index.html;
|
||||
|
||||
Reference in New Issue
Block a user