From a0a28a47777db9ca9e60a5134c8d86503c060e79 Mon Sep 17 00:00:00 2001 From: Paul Nothaft Date: Sun, 5 Jul 2026 23:37:16 +0200 Subject: [PATCH] fix(og): broaden social-crawler coverage (Bluesky Cardyb, WeChat-scraper, fediverse, etc.) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From alexvaltchev's field UA list on #699. Adds CRAWLER-EXCLUSIVE tokens to both the nginx UA regex and SOCIAL_CRAWLER_PATTERNS (kept in sync): Cardyb (Bluesky's actual link-card fetcher), facebookcatalog, Signal, Misskey, Pleroma, Synapse, Nextcloud, Rocket.Chat, kakaotalk-scrap, Google-PageRenderer, OdklBot, ZoomBot. Deliberately NOT added: UAs shared with real human in-app browsers (WeChat MicroMessenger, LINE 'Line/', Zalo) and broad strings ('InAppBrowser', 'preview', 'unfurl', 'XING' → matches 'boxing'). Our OG response is meta-only with no redirect, so matching those would serve a human the bare stub. New negative test locks that exclusion in. Verified: nginx -t passes; live harness confirms the new tokens rewrite to /og while the in-app-browser UAs still get the SPA. Backend suite 15/15. --- .../galleryOgService.shareImage.test.js | 22 ++++++++++++++++++- backend/src/services/galleryOgService.js | 20 ++++++++++++++++- frontend/nginx.conf | 7 ++++-- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/backend/src/__tests__/galleryOgService.shareImage.test.js b/backend/src/__tests__/galleryOgService.shareImage.test.js index 879db273..3f10efdb 100644 --- a/backend/src/__tests__/galleryOgService.shareImage.test.js +++ b/backend/src/__tests__/galleryOgService.shareImage.test.js @@ -343,14 +343,34 @@ describe('isSocialCrawler — extended bot coverage (#521)', () => { // 3rd-party preview services used by business-messaging stacks 'LinkPreview/1.0', 'Slack-ImgProxy/1.0', - // Viber link-preview fetcher (#699 follow-up) + // Viber + broader crawler set (#699 follow-up) 'Mozilla/5.0 (compatible; Viber)', + 'Mozilla/5.0 (compatible; Bluesky Cardyb/1.1)', + 'facebookcatalog/1.0', + 'kakaotalk-scrap/1.0', + 'Mozilla/5.0 (compatible; Synapse/1.98)', + 'Rocket.Chat/6.0', ]; for (const ua of knownBots) { expect(isSocialCrawler(ua)).toBe(true); } }); + it('does NOT match human in-app-browser UAs (our OG response is meta-only, no redirect)', () => { + // These share a token with a preview bot but are also sent by real users + // browsing inside the app's webview — matching them would serve a human + // the bare OG stub. Deliberately excluded; guard against re-adding them. + const inAppBrowsers = [ + 'Mozilla/5.0 (iPhone) AppleWebKit MicroMessenger/8.0.0', // WeChat in-app + 'Mozilla/5.0 (iPhone) AppleWebKit Line/13.0.0', // LINE in-app + 'Mozilla/5.0 (Linux; Android) Zalo', // Zalo in-app + 'Mozilla/5.0 (Macintosh) Chrome/120.0 Safari/537.36 boxing', // "XING" substring trap + ]; + for (const ua of inAppBrowsers) { + expect(isSocialCrawler(ua)).toBe(false); + } + }); + it('does not match a regular browser UA', () => { const browsers = [ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36', diff --git a/backend/src/services/galleryOgService.js b/backend/src/services/galleryOgService.js index 038c42f7..5cac9cf6 100644 --- a/backend/src/services/galleryOgService.js +++ b/backend/src/services/galleryOgService.js @@ -38,7 +38,25 @@ const SOCIAL_CRAWLER_PATTERNS = [ // Viber's link-preview fetcher — was never detected, so shared links // showed no rich preview in Viber (#699 follow-up). Keep in sync with the // UA list in frontend/nginx.conf. - /Viber/i + /Viber/i, + // Broader crawler coverage (#699 follow-up, from alexvaltchev's field list). + // IMPORTANT: only CRAWLER-EXCLUSIVE tokens are added here. Our OG response is + // meta-only (no client redirect), so a UA shared with a real human in-app + // browser would serve that human the bare stub. That rules out WeChat + // (MicroMessenger), LINE (Line/), Zalo, and generic strings like + // "InAppBrowser"/"preview"/"unfurl" — deliberately NOT added. + /Cardyb/i, // Bluesky's link-card service (the actual fetcher UA) + /facebookcatalog/i, // Facebook catalog crawler + /Signal/i, // Signal link preview + /Misskey/i, // fediverse (server-side preview fetch) + /Pleroma/i, // fediverse + /Synapse/i, // Matrix homeserver URL preview + /Nextcloud/i, // Nextcloud Talk/News link crawler + /Rocket\.Chat/i, // Rocket.Chat server preview + /kakaotalk-scrap/i, // KakaoTalk's scraper (NOT the in-app browser UA) + /Google-PageRenderer/i, // Google Chat previews (not Search) + /OdklBot/i, // Odnoklassniki + /ZoomBot/i // Zoom Team Chat ]; function isSocialCrawler(userAgent) { diff --git a/frontend/nginx.conf b/frontend/nginx.conf index c0f70c57..442a4306 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -251,8 +251,11 @@ server { # backend/src/services/galleryOgService.js. WhatsAppBot / wa-bot # and LinkPreview / Slack-ImgProxy added in #521 to catch # business-API preview fetchers that aren't the main WhatsApp app. - # Viber added in #699 follow-up (its preview fetcher was never detected). - if ($http_user_agent ~* "(facebookexternalhit|facebot|Twitterbot|WhatsApp|WhatsAppBot|wa-bot|Slackbot|Slack-ImgProxy|TelegramBot|SkypeUriPreview|Discordbot|LinkedInBot|Pinterest|vkShare|redditbot|Embedly|iframely|Snapchat|Applebot|Mastodon|Bluesky|OpenGraph|LinkPreview|Viber)") { + # Viber + the broader set below added in #699 follow-up. Only + # CRAWLER-EXCLUSIVE tokens — the backend OG response is meta-only (no + # redirect), so UAs shared with real human in-app browsers (WeChat's + # MicroMessenger, LINE's "Line/", Zalo, "InAppBrowser") are NOT added. + if ($http_user_agent ~* "(facebookexternalhit|facebookcatalog|facebot|Twitterbot|WhatsApp|WhatsAppBot|wa-bot|Slackbot|Slack-ImgProxy|TelegramBot|SkypeUriPreview|Discordbot|LinkedInBot|Pinterest|vkShare|redditbot|Embedly|iframely|Snapchat|Applebot|Mastodon|Bluesky|Cardyb|OpenGraph|LinkPreview|Viber|Signal|Misskey|Pleroma|Synapse|Nextcloud|Rocket\.Chat|kakaotalk-scrap|Google-PageRenderer|OdklBot|ZoomBot)") { rewrite ^ /og/gallery/$gallery_slug last; } try_files $uri $uri/ /index.html;