diff --git a/.gitignore b/.gitignore index 4a6603f5..aead7c1f 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,6 @@ test-results/ # Development docker compose docker-compose.dev.yml + +# New layout development files +new-layouts/ diff --git a/backend/migrations/core/068_add_seo_robots_settings.js b/backend/migrations/core/068_add_seo_robots_settings.js new file mode 100644 index 00000000..7cd59048 --- /dev/null +++ b/backend/migrations/core/068_add_seo_robots_settings.js @@ -0,0 +1,54 @@ +const DEFAULT_AI_AGENTS = [ + 'GPTBot', + 'ChatGPT-User', + 'Google-Extended', + 'Claude-Web', + 'Anthropic-AI', + 'CCBot', + 'Bytespider', + 'FacebookBot', + 'Omgilibot', + 'Diffbot', + 'PetalBot', + 'Amazonbot', + 'PerplexityBot', + 'YouBot', + 'Applebot-Extended' +]; + +exports.up = async function(knex) { + const defaults = [ + { setting_key: 'seo_allow_indexing', setting_value: JSON.stringify(false), setting_type: 'seo' }, + { setting_key: 'seo_block_ai_crawlers', setting_value: JSON.stringify(true), setting_type: 'seo' }, + { setting_key: 'seo_block_social_bots', setting_value: JSON.stringify(false), setting_type: 'seo' }, + { setting_key: 'seo_blocked_ai_agents', setting_value: JSON.stringify(DEFAULT_AI_AGENTS), setting_type: 'seo' }, + { setting_key: 'seo_custom_rules', setting_value: JSON.stringify([]), setting_type: 'seo' }, + { setting_key: 'seo_meta_noindex', setting_value: JSON.stringify(true), setting_type: 'seo' }, + { setting_key: 'seo_meta_nofollow', setting_value: JSON.stringify(false), setting_type: 'seo' }, + { setting_key: 'seo_meta_noai', setting_value: JSON.stringify(true), setting_type: 'seo' }, + { setting_key: 'seo_sitemap_url', setting_value: JSON.stringify(''), setting_type: 'seo' } + ]; + + for (const setting of defaults) { + const exists = await knex('app_settings').where('setting_key', setting.setting_key).first(); + if (!exists) { + await knex('app_settings').insert({ ...setting, updated_at: knex.fn.now() }); + } + } +}; + +exports.down = async function(knex) { + await knex('app_settings') + .whereIn('setting_key', [ + 'seo_allow_indexing', + 'seo_block_ai_crawlers', + 'seo_block_social_bots', + 'seo_blocked_ai_agents', + 'seo_custom_rules', + 'seo_meta_noindex', + 'seo_meta_nofollow', + 'seo_meta_noai', + 'seo_sitemap_url' + ]) + .del(); +}; diff --git a/backend/server.js b/backend/server.js index f2c3bf59..5ec4d19b 100644 --- a/backend/server.js +++ b/backend/server.js @@ -252,10 +252,29 @@ function renderBrandFooter(branding) { `; } +function buildSeoMetaTags(seoSettings) { + const tags = []; + const robotsDirectives = []; + + if (seoSettings.seo_meta_noindex) robotsDirectives.push('noindex'); + if (seoSettings.seo_meta_nofollow) robotsDirectives.push('nofollow'); + + if (robotsDirectives.length > 0) { + tags.push(``); + } + + if (seoSettings.seo_meta_noai) { + tags.push(''); + } + + return tags.join('\n '); +} + function buildPublicSiteDocument(payload) { const inlineStyles = composeInlineStyles(payload); const header = renderBrandHeader(payload.branding); const footer = renderBrandFooter(payload.branding); + const seoMeta = payload.seoSettings ? buildSeoMetaTags(payload.seoSettings) : ''; return ` @@ -265,6 +284,7 @@ function buildPublicSiteDocument(payload) { ${payload.title} + ${seoMeta} @@ -296,6 +316,21 @@ async function handlePublicSiteRequest(req, res, next) { return; } + // Inject SEO meta settings into payload + try { + const seoRows = await db('app_settings') + .where('setting_type', 'seo') + .whereIn('setting_key', ['seo_meta_noindex', 'seo_meta_nofollow', 'seo_meta_noai']) + .select('setting_key', 'setting_value'); + const seoSettings = {}; + for (const row of seoRows) { + let val = row.setting_value; + if (typeof val === 'string') { try { val = JSON.parse(val); } catch {} } + seoSettings[row.setting_key] = val; + } + payload.seoSettings = seoSettings; + } catch {} + const document = buildPublicSiteDocument(payload); res.setHeader('Content-Type', 'text/html; charset=utf-8'); @@ -395,6 +430,22 @@ if (process.env.NODE_ENV === 'development') { }); } +// robots.txt endpoint (dynamic, served from DB settings) +const { generateRobotsTxt } = require('./src/services/robotsTxtService'); +app.get('/robots.txt', async (req, res) => { + try { + const robotsTxt = await generateRobotsTxt(); + res.setHeader('Content-Type', 'text/plain'); + res.setHeader('Cache-Control', 'public, max-age=3600'); + res.status(200).send(robotsTxt); + } catch (error) { + logger.error('Failed to generate robots.txt', { error: error.message }); + // Safe default for a private photo platform + res.setHeader('Content-Type', 'text/plain'); + res.status(200).send('User-agent: *\nDisallow: /\n'); + } +}); + // Health check endpoint app.get('/health', async (req, res) => { try { diff --git a/backend/src/routes/adminSettings.js b/backend/src/routes/adminSettings.js index f650fc12..36d83a75 100644 --- a/backend/src/routes/adminSettings.js +++ b/backend/src/routes/adminSettings.js @@ -733,6 +733,70 @@ router.put('/analytics', adminAuth, requirePermission('settings.edit'), async (r } }); +// Update SEO settings +router.put('/seo', adminAuth, requirePermission('settings.edit'), async (req, res) => { + try { + const settings = req.body; + + // Validate seo_blocked_ai_agents is an array of strings + if (settings.seo_blocked_ai_agents !== undefined) { + if (!Array.isArray(settings.seo_blocked_ai_agents) || + !settings.seo_blocked_ai_agents.every(a => typeof a === 'string')) { + return res.status(400).json({ error: 'seo_blocked_ai_agents must be an array of strings' }); + } + } + + // Validate seo_custom_rules structure + if (settings.seo_custom_rules !== undefined) { + if (!Array.isArray(settings.seo_custom_rules)) { + return res.status(400).json({ error: 'seo_custom_rules must be an array' }); + } + for (const rule of settings.seo_custom_rules) { + if (!rule.userAgent || typeof rule.userAgent !== 'string') { + return res.status(400).json({ error: 'Each custom rule must have a userAgent string' }); + } + if (!Array.isArray(rule.disallow) || !rule.disallow.every(d => typeof d === 'string')) { + return res.status(400).json({ error: 'Each custom rule must have a disallow array of strings' }); + } + } + } + + // Update or insert each setting + for (const [key, value] of Object.entries(settings)) { + await db('app_settings') + .insert({ + setting_key: key, + setting_value: JSON.stringify(value), + setting_type: 'seo', + updated_at: new Date() + }) + .onConflict('setting_key') + .merge({ + setting_value: JSON.stringify(value), + updated_at: new Date() + }); + } + + // Clear robots.txt cache + const { clearRobotsTxtCache } = require('../services/robotsTxtService'); + clearRobotsTxtCache(); + + // Log activity + await db('activity_logs').insert({ + activity_type: 'seo_settings_updated', + actor_type: 'admin', + actor_id: req.admin.id, + actor_name: req.admin.username, + metadata: JSON.stringify({ settings_count: Object.keys(settings).length }) + }); + + res.json({ message: 'SEO settings updated successfully' }); + } catch (error) { + console.error('SEO settings update error:', error); + res.status(500).json({ error: 'Failed to update SEO settings' }); + } +}); + // Get storage info router.get('/storage/info', adminAuth, requirePermission('settings.view'), async (req, res) => { try { diff --git a/backend/src/routes/publicSettings.js b/backend/src/routes/publicSettings.js index 389860f4..1de8a673 100644 --- a/backend/src/routes/publicSettings.js +++ b/backend/src/routes/publicSettings.js @@ -12,7 +12,8 @@ router.get('/', async (req, res) => { .where(function() { this.whereIn('setting_type', ['branding', 'theme', 'general', 'security', 'analytics', 'boolean']) .orWhere('setting_key', 'like', 'analytics_%') - .orWhere('setting_key', 'like', 'event_require_%'); + .orWhere('setting_key', 'like', 'event_require_%') + .orWhereIn('setting_key', ['seo_meta_noindex', 'seo_meta_nofollow', 'seo_meta_noai']); }) .select('setting_key', 'setting_value'); }); @@ -76,7 +77,11 @@ router.get('/', async (req, res) => { event_require_customer_email: settingsObject.event_require_customer_email !== false, event_require_admin_email: settingsObject.event_require_admin_email !== false, event_require_event_date: settingsObject.event_require_event_date !== false, - event_require_expiration: settingsObject.event_require_expiration !== false + event_require_expiration: settingsObject.event_require_expiration !== false, + // SEO meta tag flags (safe to expose - these are intended for crawlers) + seo_meta_noindex: settingsObject.seo_meta_noindex === true, + seo_meta_nofollow: settingsObject.seo_meta_nofollow === true, + seo_meta_noai: settingsObject.seo_meta_noai === true }; res.json(publicSettings); diff --git a/backend/src/services/robotsTxtService.js b/backend/src/services/robotsTxtService.js new file mode 100644 index 00000000..dfa67995 --- /dev/null +++ b/backend/src/services/robotsTxtService.js @@ -0,0 +1,128 @@ +const { db } = require('../database/db'); + +let cachedRobotsTxt = null; +let cacheTimestamp = 0; +const CACHE_TTL_MS = 60 * 1000; // 60 seconds + +function parseSetting(raw) { + if (raw === null || raw === undefined) return null; + if (typeof raw !== 'string') return raw; + try { + return JSON.parse(raw); + } catch { + return raw; + } +} + +async function getSeoSettings() { + const rows = await db('app_settings') + .where('setting_type', 'seo') + .select('setting_key', 'setting_value'); + + const settings = {}; + for (const row of rows) { + settings[row.setting_key] = parseSetting(row.setting_value); + } + return settings; +} + +const SOCIAL_BOTS = [ + 'Twitterbot', + 'facebookexternalhit', + 'LinkedInBot', + 'Slackbot', + 'WhatsApp', + 'TelegramBot', + 'Discordbot' +]; + +async function generateRobotsTxt() { + const now = Date.now(); + if (cachedRobotsTxt && (now - cacheTimestamp) < CACHE_TTL_MS) { + return cachedRobotsTxt; + } + + const settings = await getSeoSettings(); + const allowIndexing = settings.seo_allow_indexing === true; + const blockAiCrawlers = settings.seo_block_ai_crawlers !== false; + const blockSocialBots = settings.seo_block_social_bots === true; + const aiAgents = Array.isArray(settings.seo_blocked_ai_agents) + ? settings.seo_blocked_ai_agents + : []; + const customRules = Array.isArray(settings.seo_custom_rules) + ? settings.seo_custom_rules + : []; + const sitemapUrl = settings.seo_sitemap_url || ''; + + const lines = []; + + // Always block admin and API paths for all agents + lines.push('# Protected paths'); + lines.push('User-agent: *'); + lines.push('Disallow: /admin'); + lines.push('Disallow: /api'); + lines.push(''); + + if (!allowIndexing) { + // Block everything for all agents + lines.push('# Indexing disabled - block all crawlers'); + lines.push('User-agent: *'); + lines.push('Disallow: /'); + lines.push(''); + } + + // Block AI crawlers if enabled + if (blockAiCrawlers && aiAgents.length > 0) { + lines.push('# AI/LLM crawler blocking'); + for (const agent of aiAgents) { + lines.push(`User-agent: ${agent}`); + lines.push('Disallow: /'); + lines.push(''); + } + } + + // Block social bots if enabled + if (blockSocialBots) { + lines.push('# Social media bot blocking'); + for (const bot of SOCIAL_BOTS) { + lines.push(`User-agent: ${bot}`); + lines.push('Disallow: /'); + lines.push(''); + } + } + + // Custom rules + if (customRules.length > 0) { + lines.push('# Custom rules'); + for (const rule of customRules) { + if (rule.userAgent && Array.isArray(rule.disallow)) { + lines.push(`User-agent: ${rule.userAgent}`); + for (const path of rule.disallow) { + lines.push(`Disallow: ${path}`); + } + lines.push(''); + } + } + } + + // Sitemap + if (sitemapUrl) { + lines.push(`Sitemap: ${sitemapUrl}`); + lines.push(''); + } + + const result = lines.join('\n'); + cachedRobotsTxt = result; + cacheTimestamp = now; + return result; +} + +function clearRobotsTxtCache() { + cachedRobotsTxt = null; + cacheTimestamp = 0; +} + +module.exports = { + generateRobotsTxt, + clearRobotsTxtCache +}; diff --git a/frontend/nginx.conf b/frontend/nginx.conf index e0bf6905..dddd2535 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -111,6 +111,17 @@ server { proxy_cache_valid 404 1m; } + # Dynamic robots.txt served by backend + location = /robots.txt { + set $backend_upstream backend; + proxy_pass http://$backend_upstream:3000/robots.txt; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + # Delegate root requests to backend for public landing page handling location = / { # Use variable to force DNS resolution per request (required for Docker Swarm) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index f27cd917..70f267bb 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,4 +1,4 @@ -import { useEffect } from 'react'; +import { useEffect, useState } from 'react'; import { BrowserRouter as Router, Routes, Route, Navigate } from 'react-router-dom'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { ToastContainer } from 'react-toastify'; @@ -29,7 +29,7 @@ import { } from './pages/admin'; import { AcceptInvitePage } from './pages/public/AcceptInvitePage'; import { AdminLayout, AdminAuthWrapper } from './components/admin'; -import { PageErrorBoundary, OfflineIndicator, SkipLink, DynamicFavicon } from './components/common'; +import { PageErrorBoundary, OfflineIndicator, SkipLink, DynamicFavicon, RobotsMetaTags } from './components/common'; import { MaintenanceWrapper } from './components/MaintenanceWrapper'; import { GlobalThemeProvider } from './components/GlobalThemeProvider'; import { getApiBaseUrl } from './utils/url'; @@ -45,6 +45,17 @@ const queryClient = new QueryClient({ }); function App() { + // Track dark mode for toast theming + const [toastTheme, setToastTheme] = useState<'light' | 'dark'>('light'); + + useEffect(() => { + const observer = new MutationObserver(() => { + setToastTheme(document.documentElement.classList.contains('dark') ? 'dark' : 'light'); + }); + observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class'] }); + return () => observer.disconnect(); + }, []); + // Initialize Umami Analytics based on settings useEffect(() => { const initializeAnalytics = async () => { @@ -103,6 +114,7 @@ function App() { + @@ -165,7 +177,7 @@ function App() { pauseOnFocusLoss draggable pauseOnHover - theme="light" + theme={toastTheme} /> diff --git a/frontend/src/components/admin/AdminAuthWrapper.tsx b/frontend/src/components/admin/AdminAuthWrapper.tsx index b0d4cb98..6fba2d33 100644 --- a/frontend/src/components/admin/AdminAuthWrapper.tsx +++ b/frontend/src/components/admin/AdminAuthWrapper.tsx @@ -1,12 +1,15 @@ import React from 'react'; import { Outlet } from 'react-router-dom'; import { AdminAuthProvider, PermissionsProvider } from '../../contexts'; +import { AdminDarkModeProvider } from '../../contexts/AdminDarkModeContext'; export const AdminAuthWrapper: React.FC = () => { return ( - + + + ); diff --git a/frontend/src/components/admin/AdminHeader.tsx b/frontend/src/components/admin/AdminHeader.tsx index 157c77d9..275e04d5 100644 --- a/frontend/src/components/admin/AdminHeader.tsx +++ b/frontend/src/components/admin/AdminHeader.tsx @@ -1,12 +1,13 @@ import React, { useState, useRef } from 'react'; import { useNavigate } from 'react-router-dom'; -import { Menu, User, LogOut, Settings, Bell, Lock, CheckCircle, Trash2 } from 'lucide-react'; +import { Menu, User, LogOut, Settings, Bell, Lock, CheckCircle, Trash2, Sun, Moon } from 'lucide-react'; import { useTranslation } from 'react-i18next'; import { useLocalizedDate } from '../../hooks/useLocalizedDate'; import { useLocalizedTimeAgo } from '../../hooks/useLocalizedTimeAgo'; import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; import { useAdminAuth } from '../../contexts'; +import { useAdminDarkMode } from '../../contexts/AdminDarkModeContext'; import { useOnClickOutside } from '../../hooks/useOnClickOutside'; import { PasswordChangeModal } from './PasswordChangeModal'; import { LanguageSelector } from '../common'; @@ -20,6 +21,7 @@ interface AdminHeaderProps { export const AdminHeader: React.FC = ({ onMenuClick }) => { const navigate = useNavigate(); const { user, logout } = useAdminAuth(); + const { isDark, toggle: toggleDarkMode } = useAdminDarkMode(); const { t } = useTranslation(); const { format } = useLocalizedDate(); const { formatTimeAgo } = useLocalizedTimeAgo(); @@ -68,7 +70,7 @@ export const AdminHeader: React.FC = ({ onMenuClick }) => { const unreadCount = notificationsData?.unreadCount || 0; return ( -
+
{/* Left side - Menu button, Logo, and Date */} @@ -87,8 +89,8 @@ export const AdminHeader: React.FC = ({ onMenuClick }) => {
{/* Date display - hidden on smaller screens */} -
-

+

+

{format(new Date(), 'PPPP')}

@@ -98,12 +100,21 @@ export const AdminHeader: React.FC = ({ onMenuClick }) => {
{/* Language Selector */} - + + {/* Dark Mode Toggle */} + + {/* Notifications */}
{notifications.length === 0 ? ( -
+
{t('admin.noNotificationsMessage')}
) : ( @@ -148,7 +159,7 @@ export const AdminHeader: React.FC = ({ onMenuClick }) => { return (
@@ -157,10 +168,10 @@ export const AdminHeader: React.FC = ({ onMenuClick }) => {
-

+

{notificationsService.formatNotificationMessage(notification)}

-

+

{formatTimeAgo(notification.createdAt)}

@@ -171,10 +182,10 @@ export const AdminHeader: React.FC = ({ onMenuClick }) => { )}
{notifications.length > 0 && ( -
- @@ -188,11 +199,11 @@ export const AdminHeader: React.FC = ({ onMenuClick }) => {
-
+
{t('gallery.photosCount', { count: photos.length })}
@@ -234,7 +234,7 @@ export const AdminPhotoGrid: React.FC = ({
!isDeleting && onPhotoClick(photo, index)} @@ -353,7 +353,7 @@ export const AdminPhotoGrid: React.FC = ({ {photos.length === 0 && (
-

{t('gallery.noMedia', 'No media uploaded yet')}

+

{t('gallery.noMedia', 'No media uploaded yet')}

)} diff --git a/frontend/src/components/admin/AdminSidebar.tsx b/frontend/src/components/admin/AdminSidebar.tsx index 44a43d7a..adaa9b84 100644 --- a/frontend/src/components/admin/AdminSidebar.tsx +++ b/frontend/src/components/admin/AdminSidebar.tsx @@ -59,15 +59,15 @@ export const AdminSidebar: React.FC = ({ isOpen, onClose }) = return (
{/* Brand */} -
+
- {t('admin.title')} + {t('admin.title')}
); })} @@ -205,7 +205,7 @@ export const BackupConfiguration = ({ config, onSave, isSaving }) => { {formData.backup_destination_type === 'local' && ( <>
-
@@ -226,7 +226,7 @@ export const BackupConfiguration = ({ config, onSave, isSaving }) => { <>
-
-
-
-
@@ -292,7 +292,7 @@ export const BackupConfiguration = ({ config, onSave, isSaving }) => { {formData.backup_destination_type === 's3' && ( <>
-
-
-
-
-