User-agent: * Disallow: / # Allow twitter crawler to index the site User-agent: Twitterbot Allow: /photos/ # Allow meta crawler to index the site User-agent: Meta-external-bot Allow: /photos/ # Allow meta crawler to index the site User-agent: facebookexternalhit Allow: /photos/