Add routine to enforce rules in robots.txt if corresponding setting is enabled

code/HopObject/HopObject.js

@@ -140,10 +140,18 @@ HopObject.prototype.onRequest = function() {
     }
   }
 
+  // Set up layout handler and skin path
   HopObject.confirmConstructor(Layout);
   res.handlers.layout = res.handlers.site.layout || new Layout;
   res.skinpath = res.handlers.layout.getSkinPath();
 
+  if (res.handlers.site.enforceRobotsTxt()) {
+    res.status = 403
+    res.data.error = "Robots.txt forbids access to this page.";
+    root.error_action();
+    res.stop();
+  }
+
   if (!this.getPermission(req.action)) {
     if (!session.user) {
       User.setLocation(root.href() + req.path);

code/Site/Site.js

@@ -1132,3 +1132,22 @@ Site.prototype.callback = function(ref) {
   }
   return;
 }
+
+Site.prototype.enforceRobotsTxt = function() {
+  if (this.robotsTxtMode !== Site.ENFORCED) {
+    return false;
+  }
+
+  // Override some patterns to prevent a site from becoming inaccessible even for the owner
+  const overrides = [
+    'User-agent: mozilla',
+    'Allow: */edit$',
+    'Allow: */layout',
+    'Allow: */main.*$',
+    'Allow: */members'
+  ];
+
+  const robotsTxt = root.renderSkinAsString('Site#robots');
+  const robots = new Robots(this.href('robots.txt'), robotsTxt + overrides.join('\n'));
+  return !robots.isAllowed(path.href() + req.action, req.getHeader('user-agent'));
+}