From f25b3c0b7658cbced29e04e0536bbeb495fec04f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20Sch=C3=A4fer?= Date: Sun, 25 May 2025 16:58:11 +0200 Subject: [PATCH 01/10] Simplify repository workflows Use deploy workflow for staging, too --- .github/workflows/deploy.yml | 47 ++++++++++++++++++++++++++++------ .github/workflows/stage.yml | 49 ------------------------------------ 2 files changed, 40 insertions(+), 56 deletions(-) delete mode 100644 .github/workflows/stage.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 98ca7656..0bc7cbb7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,15 +1,48 @@ -name: Deploy (Production) +name: Deploy -on: workflow_dispatch +on: + workflow_dispatch: + inputs: + hostname: + description: Hostname + type: string + required: true + default: antville.org jobs: - deploy: + stage: runs-on: antville environment: - name: antville.org - url: https://antville.org + name: production + url: ${{ inputs.hostname }} steps: - - name: Copy files to production server - run: ssh staging-server deploy-antville + - uses: actions/checkout@v4 + + - name: Build with Gradle + run: ./gradlew :build + + - name: Copy files to server + run: | + rsync ./build/install/antville/ ${{ inputs.hostname }}:./apps/antville/ \ + --archive --compress --delete --verbose \ + --filter '+ /claustra' \ + --filter '+ /code' \ + --filter '+ /compat' \ + --filter '+ /db' \ + --filter '+ /i18n' \ + --filter '+ /lib' \ + --filter '- /*' + rsync ./build/install/antville/static/ ${{ inputs.hostname }}:./apps/antville/static/ \ + --archive --compress --verbose \ + --filter '+ /fonts' \ + --filter '+ /formica.html' \ + --filter '+ /img' \ + --filter '+ /scripts' \ + --filter '+ /styles' \ + --filter '- /*' + + - name: Restart Helma + run: ssh ${{ inputs.hostname }} restart + diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml deleted file mode 100644 index ebadb0cd..00000000 --- a/.github/workflows/stage.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Deploy (Staging) - -on: - workflow_dispatch: - inputs: - hostname: - description: Hostname - type: string - required: true - default: antville.org - -jobs: - stage: - runs-on: antville - - environment: - name: stage - url: ${{ inputs.hostname }} - - steps: - - uses: actions/checkout@v4 - - - name: Build with Gradle - run: ./gradlew :build - - - name: Copy files to server - # The rsync command applies the same filters as the one in tools/extras/deploy.sh - run: | - rsync ./build/install/antville/ ${{ inputs.hostname }}:./apps/antville/ \ - --archive --compress --delete --verbose \ - --filter '+ /claustra' \ - --filter '+ /code' \ - --filter '+ /compat' \ - --filter '+ /db' \ - --filter '+ /i18n' \ - --filter '+ /lib' \ - --filter '- /*' - rsync ./build/install/antville/static/ ${{ inputs.hostname }}:./apps/antville/static/ \ - --archive --compress --verbose \ - --filter '+ /fonts' \ - --filter '+ /formica.html' \ - --filter '+ /img' \ - --filter '+ /scripts' \ - --filter '+ /styles' \ - --filter '- /*' - - - name: Restart Helma - run: ssh ${{ inputs.hostname }} restart - From a7cabf0d63bccaa3f2622d7876b23501d54a0253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20Sch=C3=A4fer?= Date: Sat, 10 May 2025 21:44:19 +0200 Subject: [PATCH 02/10] Add third-party robots parser, including unit tests --- code/Global/Robots.js | 491 +++++++++++++++++++++++ tests/robots.js | 903 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1394 insertions(+) create mode 100644 code/Global/Robots.js create mode 100644 tests/robots.js diff --git a/code/Global/Robots.js b/code/Global/Robots.js new file mode 100644 index 00000000..b9835e0c --- /dev/null +++ b/code/Global/Robots.js @@ -0,0 +1,491 @@ +/** + * Trims the white space from the start and end of the line. + * + * If the line is an array it will strip the white space from + * the start and end of each element of the array. + * + * @param {string|Array} line + * @return {string|Array} + * @private + */ +function trimLine(line) { + if (!line) { + return null; + } + + if (Array.isArray(line)) { + return line.map(trimLine); + } + + return String(line).trim(); +} + +/** + * Remove comments from lines + * + * @param {string} line + * @return {string} + * @private + */ +function removeComments(line) { + var commentStartIndex = line.indexOf('#'); + if (commentStartIndex > -1) { + return line.substr(0, commentStartIndex); + } + + return line; +} + +/** + * Splits a line at the first occurrence of : + * + * @param {string} line + * @return {Array.} + * @private + */ +function splitLine(line) { + var idx = String(line).indexOf(':'); + + if (!line || idx < 0) { + return null; + } + + return [line.slice(0, idx), line.slice(idx + 1)]; +} + +/** + * Normalises the user-agent string by converting it to + * lower case and removing any version numbers. + * + * @param {string} userAgent + * @return {string} + * @private + */ +function formatUserAgent(userAgent) { + var formattedUserAgent = userAgent.toLowerCase(); + + // Strip the version number from robot/1.0 user agents + var idx = formattedUserAgent.indexOf('/'); + if (idx > -1) { + formattedUserAgent = formattedUserAgent.substr(0, idx); + } + + return formattedUserAgent.trim(); +} + +/** + * Normalises the URL encoding of a path by encoding + * unicode characters. + * + * @param {string} path + * @return {string} + * @private + */ +function normaliseEncoding(path) { + try { + return urlEncodeToUpper(encodeURI(path).replace(/%25/g, '%')); + } catch (e) { + return path; + } +} + +/** + * Convert URL encodings to support case. + * + * e.g.: %2a%ef becomes %2A%EF + * + * @param {string} path + * @return {string} + * @private + */ +function urlEncodeToUpper(path) { + return path.replace(/%[0-9a-fA-F]{2}/g, function (match) { + return match.toUpperCase(); + }); +} + +/** + * Matches a pattern with the specified path + * + * Uses same algorithm to match patterns as the Google implementation in + * google/robotstxt so it should be consistent with the spec. + * + * @see https://github.com/google/robotstxt/blob/f465f0ede81099dd8bc4aeb2966b3a892bd488b3/robots.cc#L74 + * @param {string} pattern + * @param {string} path + * @return {boolean} + * @private + */ +function matches(pattern, path) { + // I've added extra comments to try make this easier to understand + + // Stores the lengths of all the current matching substrings. + // Maximum number of possible matching lengths is every length in path plus + // 1 to handle 0 length too (if pattern starts with * which is zero or more) + var matchingLengths = new Array(path.length + 1); + var numMatchingLengths = 1; + + // Initially longest match is 0 + matchingLengths[0] = 0; + + for (var p = 0; p < pattern.length; p++) { + // If $ is at the end of pattern then we must match the whole path. + // Which is true if the longest matching length matches path length + if (pattern[p] === '$' && p + 1 === pattern.length) { + return matchingLengths[numMatchingLengths - 1] === path.length; + } + + // Handle wildcards + if (pattern[p] == '*') { + // Wildcard so all substrings minus the current smallest matching + // length are matches + numMatchingLengths = path.length - matchingLengths[0] + 1; + + // Update matching lengths to include the smallest all the way up + // to numMatchingLengths + // Don't update smallest possible match as * matches zero or more + // so the smallest current match is also valid + for (var i = 1; i < numMatchingLengths; i++) { + matchingLengths[i] = matchingLengths[i - 1] + 1; + } + } else { + // Check the char at the matching length matches the pattern, if it + // does increment it and add it as a valid length, ignore if not. + var numMatches = 0; + for (var i = 0; i < numMatchingLengths; i++) { + if ( + matchingLengths[i] < path.length && + path[matchingLengths[i]] === pattern[p] + ) { + matchingLengths[numMatches++] = matchingLengths[i] + 1; + } + } + + // No paths matched the current pattern char so not a match + if (numMatches == 0) { + return false; + } + + numMatchingLengths = numMatches; + } + } + + return true; +} + +function parseRobots(contents, robots) { + var newlineRegex = /\r\n|\r|\n/; + var lines = contents + .split(newlineRegex) + .map(removeComments) + .map(splitLine) + .map(trimLine); + + var currentUserAgents = []; + var isNoneUserAgentState = true; + for (var i = 0; i < lines.length; i++) { + var line = lines[i]; + + if (!line || !line[0]) { + continue; + } + + switch (line[0].toLowerCase()) { + case 'user-agent': + if (isNoneUserAgentState) { + currentUserAgents.length = 0; + } + + if (line[1]) { + currentUserAgents.push(formatUserAgent(line[1])); + } + break; + case 'disallow': + robots.addRule(currentUserAgents, line[1], false, i + 1); + break; + case 'allow': + robots.addRule(currentUserAgents, line[1], true, i + 1); + break; + case 'crawl-delay': + robots.setCrawlDelay(currentUserAgents, line[1]); + break; + case 'sitemap': + if (line[1]) { + robots.addSitemap(line[1]); + } + break; + case 'host': + if (line[1]) { + robots.setPreferredHost(line[1].toLowerCase()); + } + break; + } + + isNoneUserAgentState = line[0].toLowerCase() !== 'user-agent'; + } +} + +/** + * Returns if a pattern is allowed by the specified rules. + * + * @param {string} path + * @param {Array.} rules + * @return {Object?} + * @private + */ +function findRule(path, rules) { + var matchedRule = null; + + for (var i = 0; i < rules.length; i++) { + var rule = rules[i]; + + if (!matches(rule.pattern, path)) { + continue; + } + + // The longest matching rule takes precedence + // If rules are the same length then allow takes precedence + if (!matchedRule || rule.pattern.length > matchedRule.pattern.length) { + matchedRule = rule; + } else if ( + rule.pattern.length == matchedRule.pattern.length && + rule.allow && + !matchedRule.allow + ) { + matchedRule = rule; + } + } + + return matchedRule; +} + +/** + * Converts provided string into an URL object. + * + * Will return null if provided string is not a valid URL. + * + * @param {string} url + * @return {?URL} + * @private + */ +function parseUrl(url) { + try { + // Specify a URL to be used with relative paths + // Using non-existent subdomain so can never cause conflict unless + // trying to crawl it but doesn't exist and even if tried worst that can + // happen is it allows relative URLs on it. + var url = new URL(url, 'http://robots-relative.samclarke.com/'); + + if (!url.port) { + url.port = url.protocol === 'https:' ? 443 : 80; + } + + return url; + } catch (e) { + return null; + } +} + +function Robots(url, contents) { + this._url = parseUrl(url) || {}; + this._rules = Object.create(null); + this._sitemaps = []; + this._preferredHost = null; + + parseRobots(contents || '', this); +} + +/** + * Adds the specified allow/deny rule to the rules + * for the specified user-agents. + * + * @param {Array.} userAgents + * @param {string} pattern + * @param {boolean} allow + * @param {number} [lineNumber] Should use 1-based indexing + */ +Robots.prototype.addRule = function (userAgents, pattern, allow, lineNumber) { + var rules = this._rules; + + userAgents.forEach(function (userAgent) { + rules[userAgent] = rules[userAgent] || []; + + if (!pattern) { + return; + } + + rules[userAgent].push({ + pattern: normaliseEncoding(pattern), + allow: allow, + lineNumber: lineNumber + }); + }); +}; + +/** + * Adds the specified delay to the specified user agents. + * + * @param {Array.} userAgents + * @param {string} delayStr + */ +Robots.prototype.setCrawlDelay = function (userAgents, delayStr) { + var rules = this._rules; + var delay = Number(delayStr); + + userAgents.forEach(function (userAgent) { + rules[userAgent] = rules[userAgent] || []; + + if (isNaN(delay)) { + return; + } + + rules[userAgent].crawlDelay = delay; + }); +}; + +/** + * Add a sitemap + * + * @param {string} url + */ +Robots.prototype.addSitemap = function (url) { + this._sitemaps.push(url); +}; + +/** + * Sets the preferred host name + * + * @param {string} url + */ +Robots.prototype.setPreferredHost = function (url) { + this._preferredHost = url; +}; + +Robots.prototype._getRule = function (url, ua, explicit) { + var parsedUrl = parseUrl(url) || {}; + var userAgent = formatUserAgent(ua || '*'); + + // The base URL must match otherwise this robots.txt is not valid for it. + if ( + parsedUrl.protocol !== this._url.protocol || + parsedUrl.hostname !== this._url.hostname || + parsedUrl.port !== this._url.port + ) { + return; + } + + var rules = this._rules[userAgent]; + if (!explicit) { + rules = rules || this._rules['*']; + } + rules = rules || []; + + var path = urlEncodeToUpper(parsedUrl.pathname + parsedUrl.search); + var rule = findRule(path, rules); + + return rule; +}; + +/** + * Returns true if allowed, false if not allowed. + * + * Will return undefined if the URL is not valid for + * this robots.txt file. + * + * @param {string} url + * @param {string?} ua + * @return {boolean?} + */ +Robots.prototype.isAllowed = function (url, ua) { + var rule = this._getRule(url, ua, false); + + if (typeof rule === 'undefined') { + return; + } + + return !rule || rule.allow; +}; + +/** + * Returns the line number of the matching directive for the specified + * URL and user-agent if any. + * + * The line numbers start at 1 and go up (1-based indexing). + * + * Return -1 if there is no matching directive. If a rule is manually + * added without a lineNumber then this will return undefined for that + * rule. + * + * @param {string} url + * @param {string?} ua + * @return {number?} + */ +Robots.prototype.getMatchingLineNumber = function (url, ua) { + var rule = this._getRule(url, ua, false); + + return rule ? rule.lineNumber : -1; +}; + +/** + * Returns the opposite of isAllowed() + * + * @param {string} url + * @param {string?} ua + * @return {boolean} + */ +Robots.prototype.isDisallowed = function (url, ua) { + return !this.isAllowed(url, ua); +}; + +/** + * Returns trues if explicitly disallowed + * for the specified user agent (User Agent wildcards are discarded). + * + * This will return undefined if the URL is not valid for this robots.txt file. + * + * @param {string} url + * @param {string} ua + * @return {boolean?} + */ +Robots.prototype.isExplicitlyDisallowed = function (url, ua) { + var rule = this._getRule(url, ua, true); + if (typeof rule === 'undefined') { + return; + } + + return !(!rule || rule.allow); +}; + +/** + * Gets the crawl delay if there is one. + * + * Will return undefined if there is no crawl delay set. + * + * @param {string} ua + * @return {number?} + */ +Robots.prototype.getCrawlDelay = function (ua) { + var userAgent = formatUserAgent(ua || '*'); + + return (this._rules[userAgent] || this._rules['*'] || {}).crawlDelay; +}; + +/** + * Returns the preferred host if there is one. + * + * @return {string?} + */ +Robots.prototype.getPreferredHost = function () { + return this._preferredHost; +}; + +/** + * Returns an array of sitemap URLs if there are any. + * + * @return {Array.} + */ +Robots.prototype.getSitemaps = function () { + return this._sitemaps.slice(0); +}; + +module.exports = Robots; \ No newline at end of file diff --git a/tests/robots.js b/tests/robots.js new file mode 100644 index 00000000..dd501c38 --- /dev/null +++ b/tests/robots.js @@ -0,0 +1,903 @@ +var robotsParser = require('../index'); +var expect = require('chai').expect; + + +function testRobots(url, contents, allowed, disallowed) { + var robots = robotsParser(url, contents); + + allowed.forEach(function (url) { + expect(robots.isAllowed(url)).to.equal(true); + }); + + disallowed.forEach(function (url) { + expect(robots.isDisallowed(url)).to.equal(true); + }); +} + +describe('Robots', function () { + it('should parse the disallow directive', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish/', + 'Disallow: /test.html' + ].join('\n'); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + 'http://www.example.com/test.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should parse the allow directive', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish/', + 'Disallow: /test.html', + 'Allow: /fish/test.html', + 'Allow: /test.html' + ].join('\n'); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/fish/test.html', + 'http://www.example.com/Test.html', + 'http://www.example.com/test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should parse patterns', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish*.php', + 'Disallow: /*.dext$', + 'Disallow: /dir*' + ].join('\n'); + + var allowed = [ + 'http://www.example.com/Fish.PHP', + 'http://www.example.com/Fish.dext1', + 'http://www.example.com/folder/dir.html', + 'http://www.example.com/folder/dir/test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish.php', + 'http://www.example.com/fishheads/catfish.php?parameters', + 'http://www.example.com/AnYthInG.dext', + 'http://www.example.com/Fish.dext.dext', + 'http://www.example.com/dir/test.html', + 'http://www.example.com/directory.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should have the correct order precedence for allow and disallow', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish*.php', + 'Allow: /fish/index.php', + 'Disallow: /test', + 'Allow: /test/', + 'Disallow: /aa/', + 'Allow: /aa/', + 'Allow: /bb/', + 'Disallow: /bb/', + ].join('\n'); + + var allowed = [ + 'http://www.example.com/test/index.html', + 'http://www.example.com/fish/index.php', + 'http://www.example.com/test/', + 'http://www.example.com/aa/', + 'http://www.example.com/bb/', + 'http://www.example.com/x/' + ]; + + var disallowed = [ + 'http://www.example.com/fish.php', + 'http://www.example.com/fishheads/catfish.php?parameters', + 'http://www.example.com/test' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should have the correct order precedence for wildcards', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /*/', + 'Allow: /x/', + ].join('\n'); + + var allowed = [ + 'http://www.example.com/x/', + 'http://www.example.com/fish.php', + 'http://www.example.com/test' + ]; + + var disallowed = [ + 'http://www.example.com/a/', + 'http://www.example.com/xx/', + 'http://www.example.com/test/index.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should parse lines delimitated by \\r', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish/', + 'Disallow: /test.html' + ].join('\r'); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + 'http://www.example.com/test.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should parse lines delimitated by \\r\\n', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish/', + 'Disallow: /test.html' + ].join('\r\n'); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + 'http://www.example.com/test.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + + it('should parse lines delimitated by mixed line endings', function () { + var contents = [ + 'User-agent: *\r', + 'Disallow: /fish/\r\n', + 'Disallow: /test.html\n\n' + ].join(''); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + 'http://www.example.com/test.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should ignore rules that are not in a group', function () { + var contents = [ + 'Disallow: /secret.html', + 'Disallow: /test', + ].join('\n'); + + var allowed = [ + 'http://www.example.com/secret.html', + 'http://www.example.com/test/index.html', + 'http://www.example.com/test/' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, []); + }); + + + it('should ignore comments', function () { + var contents = [ + '#', + '# This is a comment', + '#', + 'User-agent: *', + '# This is a comment', + 'Disallow: /fish/ # ignore', + '# Disallow: fish', + 'Disallow: /test.html' + ].join('\n'); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + 'http://www.example.com/test.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should ignore invalid lines', function () { + var contents = [ + 'invalid line', + 'User-agent: *', + 'Disallow: /fish/', + ':::::another invalid line:::::', + 'Disallow: /test.html', + 'Unknown: tule' + ].join('\n'); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + 'http://www.example.com/test.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should ignore empty user-agent lines', function () { + var contents = [ + 'User-agent:', + 'Disallow: /fish/', + 'Disallow: /test.html' + ].join('\n'); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html', + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + 'http://www.example.com/test.html' + ]; + + var disallowed = []; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should support groups with multiple user agents (case insensitive)', function () { + var contents = [ + 'User-agent: agenta', + 'User-agent: agentb', + 'Disallow: /fish', + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.isAllowed("http://www.example.com/fish", "agenta")).to.equal(false); + }); + + it('should return undefined for invalid urls', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /secret.html', + 'Disallow: /test', + ].join('\n'); + + var invalidUrls = [ + 'http://example.com/secret.html', + 'http://ex ample.com/secret.html', + 'http://www.example.net/test/index.html', + 'http://www.examsple.com/test/', + 'example.com/test/', + ':::::;;`\\|/.example.com/test/' + ]; + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + invalidUrls.forEach(function (url) { + expect(robots.isAllowed(url)).to.equal(undefined); + }); + }); + + it('should handle Unicode, urlencoded and punycode URLs', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /secret.html', + 'Disallow: /test', + ].join('\n'); + + var allowed = [ + 'http://www.münich.com/index.html', + 'http://www.xn--mnich-kva.com/index.html', + 'http://www.m%C3%BCnich.com/index.html' + ]; + + var disallowed = [ + 'http://www.münich.com/secret.html', + 'http://www.xn--mnich-kva.com/secret.html', + 'http://www.m%C3%BCnich.com/secret.html' + ]; + + testRobots('http://www.münich.com/robots.txt', contents, allowed, disallowed); + testRobots('http://www.xn--mnich-kva.com/robots.txt', contents, allowed, disallowed); + testRobots('http://www.m%C3%BCnich.com/robots.txt', contents, allowed, disallowed); + }); + + it('should handle Unicode and urlencoded paths', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /%CF%80', + 'Disallow: /%e2%9d%83', + 'Disallow: /%a%a', + 'Disallow: /💩', + 'Disallow: /✼*t$', + 'Disallow: /%E2%9C%A4*t$', + 'Disallow: /✿%a', + 'Disallow: /http%3A%2F%2Fexample.org' + ].join('\n'); + + var allowed = [ + 'http://www.example.com/✼testing', + 'http://www.example.com/%E2%9C%BCtesting', + 'http://www.example.com/✤testing', + 'http://www.example.com/%E2%9C%A4testing', + 'http://www.example.com/http://example.org', + 'http://www.example.com/http:%2F%2Fexample.org' + ]; + + var disallowed = [ + 'http://www.example.com/%CF%80', + 'http://www.example.com/%CF%80/index.html', + 'http://www.example.com/π', + 'http://www.example.com/π/index.html', + 'http://www.example.com/%e2%9d%83', + 'http://www.example.com/%E2%9D%83/index.html', + 'http://www.example.com/❃', + 'http://www.example.com/❃/index.html', + 'http://www.example.com/%F0%9F%92%A9', + 'http://www.example.com/%F0%9F%92%A9/index.html', + 'http://www.example.com/💩', + 'http://www.example.com/💩/index.html', + 'http://www.example.com/%a%a', + 'http://www.example.com/%a%a/index.html', + 'http://www.example.com/✼test', + 'http://www.example.com/%E2%9C%BCtest', + 'http://www.example.com/✤test', + 'http://www.example.com/%E2%9C%A4testt', + 'http://www.example.com/✿%a', + 'http://www.example.com/%E2%9C%BF%atest', + 'http://www.example.com/http%3A%2F%2Fexample.org' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should handle lone high / low surrogates', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /\uD800', + 'Disallow: /\uDC00' + ].join('\n'); + + // These are invalid so can't be disallowed + var allowed = [ + 'http://www.example.com/\uDC00', + 'http://www.example.com/\uD800' + ]; + + var disallowed = []; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should ignore host case', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /secret.html', + 'Disallow: /test', + ].join('\n'); + + var allowed = [ + 'http://www.example.com/index.html', + 'http://www.ExAmPlE.com/index.html', + 'http://www.EXAMPLE.com/index.html' + ]; + + var disallowed = [ + 'http://www.example.com/secret.html', + 'http://www.ExAmPlE.com/secret.html', + 'http://www.EXAMPLE.com/secret.html' + ]; + + testRobots('http://www.eXample.com/robots.txt', contents, allowed, disallowed); + }); + + it('should handle relative paths', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish', + 'Allow: /fish/test', + ].join('\n'); + + var robots = robotsParser('/robots.txt', contents); + expect(robots.isAllowed('/fish/test')).to.equal(true); + expect(robots.isAllowed('/fish')).to.equal(false); + }); + + it('should not allow relative paths if domain specified', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish', + 'Allow: /fish/test', + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + expect(robots.isAllowed('/fish/test')).to.equal(undefined); + expect(robots.isAllowed('/fish')).to.equal(undefined); + }); + + it('should not treat invalid robots.txt URLs as relative', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish', + 'Allow: /fish/test', + ].join('\n'); + + var robots = robotsParser('https://ex ample.com/robots.txt', contents); + expect(robots.isAllowed('/fish/test')).to.equal(undefined); + expect(robots.isAllowed('/fish')).to.equal(undefined); + }); + + it('should not allow URls if domain specified and robots.txt is relative', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish', + 'Allow: /fish/test', + ].join('\n'); + + var robots = robotsParser('/robots.txt', contents); + expect(robots.isAllowed('http://www.example.com/fish/test')).to.equal(undefined); + expect(robots.isAllowed('http://www.example.com/fish')).to.equal(undefined); + }); + + it('should allow all if empty robots.txt', function () { + var allowed = [ + 'http://www.example.com/secret.html', + 'http://www.example.com/test/index.html', + 'http://www.example.com/test/' + ]; + + var robots = robotsParser('http://www.example.com/robots.txt', ''); + + allowed.forEach(function (url) { + expect(robots.isAllowed(url)).to.equal(true); + }); + }); + + it('should treat null as allowing all', function () { + var robots = robotsParser('http://www.example.com/robots.txt', null); + + expect(robots.isAllowed("http://www.example.com/", "userAgent")).to.equal(true); + expect(robots.isAllowed("http://www.example.com/")).to.equal(true); + }); + + it('should handle invalid robots.txt urls', function () { + var contents = [ + 'user-agent: *', + 'disallow: /', + + 'host: www.example.com', + 'sitemap: /sitemap.xml' + ].join('\n'); + + var sitemapUrls = [ + undefined, + null, + 'null', + ':/wom/test/' + ]; + + sitemapUrls.forEach(function (url) { + var robots = robotsParser(url, contents); + expect(robots.isAllowed('http://www.example.com/index.html')).to.equal(undefined); + expect(robots.getPreferredHost()).to.equal('www.example.com'); + expect(robots.getSitemaps()).to.eql(['/sitemap.xml']); + }); + }); + + it('should parse the crawl-delay directive', function () { + var contents = [ + 'user-agent: a', + 'crawl-delay: 1', + + 'user-agent: b', + 'disallow: /d', + + 'user-agent: c', + 'user-agent: d', + 'crawl-delay: 10' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getCrawlDelay('a')).to.equal(1); + expect(robots.getCrawlDelay('b')).to.equal(undefined); + expect(robots.getCrawlDelay('c')).to.equal(10); + expect(robots.getCrawlDelay('d')).to.equal(10); + expect(robots.getCrawlDelay()).to.equal(undefined); + }); + + it('should ignore invalid crawl-delay directives', function () { + var contents = [ + 'user-agent: a', + 'crawl-delay: 1.2.1', + + 'user-agent: b', + 'crawl-delay: 1.a0', + + 'user-agent: c', + 'user-agent: d', + 'crawl-delay: 10a' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getCrawlDelay('a')).to.equal(undefined); + expect(robots.getCrawlDelay('b')).to.equal(undefined); + expect(robots.getCrawlDelay('c')).to.equal(undefined); + expect(robots.getCrawlDelay('d')).to.equal(undefined); + }); + + it('should parse the sitemap directive', function () { + var contents = [ + 'user-agent: a', + 'crawl-delay: 1', + 'sitemap: http://example.com/test.xml', + + 'user-agent: b', + 'disallow: /d', + + 'sitemap: /sitemap.xml', + 'sitemap: http://example.com/test/sitemap.xml ' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getSitemaps()).to.eql([ + 'http://example.com/test.xml', + '/sitemap.xml', + 'http://example.com/test/sitemap.xml' + ]); + }); + + it('should parse the host directive', function () { + var contents = [ + 'user-agent: a', + 'crawl-delay: 1', + 'host: www.example.net', + + 'user-agent: b', + 'disallow: /d', + + 'host: example.com' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getPreferredHost()).to.equal('example.com'); + }); + + it('should parse empty and invalid directives', function () { + var contents = [ + 'user-agent:', + 'user-agent:::: a::', + 'crawl-delay:', + 'crawl-delay:::: 0:', + 'host:', + 'host:: example.com', + 'sitemap:', + 'sitemap:: site:map.xml', + 'disallow:', + 'disallow::: /:', + 'allow:', + 'allow::: /:', + ].join('\n'); + + robotsParser('http://www.example.com/robots.txt', contents); + }); + + it('should treat only the last host directive as valid', function () { + var contents = [ + 'user-agent: a', + 'crawl-delay: 1', + 'host: www.example.net', + + 'user-agent: b', + 'disallow: /d', + + 'host: example.net', + 'host: example.com' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getPreferredHost()).to.equal('example.com'); + }); + + it('should return null when there is no host directive', function () { + var contents = [ + 'user-agent: a', + 'crawl-delay: 1', + + 'user-agent: b', + 'disallow: /d', + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getPreferredHost()).to.equal(null); + }); + + it('should fallback to * when a UA has no rules of its own', function () { + var contents = [ + 'user-agent: *', + 'crawl-delay: 1', + + 'user-agent: b', + 'crawl-delay: 12', + + 'user-agent: c', + 'user-agent: d', + 'crawl-delay: 10' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getCrawlDelay('should-fall-back')).to.equal(1); + expect(robots.getCrawlDelay('d')).to.equal(10); + expect(robots.getCrawlDelay('dd')).to.equal(1); + }); + + it('should not fallback to * when a UA has rules', function () { + var contents = [ + 'user-agent: *', + 'crawl-delay: 1', + + 'user-agent: b', + 'disallow:' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getCrawlDelay('b')).to.equal(undefined); + }); + + it('should handle UAs with object property names', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish', + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + expect(robots.isAllowed('http://www.example.com/fish', 'constructor')).to.equal(false); + expect(robots.isAllowed('http://www.example.com/fish', '__proto__')).to.equal(false); + }); + + it('should ignore version numbers in the UA string', function () { + var contents = [ + 'user-agent: *', + 'crawl-delay: 1', + + 'user-agent: b', + 'crawl-delay: 12', + + 'user-agent: c', + 'user-agent: d', + 'crawl-delay: 10' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getCrawlDelay('should-fall-back/1.0.0')).to.equal(1); + expect(robots.getCrawlDelay('d/12')).to.equal(10); + expect(robots.getCrawlDelay('dd / 0-32-3')).to.equal(1); + expect(robots.getCrawlDelay('b / 1.0')).to.equal(12); + }); + + + it('should return the line number of the matching directive', function () { + var contents = [ + '', + 'User-agent: *', + '', + 'Disallow: /fish/', + 'Disallow: /test.html', + 'Allow: /fish/test.html', + 'Allow: /test.html', + '', + 'User-agent: a', + 'allow: /', + '', + 'User-agent: b', + 'disallow: /test', + 'disallow: /t*t', + '', + 'User-agent: c', + 'Disallow: /fish*.php', + 'Allow: /fish/index.php' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getMatchingLineNumber('http://www.example.com/fish')).to.equal(-1); + expect(robots.getMatchingLineNumber('http://www.example.com/fish/test.html')).to.equal(6); + expect(robots.getMatchingLineNumber('http://www.example.com/Test.html')).to.equal(-1); + + expect(robots.getMatchingLineNumber('http://www.example.com/fish/index.php')).to.equal(4); + expect(robots.getMatchingLineNumber('http://www.example.com/fish/')).to.equal(4); + expect(robots.getMatchingLineNumber('http://www.example.com/test.html')).to.equal(7); + + expect(robots.getMatchingLineNumber('http://www.example.com/test.html', 'a')).to.equal(10); + + expect(robots.getMatchingLineNumber('http://www.example.com/fish.php', 'c')).to.equal(17); + expect(robots.getMatchingLineNumber('http://www.example.com/fish/index.php', 'c')).to.equal(18); + }); + + it('should handle large wildcards efficiently', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /' + '*'.repeat(2048) + '.html', + ].join('\n'); + + var allowed = [ + 'http://www.example.com/' + 'sub'.repeat(2048) + 'folder/index.php', + ]; + + var disallowed = [ + 'http://www.example.com/secret.html' + ]; + + const start = Date.now(); + testRobots('http://www.eXample.com/robots.txt', contents, allowed, disallowed); + const end = Date.now(); + + // Should take less than 500 ms (high to allow for variableness of + // machines running the test, should normally be much less) + expect(end - start).to.be.lessThan(500); + }); + + it('should honor given port number', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish/', + 'Disallow: /test.html' + ].join('\n'); + + var allowed = [ + 'http://www.example.com:8080/fish', + 'http://www.example.com:8080/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html', + 'http://www.example.com:80/fish', + 'http://www.example.com:80/Test.html' + ]; + + testRobots('http://www.example.com:8080/robots.txt', contents, allowed, disallowed); + }); + + it('should default to port 80 for http: if no port given', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish/', + 'Disallow: /test.html' + ].join('\n'); + + var allowed = [ + 'http://www.example.com:80/fish', + 'http://www.example.com:80/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com:443/fish', + 'http://www.example.com:443/Test.html', + 'http://www.example.com:80/fish/index.php', + 'http://www.example.com:80/fish/', + 'http://www.example.com:80/test.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should default to port 443 for https: if no port given', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish/', + 'Disallow: /test.html' + ].join('\n'); + + var allowed = [ + 'https://www.example.com:443/fish', + 'https://www.example.com:443/Test.html', + 'https://www.example.com/fish', + 'https://www.example.com/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com:80/fish', + 'http://www.example.com:80/Test.html', + 'http://www.example.com:443/fish/index.php', + 'http://www.example.com:443/fish/', + 'http://www.example.com:443/test.html' + ]; + + testRobots('https://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should not be disallowed when wildcard is used in explicit mode', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /', + ].join('\n') + + var url = 'https://www.example.com/hello' + var userAgent = 'SomeBot'; + var robots = robotsParser(url, contents); + + expect(robots.isExplicitlyDisallowed(url, userAgent)).to.equal(false) + }); + + it('should be disallowed when user agent equal robots rule in explicit mode', function () { + var contents = [ + 'User-agent: SomeBot', + 'Disallow: /', + ].join('\n') + + var url = 'https://www.example.com/hello' + var userAgent = 'SomeBot'; + var robots = robotsParser(url, contents); + + expect(robots.isExplicitlyDisallowed(url, userAgent)).to.equal(true) + }); + + it('should return undefined when given an invalid URL in explicit mode', function () { + var contents = [ + 'User-agent: SomeBot', + 'Disallow: /', + ].join('\n') + + var url = 'https://www.example.com/hello' + var userAgent = 'SomeBot'; + var robots = robotsParser('http://example.com', contents); + + expect(robots.isExplicitlyDisallowed(url, userAgent)).to.equal(undefined) + }); +}); From 362ca05ab88589eaad9116f1a8ed03b24f81cbee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20Sch=C3=A4fer?= Date: Sat, 10 May 2025 21:58:07 +0200 Subject: [PATCH 03/10] Adapt robots parser and its tests for Rhino --- code/Global/Robots.js | 1028 ++++++++++++----------- tests/robots.js | 1797 +++++++++++++++++++++-------------------- 2 files changed, 1443 insertions(+), 1382 deletions(-) diff --git a/code/Global/Robots.js b/code/Global/Robots.js index b9835e0c..04c69d67 100644 --- a/code/Global/Robots.js +++ b/code/Global/Robots.js @@ -1,491 +1,543 @@ -/** - * Trims the white space from the start and end of the line. - * - * If the line is an array it will strip the white space from - * the start and end of each element of the array. - * - * @param {string|Array} line - * @return {string|Array} - * @private - */ -function trimLine(line) { - if (!line) { - return null; - } +// Robots parser adapted for Rhino-compatible JavaScript +// Source: +// Copyright (c) 2014 Sam Clarke +// Copyright (c) 2025 Antville.org +// MIT License (MIT) - if (Array.isArray(line)) { - return line.map(trimLine); - } +// Transformation steps: +// 1. Add IIFE around the code +// 2. Replace module.exports with return statement +// 3. Add conditional module.exports for CommonJS support +// 4. Add URL class imitation - return String(line).trim(); +var Robots = (() => { + /** + * Half-baked (read-only) imitation of the URL class of Node.js + */ + function nodeJsUrl(str, base) { + if (!str.includes('://')) { + str = (base || 'http://localhost') + str; + } + + const url = new java.net.URL(str); + const port = url.port < 0 ? '' : url.port; + const userInfo = (url.getUserInfo() || "").split(':'); + + return { + hash: url.ref ? '#' + url.ref : '', + href: url.toString(), + host: url.host + (port ? ':' + port : port), + hostname: url.host, + password: userInfo[1] || "", + pathname: url.path, + origin: url.protocol + '://' + url.host + (port ? ':' + port : port), + port, + protocol: url.protocol, + search: url.queryy ? '?' + url.query : '', + searchParams: { + get: () => null, + set: () => null + }, + username: userInfo[0] || "", + }; + } + + if (typeof URL === 'undefined') { + globalThis.URL = nodeJsUrl; + } + + /** + * Trims the white space from the start and end of the line. + * + * If the line is an array it will strip the white space from + * the start and end of each element of the array. + * + * @param {string|Array} line + * @return {string|Array} + * @private + */ + function trimLine(line) { + if (!line) { + return null; + } + + if (Array.isArray(line)) { + return line.map(trimLine); + } + + return String(line).trim(); + } + + /** + * Remove comments from lines + * + * @param {string} line + * @return {string} + * @private + */ + function removeComments(line) { + var commentStartIndex = line.indexOf('#'); + if (commentStartIndex > -1) { + return line.substr(0, commentStartIndex); + } + + return line; + } + + /** + * Splits a line at the first occurrence of : + * + * @param {string} line + * @return {Array.} + * @private + */ + function splitLine(line) { + var idx = String(line).indexOf(':'); + + if (!line || idx < 0) { + return null; + } + + return [line.slice(0, idx), line.slice(idx + 1)]; + } + + /** + * Normalises the user-agent string by converting it to + * lower case and removing any version numbers. + * + * @param {string} userAgent + * @return {string} + * @private + */ + function formatUserAgent(userAgent) { + var formattedUserAgent = userAgent.toLowerCase(); + + // Strip the version number from robot/1.0 user agents + var idx = formattedUserAgent.indexOf('/'); + if (idx > -1) { + formattedUserAgent = formattedUserAgent.substr(0, idx); + } + + return formattedUserAgent.trim(); + } + + /** + * Normalises the URL encoding of a path by encoding + * unicode characters. + * + * @param {string} path + * @return {string} + * @private + */ + function normaliseEncoding(path) { + try { + return urlEncodeToUpper(encodeURI(path).replace(/%25/g, '%')); + } catch (e) { + return path; + } + } + + /** + * Convert URL encodings to support case. + * + * e.g.: %2a%ef becomes %2A%EF + * + * @param {string} path + * @return {string} + * @private + */ + function urlEncodeToUpper(path) { + return path.replace(/%[0-9a-fA-F]{2}/g, function (match) { + return match.toUpperCase(); + }); + } + + /** + * Matches a pattern with the specified path + * + * Uses same algorithm to match patterns as the Google implementation in + * google/robotstxt so it should be consistent with the spec. + * + * @see https://github.com/google/robotstxt/blob/f465f0ede81099dd8bc4aeb2966b3a892bd488b3/robots.cc#L74 + * @param {string} pattern + * @param {string} path + * @return {boolean} + * @private + */ + function matches(pattern, path) { + // I've added extra comments to try make this easier to understand + + // Stores the lengths of all the current matching substrings. + // Maximum number of possible matching lengths is every length in path plus + // 1 to handle 0 length too (if pattern starts with * which is zero or more) + var matchingLengths = new Array(path.length + 1); + var numMatchingLengths = 1; + + // Initially longest match is 0 + matchingLengths[0] = 0; + + for (var p = 0; p < pattern.length; p++) { + // If $ is at the end of pattern then we must match the whole path. + // Which is true if the longest matching length matches path length + if (pattern[p] === '$' && p + 1 === pattern.length) { + return matchingLengths[numMatchingLengths - 1] === path.length; + } + + // Handle wildcards + if (pattern[p] == '*') { + // Wildcard so all substrings minus the current smallest matching + // length are matches + numMatchingLengths = path.length - matchingLengths[0] + 1; + + // Update matching lengths to include the smallest all the way up + // to numMatchingLengths + // Don't update smallest possible match as * matches zero or more + // so the smallest current match is also valid + for (var i = 1; i < numMatchingLengths; i++) { + matchingLengths[i] = matchingLengths[i - 1] + 1; + } + } else { + // Check the char at the matching length matches the pattern, if it + // does increment it and add it as a valid length, ignore if not. + var numMatches = 0; + for (var i = 0; i < numMatchingLengths; i++) { + if ( + matchingLengths[i] < path.length && + path[matchingLengths[i]] === pattern[p] + ) { + matchingLengths[numMatches++] = matchingLengths[i] + 1; + } + } + + // No paths matched the current pattern char so not a match + if (numMatches == 0) { + return false; + } + + numMatchingLengths = numMatches; + } + } + + return true; + } + + function parseRobots(contents, robots) { + var newlineRegex = /\r\n|\r|\n/; + var lines = contents + .split(newlineRegex) + .map(removeComments) + .map(splitLine) + .map(trimLine); + + var currentUserAgents = []; + var isNoneUserAgentState = true; + for (var i = 0; i < lines.length; i++) { + var line = lines[i]; + + if (!line || !line[0]) { + continue; + } + + switch (line[0].toLowerCase()) { + case 'user-agent': + if (isNoneUserAgentState) { + currentUserAgents.length = 0; + } + + if (line[1]) { + currentUserAgents.push(formatUserAgent(line[1])); + } + break; + case 'disallow': + robots.addRule(currentUserAgents, line[1], false, i + 1); + break; + case 'allow': + robots.addRule(currentUserAgents, line[1], true, i + 1); + break; + case 'crawl-delay': + robots.setCrawlDelay(currentUserAgents, line[1]); + break; + case 'sitemap': + if (line[1]) { + robots.addSitemap(line[1]); + } + break; + case 'host': + if (line[1]) { + robots.setPreferredHost(line[1].toLowerCase()); + } + break; + } + + isNoneUserAgentState = line[0].toLowerCase() !== 'user-agent'; + } + } + + /** + * Returns if a pattern is allowed by the specified rules. + * + * @param {string} path + * @param {Array.} rules + * @return {Object?} + * @private + */ + function findRule(path, rules) { + var matchedRule = null; + + for (var i = 0; i < rules.length; i++) { + var rule = rules[i]; + + if (!matches(rule.pattern, path)) { + continue; + } + + // The longest matching rule takes precedence + // If rules are the same length then allow takes precedence + if (!matchedRule || rule.pattern.length > matchedRule.pattern.length) { + matchedRule = rule; + } else if ( + rule.pattern.length == matchedRule.pattern.length && + rule.allow && + !matchedRule.allow + ) { + matchedRule = rule; + } + } + + return matchedRule; + } + + /** + * Converts provided string into an URL object. + * + * Will return null if provided string is not a valid URL. + * + * @param {string} url + * @return {?URL} + * @private + */ + function parseUrl(url) { + try { + // Specify a URL to be used with relative paths + // Using non-existent subdomain so can never cause conflict unless + // trying to crawl it but doesn't exist and even if tried worst that can + // happen is it allows relative URLs on it. + var url = new URL(url, 'http://robots-relative.samclarke.com/'); + + if (!url.port) { + url.port = url.protocol === 'https:' ? 443 : 80; + } + + return url; + } catch (e) { + return null; + } + } + + function Robots(url, contents) { + this._url = parseUrl(url) || {}; + this._rules = Object.create(null); + this._sitemaps = []; + this._preferredHost = null; + + parseRobots(contents || '', this); + } + + /** + * Adds the specified allow/deny rule to the rules + * for the specified user-agents. + * + * @param {Array.} userAgents + * @param {string} pattern + * @param {boolean} allow + * @param {number} [lineNumber] Should use 1-based indexing + */ + Robots.prototype.addRule = function (userAgents, pattern, allow, lineNumber) { + var rules = this._rules; + + userAgents.forEach(function (userAgent) { + rules[userAgent] = rules[userAgent] || []; + + if (!pattern) { + return; + } + + rules[userAgent].push({ + pattern: normaliseEncoding(pattern), + allow: allow, + lineNumber: lineNumber + }); + }); + }; + + /** + * Adds the specified delay to the specified user agents. + * + * @param {Array.} userAgents + * @param {string} delayStr + */ + Robots.prototype.setCrawlDelay = function (userAgents, delayStr) { + var rules = this._rules; + var delay = Number(delayStr); + + userAgents.forEach(function (userAgent) { + rules[userAgent] = rules[userAgent] || []; + + if (isNaN(delay)) { + return; + } + + rules[userAgent].crawlDelay = delay; + }); + }; + + /** + * Add a sitemap + * + * @param {string} url + */ + Robots.prototype.addSitemap = function (url) { + this._sitemaps.push(url); + }; + + /** + * Sets the preferred host name + * + * @param {string} url + */ + Robots.prototype.setPreferredHost = function (url) { + this._preferredHost = url; + }; + + Robots.prototype._getRule = function (url, ua, explicit) { + var parsedUrl = parseUrl(url) || {}; + var userAgent = formatUserAgent(ua || '*'); + + // The base URL must match otherwise this robots.txt is not valid for it. + if ( + parsedUrl.protocol !== this._url.protocol || + parsedUrl.hostname !== this._url.hostname || + parsedUrl.port !== this._url.port + ) { + return; + } + + var rules = this._rules[userAgent]; + if (!explicit) { + rules = rules || this._rules['*']; + } + rules = rules || []; + var path = urlEncodeToUpper(parsedUrl.pathname + parsedUrl.search); + var rule = findRule(path, rules); + + return rule; + }; + + /** + * Returns true if allowed, false if not allowed. + * + * Will return undefined if the URL is not valid for + * this robots.txt file. + * + * @param {string} url + * @param {string?} ua + * @return {boolean?} + */ + Robots.prototype.isAllowed = function (url, ua) { + var rule = this._getRule(url, ua, false); + + if (typeof rule === 'undefined') { + return; + } + + return !rule || rule.allow; + }; + + /** + * Returns the line number of the matching directive for the specified + * URL and user-agent if any. + * + * The line numbers start at 1 and go up (1-based indexing). + * + * Return -1 if there is no matching directive. If a rule is manually + * added without a lineNumber then this will return undefined for that + * rule. + * + * @param {string} url + * @param {string?} ua + * @return {number?} + */ + Robots.prototype.getMatchingLineNumber = function (url, ua) { + var rule = this._getRule(url, ua, false); + + return rule ? rule.lineNumber : -1; + }; + + /** + * Returns the opposite of isAllowed() + * + * @param {string} url + * @param {string?} ua + * @return {boolean} + */ + Robots.prototype.isDisallowed = function (url, ua) { + return !this.isAllowed(url, ua); + }; + + /** + * Returns trues if explicitly disallowed + * for the specified user agent (User Agent wildcards are discarded). + * + * This will return undefined if the URL is not valid for this robots.txt file. + * + * @param {string} url + * @param {string} ua + * @return {boolean?} + */ + Robots.prototype.isExplicitlyDisallowed = function (url, ua) { + var rule = this._getRule(url, ua, true); + if (typeof rule === 'undefined') { + return; + } + + return !(!rule || rule.allow); + }; + + /** + * Gets the crawl delay if there is one. + * + * Will return undefined if there is no crawl delay set. + * + * @param {string} ua + * @return {number?} + */ + Robots.prototype.getCrawlDelay = function (ua) { + var userAgent = formatUserAgent(ua || '*'); + + return (this._rules[userAgent] || this._rules['*'] || {}).crawlDelay; + }; + + /** + * Returns the preferred host if there is one. + * + * @return {string?} + */ + Robots.prototype.getPreferredHost = function () { + return this._preferredHost; + }; + + /** + * Returns an array of sitemap URLs if there are any. + * + * @return {Array.} + */ + Robots.prototype.getSitemaps = function () { + return this._sitemaps.slice(0); + }; + + return Robots; +})(); + +if (typeof module !== 'undefined' && module.exports) { + module.exports = Robots; } - -/** - * Remove comments from lines - * - * @param {string} line - * @return {string} - * @private - */ -function removeComments(line) { - var commentStartIndex = line.indexOf('#'); - if (commentStartIndex > -1) { - return line.substr(0, commentStartIndex); - } - - return line; -} - -/** - * Splits a line at the first occurrence of : - * - * @param {string} line - * @return {Array.} - * @private - */ -function splitLine(line) { - var idx = String(line).indexOf(':'); - - if (!line || idx < 0) { - return null; - } - - return [line.slice(0, idx), line.slice(idx + 1)]; -} - -/** - * Normalises the user-agent string by converting it to - * lower case and removing any version numbers. - * - * @param {string} userAgent - * @return {string} - * @private - */ -function formatUserAgent(userAgent) { - var formattedUserAgent = userAgent.toLowerCase(); - - // Strip the version number from robot/1.0 user agents - var idx = formattedUserAgent.indexOf('/'); - if (idx > -1) { - formattedUserAgent = formattedUserAgent.substr(0, idx); - } - - return formattedUserAgent.trim(); -} - -/** - * Normalises the URL encoding of a path by encoding - * unicode characters. - * - * @param {string} path - * @return {string} - * @private - */ -function normaliseEncoding(path) { - try { - return urlEncodeToUpper(encodeURI(path).replace(/%25/g, '%')); - } catch (e) { - return path; - } -} - -/** - * Convert URL encodings to support case. - * - * e.g.: %2a%ef becomes %2A%EF - * - * @param {string} path - * @return {string} - * @private - */ -function urlEncodeToUpper(path) { - return path.replace(/%[0-9a-fA-F]{2}/g, function (match) { - return match.toUpperCase(); - }); -} - -/** - * Matches a pattern with the specified path - * - * Uses same algorithm to match patterns as the Google implementation in - * google/robotstxt so it should be consistent with the spec. - * - * @see https://github.com/google/robotstxt/blob/f465f0ede81099dd8bc4aeb2966b3a892bd488b3/robots.cc#L74 - * @param {string} pattern - * @param {string} path - * @return {boolean} - * @private - */ -function matches(pattern, path) { - // I've added extra comments to try make this easier to understand - - // Stores the lengths of all the current matching substrings. - // Maximum number of possible matching lengths is every length in path plus - // 1 to handle 0 length too (if pattern starts with * which is zero or more) - var matchingLengths = new Array(path.length + 1); - var numMatchingLengths = 1; - - // Initially longest match is 0 - matchingLengths[0] = 0; - - for (var p = 0; p < pattern.length; p++) { - // If $ is at the end of pattern then we must match the whole path. - // Which is true if the longest matching length matches path length - if (pattern[p] === '$' && p + 1 === pattern.length) { - return matchingLengths[numMatchingLengths - 1] === path.length; - } - - // Handle wildcards - if (pattern[p] == '*') { - // Wildcard so all substrings minus the current smallest matching - // length are matches - numMatchingLengths = path.length - matchingLengths[0] + 1; - - // Update matching lengths to include the smallest all the way up - // to numMatchingLengths - // Don't update smallest possible match as * matches zero or more - // so the smallest current match is also valid - for (var i = 1; i < numMatchingLengths; i++) { - matchingLengths[i] = matchingLengths[i - 1] + 1; - } - } else { - // Check the char at the matching length matches the pattern, if it - // does increment it and add it as a valid length, ignore if not. - var numMatches = 0; - for (var i = 0; i < numMatchingLengths; i++) { - if ( - matchingLengths[i] < path.length && - path[matchingLengths[i]] === pattern[p] - ) { - matchingLengths[numMatches++] = matchingLengths[i] + 1; - } - } - - // No paths matched the current pattern char so not a match - if (numMatches == 0) { - return false; - } - - numMatchingLengths = numMatches; - } - } - - return true; -} - -function parseRobots(contents, robots) { - var newlineRegex = /\r\n|\r|\n/; - var lines = contents - .split(newlineRegex) - .map(removeComments) - .map(splitLine) - .map(trimLine); - - var currentUserAgents = []; - var isNoneUserAgentState = true; - for (var i = 0; i < lines.length; i++) { - var line = lines[i]; - - if (!line || !line[0]) { - continue; - } - - switch (line[0].toLowerCase()) { - case 'user-agent': - if (isNoneUserAgentState) { - currentUserAgents.length = 0; - } - - if (line[1]) { - currentUserAgents.push(formatUserAgent(line[1])); - } - break; - case 'disallow': - robots.addRule(currentUserAgents, line[1], false, i + 1); - break; - case 'allow': - robots.addRule(currentUserAgents, line[1], true, i + 1); - break; - case 'crawl-delay': - robots.setCrawlDelay(currentUserAgents, line[1]); - break; - case 'sitemap': - if (line[1]) { - robots.addSitemap(line[1]); - } - break; - case 'host': - if (line[1]) { - robots.setPreferredHost(line[1].toLowerCase()); - } - break; - } - - isNoneUserAgentState = line[0].toLowerCase() !== 'user-agent'; - } -} - -/** - * Returns if a pattern is allowed by the specified rules. - * - * @param {string} path - * @param {Array.} rules - * @return {Object?} - * @private - */ -function findRule(path, rules) { - var matchedRule = null; - - for (var i = 0; i < rules.length; i++) { - var rule = rules[i]; - - if (!matches(rule.pattern, path)) { - continue; - } - - // The longest matching rule takes precedence - // If rules are the same length then allow takes precedence - if (!matchedRule || rule.pattern.length > matchedRule.pattern.length) { - matchedRule = rule; - } else if ( - rule.pattern.length == matchedRule.pattern.length && - rule.allow && - !matchedRule.allow - ) { - matchedRule = rule; - } - } - - return matchedRule; -} - -/** - * Converts provided string into an URL object. - * - * Will return null if provided string is not a valid URL. - * - * @param {string} url - * @return {?URL} - * @private - */ -function parseUrl(url) { - try { - // Specify a URL to be used with relative paths - // Using non-existent subdomain so can never cause conflict unless - // trying to crawl it but doesn't exist and even if tried worst that can - // happen is it allows relative URLs on it. - var url = new URL(url, 'http://robots-relative.samclarke.com/'); - - if (!url.port) { - url.port = url.protocol === 'https:' ? 443 : 80; - } - - return url; - } catch (e) { - return null; - } -} - -function Robots(url, contents) { - this._url = parseUrl(url) || {}; - this._rules = Object.create(null); - this._sitemaps = []; - this._preferredHost = null; - - parseRobots(contents || '', this); -} - -/** - * Adds the specified allow/deny rule to the rules - * for the specified user-agents. - * - * @param {Array.} userAgents - * @param {string} pattern - * @param {boolean} allow - * @param {number} [lineNumber] Should use 1-based indexing - */ -Robots.prototype.addRule = function (userAgents, pattern, allow, lineNumber) { - var rules = this._rules; - - userAgents.forEach(function (userAgent) { - rules[userAgent] = rules[userAgent] || []; - - if (!pattern) { - return; - } - - rules[userAgent].push({ - pattern: normaliseEncoding(pattern), - allow: allow, - lineNumber: lineNumber - }); - }); -}; - -/** - * Adds the specified delay to the specified user agents. - * - * @param {Array.} userAgents - * @param {string} delayStr - */ -Robots.prototype.setCrawlDelay = function (userAgents, delayStr) { - var rules = this._rules; - var delay = Number(delayStr); - - userAgents.forEach(function (userAgent) { - rules[userAgent] = rules[userAgent] || []; - - if (isNaN(delay)) { - return; - } - - rules[userAgent].crawlDelay = delay; - }); -}; - -/** - * Add a sitemap - * - * @param {string} url - */ -Robots.prototype.addSitemap = function (url) { - this._sitemaps.push(url); -}; - -/** - * Sets the preferred host name - * - * @param {string} url - */ -Robots.prototype.setPreferredHost = function (url) { - this._preferredHost = url; -}; - -Robots.prototype._getRule = function (url, ua, explicit) { - var parsedUrl = parseUrl(url) || {}; - var userAgent = formatUserAgent(ua || '*'); - - // The base URL must match otherwise this robots.txt is not valid for it. - if ( - parsedUrl.protocol !== this._url.protocol || - parsedUrl.hostname !== this._url.hostname || - parsedUrl.port !== this._url.port - ) { - return; - } - - var rules = this._rules[userAgent]; - if (!explicit) { - rules = rules || this._rules['*']; - } - rules = rules || []; - - var path = urlEncodeToUpper(parsedUrl.pathname + parsedUrl.search); - var rule = findRule(path, rules); - - return rule; -}; - -/** - * Returns true if allowed, false if not allowed. - * - * Will return undefined if the URL is not valid for - * this robots.txt file. - * - * @param {string} url - * @param {string?} ua - * @return {boolean?} - */ -Robots.prototype.isAllowed = function (url, ua) { - var rule = this._getRule(url, ua, false); - - if (typeof rule === 'undefined') { - return; - } - - return !rule || rule.allow; -}; - -/** - * Returns the line number of the matching directive for the specified - * URL and user-agent if any. - * - * The line numbers start at 1 and go up (1-based indexing). - * - * Return -1 if there is no matching directive. If a rule is manually - * added without a lineNumber then this will return undefined for that - * rule. - * - * @param {string} url - * @param {string?} ua - * @return {number?} - */ -Robots.prototype.getMatchingLineNumber = function (url, ua) { - var rule = this._getRule(url, ua, false); - - return rule ? rule.lineNumber : -1; -}; - -/** - * Returns the opposite of isAllowed() - * - * @param {string} url - * @param {string?} ua - * @return {boolean} - */ -Robots.prototype.isDisallowed = function (url, ua) { - return !this.isAllowed(url, ua); -}; - -/** - * Returns trues if explicitly disallowed - * for the specified user agent (User Agent wildcards are discarded). - * - * This will return undefined if the URL is not valid for this robots.txt file. - * - * @param {string} url - * @param {string} ua - * @return {boolean?} - */ -Robots.prototype.isExplicitlyDisallowed = function (url, ua) { - var rule = this._getRule(url, ua, true); - if (typeof rule === 'undefined') { - return; - } - - return !(!rule || rule.allow); -}; - -/** - * Gets the crawl delay if there is one. - * - * Will return undefined if there is no crawl delay set. - * - * @param {string} ua - * @return {number?} - */ -Robots.prototype.getCrawlDelay = function (ua) { - var userAgent = formatUserAgent(ua || '*'); - - return (this._rules[userAgent] || this._rules['*'] || {}).crawlDelay; -}; - -/** - * Returns the preferred host if there is one. - * - * @return {string?} - */ -Robots.prototype.getPreferredHost = function () { - return this._preferredHost; -}; - -/** - * Returns an array of sitemap URLs if there are any. - * - * @return {Array.} - */ -Robots.prototype.getSitemaps = function () { - return this._sitemaps.slice(0); -}; - -module.exports = Robots; \ No newline at end of file diff --git a/tests/robots.js b/tests/robots.js index dd501c38..7a138549 100644 --- a/tests/robots.js +++ b/tests/robots.js @@ -1,903 +1,912 @@ -var robotsParser = require('../index'); -var expect = require('chai').expect; +// Unit tests of the robots parser +// Source: +// Copyright (c) 2014 Sam Clarke +// MIT License (MIT) +// Run with `npx nyc --reporter=text-summary --reporter=html --reporter=lcovonly mocha tests/robots.js` + +// Set up the test environment with Antville’s version of the robots parser +const Robots = require('../code/Global/Robots.js'); +const robotsParser = (url, contents) => new Robots(url, contents); + +const { expect } = require('chai'); function testRobots(url, contents, allowed, disallowed) { - var robots = robotsParser(url, contents); + var robots = robotsParser(url, contents); - allowed.forEach(function (url) { - expect(robots.isAllowed(url)).to.equal(true); - }); + allowed.forEach(function (url) { + expect(robots.isAllowed(url)).to.equal(true); + }); - disallowed.forEach(function (url) { - expect(robots.isDisallowed(url)).to.equal(true); - }); + disallowed.forEach(function (url) { + expect(robots.isDisallowed(url)).to.equal(true); + }); } describe('Robots', function () { - it('should parse the disallow directive', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /fish/', - 'Disallow: /test.html' - ].join('\n'); - - var allowed = [ - 'http://www.example.com/fish', - 'http://www.example.com/Test.html' - ]; - - var disallowed = [ - 'http://www.example.com/fish/index.php', - 'http://www.example.com/fish/', - 'http://www.example.com/test.html' - ]; - - testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); - }); - - it('should parse the allow directive', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /fish/', - 'Disallow: /test.html', - 'Allow: /fish/test.html', - 'Allow: /test.html' - ].join('\n'); - - var allowed = [ - 'http://www.example.com/fish', - 'http://www.example.com/fish/test.html', - 'http://www.example.com/Test.html', - 'http://www.example.com/test.html' - ]; - - var disallowed = [ - 'http://www.example.com/fish/index.php', - 'http://www.example.com/fish/', - ]; - - testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); - }); - - it('should parse patterns', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /fish*.php', - 'Disallow: /*.dext$', - 'Disallow: /dir*' - ].join('\n'); - - var allowed = [ - 'http://www.example.com/Fish.PHP', - 'http://www.example.com/Fish.dext1', - 'http://www.example.com/folder/dir.html', - 'http://www.example.com/folder/dir/test.html' - ]; - - var disallowed = [ - 'http://www.example.com/fish.php', - 'http://www.example.com/fishheads/catfish.php?parameters', - 'http://www.example.com/AnYthInG.dext', - 'http://www.example.com/Fish.dext.dext', - 'http://www.example.com/dir/test.html', - 'http://www.example.com/directory.html' - ]; - - testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); - }); - - it('should have the correct order precedence for allow and disallow', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /fish*.php', - 'Allow: /fish/index.php', - 'Disallow: /test', - 'Allow: /test/', - 'Disallow: /aa/', - 'Allow: /aa/', - 'Allow: /bb/', - 'Disallow: /bb/', - ].join('\n'); - - var allowed = [ - 'http://www.example.com/test/index.html', - 'http://www.example.com/fish/index.php', - 'http://www.example.com/test/', - 'http://www.example.com/aa/', - 'http://www.example.com/bb/', - 'http://www.example.com/x/' - ]; - - var disallowed = [ - 'http://www.example.com/fish.php', - 'http://www.example.com/fishheads/catfish.php?parameters', - 'http://www.example.com/test' - ]; - - testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); - }); - - it('should have the correct order precedence for wildcards', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /*/', - 'Allow: /x/', - ].join('\n'); - - var allowed = [ - 'http://www.example.com/x/', - 'http://www.example.com/fish.php', - 'http://www.example.com/test' - ]; - - var disallowed = [ - 'http://www.example.com/a/', - 'http://www.example.com/xx/', - 'http://www.example.com/test/index.html' - ]; - - testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); - }); - - it('should parse lines delimitated by \\r', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /fish/', - 'Disallow: /test.html' - ].join('\r'); - - var allowed = [ - 'http://www.example.com/fish', - 'http://www.example.com/Test.html' - ]; - - var disallowed = [ - 'http://www.example.com/fish/index.php', - 'http://www.example.com/fish/', - 'http://www.example.com/test.html' - ]; - - testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); - }); - - it('should parse lines delimitated by \\r\\n', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /fish/', - 'Disallow: /test.html' - ].join('\r\n'); - - var allowed = [ - 'http://www.example.com/fish', - 'http://www.example.com/Test.html' - ]; - - var disallowed = [ - 'http://www.example.com/fish/index.php', - 'http://www.example.com/fish/', - 'http://www.example.com/test.html' - ]; - - testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); - }); - - - it('should parse lines delimitated by mixed line endings', function () { - var contents = [ - 'User-agent: *\r', - 'Disallow: /fish/\r\n', - 'Disallow: /test.html\n\n' - ].join(''); - - var allowed = [ - 'http://www.example.com/fish', - 'http://www.example.com/Test.html' - ]; - - var disallowed = [ - 'http://www.example.com/fish/index.php', - 'http://www.example.com/fish/', - 'http://www.example.com/test.html' - ]; - - testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); - }); - - it('should ignore rules that are not in a group', function () { - var contents = [ - 'Disallow: /secret.html', - 'Disallow: /test', - ].join('\n'); - - var allowed = [ - 'http://www.example.com/secret.html', - 'http://www.example.com/test/index.html', - 'http://www.example.com/test/' - ]; - - testRobots('http://www.example.com/robots.txt', contents, allowed, []); - }); - - - it('should ignore comments', function () { - var contents = [ - '#', - '# This is a comment', - '#', - 'User-agent: *', - '# This is a comment', - 'Disallow: /fish/ # ignore', - '# Disallow: fish', - 'Disallow: /test.html' - ].join('\n'); - - var allowed = [ - 'http://www.example.com/fish', - 'http://www.example.com/Test.html' - ]; - - var disallowed = [ - 'http://www.example.com/fish/index.php', - 'http://www.example.com/fish/', - 'http://www.example.com/test.html' - ]; - - testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); - }); - - it('should ignore invalid lines', function () { - var contents = [ - 'invalid line', - 'User-agent: *', - 'Disallow: /fish/', - ':::::another invalid line:::::', - 'Disallow: /test.html', - 'Unknown: tule' - ].join('\n'); - - var allowed = [ - 'http://www.example.com/fish', - 'http://www.example.com/Test.html' - ]; - - var disallowed = [ - 'http://www.example.com/fish/index.php', - 'http://www.example.com/fish/', - 'http://www.example.com/test.html' - ]; - - testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); - }); - - it('should ignore empty user-agent lines', function () { - var contents = [ - 'User-agent:', - 'Disallow: /fish/', - 'Disallow: /test.html' - ].join('\n'); - - var allowed = [ - 'http://www.example.com/fish', - 'http://www.example.com/Test.html', - 'http://www.example.com/fish/index.php', - 'http://www.example.com/fish/', - 'http://www.example.com/test.html' - ]; - - var disallowed = []; - - testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); - }); - - it('should support groups with multiple user agents (case insensitive)', function () { - var contents = [ - 'User-agent: agenta', - 'User-agent: agentb', - 'Disallow: /fish', - ].join('\n'); - - var robots = robotsParser('http://www.example.com/robots.txt', contents); - - expect(robots.isAllowed("http://www.example.com/fish", "agenta")).to.equal(false); - }); - - it('should return undefined for invalid urls', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /secret.html', - 'Disallow: /test', - ].join('\n'); - - var invalidUrls = [ - 'http://example.com/secret.html', - 'http://ex ample.com/secret.html', - 'http://www.example.net/test/index.html', - 'http://www.examsple.com/test/', - 'example.com/test/', - ':::::;;`\\|/.example.com/test/' - ]; - - var robots = robotsParser('http://www.example.com/robots.txt', contents); - - invalidUrls.forEach(function (url) { - expect(robots.isAllowed(url)).to.equal(undefined); - }); - }); - - it('should handle Unicode, urlencoded and punycode URLs', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /secret.html', - 'Disallow: /test', - ].join('\n'); - - var allowed = [ - 'http://www.münich.com/index.html', - 'http://www.xn--mnich-kva.com/index.html', - 'http://www.m%C3%BCnich.com/index.html' - ]; - - var disallowed = [ - 'http://www.münich.com/secret.html', - 'http://www.xn--mnich-kva.com/secret.html', - 'http://www.m%C3%BCnich.com/secret.html' - ]; - - testRobots('http://www.münich.com/robots.txt', contents, allowed, disallowed); - testRobots('http://www.xn--mnich-kva.com/robots.txt', contents, allowed, disallowed); - testRobots('http://www.m%C3%BCnich.com/robots.txt', contents, allowed, disallowed); - }); - - it('should handle Unicode and urlencoded paths', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /%CF%80', - 'Disallow: /%e2%9d%83', - 'Disallow: /%a%a', - 'Disallow: /💩', - 'Disallow: /✼*t$', - 'Disallow: /%E2%9C%A4*t$', - 'Disallow: /✿%a', - 'Disallow: /http%3A%2F%2Fexample.org' - ].join('\n'); - - var allowed = [ - 'http://www.example.com/✼testing', - 'http://www.example.com/%E2%9C%BCtesting', - 'http://www.example.com/✤testing', - 'http://www.example.com/%E2%9C%A4testing', - 'http://www.example.com/http://example.org', - 'http://www.example.com/http:%2F%2Fexample.org' - ]; - - var disallowed = [ - 'http://www.example.com/%CF%80', - 'http://www.example.com/%CF%80/index.html', - 'http://www.example.com/π', - 'http://www.example.com/π/index.html', - 'http://www.example.com/%e2%9d%83', - 'http://www.example.com/%E2%9D%83/index.html', - 'http://www.example.com/❃', - 'http://www.example.com/❃/index.html', - 'http://www.example.com/%F0%9F%92%A9', - 'http://www.example.com/%F0%9F%92%A9/index.html', - 'http://www.example.com/💩', - 'http://www.example.com/💩/index.html', - 'http://www.example.com/%a%a', - 'http://www.example.com/%a%a/index.html', - 'http://www.example.com/✼test', - 'http://www.example.com/%E2%9C%BCtest', - 'http://www.example.com/✤test', - 'http://www.example.com/%E2%9C%A4testt', - 'http://www.example.com/✿%a', - 'http://www.example.com/%E2%9C%BF%atest', - 'http://www.example.com/http%3A%2F%2Fexample.org' - ]; - - testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); - }); - - it('should handle lone high / low surrogates', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /\uD800', - 'Disallow: /\uDC00' - ].join('\n'); - - // These are invalid so can't be disallowed - var allowed = [ - 'http://www.example.com/\uDC00', - 'http://www.example.com/\uD800' - ]; - - var disallowed = []; - - testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); - }); - - it('should ignore host case', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /secret.html', - 'Disallow: /test', - ].join('\n'); - - var allowed = [ - 'http://www.example.com/index.html', - 'http://www.ExAmPlE.com/index.html', - 'http://www.EXAMPLE.com/index.html' - ]; - - var disallowed = [ - 'http://www.example.com/secret.html', - 'http://www.ExAmPlE.com/secret.html', - 'http://www.EXAMPLE.com/secret.html' - ]; - - testRobots('http://www.eXample.com/robots.txt', contents, allowed, disallowed); - }); - - it('should handle relative paths', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /fish', - 'Allow: /fish/test', - ].join('\n'); - - var robots = robotsParser('/robots.txt', contents); - expect(robots.isAllowed('/fish/test')).to.equal(true); - expect(robots.isAllowed('/fish')).to.equal(false); - }); - - it('should not allow relative paths if domain specified', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /fish', - 'Allow: /fish/test', - ].join('\n'); - - var robots = robotsParser('http://www.example.com/robots.txt', contents); - expect(robots.isAllowed('/fish/test')).to.equal(undefined); - expect(robots.isAllowed('/fish')).to.equal(undefined); - }); - - it('should not treat invalid robots.txt URLs as relative', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /fish', - 'Allow: /fish/test', - ].join('\n'); - - var robots = robotsParser('https://ex ample.com/robots.txt', contents); - expect(robots.isAllowed('/fish/test')).to.equal(undefined); - expect(robots.isAllowed('/fish')).to.equal(undefined); - }); - - it('should not allow URls if domain specified and robots.txt is relative', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /fish', - 'Allow: /fish/test', - ].join('\n'); - - var robots = robotsParser('/robots.txt', contents); - expect(robots.isAllowed('http://www.example.com/fish/test')).to.equal(undefined); - expect(robots.isAllowed('http://www.example.com/fish')).to.equal(undefined); - }); - - it('should allow all if empty robots.txt', function () { - var allowed = [ - 'http://www.example.com/secret.html', - 'http://www.example.com/test/index.html', - 'http://www.example.com/test/' - ]; - - var robots = robotsParser('http://www.example.com/robots.txt', ''); - - allowed.forEach(function (url) { - expect(robots.isAllowed(url)).to.equal(true); - }); - }); - - it('should treat null as allowing all', function () { - var robots = robotsParser('http://www.example.com/robots.txt', null); - - expect(robots.isAllowed("http://www.example.com/", "userAgent")).to.equal(true); - expect(robots.isAllowed("http://www.example.com/")).to.equal(true); - }); - - it('should handle invalid robots.txt urls', function () { - var contents = [ - 'user-agent: *', - 'disallow: /', - - 'host: www.example.com', - 'sitemap: /sitemap.xml' - ].join('\n'); - - var sitemapUrls = [ - undefined, - null, - 'null', - ':/wom/test/' - ]; - - sitemapUrls.forEach(function (url) { - var robots = robotsParser(url, contents); - expect(robots.isAllowed('http://www.example.com/index.html')).to.equal(undefined); - expect(robots.getPreferredHost()).to.equal('www.example.com'); - expect(robots.getSitemaps()).to.eql(['/sitemap.xml']); - }); - }); - - it('should parse the crawl-delay directive', function () { - var contents = [ - 'user-agent: a', - 'crawl-delay: 1', - - 'user-agent: b', - 'disallow: /d', - - 'user-agent: c', - 'user-agent: d', - 'crawl-delay: 10' - ].join('\n'); - - var robots = robotsParser('http://www.example.com/robots.txt', contents); - - expect(robots.getCrawlDelay('a')).to.equal(1); - expect(robots.getCrawlDelay('b')).to.equal(undefined); - expect(robots.getCrawlDelay('c')).to.equal(10); - expect(robots.getCrawlDelay('d')).to.equal(10); - expect(robots.getCrawlDelay()).to.equal(undefined); - }); - - it('should ignore invalid crawl-delay directives', function () { - var contents = [ - 'user-agent: a', - 'crawl-delay: 1.2.1', - - 'user-agent: b', - 'crawl-delay: 1.a0', - - 'user-agent: c', - 'user-agent: d', - 'crawl-delay: 10a' - ].join('\n'); - - var robots = robotsParser('http://www.example.com/robots.txt', contents); - - expect(robots.getCrawlDelay('a')).to.equal(undefined); - expect(robots.getCrawlDelay('b')).to.equal(undefined); - expect(robots.getCrawlDelay('c')).to.equal(undefined); - expect(robots.getCrawlDelay('d')).to.equal(undefined); - }); - - it('should parse the sitemap directive', function () { - var contents = [ - 'user-agent: a', - 'crawl-delay: 1', - 'sitemap: http://example.com/test.xml', - - 'user-agent: b', - 'disallow: /d', - - 'sitemap: /sitemap.xml', - 'sitemap: http://example.com/test/sitemap.xml ' - ].join('\n'); - - var robots = robotsParser('http://www.example.com/robots.txt', contents); - - expect(robots.getSitemaps()).to.eql([ - 'http://example.com/test.xml', - '/sitemap.xml', - 'http://example.com/test/sitemap.xml' - ]); - }); - - it('should parse the host directive', function () { - var contents = [ - 'user-agent: a', - 'crawl-delay: 1', - 'host: www.example.net', - - 'user-agent: b', - 'disallow: /d', - - 'host: example.com' - ].join('\n'); - - var robots = robotsParser('http://www.example.com/robots.txt', contents); - - expect(robots.getPreferredHost()).to.equal('example.com'); - }); - - it('should parse empty and invalid directives', function () { - var contents = [ - 'user-agent:', - 'user-agent:::: a::', - 'crawl-delay:', - 'crawl-delay:::: 0:', - 'host:', - 'host:: example.com', - 'sitemap:', - 'sitemap:: site:map.xml', - 'disallow:', - 'disallow::: /:', - 'allow:', - 'allow::: /:', - ].join('\n'); - - robotsParser('http://www.example.com/robots.txt', contents); - }); - - it('should treat only the last host directive as valid', function () { - var contents = [ - 'user-agent: a', - 'crawl-delay: 1', - 'host: www.example.net', - - 'user-agent: b', - 'disallow: /d', - - 'host: example.net', - 'host: example.com' - ].join('\n'); - - var robots = robotsParser('http://www.example.com/robots.txt', contents); - - expect(robots.getPreferredHost()).to.equal('example.com'); - }); - - it('should return null when there is no host directive', function () { - var contents = [ - 'user-agent: a', - 'crawl-delay: 1', - - 'user-agent: b', - 'disallow: /d', - ].join('\n'); - - var robots = robotsParser('http://www.example.com/robots.txt', contents); - - expect(robots.getPreferredHost()).to.equal(null); - }); - - it('should fallback to * when a UA has no rules of its own', function () { - var contents = [ - 'user-agent: *', - 'crawl-delay: 1', - - 'user-agent: b', - 'crawl-delay: 12', - - 'user-agent: c', - 'user-agent: d', - 'crawl-delay: 10' - ].join('\n'); - - var robots = robotsParser('http://www.example.com/robots.txt', contents); - - expect(robots.getCrawlDelay('should-fall-back')).to.equal(1); - expect(robots.getCrawlDelay('d')).to.equal(10); - expect(robots.getCrawlDelay('dd')).to.equal(1); - }); - - it('should not fallback to * when a UA has rules', function () { - var contents = [ - 'user-agent: *', - 'crawl-delay: 1', - - 'user-agent: b', - 'disallow:' - ].join('\n'); - - var robots = robotsParser('http://www.example.com/robots.txt', contents); - - expect(robots.getCrawlDelay('b')).to.equal(undefined); - }); - - it('should handle UAs with object property names', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /fish', - ].join('\n'); - - var robots = robotsParser('http://www.example.com/robots.txt', contents); - expect(robots.isAllowed('http://www.example.com/fish', 'constructor')).to.equal(false); - expect(robots.isAllowed('http://www.example.com/fish', '__proto__')).to.equal(false); - }); - - it('should ignore version numbers in the UA string', function () { - var contents = [ - 'user-agent: *', - 'crawl-delay: 1', - - 'user-agent: b', - 'crawl-delay: 12', - - 'user-agent: c', - 'user-agent: d', - 'crawl-delay: 10' - ].join('\n'); - - var robots = robotsParser('http://www.example.com/robots.txt', contents); - - expect(robots.getCrawlDelay('should-fall-back/1.0.0')).to.equal(1); - expect(robots.getCrawlDelay('d/12')).to.equal(10); - expect(robots.getCrawlDelay('dd / 0-32-3')).to.equal(1); - expect(robots.getCrawlDelay('b / 1.0')).to.equal(12); - }); - - - it('should return the line number of the matching directive', function () { - var contents = [ - '', - 'User-agent: *', - '', - 'Disallow: /fish/', - 'Disallow: /test.html', - 'Allow: /fish/test.html', - 'Allow: /test.html', - '', - 'User-agent: a', - 'allow: /', - '', - 'User-agent: b', - 'disallow: /test', - 'disallow: /t*t', - '', - 'User-agent: c', - 'Disallow: /fish*.php', - 'Allow: /fish/index.php' - ].join('\n'); - - var robots = robotsParser('http://www.example.com/robots.txt', contents); - - expect(robots.getMatchingLineNumber('http://www.example.com/fish')).to.equal(-1); - expect(robots.getMatchingLineNumber('http://www.example.com/fish/test.html')).to.equal(6); - expect(robots.getMatchingLineNumber('http://www.example.com/Test.html')).to.equal(-1); - - expect(robots.getMatchingLineNumber('http://www.example.com/fish/index.php')).to.equal(4); - expect(robots.getMatchingLineNumber('http://www.example.com/fish/')).to.equal(4); - expect(robots.getMatchingLineNumber('http://www.example.com/test.html')).to.equal(7); - - expect(robots.getMatchingLineNumber('http://www.example.com/test.html', 'a')).to.equal(10); - - expect(robots.getMatchingLineNumber('http://www.example.com/fish.php', 'c')).to.equal(17); - expect(robots.getMatchingLineNumber('http://www.example.com/fish/index.php', 'c')).to.equal(18); - }); - - it('should handle large wildcards efficiently', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /' + '*'.repeat(2048) + '.html', - ].join('\n'); - - var allowed = [ - 'http://www.example.com/' + 'sub'.repeat(2048) + 'folder/index.php', - ]; - - var disallowed = [ - 'http://www.example.com/secret.html' - ]; - - const start = Date.now(); - testRobots('http://www.eXample.com/robots.txt', contents, allowed, disallowed); - const end = Date.now(); - - // Should take less than 500 ms (high to allow for variableness of - // machines running the test, should normally be much less) - expect(end - start).to.be.lessThan(500); - }); - - it('should honor given port number', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /fish/', - 'Disallow: /test.html' - ].join('\n'); - - var allowed = [ - 'http://www.example.com:8080/fish', - 'http://www.example.com:8080/Test.html' - ]; - - var disallowed = [ - 'http://www.example.com/fish', - 'http://www.example.com/Test.html', - 'http://www.example.com:80/fish', - 'http://www.example.com:80/Test.html' - ]; - - testRobots('http://www.example.com:8080/robots.txt', contents, allowed, disallowed); - }); - - it('should default to port 80 for http: if no port given', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /fish/', - 'Disallow: /test.html' - ].join('\n'); - - var allowed = [ - 'http://www.example.com:80/fish', - 'http://www.example.com:80/Test.html' - ]; - - var disallowed = [ - 'http://www.example.com:443/fish', - 'http://www.example.com:443/Test.html', - 'http://www.example.com:80/fish/index.php', - 'http://www.example.com:80/fish/', - 'http://www.example.com:80/test.html' - ]; - - testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); - }); - - it('should default to port 443 for https: if no port given', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /fish/', - 'Disallow: /test.html' - ].join('\n'); - - var allowed = [ - 'https://www.example.com:443/fish', - 'https://www.example.com:443/Test.html', - 'https://www.example.com/fish', - 'https://www.example.com/Test.html' - ]; - - var disallowed = [ - 'http://www.example.com:80/fish', - 'http://www.example.com:80/Test.html', - 'http://www.example.com:443/fish/index.php', - 'http://www.example.com:443/fish/', - 'http://www.example.com:443/test.html' - ]; - - testRobots('https://www.example.com/robots.txt', contents, allowed, disallowed); - }); - - it('should not be disallowed when wildcard is used in explicit mode', function () { - var contents = [ - 'User-agent: *', - 'Disallow: /', - ].join('\n') - - var url = 'https://www.example.com/hello' - var userAgent = 'SomeBot'; - var robots = robotsParser(url, contents); - - expect(robots.isExplicitlyDisallowed(url, userAgent)).to.equal(false) - }); - - it('should be disallowed when user agent equal robots rule in explicit mode', function () { - var contents = [ - 'User-agent: SomeBot', - 'Disallow: /', - ].join('\n') - - var url = 'https://www.example.com/hello' - var userAgent = 'SomeBot'; - var robots = robotsParser(url, contents); - - expect(robots.isExplicitlyDisallowed(url, userAgent)).to.equal(true) - }); - - it('should return undefined when given an invalid URL in explicit mode', function () { - var contents = [ - 'User-agent: SomeBot', - 'Disallow: /', - ].join('\n') - - var url = 'https://www.example.com/hello' - var userAgent = 'SomeBot'; - var robots = robotsParser('http://example.com', contents); - - expect(robots.isExplicitlyDisallowed(url, userAgent)).to.equal(undefined) - }); + it('should parse the disallow directive', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish/', + 'Disallow: /test.html' + ].join('\n'); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + 'http://www.example.com/test.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should parse the allow directive', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish/', + 'Disallow: /test.html', + 'Allow: /fish/test.html', + 'Allow: /test.html' + ].join('\n'); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/fish/test.html', + 'http://www.example.com/Test.html', + 'http://www.example.com/test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should parse patterns', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish*.php', + 'Disallow: /*.dext$', + 'Disallow: /dir*' + ].join('\n'); + + var allowed = [ + 'http://www.example.com/Fish.PHP', + 'http://www.example.com/Fish.dext1', + 'http://www.example.com/folder/dir.html', + 'http://www.example.com/folder/dir/test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish.php', + 'http://www.example.com/fishheads/catfish.php?parameters', + 'http://www.example.com/AnYthInG.dext', + 'http://www.example.com/Fish.dext.dext', + 'http://www.example.com/dir/test.html', + 'http://www.example.com/directory.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should have the correct order precedence for allow and disallow', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish*.php', + 'Allow: /fish/index.php', + 'Disallow: /test', + 'Allow: /test/', + 'Disallow: /aa/', + 'Allow: /aa/', + 'Allow: /bb/', + 'Disallow: /bb/', + ].join('\n'); + + var allowed = [ + 'http://www.example.com/test/index.html', + 'http://www.example.com/fish/index.php', + 'http://www.example.com/test/', + 'http://www.example.com/aa/', + 'http://www.example.com/bb/', + 'http://www.example.com/x/' + ]; + + var disallowed = [ + 'http://www.example.com/fish.php', + 'http://www.example.com/fishheads/catfish.php?parameters', + 'http://www.example.com/test' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should have the correct order precedence for wildcards', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /*/', + 'Allow: /x/', + ].join('\n'); + + var allowed = [ + 'http://www.example.com/x/', + 'http://www.example.com/fish.php', + 'http://www.example.com/test' + ]; + + var disallowed = [ + 'http://www.example.com/a/', + 'http://www.example.com/xx/', + 'http://www.example.com/test/index.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should parse lines delimitated by \\r', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish/', + 'Disallow: /test.html' + ].join('\r'); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + 'http://www.example.com/test.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should parse lines delimitated by \\r\\n', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish/', + 'Disallow: /test.html' + ].join('\r\n'); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + 'http://www.example.com/test.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + + it('should parse lines delimitated by mixed line endings', function () { + var contents = [ + 'User-agent: *\r', + 'Disallow: /fish/\r\n', + 'Disallow: /test.html\n\n' + ].join(''); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + 'http://www.example.com/test.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should ignore rules that are not in a group', function () { + var contents = [ + 'Disallow: /secret.html', + 'Disallow: /test', + ].join('\n'); + + var allowed = [ + 'http://www.example.com/secret.html', + 'http://www.example.com/test/index.html', + 'http://www.example.com/test/' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, []); + }); + + + it('should ignore comments', function () { + var contents = [ + '#', + '# This is a comment', + '#', + 'User-agent: *', + '# This is a comment', + 'Disallow: /fish/ # ignore', + '# Disallow: fish', + 'Disallow: /test.html' + ].join('\n'); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + 'http://www.example.com/test.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should ignore invalid lines', function () { + var contents = [ + 'invalid line', + 'User-agent: *', + 'Disallow: /fish/', + ':::::another invalid line:::::', + 'Disallow: /test.html', + 'Unknown: tule' + ].join('\n'); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + 'http://www.example.com/test.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should ignore empty user-agent lines', function () { + var contents = [ + 'User-agent:', + 'Disallow: /fish/', + 'Disallow: /test.html' + ].join('\n'); + + var allowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html', + 'http://www.example.com/fish/index.php', + 'http://www.example.com/fish/', + 'http://www.example.com/test.html' + ]; + + var disallowed = []; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should support groups with multiple user agents (case insensitive)', function () { + var contents = [ + 'User-agent: agenta', + 'User-agent: agentb', + 'Disallow: /fish', + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.isAllowed("http://www.example.com/fish", "agenta")).to.equal(false); + }); + + it('should return undefined for invalid urls', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /secret.html', + 'Disallow: /test', + ].join('\n'); + + var invalidUrls = [ + 'http://example.com/secret.html', + 'http://ex ample.com/secret.html', + 'http://www.example.net/test/index.html', + 'http://www.examsple.com/test/', + 'example.com/test/', + ':::::;;`\\|/.example.com/test/' + ]; + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + invalidUrls.forEach(function (url) { + expect(robots.isAllowed(url)).to.equal(undefined); + }); + }); + + it('should handle Unicode, urlencoded and punycode URLs', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /secret.html', + 'Disallow: /test', + ].join('\n'); + + var allowed = [ + 'http://www.münich.com/index.html', + 'http://www.xn--mnich-kva.com/index.html', + 'http://www.m%C3%BCnich.com/index.html' + ]; + + var disallowed = [ + 'http://www.münich.com/secret.html', + 'http://www.xn--mnich-kva.com/secret.html', + 'http://www.m%C3%BCnich.com/secret.html' + ]; + + testRobots('http://www.münich.com/robots.txt', contents, allowed, disallowed); + testRobots('http://www.xn--mnich-kva.com/robots.txt', contents, allowed, disallowed); + testRobots('http://www.m%C3%BCnich.com/robots.txt', contents, allowed, disallowed); + }); + + it('should handle Unicode and urlencoded paths', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /%CF%80', + 'Disallow: /%e2%9d%83', + 'Disallow: /%a%a', + 'Disallow: /💩', + 'Disallow: /✼*t$', + 'Disallow: /%E2%9C%A4*t$', + 'Disallow: /✿%a', + 'Disallow: /http%3A%2F%2Fexample.org' + ].join('\n'); + + var allowed = [ + 'http://www.example.com/✼testing', + 'http://www.example.com/%E2%9C%BCtesting', + 'http://www.example.com/✤testing', + 'http://www.example.com/%E2%9C%A4testing', + 'http://www.example.com/http://example.org', + 'http://www.example.com/http:%2F%2Fexample.org' + ]; + + var disallowed = [ + 'http://www.example.com/%CF%80', + 'http://www.example.com/%CF%80/index.html', + 'http://www.example.com/π', + 'http://www.example.com/π/index.html', + 'http://www.example.com/%e2%9d%83', + 'http://www.example.com/%E2%9D%83/index.html', + 'http://www.example.com/❃', + 'http://www.example.com/❃/index.html', + 'http://www.example.com/%F0%9F%92%A9', + 'http://www.example.com/%F0%9F%92%A9/index.html', + 'http://www.example.com/💩', + 'http://www.example.com/💩/index.html', + 'http://www.example.com/%a%a', + 'http://www.example.com/%a%a/index.html', + 'http://www.example.com/✼test', + 'http://www.example.com/%E2%9C%BCtest', + 'http://www.example.com/✤test', + 'http://www.example.com/%E2%9C%A4testt', + 'http://www.example.com/✿%a', + 'http://www.example.com/%E2%9C%BF%atest', + 'http://www.example.com/http%3A%2F%2Fexample.org' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should handle lone high / low surrogates', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /\uD800', + 'Disallow: /\uDC00' + ].join('\n'); + + // These are invalid so can't be disallowed + var allowed = [ + 'http://www.example.com/\uDC00', + 'http://www.example.com/\uD800' + ]; + + var disallowed = []; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should ignore host case', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /secret.html', + 'Disallow: /test', + ].join('\n'); + + var allowed = [ + 'http://www.example.com/index.html', + 'http://www.ExAmPlE.com/index.html', + 'http://www.EXAMPLE.com/index.html' + ]; + + var disallowed = [ + 'http://www.example.com/secret.html', + 'http://www.ExAmPlE.com/secret.html', + 'http://www.EXAMPLE.com/secret.html' + ]; + + testRobots('http://www.eXample.com/robots.txt', contents, allowed, disallowed); + }); + + it('should handle relative paths', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish', + 'Allow: /fish/test', + ].join('\n'); + + var robots = robotsParser('/robots.txt', contents); + expect(robots.isAllowed('/fish/test')).to.equal(true); + expect(robots.isAllowed('/fish')).to.equal(false); + }); + + it('should not allow relative paths if domain specified', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish', + 'Allow: /fish/test', + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + expect(robots.isAllowed('/fish/test')).to.equal(undefined); + expect(robots.isAllowed('/fish')).to.equal(undefined); + }); + + it('should not treat invalid robots.txt URLs as relative', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish', + 'Allow: /fish/test', + ].join('\n'); + + var robots = robotsParser('https://ex ample.com/robots.txt', contents); + expect(robots.isAllowed('/fish/test')).to.equal(undefined); + expect(robots.isAllowed('/fish')).to.equal(undefined); + }); + + it('should not allow URls if domain specified and robots.txt is relative', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish', + 'Allow: /fish/test', + ].join('\n'); + + var robots = robotsParser('/robots.txt', contents); + expect(robots.isAllowed('http://www.example.com/fish/test')).to.equal(undefined); + expect(robots.isAllowed('http://www.example.com/fish')).to.equal(undefined); + }); + + it('should allow all if empty robots.txt', function () { + var allowed = [ + 'http://www.example.com/secret.html', + 'http://www.example.com/test/index.html', + 'http://www.example.com/test/' + ]; + + var robots = robotsParser('http://www.example.com/robots.txt', ''); + + allowed.forEach(function (url) { + expect(robots.isAllowed(url)).to.equal(true); + }); + }); + + it('should treat null as allowing all', function () { + var robots = robotsParser('http://www.example.com/robots.txt', null); + + expect(robots.isAllowed("http://www.example.com/", "userAgent")).to.equal(true); + expect(robots.isAllowed("http://www.example.com/")).to.equal(true); + }); + + it('should handle invalid robots.txt urls', function () { + var contents = [ + 'user-agent: *', + 'disallow: /', + + 'host: www.example.com', + 'sitemap: /sitemap.xml' + ].join('\n'); + + var sitemapUrls = [ + undefined, + null, + 'null', + ':/wom/test/' + ]; + + sitemapUrls.forEach(function (url) { + var robots = robotsParser(url, contents); + expect(robots.isAllowed('http://www.example.com/index.html')).to.equal(undefined); + expect(robots.getPreferredHost()).to.equal('www.example.com'); + expect(robots.getSitemaps()).to.eql(['/sitemap.xml']); + }); + }); + + it('should parse the crawl-delay directive', function () { + var contents = [ + 'user-agent: a', + 'crawl-delay: 1', + + 'user-agent: b', + 'disallow: /d', + + 'user-agent: c', + 'user-agent: d', + 'crawl-delay: 10' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getCrawlDelay('a')).to.equal(1); + expect(robots.getCrawlDelay('b')).to.equal(undefined); + expect(robots.getCrawlDelay('c')).to.equal(10); + expect(robots.getCrawlDelay('d')).to.equal(10); + expect(robots.getCrawlDelay()).to.equal(undefined); + }); + + it('should ignore invalid crawl-delay directives', function () { + var contents = [ + 'user-agent: a', + 'crawl-delay: 1.2.1', + + 'user-agent: b', + 'crawl-delay: 1.a0', + + 'user-agent: c', + 'user-agent: d', + 'crawl-delay: 10a' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getCrawlDelay('a')).to.equal(undefined); + expect(robots.getCrawlDelay('b')).to.equal(undefined); + expect(robots.getCrawlDelay('c')).to.equal(undefined); + expect(robots.getCrawlDelay('d')).to.equal(undefined); + }); + + it('should parse the sitemap directive', function () { + var contents = [ + 'user-agent: a', + 'crawl-delay: 1', + 'sitemap: http://example.com/test.xml', + + 'user-agent: b', + 'disallow: /d', + + 'sitemap: /sitemap.xml', + 'sitemap: http://example.com/test/sitemap.xml ' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getSitemaps()).to.eql([ + 'http://example.com/test.xml', + '/sitemap.xml', + 'http://example.com/test/sitemap.xml' + ]); + }); + + it('should parse the host directive', function () { + var contents = [ + 'user-agent: a', + 'crawl-delay: 1', + 'host: www.example.net', + + 'user-agent: b', + 'disallow: /d', + + 'host: example.com' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getPreferredHost()).to.equal('example.com'); + }); + + it('should parse empty and invalid directives', function () { + var contents = [ + 'user-agent:', + 'user-agent:::: a::', + 'crawl-delay:', + 'crawl-delay:::: 0:', + 'host:', + 'host:: example.com', + 'sitemap:', + 'sitemap:: site:map.xml', + 'disallow:', + 'disallow::: /:', + 'allow:', + 'allow::: /:', + ].join('\n'); + + robotsParser('http://www.example.com/robots.txt', contents); + }); + + it('should treat only the last host directive as valid', function () { + var contents = [ + 'user-agent: a', + 'crawl-delay: 1', + 'host: www.example.net', + + 'user-agent: b', + 'disallow: /d', + + 'host: example.net', + 'host: example.com' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getPreferredHost()).to.equal('example.com'); + }); + + it('should return null when there is no host directive', function () { + var contents = [ + 'user-agent: a', + 'crawl-delay: 1', + + 'user-agent: b', + 'disallow: /d', + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getPreferredHost()).to.equal(null); + }); + + it('should fallback to * when a UA has no rules of its own', function () { + var contents = [ + 'user-agent: *', + 'crawl-delay: 1', + + 'user-agent: b', + 'crawl-delay: 12', + + 'user-agent: c', + 'user-agent: d', + 'crawl-delay: 10' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getCrawlDelay('should-fall-back')).to.equal(1); + expect(robots.getCrawlDelay('d')).to.equal(10); + expect(robots.getCrawlDelay('dd')).to.equal(1); + }); + + it('should not fallback to * when a UA has rules', function () { + var contents = [ + 'user-agent: *', + 'crawl-delay: 1', + + 'user-agent: b', + 'disallow:' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getCrawlDelay('b')).to.equal(undefined); + }); + + it('should handle UAs with object property names', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish', + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + expect(robots.isAllowed('http://www.example.com/fish', 'constructor')).to.equal(false); + expect(robots.isAllowed('http://www.example.com/fish', '__proto__')).to.equal(false); + }); + + it('should ignore version numbers in the UA string', function () { + var contents = [ + 'user-agent: *', + 'crawl-delay: 1', + + 'user-agent: b', + 'crawl-delay: 12', + + 'user-agent: c', + 'user-agent: d', + 'crawl-delay: 10' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getCrawlDelay('should-fall-back/1.0.0')).to.equal(1); + expect(robots.getCrawlDelay('d/12')).to.equal(10); + expect(robots.getCrawlDelay('dd / 0-32-3')).to.equal(1); + expect(robots.getCrawlDelay('b / 1.0')).to.equal(12); + }); + + + it('should return the line number of the matching directive', function () { + var contents = [ + '', + 'User-agent: *', + '', + 'Disallow: /fish/', + 'Disallow: /test.html', + 'Allow: /fish/test.html', + 'Allow: /test.html', + '', + 'User-agent: a', + 'allow: /', + '', + 'User-agent: b', + 'disallow: /test', + 'disallow: /t*t', + '', + 'User-agent: c', + 'Disallow: /fish*.php', + 'Allow: /fish/index.php' + ].join('\n'); + + var robots = robotsParser('http://www.example.com/robots.txt', contents); + + expect(robots.getMatchingLineNumber('http://www.example.com/fish')).to.equal(-1); + expect(robots.getMatchingLineNumber('http://www.example.com/fish/test.html')).to.equal(6); + expect(robots.getMatchingLineNumber('http://www.example.com/Test.html')).to.equal(-1); + + expect(robots.getMatchingLineNumber('http://www.example.com/fish/index.php')).to.equal(4); + expect(robots.getMatchingLineNumber('http://www.example.com/fish/')).to.equal(4); + expect(robots.getMatchingLineNumber('http://www.example.com/test.html')).to.equal(7); + + expect(robots.getMatchingLineNumber('http://www.example.com/test.html', 'a')).to.equal(10); + + expect(robots.getMatchingLineNumber('http://www.example.com/fish.php', 'c')).to.equal(17); + expect(robots.getMatchingLineNumber('http://www.example.com/fish/index.php', 'c')).to.equal(18); + }); + + it('should handle large wildcards efficiently', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /' + '*'.repeat(2048) + '.html', + ].join('\n'); + + var allowed = [ + 'http://www.example.com/' + 'sub'.repeat(2048) + 'folder/index.php', + ]; + + var disallowed = [ + 'http://www.example.com/secret.html' + ]; + + const start = Date.now(); + testRobots('http://www.eXample.com/robots.txt', contents, allowed, disallowed); + const end = Date.now(); + + // Should take less than 500 ms (high to allow for variableness of + // machines running the test, should normally be much less) + expect(end - start).to.be.lessThan(500); + }); + + it('should honor given port number', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish/', + 'Disallow: /test.html' + ].join('\n'); + + var allowed = [ + 'http://www.example.com:8080/fish', + 'http://www.example.com:8080/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com/fish', + 'http://www.example.com/Test.html', + 'http://www.example.com:80/fish', + 'http://www.example.com:80/Test.html' + ]; + + testRobots('http://www.example.com:8080/robots.txt', contents, allowed, disallowed); + }); + + it('should default to port 80 for http: if no port given', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish/', + 'Disallow: /test.html' + ].join('\n'); + + var allowed = [ + 'http://www.example.com:80/fish', + 'http://www.example.com:80/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com:443/fish', + 'http://www.example.com:443/Test.html', + 'http://www.example.com:80/fish/index.php', + 'http://www.example.com:80/fish/', + 'http://www.example.com:80/test.html' + ]; + + testRobots('http://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should default to port 443 for https: if no port given', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /fish/', + 'Disallow: /test.html' + ].join('\n'); + + var allowed = [ + 'https://www.example.com:443/fish', + 'https://www.example.com:443/Test.html', + 'https://www.example.com/fish', + 'https://www.example.com/Test.html' + ]; + + var disallowed = [ + 'http://www.example.com:80/fish', + 'http://www.example.com:80/Test.html', + 'http://www.example.com:443/fish/index.php', + 'http://www.example.com:443/fish/', + 'http://www.example.com:443/test.html' + ]; + + testRobots('https://www.example.com/robots.txt', contents, allowed, disallowed); + }); + + it('should not be disallowed when wildcard is used in explicit mode', function () { + var contents = [ + 'User-agent: *', + 'Disallow: /', + ].join('\n') + + var url = 'https://www.example.com/hello' + var userAgent = 'SomeBot'; + var robots = robotsParser(url, contents); + + expect(robots.isExplicitlyDisallowed(url, userAgent)).to.equal(false) + }); + + it('should be disallowed when user agent equal robots rule in explicit mode', function () { + var contents = [ + 'User-agent: SomeBot', + 'Disallow: /', + ].join('\n') + + var url = 'https://www.example.com/hello' + var userAgent = 'SomeBot'; + var robots = robotsParser(url, contents); + + expect(robots.isExplicitlyDisallowed(url, userAgent)).to.equal(true) + }); + + it('should return undefined when given an invalid URL in explicit mode', function () { + var contents = [ + 'User-agent: SomeBot', + 'Disallow: /', + ].join('\n') + + var url = 'https://www.example.com/hello' + var userAgent = 'SomeBot'; + var robots = robotsParser('http://example.com', contents); + + expect(robots.isExplicitlyDisallowed(url, userAgent)).to.equal(undefined) + }); }); From c5b9a613a8929ae537b619450a219118b4104251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20Sch=C3=A4fer?= Date: Sat, 10 May 2025 21:59:10 +0200 Subject: [PATCH 04/10] Add site setting for enforcing rules in robots.txt --- code/Site/$Site.skin | 16 ++++++++++++++++ code/Site/Site.js | 16 ++++++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/code/Site/$Site.skin b/code/Site/$Site.skin index a5098f36..5452557c 100644 --- a/code/Site/$Site.skin +++ b/code/Site/$Site.skin @@ -143,6 +143,22 @@ +
+ + +
+ +

+ <% gettext 'Edit the rules in the robots.txt skin.' | replace '%s/' <% site.layout.skins.href %> %> +

+
+
+

- <% gettext 'Edit the rules in the robots.txt skin.' | replace '%s/' <% site.layout.skins.href %> %> + <% gettext 'Edit the rules in the robots.txt skin.' <% site.layout.skins.href %> %>

diff --git a/code/Site/Site.js b/code/Site/Site.js index 92bcea8e..d638d1d2 100644 --- a/code/Site/Site.js +++ b/code/Site/Site.js @@ -95,9 +95,13 @@ Site.getNotificationModes = defineConstants(Site, markgettext('Nobody'), */ Site.getCallbackModes = defineConstants(Site, markgettext('disabled'), markgettext('enabled')); - -Site.getRobotsTxtModes = defineConstants(Site, markgettext('relaxed'), - markgettext('enforced')); +/** + * @function + * @returns {String[]} + * @see defineConstants + */ +Site.getRobotsTxtModes = defineConstants(Site, markgettext('suggest'), + markgettext('enforce')); /** * @param {String} name A unique identifier also used in the URL of a site @@ -136,7 +140,7 @@ Site.add = function(data, user) { configured: now, created: now, creator: user, - robotsTxtMode: Site.RELAXED, + robotsTxtMode: Site.SUGGEST, modified: now, modifier: user, status: user.status === User.PRIVILEGED ? Site.TRUSTED : user.status, @@ -1134,20 +1138,25 @@ Site.prototype.callback = function(ref) { } Site.prototype.enforceRobotsTxt = function() { - if (this.robotsTxtMode !== Site.ENFORCED) { + if (this.robotsTxtMode !== Site.ENFORCE) { return false; } - // Override some patterns to prevent a site from becoming inaccessible even for the owner + // Override some URLs to prevent a site from becoming inaccessible even for the owner const overrides = [ - 'User-agent: mozilla', - 'Allow: */edit$', - 'Allow: */layout', - 'Allow: */main.*$', - 'Allow: */members' + this.href('edit'), + this.layout.href(), + this.href('main.css'), + this.href('main.js'), + this.members.href() ]; const robotsTxt = root.renderSkinAsString('Site#robots'); - const robots = new Robots(this.href('robots.txt'), robotsTxt + overrides.join('\n')); - return !robots.isAllowed(path.href() + req.action, req.getHeader('user-agent')); + const robots = new Robots(this.href('robots.txt'), robotsTxt); + + const href = path.href(req.action); + const fullUrl = (href.includes('://') ? '' : this.href()) + href.slice(1); + + return !overrides.some(href => fullUrl.includes(href)) + && !robots.isAllowed(fullUrl, req.getHeader('user-agent')); } diff --git a/i18n/antville.pot b/i18n/antville.pot index 220ec331..076cfc8d 100644 --- a/i18n/antville.pot +++ b/i18n/antville.pot @@ -2,7 +2,7 @@ # The Antville Project # http://code.google.com/p/antville # -# Copyright 2001-2024 by the Workers of Antville. +# Copyright 2001-2025 by the Workers of Antville. # # Licensed under the Apache License, Version 2.0 (the ``License'' # you may not use this file except in compliance with the License. @@ -20,17 +20,17 @@ #, fuzzy msgid "" msgstr "" -"Project-Id-Version: Antville-1\n" +"Project-Id-Version: Antville-0\n" "Report-Msgid-Bugs-To: mail@antville.org\n" -"POT-Creation-Date: 2024-06-11 21:10+0200\n" -"PO-Revision-Date: 2024-06-11 21:10+0200\n" +"POT-Creation-Date: 2025-05-11 00:27+0200\n" +"PO-Revision-Date: 2025-05-11 00:27+0200\n" "Language-Team: The Antville People \n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" -#: Global/Global.js:987 +#: Global/Global.js:989 #, java-format msgid "({0} character)" msgid_plural "({0} characters)" @@ -141,7 +141,7 @@ msgid "Admin" msgstr "" #: User/$User.skin:110 -#: Site/$Site.skin:279 +#: Site/$Site.skin:295 msgid "Administration" msgstr "" @@ -168,7 +168,7 @@ msgid "All Skins" msgstr "" #: User/$User.skin:178 -#: Site/$Site.skin:315 +#: Site/$Site.skin:331 msgid "All of this will be deleted irreversibly." msgstr "" @@ -193,7 +193,7 @@ msgstr "" msgid "Antville supports the following application programming interfaces:" msgstr "" -#: Root/$Root.skin:128 +#: Root/$Root.skin:131 msgid "Application Server" msgstr "" @@ -201,12 +201,12 @@ msgstr "" msgid "Archive" msgstr "" -#: Site/$Site.skin:1184 +#: Site/$Site.skin:1200 msgid "Are you sure you want to add this URL to the referrer filter? Edit it below to filter a pattern only." msgstr "" #: User/$User.skin:179 -#: Site/$Site.skin:316 +#: Site/$Site.skin:332 msgid "Are you sure you want to proceed?" msgstr "" @@ -227,8 +227,8 @@ msgstr "" #: Membership/$Membership.skin:134 #: Membership/$Membership.skin:155 #: Membership/$Membership.skin:145 -#: Site/$Site.skin:1307 -#: Site/$Site.skin:1317 +#: Site/$Site.skin:1323 +#: Site/$Site.skin:1333 #: HopObject/$HopObject.skin:37 #: HopObject/$HopObject.skin:46 msgid "Best regards." @@ -237,11 +237,11 @@ msgstr "" #: User/User.js:197 #: Admin/$Admin.skin:417 #: Admin/$Admin.skin:396 -#: Site/Site.js:53 +#: Site/Site.js:54 msgid "Blocked" msgstr "" -#: Site/$Site.skin:185 +#: Site/$Site.skin:201 msgid "Bookmarklet" msgstr "" @@ -255,7 +255,7 @@ msgstr "" #: Api/Api.js:123 #: Api/$Api.skin:13 -#: Site/$Site.skin:171 +#: Site/$Site.skin:187 msgid "Callback URL" msgstr "" @@ -274,10 +274,10 @@ msgstr "" #: Membership/$Membership.skin:78 #: Admin/$Admin.skin:139 #: Story/Story.skin:83 -#: Site/$Site.skin:1296 -#: Site/$Site.skin:327 -#: Site/$Site.skin:1273 -#: Site/$Site.skin:212 +#: Site/$Site.skin:1312 +#: Site/$Site.skin:343 +#: Site/$Site.skin:1289 +#: Site/$Site.skin:228 #: Comment/Comment.skin:70 #: File/$File.skin:64 #: Image/$Image.skin:90 @@ -315,7 +315,7 @@ msgstr "" msgid "Close" msgstr "" -#: Site/Site.js:60 +#: Site/Site.js:61 #: Stories/$Stories.skin:8 msgid "Closed" msgstr "" @@ -326,7 +326,7 @@ msgstr "" #: Comment/Comment.js:22 #: Comment/Comment.skin:62 -#: HopObject/HopObject.js:173 +#: HopObject/HopObject.js:181 msgid "Comment" msgstr "" @@ -359,7 +359,7 @@ msgstr "" msgid "Compare {0}" msgstr "" -#: HopObject/HopObject.js:207 +#: HopObject/HopObject.js:215 msgid "Confirm Deletion" msgstr "" @@ -368,7 +368,7 @@ msgstr "" msgid "Confirm Reset" msgstr "" -#: Site/Site.js:736 +#: Site/Site.js:748 msgid "Confirm Unsubscribe" msgstr "" @@ -393,7 +393,7 @@ msgid "Content of Member {0}" msgstr "" #: Membership/Membership.js:54 -#: Site/Site.js:88 +#: Site/Site.js:89 msgid "Contributor" msgstr "" @@ -439,11 +439,11 @@ msgstr "" msgid "Create a site. It only takes a few clicks." msgstr "" -#: Global/Global.js:552 +#: Global/Global.js:553 msgid "Create missing file" msgstr "" -#: Global/Global.js:599 +#: Global/Global.js:600 msgid "Create missing image" msgstr "" @@ -469,7 +469,7 @@ msgstr "" msgid "Created on {0}" msgstr "" -#: Site/$Site.skin:372 +#: Site/$Site.skin:388 #, java-format msgid "Created {0}" msgstr "" @@ -490,7 +490,7 @@ msgstr "" #: User/$User.skin:103 #: Poll/$Poll.skin:121 #: Story/Story.skin:82 -#: Site/$Site.skin:211 +#: Site/$Site.skin:227 #: File/$File.skin:63 #: Image/$Image.skin:89 msgid "Delete" @@ -499,7 +499,7 @@ msgstr "" #: User/User.js:197 #: Admin/$Admin.skin:402 #: Admin/$Admin.skin:414 -#: Site/Site.js:53 +#: Site/Site.js:54 msgid "Deleted" msgstr "" @@ -527,7 +527,7 @@ msgstr "" msgid "Dimensions" msgstr "" -#: Site/$Site.skin:1114 +#: Site/$Site.skin:1130 msgid "Disable filter" msgstr "" @@ -539,7 +539,7 @@ msgstr "" msgid "Disk Quota" msgstr "" -#: Site/$Site.skin:195 +#: Site/$Site.skin:211 msgid "Disk Space" msgstr "" @@ -553,7 +553,7 @@ msgstr "" msgid "Do Androids dream of electric sheep?" msgstr "" -#: Site/$Site.skin:189 +#: Site/$Site.skin:205 msgid "Drag to Bookmarks Bar" msgstr "" @@ -603,10 +603,15 @@ msgstr "" msgid "Edit Story" msgstr "" -#: Site/$Site.skin:1115 +#: Site/$Site.skin:1131 msgid "Edit the filter in the site settings." msgstr "" +#: Site/$Site.skin:157 +#, java-format +msgid "Edit the rules in the robots.txt skin." +msgstr "" + #: Skin/Skin.js:169 #, java-format msgid "Edit {0}.{1}" @@ -616,7 +621,7 @@ msgstr "" msgid "Enabled" msgstr "" -#: Site/$Site.skin:164 +#: Site/$Site.skin:180 #, java-format msgid "Enter one filter {0}pattern{1} per line to be applied on every URL in the referrer and backlink lists." msgstr "" @@ -641,7 +646,7 @@ msgstr "" #: User/$User.skin:52 #: User/$User.skin:146 #: Layout/$Layout.skin:10 -#: Site/$Site.skin:1273 +#: Site/$Site.skin:1289 #: Site/$Site.skin:26 msgid "Export" msgstr "" @@ -650,16 +655,16 @@ msgstr "" msgid "Export Account Data" msgstr "" -#: Site/$Site.skin:1267 +#: Site/$Site.skin:1283 msgid "Export Site Data" msgstr "" #: Story/$Story.skin:96 -#: Site/$Site.skin:1286 +#: Site/$Site.skin:1302 #: File/$File.skin:30 #: File/File.js:22 #: Image/$Image.skin:47 -#: HopObject/HopObject.js:174 +#: HopObject/HopObject.js:182 msgid "File" msgstr "" @@ -686,14 +691,14 @@ msgid "Filter" msgstr "" #: Members/$Members.skin:210 -#: Site/$Site.skin:325 +#: Site/$Site.skin:341 #: ../compat/Site/$Site.skin:8 msgid "Find" msgstr "" #: Members/Members.js:327 -#: Site/Site.js:692 -#: Site/Site.js:972 +#: Site/Site.js:704 +#: Site/Site.js:984 #, java-format msgid "Found more than {0} results. Please try a more specific query." msgstr "" @@ -727,7 +732,7 @@ msgstr "" msgid "Have fun!" msgstr "" -#: Root/Root.js:306 +#: Root/Root.js:317 #: Root/Site.skin:31 msgid "Health" msgstr "" @@ -736,8 +741,8 @@ msgstr "" #: Membership/$Membership.skin:127 #: Membership/$Membership.skin:149 #: Membership/$Membership.skin:138 -#: Site/$Site.skin:1301 -#: Site/$Site.skin:1311 +#: Site/$Site.skin:1317 +#: Site/$Site.skin:1327 #: HopObject/$HopObject.skin:32 #: HopObject/$HopObject.skin:41 #, java-format @@ -789,7 +794,7 @@ msgid "If you want to resize the image please specify your desired maximum width msgstr "" #: Image/Image.js:24 -#: HopObject/HopObject.js:175 +#: HopObject/HopObject.js:183 msgid "Image" msgstr "" @@ -823,14 +828,14 @@ msgstr "" msgid "Import Layout" msgstr "" -#: Site/$Site.skin:1279 +#: Site/$Site.skin:1295 msgid "Import Site Data" msgstr "" #: User/$User.skin:121 #: Poll/$Poll.skin:107 #: Story/Story.skin:71 -#: Site/$Site.skin:290 +#: Site/$Site.skin:306 #: File/$File.skin:54 #: Image/$Image.skin:80 msgid "Information" @@ -874,7 +879,7 @@ msgstr "" msgid "Last modified on {0}" msgstr "" -#: Site/$Site.skin:373 +#: Site/$Site.skin:389 #, java-format msgid "Last modified {0}" msgstr "" @@ -916,7 +921,7 @@ msgid "Logout // verb" msgstr "" #: Membership/Membership.js:54 -#: Site/Site.js:88 +#: Site/Site.js:89 msgid "Manager" msgstr "" @@ -941,7 +946,7 @@ msgid "Members" msgstr "" #: Membership/Membership.js:22 -#: HopObject/HopObject.js:176 +#: HopObject/HopObject.js:184 msgid "Membership" msgstr "" @@ -1018,7 +1023,7 @@ msgstr "" msgid "No differences were found." msgstr "" -#: Site/Site.js:87 +#: Site/Site.js:88 msgid "Nobody" msgstr "" @@ -1036,7 +1041,7 @@ msgid "Note" msgstr "" #: User/$User.skin:131 -#: Site/$Site.skin:299 +#: Site/$Site.skin:315 msgid "Notes" msgstr "" @@ -1066,7 +1071,7 @@ msgstr "" msgid "Of course, you can now also start to add stories, upload some images or files, create your first poll or get a glimpse of Antville’s wonderful customization possibilities; just take a look at the layout section where you can modify the appearance of your site according to your needs." msgstr "" -#: Site/Site.js:61 +#: Site/Site.js:62 msgid "Open" msgstr "" @@ -1079,7 +1084,7 @@ msgid "Original skin" msgstr "" #: Membership/Membership.js:54 -#: Site/Site.js:88 +#: Site/Site.js:89 msgid "Owner" msgstr "" @@ -1130,7 +1135,7 @@ msgstr "" #: User/User.js:224 #: Skins/Skins.js:93 -#: Site/Site.js:122 +#: Site/Site.js:130 msgid "Please avoid special characters or HTML code in the name field." msgstr "" @@ -1151,11 +1156,11 @@ msgstr "" msgid "Please contact an administrator for further information." msgstr "" -#: Site/$Site.skin:1323 +#: Site/$Site.skin:1339 msgid "Please enable JavaScript in your browser for improved functionality." msgstr "" -#: Site/Site.js:111 +#: Site/Site.js:119 msgid "Please enter a name for your new site." msgstr "" @@ -1168,7 +1173,7 @@ msgid "Please enter a new password." msgstr "" #: Members/Members.js:321 -#: Site/Site.js:959 +#: Site/Site.js:971 msgid "Please enter a query in the search form." msgstr "" @@ -1214,7 +1219,7 @@ msgid "Please fill out the whole form to create a valid poll." msgstr "" #: Story/Story.js:312 -#: HopObject/HopObject.js:150 +#: HopObject/HopObject.js:158 msgid "Please login first." msgstr "" @@ -1228,7 +1233,7 @@ msgstr "" #: Poll/Poll.js:22 #: Story/$Story.skin:101 -#: HopObject/HopObject.js:177 +#: HopObject/HopObject.js:185 msgid "Poll" msgstr "" @@ -1252,7 +1257,7 @@ msgstr "" msgid "Polls by {0}" msgstr "" -#: Site/$Site.skin:189 +#: Site/$Site.skin:205 #, java-format msgid "Post to {0}" msgstr "" @@ -1302,7 +1307,7 @@ msgstr "" msgid "Proceed" msgstr "" -#: Site/Site.js:61 +#: Site/Site.js:62 msgid "Public" msgstr "" @@ -1334,15 +1339,15 @@ msgstr "" msgid "Reference" msgstr "" -#: Site/$Site.skin:1128 +#: Site/$Site.skin:1144 msgid "Referrer" msgstr "" -#: Site/$Site.skin:158 +#: Site/$Site.skin:174 msgid "Referrer Filter" msgstr "" -#: Site/Site.js:678 +#: Site/Site.js:690 #: Site/Site.skin:34 #: Root/Site.skin:37 msgid "Referrers" @@ -1370,7 +1375,7 @@ msgstr "" #: Admin/Admin.js:94 #: Admin/Admin.js:108 #: Admin/Admin.js:115 -#: Site/Site.js:54 +#: Site/Site.js:55 msgid "Regular" msgstr "" @@ -1395,7 +1400,7 @@ msgstr "" #: Admin/$Admin.skin:211 #: Admin/$Admin.skin:257 #: Admin/$Admin.skin:164 -#: Site/$Site.skin:1109 +#: Site/$Site.skin:1125 #: Skin/$Skin.skin:71 msgid "Reset" msgstr "" @@ -1409,7 +1414,7 @@ msgid "Resource type (e.g. Story or Comment)" msgstr "" #: Admin/Admin.js:101 -#: Site/Site.js:60 +#: Site/Site.js:61 msgid "Restricted" msgstr "" @@ -1417,6 +1422,10 @@ msgstr "" msgid "Results" msgstr "" +#: Site/$Site.skin:148 +msgid "Robot rules" +msgstr "" + #: User/$User.skin:34 #: Members/$Members.skin:16 #: Members/$Members.skin:227 @@ -1439,7 +1448,7 @@ msgstr "" #: Membership/$Membership.skin:16 #: Admin/$Admin.skin:137 #: Story/Story.skin:80 -#: Site/$Site.skin:208 +#: Site/$Site.skin:224 #: Comment/Comment.skin:69 #: File/$File.skin:61 #: Image/$Image.skin:87 @@ -1451,19 +1460,19 @@ msgstr "" msgid "Save and Run" msgstr "" -#: Root/$Root.skin:130 +#: Root/$Root.skin:138 msgid "Scripting Engine" msgstr "" -#: Site/Site.js:704 +#: Site/Site.js:716 #: Site/Site.skin:41 -#: Site/$Site.skin:1107 -#: Site/$Site.skin:1207 +#: Site/$Site.skin:1123 +#: Site/$Site.skin:1223 #: ../compat/Global/aspects.js:246 msgid "Search" msgstr "" -#: Site/$Site.skin:340 +#: Site/$Site.skin:356 #, java-format msgid "Search with {0}" msgstr "" @@ -1485,7 +1494,7 @@ msgstr "" msgid "Separated by commas" msgstr "" -#: Root/$Root.skin:134 +#: Root/$Root.skin:142 msgid "Servlet Interface" msgstr "" @@ -1494,9 +1503,9 @@ msgid "Sessions" msgstr "" #: Layout/$Layout.skin:73 -#: Site/Site.js:338 +#: Site/Site.js:347 #: Site/Site.skin:31 -#: Site/$Site.skin:1275 +#: Site/$Site.skin:1291 #: Root/Site.skin:35 msgid "Settings" msgstr "" @@ -1526,7 +1535,7 @@ msgstr "" #: Admin/$Admin.skin:216 #: Admin/$Admin.skin:262 #: Admin/$Admin.skin:169 -#: Site/$Site.skin:331 +#: Site/$Site.skin:347 #, java-format msgid "Showing {0} result" msgid_plural "Showing {0} results" @@ -1557,7 +1566,7 @@ msgstr "" msgid "Site Phase-Out" msgstr "" -#: Site/Site.js:803 +#: Site/Site.js:815 msgid "Site is scheduled for import." msgstr "" @@ -1581,7 +1590,7 @@ msgstr "" msgid "Skins" msgstr "" -#: Site/Site.js:835 +#: Site/Site.js:847 msgid "Something went wrong." msgstr "" @@ -1625,7 +1634,7 @@ msgid "Source: {0}" msgstr "" #: Site/Site.skin:18 -#: Site/$Site.skin:1294 +#: Site/$Site.skin:1310 #: Root/Site.skin:24 msgid "Start" msgstr "" @@ -1637,13 +1646,13 @@ msgstr "" #: User/$User.skin:113 #: Admin/$Admin.skin:224 #: Admin/$Admin.skin:270 -#: Site/$Site.skin:282 +#: Site/$Site.skin:298 #: Root/$Root.skin:70 msgid "Status" msgstr "" #: Poll/$Poll.skin:117 -#: Site/$Site.skin:1294 +#: Site/$Site.skin:1310 msgid "Stop" msgstr "" @@ -1659,7 +1668,7 @@ msgid "Stories by {0}" msgstr "" #: Story/Story.js:22 -#: HopObject/HopObject.js:178 +#: HopObject/HopObject.js:186 msgid "Story" msgstr "" @@ -1689,7 +1698,7 @@ msgid "Subscribed" msgstr "" #: Membership/Membership.js:53 -#: Site/Site.js:89 +#: Site/Site.js:90 msgid "Subscriber" msgstr "" @@ -1713,12 +1722,12 @@ msgstr "" msgid "Successfully created your site." msgstr "" -#: Site/Site.js:713 +#: Site/Site.js:725 #, java-format msgid "Successfully subscribed to site {0}." msgstr "" -#: Site/Site.js:727 +#: Site/Site.js:739 #, java-format msgid "Successfully unsubscribed from site {0}." msgstr "" @@ -1731,7 +1740,7 @@ msgstr "" msgid "Successfully updated the setup." msgstr "" -#: Root/Root.skin:2 +#: Root/Root.skin:3 msgid "System is up and running." msgstr "" @@ -1770,8 +1779,8 @@ msgstr "" #: Membership/$Membership.skin:124 #: Membership/$Membership.skin:156 #: Membership/$Membership.skin:146 -#: Site/$Site.skin:1308 -#: Site/$Site.skin:1318 +#: Site/$Site.skin:1324 +#: Site/$Site.skin:1334 #: HopObject/$HopObject.skin:38 #: HopObject/$HopObject.skin:47 msgid "The Management" @@ -1801,14 +1810,14 @@ msgstr "" #: User/User.js:523 #: Skins/Skins.js:99 #: Membership/Membership.js:159 -#: Site/Site.js:329 +#: Site/Site.js:338 #: File/File.js:204 #: Image/Image.js:203 #: Skin/Skin.js:157 msgid "The changes were saved successfully." msgstr "" -#: Site/Site.js:115 +#: Site/Site.js:123 msgid "The chosen name is too long. Please enter a shorter one." msgstr "" @@ -1861,15 +1870,15 @@ msgstr "" msgid "The poll was updated successfully." msgstr "" -#: Site/Site.js:768 +#: Site/Site.js:780 msgid "The site data will be available for download from here, soon." msgstr "" -#: Site/Site.js:756 +#: Site/Site.js:768 msgid "The site is queued for export." msgstr "" -#: Site/$Site.skin:1281 +#: Site/$Site.skin:1297 #, java-format msgid "The site is scheduled for importing the file {0}. The imported site data will be available within 24 hours." msgstr "" @@ -1878,17 +1887,17 @@ msgstr "" msgid "The site you requested has been blocked." msgstr "" -#: Site/$Site.skin:1303 +#: Site/$Site.skin:1319 #, java-format msgid "The site {0} at {1} will be blocked in {2} because it is being restricted for too long." msgstr "" -#: Site/$Site.skin:1313 +#: Site/$Site.skin:1329 #, java-format msgid "The site {0} at {1} will be deleted in {2} because it has been considered as abandoned." msgstr "" -#: Site/Site.js:352 +#: Site/Site.js:361 #, java-format msgid "The site {0} is being deleted." msgstr "" @@ -1925,7 +1934,7 @@ msgstr "" msgid "The {0} macro is missing. It is essential for accessing the site and must be present in this skin." msgstr "" -#: Site/Site.js:124 +#: Site/Site.js:132 msgid "There already is a site with this name." msgstr "" @@ -1934,8 +1943,8 @@ msgstr "" msgid "There is already another job queued for this account: {0}" msgstr "" -#: Site/Site.js:753 -#: Site/Site.js:790 +#: Site/Site.js:765 +#: Site/Site.js:802 #, java-format msgid "There is already another job queued for this site: {0}" msgstr "" @@ -2080,7 +2089,7 @@ msgstr "" msgid "Total sites hosted here" msgstr "" -#: Site/$Site.skin:148 +#: Site/$Site.skin:164 msgid "Troll Filter" msgstr "" @@ -2090,7 +2099,7 @@ msgstr "" #: Admin/Admin.js:94 #: Admin/Admin.js:108 #: Admin/Admin.js:115 -#: Site/Site.js:54 +#: Site/Site.js:55 msgid "Trusted" msgstr "" @@ -2134,7 +2143,7 @@ msgstr "" msgid "Uptime" msgstr "" -#: HopObject/HopObject.js:179 +#: HopObject/HopObject.js:187 msgid "User" msgstr "" @@ -2150,7 +2159,7 @@ msgstr "" msgid "Via" msgstr "" -#: Root/$Root.skin:136 +#: Root/$Root.skin:148 msgid "Virtual Machine" msgstr "" @@ -2172,7 +2181,7 @@ msgstr "" msgid "We have updated our terms and conditions. Please reaffirm you understand and accept the following:" msgstr "" -#: Root/$Root.skin:132 +#: Root/$Root.skin:140 msgid "Webserver" msgstr "" @@ -2258,7 +2267,7 @@ msgstr "" msgid "You are about to delete the membership of {0}." msgstr "" -#: Site/Site.js:410 +#: Site/Site.js:421 #, java-format msgid "You are about to delete the site {0}." msgstr "" @@ -2268,7 +2277,7 @@ msgstr "" msgid "You are about to delete the whole account which currently contains {0}, {1}, {2}, {3}, {4} and {5}." msgstr "" -#: Site/$Site.skin:309 +#: Site/$Site.skin:325 #, java-format msgid "You are about to delete the whole site which currently contains {0}, {1}, {2}, {3} and {4}." msgstr "" @@ -2283,7 +2292,7 @@ msgstr "" msgid "You are about to reset the skin {0}.{1}." msgstr "" -#: Site/Site.js:738 +#: Site/Site.js:750 #, java-format msgid "You are about to unsubscribe from the site {0}." msgstr "" @@ -2292,7 +2301,7 @@ msgstr "" msgid "You are going to discard unsaved content." msgstr "" -#: HopObject/HopObject.js:155 +#: HopObject/HopObject.js:163 msgid "You are not allowed to access this part of the site." msgstr "" @@ -2338,8 +2347,8 @@ msgstr "" msgid "You did not vote, yet. You can vote until the poll is closed." msgstr "" -#: Root/Root.js:418 -#: Root/Root.js:427 +#: Root/Root.js:429 +#: Root/Root.js:438 #, java-format msgid "You need to wait {0} before you are allowed to create a new site." msgstr "" @@ -2381,7 +2390,7 @@ msgstr "" msgid "[{0}] Notification of membership change" msgstr "" -#: HopObject/HopObject.js:273 +#: HopObject/HopObject.js:281 #, java-format msgid "[{0}] Notification of site changes" msgstr "" @@ -2493,7 +2502,7 @@ msgstr "" #: Admin/$Admin.skin:198 #: Story/Story.js:78 #: Story/Story.js:92 -#: Site/Site.js:80 +#: Site/Site.js:81 msgid "closed" msgstr "" @@ -2518,7 +2527,7 @@ msgstr "" #: Admin/$Admin.skin:104 #: Admin/$Admin.skin:123 #: Admin/$Admin.skin:131 -#: Site/Site.js:67 +#: Site/Site.js:68 msgid "days" msgstr "" @@ -2534,8 +2543,8 @@ msgstr "" msgid "descending" msgstr "" -#: Site/Site.js:73 -#: Site/Site.js:95 +#: Site/Site.js:74 +#: Site/Site.js:96 msgid "disabled" msgstr "" @@ -2550,14 +2559,22 @@ msgid "e.g. {0}" msgstr "" #: Layout/$Layout.skin:23 -#: Site/Site.js:74 -#: Site/Site.js:96 +#: Site/Site.js:75 +#: Site/Site.js:97 #: Site/$Site.skin:81 #: Site/$Site.skin:94 -#: Site/$Site.skin:178 +#: Site/$Site.skin:194 msgid "enabled" msgstr "" +#: Site/Site.js:104 +msgid "enforce" +msgstr "" + +#: Site/$Site.skin:154 +msgid "enforced" +msgstr "" + #: Admin/Admin.js:22 msgid "export" msgstr "" @@ -2575,7 +2592,7 @@ msgstr "" msgid "files" msgstr "" -#: Site/Site.js:940 +#: Site/Site.js:952 msgid "free" msgstr "" @@ -2600,42 +2617,42 @@ msgstr "" msgid "in" msgstr "" -#: Global/Global.js:1120 +#: Global/Global.js:1122 #, java-format msgid "in {0} day" msgid_plural "in {0} days" msgstr[0] "" msgstr[1] "" -#: Global/Global.js:1116 +#: Global/Global.js:1118 #, java-format msgid "in {0} hour" msgid_plural "in {0} hours" msgstr[0] "" msgstr[1] "" -#: Global/Global.js:1114 +#: Global/Global.js:1116 #, java-format msgid "in {0} minute" msgid_plural "in {0} minutes" msgstr[0] "" msgstr[1] "" -#: Global/Global.js:1124 +#: Global/Global.js:1126 #, java-format msgid "in {0} month" msgid_plural "in {0} months" msgstr[0] "" msgstr[1] "" -#: Global/Global.js:1122 +#: Global/Global.js:1124 #, java-format msgid "in {0} week" msgid_plural "in {0} weeks" msgstr[0] "" msgstr[1] "" -#: Global/Global.js:1126 +#: Global/Global.js:1128 #, java-format msgid "in {0} year" msgid_plural "in {0} years" @@ -2708,7 +2725,7 @@ msgstr "" #: Admin/$Admin.skin:198 #: Story/Story.js:79 -#: Site/Site.js:81 +#: Site/Site.js:82 #: Comment/Comment.js:32 msgid "public" msgstr "" @@ -2725,7 +2742,7 @@ msgstr "" msgid "restricted" msgstr "" -#: Global/Global.js:1165 +#: Global/Global.js:1167 msgid "right now" msgstr "" @@ -2746,7 +2763,7 @@ msgstr "" msgid "skins" msgstr "" -#: Global/Global.js:1112 +#: Global/Global.js:1114 msgid "soon" msgstr "" @@ -2759,6 +2776,10 @@ msgstr "" msgid "story" msgstr "" +#: Site/Site.js:103 +msgid "suggest" +msgstr "" + #: Tag/Tag.js:23 msgid "tag" msgstr "" @@ -2767,7 +2788,7 @@ msgstr "" msgid "tags" msgstr "" -#: Global/Global.js:1118 +#: Global/Global.js:1120 msgid "tomorrow" msgstr "" @@ -2780,7 +2801,7 @@ msgstr "" msgid "updated // has updated" msgstr "" -#: Site/Site.js:940 +#: Site/Site.js:952 msgid "used" msgstr "" @@ -2788,12 +2809,12 @@ msgstr "" msgid "vote" msgstr "" -#: Global/Global.js:1171 +#: Global/Global.js:1173 msgid "yesterday" msgstr "" #: User/$User.skin:125 -#: Site/$Site.skin:293 +#: Site/$Site.skin:309 #, java-format msgid "{0} Comment" msgid_plural "{0} Comments" @@ -2801,7 +2822,7 @@ msgstr[0] "" msgstr[1] "" #: User/$User.skin:127 -#: Site/$Site.skin:295 +#: Site/$Site.skin:311 #, java-format msgid "{0} File" msgid_plural "{0} Files" @@ -2809,7 +2830,7 @@ msgstr[0] "" msgstr[1] "" #: User/$User.skin:126 -#: Site/$Site.skin:294 +#: Site/$Site.skin:310 #, java-format msgid "{0} Image" msgid_plural "{0} Images" @@ -2834,7 +2855,7 @@ msgstr[0] "" msgstr[1] "" #: User/$User.skin:124 -#: Site/$Site.skin:292 +#: Site/$Site.skin:308 #, java-format msgid "{0} Story" msgid_plural "{0} Stories" @@ -2872,25 +2893,25 @@ msgstr[1] "" #: User/$User.skin:174 #: Story/Story.js:574 -#: Site/$Site.skin:311 +#: Site/$Site.skin:327 #, java-format msgid "{0} comment" msgid_plural "{0} comments" msgstr[0] "" msgstr[1] "" -#: Site/$Site.skin:1305 -#: Site/$Site.skin:1315 +#: Site/$Site.skin:1321 +#: Site/$Site.skin:1331 #: Root/$Root.skin:79 -#: Root/Root.js:419 -#: Root/Root.js:428 +#: Root/Root.js:430 +#: Root/Root.js:439 #, java-format msgid "{0} day" msgid_plural "{0} days" msgstr[0] "" msgstr[1] "" -#: Global/Global.js:1173 +#: Global/Global.js:1175 #, java-format msgid "{0} day ago" msgid_plural "{0} days ago" @@ -2898,7 +2919,7 @@ msgstr[0] "" msgstr[1] "" #: User/$User.skin:176 -#: Site/$Site.skin:313 +#: Site/$Site.skin:329 #, java-format msgid "{0} file" msgid_plural "{0} files" @@ -2920,7 +2941,7 @@ msgstr "" msgid "{0} has modified {1} at the site {2}:" msgstr "" -#: Global/Global.js:1169 +#: Global/Global.js:1171 #, java-format msgid "{0} hour ago" msgid_plural "{0} hours ago" @@ -2928,7 +2949,7 @@ msgstr[0] "" msgstr[1] "" #: User/$User.skin:175 -#: Site/$Site.skin:312 +#: Site/$Site.skin:328 #, java-format msgid "{0} image" msgid_plural "{0} images" @@ -2953,14 +2974,14 @@ msgid_plural "{0} mails" msgstr[0] "" msgstr[1] "" -#: Global/Global.js:1167 +#: Global/Global.js:1169 #, java-format msgid "{0} minute ago" msgid_plural "{0} minutes ago" msgstr[0] "" msgstr[1] "" -#: Global/Global.js:1177 +#: Global/Global.js:1179 #, java-format msgid "{0} month ago" msgid_plural "{0} months ago" @@ -2978,7 +2999,7 @@ msgid "{0} per page" msgstr "" #: User/$User.skin:177 -#: Site/$Site.skin:314 +#: Site/$Site.skin:330 #, java-format msgid "{0} poll" msgid_plural "{0} polls" @@ -3008,7 +3029,7 @@ msgid "{0} sites sorted by {1} in {2} order." msgstr "" #: User/$User.skin:173 -#: Site/$Site.skin:310 +#: Site/$Site.skin:326 #, java-format msgid "{0} story" msgid_plural "{0} stories" @@ -3029,7 +3050,7 @@ msgid_plural "{0} votes" msgstr[0] "" msgstr[1] "" -#: HopObject/HopObject.js:190 +#: HopObject/HopObject.js:198 #, java-format msgid "{0} was successfully deleted." msgstr "" @@ -3040,14 +3061,14 @@ msgstr "" msgid "{0} was successfully reset." msgstr "" -#: Global/Global.js:1175 +#: Global/Global.js:1177 #, java-format msgid "{0} week ago" msgid_plural "{0} weeks ago" msgstr[0] "" msgstr[1] "" -#: Global/Global.js:1179 +#: Global/Global.js:1181 #, java-format msgid "{0} year ago" msgid_plural "{0} years ago" @@ -3076,7 +3097,7 @@ msgid "{0}% total" msgstr "" #: User/$User.skin:142 -#: Site/$Site.skin:1269 +#: Site/$Site.skin:1285 #, java-format msgid "{0}Download the archive{1} or click “Export” to create a new one." msgstr "" diff --git a/i18n/de.po b/i18n/de.po index 4eaedb00..4067b2d5 100644 --- a/i18n/de.po +++ b/i18n/de.po @@ -18,8 +18,8 @@ msgid "" msgstr "" "Project-Id-Version: Antville-1.5\n" "Report-Msgid-Bugs-To: mail@antville.org\n" -"POT-Creation-Date: 2024-06-11 21:10+0200\n" -"PO-Revision-Date: 2024-06-11 21:15+0200\n" +"POT-Creation-Date: 2025-05-11 00:27+0200\n" +"PO-Revision-Date: 2025-05-11 00:29+0200\n" "Last-Translator: Tobi Schäfer \n" "Language-Team: The Antville People \n" "Language: de\n" @@ -27,10 +27,10 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" -"X-Generator: Poedit 3.0.1\n" +"X-Generator: Poedit 3.4.2\n" "X-Poedit-SourceCharset: UTF-8\n" -#: Global/Global.js:987 +#: Global/Global.js:989 #, java-format msgid "({0} character)" msgid_plural "({0} characters)" @@ -126,7 +126,7 @@ msgstr "Stimmen Sie ab" msgid "Admin" msgstr "Verwaltung" -#: User/$User.skin:110 Site/$Site.skin:279 +#: User/$User.skin:110 Site/$Site.skin:295 msgid "Administration" msgstr "Vewaltung" @@ -148,7 +148,7 @@ msgstr "Alle Umfragen" msgid "All Skins" msgstr "Alle Skins" -#: User/$User.skin:178 Site/$Site.skin:315 +#: User/$User.skin:178 Site/$Site.skin:331 msgid "All of this will be deleted irreversibly." msgstr "All das wird unwiderruflich gelöscht werden." @@ -182,7 +182,7 @@ msgid "Antville supports the following application programming interfaces:" msgstr "" "Antville unterstützt folgende Schnittstellen zur Anwendungsprogrammierung:" -#: Root/$Root.skin:128 +#: Root/$Root.skin:131 msgid "Application Server" msgstr "Anwendungsserver" @@ -190,7 +190,7 @@ msgstr "Anwendungsserver" msgid "Archive" msgstr "Archiv" -#: Site/$Site.skin:1184 +#: Site/$Site.skin:1200 msgid "" "Are you sure you want to add this URL to the referrer filter? Edit it below " "to filter a pattern only." @@ -198,7 +198,7 @@ msgstr "" "Sind Sie sicher, dass Sie diesen Rückverweis filtern wollen? Sie können ihn " "bearbeiten, um ein Schema zu filtern." -#: User/$User.skin:179 Site/$Site.skin:316 +#: User/$User.skin:179 Site/$Site.skin:332 msgid "Are you sure you want to proceed?" msgstr "Sind Sie sicher, dass Sie fortfahren wollen?" @@ -216,16 +216,16 @@ msgstr "Grundlegende Skins" #: User/$User.skin:206 Membership/$Membership.skin:106 #: Membership/$Membership.skin:134 Membership/$Membership.skin:155 -#: Membership/$Membership.skin:145 Site/$Site.skin:1307 Site/$Site.skin:1317 +#: Membership/$Membership.skin:145 Site/$Site.skin:1323 Site/$Site.skin:1333 #: HopObject/$HopObject.skin:37 HopObject/$HopObject.skin:46 msgid "Best regards." msgstr "Beste Grüße." -#: User/User.js:197 Admin/$Admin.skin:417 Admin/$Admin.skin:396 Site/Site.js:53 +#: User/User.js:197 Admin/$Admin.skin:417 Admin/$Admin.skin:396 Site/Site.js:54 msgid "Blocked" msgstr "Gesperrt" -#: Site/$Site.skin:185 +#: Site/$Site.skin:201 msgid "Bookmarklet" msgstr "Bookmarklet" @@ -237,7 +237,7 @@ msgstr "Beides" msgid "Cache" msgstr "Zwischenspeicher" -#: Api/Api.js:123 Api/$Api.skin:13 Site/$Site.skin:171 +#: Api/Api.js:123 Api/$Api.skin:13 Site/$Site.skin:187 msgid "Callback URL" msgstr "Rückruf-Adresse" @@ -246,8 +246,8 @@ msgstr "Rückruf-Adresse" #: Members/$Members.skin:177 Members/$Members.skin:79 Members/$Members.skin:125 #: Members/$Members.skin:155 Poll/$Poll.skin:122 Membership/$Membership.skin:18 #: Membership/$Membership.skin:78 Admin/$Admin.skin:139 Story/Story.skin:83 -#: Site/$Site.skin:1296 Site/$Site.skin:327 Site/$Site.skin:1273 -#: Site/$Site.skin:212 Comment/Comment.skin:70 File/$File.skin:64 +#: Site/$Site.skin:1312 Site/$Site.skin:343 Site/$Site.skin:1289 +#: Site/$Site.skin:228 Comment/Comment.skin:70 File/$File.skin:64 #: Image/$Image.skin:90 HopObject/$HopObject.skin:22 Root/$Root.skin:33 #: Skin/$Skin.skin:72 Skin/$Skin.skin:32 ../compat/Layout/$Layout.skin:13 #: ../compat/Site/$Site.skin:10 @@ -281,7 +281,7 @@ msgstr "" msgid "Close" msgstr "Schließen" -#: Site/Site.js:60 Stories/$Stories.skin:8 +#: Site/Site.js:61 Stories/$Stories.skin:8 msgid "Closed" msgstr "Geschlossen" @@ -289,7 +289,7 @@ msgstr "Geschlossen" msgid "Closed Stories" msgstr "Geschlossene Beiträge" -#: Comment/Comment.js:22 Comment/Comment.skin:62 HopObject/HopObject.js:173 +#: Comment/Comment.js:22 Comment/Comment.skin:62 HopObject/HopObject.js:181 msgid "Comment" msgstr "Kommentar" @@ -320,7 +320,7 @@ msgstr "Vergleichen" msgid "Compare {0}" msgstr "Vergleichen {0}" -#: HopObject/HopObject.js:207 +#: HopObject/HopObject.js:215 msgid "Confirm Deletion" msgstr "Löschen bestätigen" @@ -328,7 +328,7 @@ msgstr "Löschen bestätigen" msgid "Confirm Reset" msgstr "Zurücksetzen bestätigen" -#: Site/Site.js:736 +#: Site/Site.js:748 msgid "Confirm Unsubscribe" msgstr "Stornierung bestätigen" @@ -350,7 +350,7 @@ msgstr "Inhalt" msgid "Content of Member {0}" msgstr "Beiträge von Mitglied {0}" -#: Membership/Membership.js:54 Site/Site.js:88 +#: Membership/Membership.js:54 Site/Site.js:89 msgid "Contributor" msgstr "Autorin" @@ -396,11 +396,11 @@ msgstr "Erstellen" msgid "Create a site. It only takes a few clicks." msgstr "Erstellen Sie Ihre eigene Website mit ein paar Mausklicks." -#: Global/Global.js:552 +#: Global/Global.js:553 msgid "Create missing file" msgstr "Fehlende Datei hinzufügen" -#: Global/Global.js:599 +#: Global/Global.js:600 msgid "Create missing image" msgstr "Fehlendes Bild hinzufügen" @@ -423,7 +423,7 @@ msgstr "Erstellt von {0} am {1}." msgid "Created on {0}" msgstr "Erstellt am {0}" -#: Site/$Site.skin:372 +#: Site/$Site.skin:388 #, java-format msgid "Created {0}" msgstr "Erstellt {0}" @@ -441,11 +441,11 @@ msgid "Date string in Unix timestamp format" msgstr "Datum im Unix-Format" #: User/$User.skin:103 Poll/$Poll.skin:121 Story/Story.skin:82 -#: Site/$Site.skin:211 File/$File.skin:63 Image/$Image.skin:89 +#: Site/$Site.skin:227 File/$File.skin:63 Image/$Image.skin:89 msgid "Delete" msgstr "Löschen" -#: User/User.js:197 Admin/$Admin.skin:402 Admin/$Admin.skin:414 Site/Site.js:53 +#: User/User.js:197 Admin/$Admin.skin:402 Admin/$Admin.skin:414 Site/Site.js:54 msgid "Deleted" msgstr "Gelöscht" @@ -471,7 +471,7 @@ msgstr "Entwicklung" msgid "Dimensions" msgstr "Abmessungen" -#: Site/$Site.skin:1114 +#: Site/$Site.skin:1130 msgid "Disable filter" msgstr "Filter aufheben" @@ -483,7 +483,7 @@ msgstr "Deaktiviert" msgid "Disk Quota" msgstr "Speicherplatzanteil" -#: Site/$Site.skin:195 +#: Site/$Site.skin:211 msgid "Disk Space" msgstr "Speicherplatz" @@ -495,7 +495,7 @@ msgstr "Anzeige" msgid "Do Androids dream of electric sheep?" msgstr "Zählen Androiden elektrische Schäfchen?" -#: Site/$Site.skin:189 +#: Site/$Site.skin:205 msgid "Drag to Bookmarks Bar" msgstr "In die Lesezeichenleiste ziehen" @@ -556,10 +556,18 @@ msgstr "Umfrage bearbeiten" msgid "Edit Story" msgstr "Beitrag bearbeiten" -#: Site/$Site.skin:1115 +#: Site/$Site.skin:1131 msgid "Edit the filter in the site settings." msgstr "Der Filter kann in den Einstellungen bearbeitet werden." +#: Site/$Site.skin:157 +#, java-format +msgid "" +"Edit the rules in the robots.txt skin." +msgstr "" +"Bearbeiten Sie die Regeln im robots.txt-" +"Skin." + #: Skin/Skin.js:169 #, java-format msgid "Edit {0}.{1}" @@ -569,7 +577,7 @@ msgstr "{0}.{1} bearbeiten" msgid "Enabled" msgstr "Aktiviert" -#: Site/$Site.skin:164 +#: Site/$Site.skin:180 #, java-format msgid "" "Enter one filter {0}pattern{1} per line to be applied on every URL in the " @@ -602,7 +610,7 @@ msgid "Errors" msgstr "Fehler" #: User/$User.skin:52 User/$User.skin:146 Layout/$Layout.skin:10 -#: Site/$Site.skin:1273 Site/$Site.skin:26 +#: Site/$Site.skin:1289 Site/$Site.skin:26 msgid "Export" msgstr "Exportieren" @@ -610,12 +618,12 @@ msgstr "Exportieren" msgid "Export Account Data" msgstr "Kontodaten exportieren" -#: Site/$Site.skin:1267 +#: Site/$Site.skin:1283 msgid "Export Site Data" msgstr "Site-Daten exportieren" -#: Story/$Story.skin:96 Site/$Site.skin:1286 File/$File.skin:30 File/File.js:22 -#: Image/$Image.skin:47 HopObject/HopObject.js:174 +#: Story/$Story.skin:96 Site/$Site.skin:1302 File/$File.skin:30 File/File.js:22 +#: Image/$Image.skin:47 HopObject/HopObject.js:182 msgid "File" msgstr "Datei" @@ -636,11 +644,11 @@ msgstr "Dateien von {0}" msgid "Filter" msgstr "Filtern" -#: Members/$Members.skin:210 Site/$Site.skin:325 ../compat/Site/$Site.skin:8 +#: Members/$Members.skin:210 Site/$Site.skin:341 ../compat/Site/$Site.skin:8 msgid "Find" msgstr "Finden" -#: Members/Members.js:327 Site/Site.js:692 Site/Site.js:972 +#: Members/Members.js:327 Site/Site.js:704 Site/Site.js:984 #, java-format msgid "Found more than {0} results. Please try a more specific query." msgstr "" @@ -676,13 +684,13 @@ msgstr "Viel Vergnügen!" msgid "Have fun!" msgstr "Viele Späße!" -#: Root/Root.js:306 Root/Site.skin:31 +#: Root/Root.js:317 Root/Site.skin:31 msgid "Health" msgstr "Statusmonitor" #: User/$User.skin:196 Membership/$Membership.skin:127 #: Membership/$Membership.skin:149 Membership/$Membership.skin:138 -#: Site/$Site.skin:1301 Site/$Site.skin:1311 HopObject/$HopObject.skin:32 +#: Site/$Site.skin:1317 Site/$Site.skin:1327 HopObject/$HopObject.skin:32 #: HopObject/$HopObject.skin:41 #, java-format msgid "Hello {0}." @@ -729,11 +737,11 @@ msgstr "" #: ../compat/Members/Members.js:21 #, java-format msgid "" -"If you should really have forgotten your password, you can use the password reset option." +"If you should really have forgotten your password, you can use the password reset option." msgstr "" -"Falls Sie Ihr Kennwort vergessen haben sollten, können Sie es zurücksetzen lassen." +"Falls Sie Ihr Kennwort vergessen haben sollten, können Sie es zurücksetzen lassen." #: Global/$Global.skin:59 msgid "" @@ -757,7 +765,7 @@ msgstr "" "beibehalten. Falls die Breite oder Höhe des Bildes 100 Pixel überschreitet, " "erstellt Antville außerdem automatisch ein Miniaturbild davon." -#: Image/Image.js:24 HopObject/HopObject.js:175 +#: Image/Image.js:24 HopObject/HopObject.js:183 msgid "Image" msgstr "Bild" @@ -788,12 +796,12 @@ msgstr "Importieren" msgid "Import Layout" msgstr "Layout importieren" -#: Site/$Site.skin:1279 +#: Site/$Site.skin:1295 msgid "Import Site Data" msgstr "Site-Daten importieren" #: User/$User.skin:121 Poll/$Poll.skin:107 Story/Story.skin:71 -#: Site/$Site.skin:290 File/$File.skin:54 Image/$Image.skin:80 +#: Site/$Site.skin:306 File/$File.skin:54 Image/$Image.skin:80 msgid "Information" msgstr "Information" @@ -831,7 +839,7 @@ msgstr "Zuletzt geändert von {0} am {1}" msgid "Last modified on {0}" msgstr "Zuletzt geändert am {0}" -#: Site/$Site.skin:373 +#: Site/$Site.skin:389 #, java-format msgid "Last modified {0}" msgstr "Zuletzt geändert {0}" @@ -867,7 +875,7 @@ msgstr "Anmelden" msgid "Logout // verb" msgstr "Abmelden" -#: Membership/Membership.js:54 Site/Site.js:88 +#: Membership/Membership.js:54 Site/Site.js:89 msgid "Manager" msgstr "Redakteurin" @@ -888,7 +896,7 @@ msgstr "Mitglied {0}" msgid "Members" msgstr "Mitglieder" -#: Membership/Membership.js:22 HopObject/HopObject.js:176 +#: Membership/Membership.js:22 HopObject/HopObject.js:184 msgid "Membership" msgstr "Mitgliedschaft" @@ -950,7 +958,7 @@ msgstr "Nächste Seite" msgid "No differences were found." msgstr "Es wurden keine Unterschiede gefunden." -#: Site/Site.js:87 +#: Site/Site.js:88 msgid "Nobody" msgstr "Niemand" @@ -966,7 +974,7 @@ msgstr "Noch nicht registriert?" msgid "Note" msgstr "Bemerkung" -#: User/$User.skin:131 Site/$Site.skin:299 +#: User/$User.skin:131 Site/$Site.skin:315 msgid "Notes" msgstr "Anmerkungen" @@ -1007,7 +1015,7 @@ msgstr "" "sich den Layout-Bereich anschauen, wo Sie das " "Erscheinungsbild Ihrer Website nach Ihren Wünschen ändern können." -#: Site/Site.js:61 +#: Site/Site.js:62 msgid "Open" msgstr "Offen" @@ -1019,7 +1027,7 @@ msgstr "Optionen" msgid "Original skin" msgstr "Ursprünglicher Skin" -#: Membership/Membership.js:54 Site/Site.js:88 +#: Membership/Membership.js:54 Site/Site.js:89 msgid "Owner" msgstr "Besitzerin" @@ -1066,7 +1074,7 @@ msgstr "Bitte akzeptieren Sie die Datenschutzerklärung." msgid "Please accept the terms and conditions." msgstr "Bitte akzeptieren Sie die Nutzungsbedingungen." -#: User/User.js:224 Skins/Skins.js:93 Site/Site.js:122 +#: User/User.js:224 Skins/Skins.js:93 Site/Site.js:130 msgid "Please avoid special characters or HTML code in the name field." msgstr "" "Bitte vermeiden Sie Sonderzeichen oder HTML-Code im Feld für den Namen." @@ -1093,12 +1101,12 @@ msgid "Please contact an administrator for further information." msgstr "" "Bitte wenden Sie sich an eine Administratorin für weitere Informationen." -#: Site/$Site.skin:1323 +#: Site/$Site.skin:1339 msgid "Please enable JavaScript in your browser for improved functionality." msgstr "" "Bitte aktivieren Sie für optimale Funktionalität JavaScript in Ihrem Browser." -#: Site/Site.js:111 +#: Site/Site.js:119 msgid "Please enter a name for your new site." msgstr "Bitte geben Sie einen Namen für Ihre neue Website ein." @@ -1110,7 +1118,7 @@ msgstr "Bitte geben Sie einen neuen Namen für dieses Stichwort an" msgid "Please enter a new password." msgstr "Bitte geben Sie ein neues Kennwort ein." -#: Members/Members.js:321 Site/Site.js:959 +#: Members/Members.js:321 Site/Site.js:971 msgid "Please enter a query in the search form." msgstr "Bitte geben Sie eine Suchanfrage in das Suchformular ein." @@ -1158,7 +1166,7 @@ msgstr "" "Bitte füllen Sie das gesamte Formular aus, um eine gültige Umfrage zu " "erstellen." -#: Story/Story.js:312 HopObject/HopObject.js:150 +#: Story/Story.js:312 HopObject/HopObject.js:158 msgid "Please login first." msgstr "Bitte melden Sie sich zuerst an." @@ -1172,7 +1180,7 @@ msgstr "" msgid "Please upload a zipped layout archive" msgstr "Bitte laden Sie ein Layout als ZIP-Archiv hoch" -#: Poll/Poll.js:22 Story/$Story.skin:101 HopObject/HopObject.js:177 +#: Poll/Poll.js:22 Story/$Story.skin:101 HopObject/HopObject.js:185 msgid "Poll" msgstr "Umfrage" @@ -1195,7 +1203,7 @@ msgstr "Umfragen" msgid "Polls by {0}" msgstr "Umfragen von {0}" -#: Site/$Site.skin:189 +#: Site/$Site.skin:205 #, java-format msgid "Post to {0}" msgstr "Auf {0} veröffentlichen." @@ -1236,7 +1244,7 @@ msgstr "Bewährungsfrist" msgid "Proceed" msgstr "Fortfahren" -#: Site/Site.js:61 +#: Site/Site.js:62 msgid "Public" msgstr "Öffentlich" @@ -1264,15 +1272,15 @@ msgstr "Wiederherstellung Ihres Kennworts" msgid "Reference" msgstr "Bezug" -#: Site/$Site.skin:1128 +#: Site/$Site.skin:1144 msgid "Referrer" msgstr "Rückverweis" -#: Site/$Site.skin:158 +#: Site/$Site.skin:174 msgid "Referrer Filter" msgstr "Rückverweis-Filter" -#: Site/Site.js:678 Site/Site.skin:34 Root/Site.skin:37 +#: Site/Site.js:690 Site/Site.skin:34 Root/Site.skin:37 msgid "Referrers" msgstr "Rückverweise" @@ -1293,7 +1301,7 @@ msgid "Registration & Login" msgstr "Registrierung & Anmeldung" #: User/User.js:197 Admin/Admin.js:94 Admin/Admin.js:108 Admin/Admin.js:115 -#: Site/Site.js:54 +#: Site/Site.js:55 msgid "Regular" msgstr "Normal" @@ -1314,7 +1322,7 @@ msgid "Required Account Status" msgstr "Benötigter Konto-Status" #: Layout/$Layout.skin:88 Admin/$Admin.skin:211 Admin/$Admin.skin:257 -#: Admin/$Admin.skin:164 Site/$Site.skin:1109 Skin/$Skin.skin:71 +#: Admin/$Admin.skin:164 Site/$Site.skin:1125 Skin/$Skin.skin:71 msgid "Reset" msgstr "Zurücksetzen" @@ -1326,7 +1334,7 @@ msgstr "Kennwort zurücksetzen" msgid "Resource type (e.g. Story or Comment)" msgstr "Art der Ressource (z.B. Beitrag oder Kommentar)" -#: Admin/Admin.js:101 Site/Site.js:60 +#: Admin/Admin.js:101 Site/Site.js:61 msgid "Restricted" msgstr "Eingeschränkt" @@ -1334,6 +1342,10 @@ msgstr "Eingeschränkt" msgid "Results" msgstr "Ergebnis" +#: Site/$Site.skin:148 +msgid "Robot rules" +msgstr "Regeln für Robots" + #: User/$User.skin:34 Members/$Members.skin:16 Members/$Members.skin:227 #: Membership/$Membership.skin:7 msgid "Role" @@ -1349,7 +1361,7 @@ msgstr "Laufende Umfragen" #: User/$User.skin:101 Layout/$Layout.skin:86 Members/$Members.skin:194 #: Poll/$Poll.skin:114 Membership/$Membership.skin:16 Admin/$Admin.skin:137 -#: Story/Story.skin:80 Site/$Site.skin:208 Comment/Comment.skin:69 +#: Story/Story.skin:80 Site/$Site.skin:224 Comment/Comment.skin:69 #: File/$File.skin:61 Image/$Image.skin:87 Skin/$Skin.skin:29 msgid "Save" msgstr "Speichern" @@ -1358,16 +1370,16 @@ msgstr "Speichern" msgid "Save and Run" msgstr "Speichern und starten" -#: Root/$Root.skin:130 +#: Root/$Root.skin:138 msgid "Scripting Engine" msgstr "Scripting-Umgebung" -#: Site/Site.js:704 Site/Site.skin:41 Site/$Site.skin:1107 Site/$Site.skin:1207 +#: Site/Site.js:716 Site/Site.skin:41 Site/$Site.skin:1123 Site/$Site.skin:1223 #: ../compat/Global/aspects.js:246 msgid "Search" msgstr "Suche" -#: Site/$Site.skin:340 +#: Site/$Site.skin:356 #, java-format msgid "Search with {0}" msgstr "Mit {0} suchen" @@ -1388,7 +1400,7 @@ msgstr "Anfrage senden" msgid "Separated by commas" msgstr "Durch Komma getrennt" -#: Root/$Root.skin:134 +#: Root/$Root.skin:142 msgid "Servlet Interface" msgstr "Servlet-Schnittstelle" @@ -1396,8 +1408,8 @@ msgstr "Servlet-Schnittstelle" msgid "Sessions" msgstr "Sitzungen" -#: Layout/$Layout.skin:73 Site/Site.js:338 Site/Site.skin:31 -#: Site/$Site.skin:1275 Root/Site.skin:35 +#: Layout/$Layout.skin:73 Site/Site.js:347 Site/Site.skin:31 +#: Site/$Site.skin:1291 Root/Site.skin:35 msgid "Settings" msgstr "Einstellungen" @@ -1416,7 +1428,7 @@ msgid "Show Controls" msgstr "Kontrollelemente anzeigen" #: Members/$Members.skin:217 Admin/$Admin.skin:283 Admin/$Admin.skin:216 -#: Admin/$Admin.skin:262 Admin/$Admin.skin:169 Site/$Site.skin:331 +#: Admin/$Admin.skin:262 Admin/$Admin.skin:169 Site/$Site.skin:347 #, java-format msgid "Showing {0} result" msgid_plural "Showing {0} results" @@ -1428,8 +1440,8 @@ msgstr[1] "{0} Treffer werden angezeigt" msgid "" "Since you are an administrator of this Antville installation you are " "entitled to manage sites and accounts, monitor all activity, configure the setup and much more." +"a>, monitor all activity, configure the setup and much more." msgstr "" "Da Sie Administratorin dieser Antville-Installation sind, haben Sie auch die " "Berechtigung, Websites und Konten zu " @@ -1452,7 +1464,7 @@ msgstr "Basis-Seite" msgid "Site Phase-Out" msgstr "Automatisches Löschen von Websites" -#: Site/Site.js:803 +#: Site/Site.js:815 msgid "Site is scheduled for import." msgstr "Die Website ist für den Import eingeplant." @@ -1472,7 +1484,7 @@ msgstr "Skin" msgid "Skins" msgstr "Skins" -#: Site/Site.js:835 +#: Site/Site.js:847 msgid "Something went wrong." msgstr "Irgendwas ist schiefgelaufen." @@ -1524,7 +1536,7 @@ msgstr "Leider ist unter diesem Namen kein Konto registriert." msgid "Source: {0}" msgstr "Quelle: {0}" -#: Site/Site.skin:18 Site/$Site.skin:1294 Root/Site.skin:24 +#: Site/Site.skin:18 Site/$Site.skin:1310 Root/Site.skin:24 msgid "Start" msgstr "Start" @@ -1533,11 +1545,11 @@ msgid "Start Page" msgstr "Startseite" #: User/$User.skin:113 Admin/$Admin.skin:224 Admin/$Admin.skin:270 -#: Site/$Site.skin:282 Root/$Root.skin:70 +#: Site/$Site.skin:298 Root/$Root.skin:70 msgid "Status" msgstr "Status" -#: Poll/$Poll.skin:117 Site/$Site.skin:1294 +#: Poll/$Poll.skin:117 Site/$Site.skin:1310 msgid "Stop" msgstr "Beenden" @@ -1550,7 +1562,7 @@ msgstr "Beiträge" msgid "Stories by {0}" msgstr "Beiträge von {0}" -#: Story/Story.js:22 HopObject/HopObject.js:178 +#: Story/Story.js:22 HopObject/HopObject.js:186 msgid "Story" msgstr "Beitrag" @@ -1579,7 +1591,7 @@ msgstr "Abonnieren" msgid "Subscribed" msgstr "Abonniert" -#: Membership/Membership.js:53 Site/Site.js:89 +#: Membership/Membership.js:53 Site/Site.js:90 msgid "Subscriber" msgstr "Abonnentin" @@ -1600,12 +1612,12 @@ msgstr "{0} wurde erfolgreich zur Liste der Mitglieder hinzugefügt." msgid "Successfully created your site." msgstr "Ihr Website wurde erfolgreich erstellt." -#: Site/Site.js:713 +#: Site/Site.js:725 #, java-format msgid "Successfully subscribed to site {0}." msgstr "Die Website {0} wurde erfolgreich abonniert." -#: Site/Site.js:727 +#: Site/Site.js:739 #, java-format msgid "Successfully unsubscribed from site {0}." msgstr "Das Abonnement der Website {0} wurde erfolgreich storniert." @@ -1618,7 +1630,7 @@ msgstr "Das Layout wurde erfolgreich aktualisiert." msgid "Successfully updated the setup." msgstr "Die Konfiguration wurde erfolgreich aktualisiert." -#: Root/Root.skin:2 +#: Root/Root.skin:3 msgid "System is up and running." msgstr "System ist betriebsbereit." @@ -1654,7 +1666,7 @@ msgstr "" #: User/$User.skin:207 Membership/$Membership.skin:107 #: Membership/$Membership.skin:135 Membership/$Membership.skin:124 #: Membership/$Membership.skin:156 Membership/$Membership.skin:146 -#: Site/$Site.skin:1308 Site/$Site.skin:1318 HopObject/$HopObject.skin:38 +#: Site/$Site.skin:1324 Site/$Site.skin:1334 HopObject/$HopObject.skin:38 #: HopObject/$HopObject.skin:47 msgid "The Management" msgstr "Die Direktion" @@ -1687,11 +1699,11 @@ msgstr "" "Methode aufgerufen:" #: User/User.js:523 Skins/Skins.js:99 Membership/Membership.js:159 -#: Site/Site.js:329 File/File.js:204 Image/Image.js:203 Skin/Skin.js:157 +#: Site/Site.js:338 File/File.js:204 Image/Image.js:203 Skin/Skin.js:157 msgid "The changes were saved successfully." msgstr "Die Änderungen wurden erfolgreich gespeichert." -#: Site/Site.js:115 +#: Site/Site.js:123 msgid "The chosen name is too long. Please enter a shorter one." msgstr "Der gewählte Name ist zu lang. Bitte geben Sie einen kürzeren ein." @@ -1721,14 +1733,14 @@ msgstr "Der Kommentar wurde erfolgreich aktualisiert." #: User/$User.skin:11 #, java-format msgid "" -"The easiest way to customize your site is to change its settings. You can change the language and time zone or the main " -"title of your site, open or close it and much more." +"The easiest way to customize your site is to change its settings. You can change the language and time zone or the " +"main title of your site, open or close it and much more." msgstr "" -"Sie können Ihre Website am einfachsten anpassen, indem Sie deren Einstellungen ändern. Sie können die Sprache und Zeitzone oder " -"den Titel der Website ändern, die Website veröffentlichen, einschränken und " -"vieles mehr." +"Sie können Ihre Website am einfachsten anpassen, indem Sie deren Einstellungen ändern. Sie können die Sprache und Zeitzone " +"oder den Titel der Website ändern, die Website veröffentlichen, einschränken " +"und vieles mehr." #: Files/Files.js:60 msgid "The file was successfully added." @@ -1762,15 +1774,15 @@ msgstr "Die Umfrage wurde erfolgreich erstellt." msgid "The poll was updated successfully." msgstr "Die Umfrage wurde erfolgreich aktualisiert." -#: Site/Site.js:768 +#: Site/Site.js:780 msgid "The site data will be available for download from here, soon." msgstr "Der Site-Export steht demnächst hier zum Download bereit." -#: Site/Site.js:756 +#: Site/Site.js:768 msgid "The site is queued for export." msgstr "Der Export der Site-Daten wird vorbereitet." -#: Site/$Site.skin:1281 +#: Site/$Site.skin:1297 #, java-format msgid "" "The site is scheduled for importing the file {0}. The imported site data " @@ -1783,7 +1795,7 @@ msgstr "" msgid "The site you requested has been blocked." msgstr "Die von Ihnen angeforderte Website ist gesperrt." -#: Site/$Site.skin:1303 +#: Site/$Site.skin:1319 #, java-format msgid "" "The site {0} at {1} will be blocked in {2} because it is being restricted " @@ -1792,7 +1804,7 @@ msgstr "" "Die Website {0} unter {1} wird in {2} gesperrt werden, weil sie schon für zu " "lange Zeit eingeschränkt ist." -#: Site/$Site.skin:1313 +#: Site/$Site.skin:1329 #, java-format msgid "" "The site {0} at {1} will be deleted in {2} because it has been considered as " @@ -1801,7 +1813,7 @@ msgstr "" "Die Website {0} unter {1} wird in {2} gelöscht werden, weil sie verlassen zu " "sein scheint." -#: Site/Site.js:352 +#: Site/Site.js:361 #, java-format msgid "The site {0} is being deleted." msgstr "Die Website {0} wird gelöscht." @@ -1851,7 +1863,7 @@ msgstr "" "Das {0}-Makro fehlt. Es ist erforderlich, um die Website korrekt " "darzustellen und muss unbedingt in diesem Skin enthalten sein." -#: Site/Site.js:124 +#: Site/Site.js:132 msgid "There already is a site with this name." msgstr "Es gibt bereits eine Website mit diesem Namen." @@ -1860,7 +1872,7 @@ msgstr "Es gibt bereits eine Website mit diesem Namen." msgid "There is already another job queued for this account: {0}" msgstr "Für dieses Konto wird bereits ein anderer Auftrag berarbeitet: {0}" -#: Site/Site.js:753 Site/Site.js:790 +#: Site/Site.js:765 Site/Site.js:802 #, java-format msgid "There is already another job queued for this site: {0}" msgstr "Ein anderer Prozess ist für diese Website bereits gereiht: {0}" @@ -2046,12 +2058,12 @@ msgstr "Summe" msgid "Total sites hosted here" msgstr "Summe aller Websites" -#: Site/$Site.skin:148 +#: Site/$Site.skin:164 msgid "Troll Filter" msgstr "Trollfilter" #: User/User.js:197 Admin/$Admin.skin:399 Admin/$Admin.skin:420 -#: Admin/Admin.js:94 Admin/Admin.js:108 Admin/Admin.js:115 Site/Site.js:54 +#: Admin/Admin.js:94 Admin/Admin.js:108 Admin/Admin.js:115 Site/Site.js:55 msgid "Trusted" msgstr "Vertrauenswürdig" @@ -2096,7 +2108,7 @@ msgstr "" msgid "Uptime" msgstr "Betriebszeit" -#: HopObject/HopObject.js:179 +#: HopObject/HopObject.js:187 msgid "User" msgstr "Konto" @@ -2114,7 +2126,7 @@ msgstr "Versionen" msgid "Via" msgstr "Via" -#: Root/$Root.skin:136 +#: Root/$Root.skin:148 msgid "Virtual Machine" msgstr "Virtuelle Maschine" @@ -2143,7 +2155,7 @@ msgstr "" "Wir haben unsere Nutzungsbedingungen geändert. Bitte bestätigen Sie im " "folgenden, dass Sie diese verstehen und akzeptieren:" -#: Root/$Root.skin:132 +#: Root/$Root.skin:140 msgid "Webserver" msgstr "Webserver" @@ -2232,7 +2244,7 @@ msgstr "Sie sind im Begriff, das Bild {0} zu löschen." msgid "You are about to delete the membership of {0}." msgstr "Sie sind im Begriff, die Mitgliedschaft von {0} zu löschen." -#: Site/Site.js:410 +#: Site/Site.js:421 #, java-format msgid "You are about to delete the site {0}." msgstr "Sie sind im Begriff, die Website {0} zu löschen." @@ -2246,7 +2258,7 @@ msgstr "" "Sie sind im Begriff, das komplette Konto zu löschen, das zur Zeit {0}, {1}, " "{2}, {3}, {4} und {5} umfasst." -#: Site/$Site.skin:309 +#: Site/$Site.skin:325 #, java-format msgid "" "You are about to delete the whole site which currently contains {0}, {1}, " @@ -2265,7 +2277,7 @@ msgstr "Sie sind im Begriff, das Layout der Website {0} zurückzusetzen." msgid "You are about to reset the skin {0}.{1}." msgstr "Sie sind im Begriff, den Skin {0}.{1} zurückzusetzen." -#: Site/Site.js:738 +#: Site/Site.js:750 #, java-format msgid "You are about to unsubscribe from the site {0}." msgstr "Sie sind im Begriff, das Abonnement der Website {0} zu löschen." @@ -2274,7 +2286,7 @@ msgstr "Sie sind im Begriff, das Abonnement der Website {0} zu löschen." msgid "You are going to discard unsaved content." msgstr "Sie sind im Begriff, ungesicherte Inhalte zu verwerfen." -#: HopObject/HopObject.js:155 +#: HopObject/HopObject.js:163 msgid "You are not allowed to access this part of the site." msgstr "" "Sie sind leider nicht berechtigt, auf diesen Teil der Website zuzugreifen." @@ -2336,7 +2348,7 @@ msgstr "" "Sie haben noch nicht abgestimmt. Sie können abstimmen, bis die Umfrage " "beendet ist." -#: Root/Root.js:418 Root/Root.js:427 +#: Root/Root.js:429 Root/Root.js:438 #, java-format msgid "You need to wait {0} before you are allowed to create a new site." msgstr "Sie müssen {0} warten, bevor Sie eine neue Website erstellen können." @@ -2377,7 +2389,7 @@ msgstr "[{0}] Benachrichtigung über Beendigung der Mitgliedschaft" msgid "[{0}] Notification of membership change" msgstr "[{0}] Benachrichtigung über Änderung der Mitgliedschaft" -#: HopObject/HopObject.js:273 +#: HopObject/HopObject.js:281 #, java-format msgid "[{0}] Notification of site changes" msgstr "[{0}] Benachrichtigung über Änderung der Website" @@ -2479,7 +2491,7 @@ msgid "choice" msgstr "Antwortmöglichkeit" #: Poll/Poll.js:32 Admin/$Admin.skin:198 Story/Story.js:78 Story/Story.js:92 -#: Site/Site.js:80 +#: Site/Site.js:81 msgid "closed" msgstr "geschlossen" @@ -2500,7 +2512,7 @@ msgid "created" msgstr "erstellt" #: Admin/$Admin.skin:96 Admin/$Admin.skin:104 Admin/$Admin.skin:123 -#: Admin/$Admin.skin:131 Site/Site.js:67 +#: Admin/$Admin.skin:131 Site/Site.js:68 msgid "days" msgstr "Tage" @@ -2512,7 +2524,7 @@ msgstr "gelöscht" msgid "descending" msgstr "absteigend" -#: Site/Site.js:73 Site/Site.js:95 +#: Site/Site.js:74 Site/Site.js:96 msgid "disabled" msgstr "deaktiviert" @@ -2525,11 +2537,19 @@ msgstr "E-Mail" msgid "e.g. {0}" msgstr "z.B. {0}" -#: Layout/$Layout.skin:23 Site/Site.js:74 Site/Site.js:96 Site/$Site.skin:81 -#: Site/$Site.skin:94 Site/$Site.skin:178 +#: Layout/$Layout.skin:23 Site/Site.js:75 Site/Site.js:97 Site/$Site.skin:81 +#: Site/$Site.skin:94 Site/$Site.skin:194 msgid "enabled" msgstr "aktiviert" +#: Site/Site.js:104 +msgid "enforce" +msgstr "erzwingen" + +#: Site/$Site.skin:154 +msgid "enforced" +msgstr "erzwingen" + #: Admin/Admin.js:22 msgid "export" msgstr "Exportieren" @@ -2546,7 +2566,7 @@ msgstr "Datei" msgid "files" msgstr "Dateien" -#: Site/Site.js:940 +#: Site/Site.js:952 msgid "free" msgstr "frei" @@ -2570,42 +2590,42 @@ msgstr "Importieren" msgid "in" msgstr "in" -#: Global/Global.js:1120 +#: Global/Global.js:1122 #, java-format msgid "in {0} day" msgid_plural "in {0} days" msgstr[0] "in {0} Tag" msgstr[1] "in {0} Tagen" -#: Global/Global.js:1116 +#: Global/Global.js:1118 #, java-format msgid "in {0} hour" msgid_plural "in {0} hours" msgstr[0] "in {0} Stunde" msgstr[1] "in {0} Stunden" -#: Global/Global.js:1114 +#: Global/Global.js:1116 #, java-format msgid "in {0} minute" msgid_plural "in {0} minutes" msgstr[0] "in {0} Minute" msgstr[1] "in {0} Minuten" -#: Global/Global.js:1124 +#: Global/Global.js:1126 #, java-format msgid "in {0} month" msgid_plural "in {0} months" msgstr[0] "in {0} Monat" msgstr[1] "in {0} Monaten" -#: Global/Global.js:1122 +#: Global/Global.js:1124 #, java-format msgid "in {0} week" msgid_plural "in {0} weeks" msgstr[0] "in {0} Woche" msgstr[1] "in {0} Wochen" -#: Global/Global.js:1126 +#: Global/Global.js:1128 #, java-format msgid "in {0} year" msgid_plural "in {0} years" @@ -2672,7 +2692,7 @@ msgstr "veröffentlichte" msgid "privileged" msgstr "privilegiert" -#: Admin/$Admin.skin:198 Story/Story.js:79 Site/Site.js:81 +#: Admin/$Admin.skin:198 Story/Story.js:79 Site/Site.js:82 #: Comment/Comment.js:32 msgid "public" msgstr "öffentlich" @@ -2689,7 +2709,7 @@ msgstr "löschen" msgid "restricted" msgstr "eingeschränkt" -#: Global/Global.js:1165 +#: Global/Global.js:1167 msgid "right now" msgstr "vor kurzem" @@ -2709,7 +2729,7 @@ msgstr "Skin" msgid "skins" msgstr "Skins" -#: Global/Global.js:1112 +#: Global/Global.js:1114 msgid "soon" msgstr "in Kürze" @@ -2721,6 +2741,10 @@ msgstr "Beiträge" msgid "story" msgstr "Beitrag" +#: Site/Site.js:103 +msgid "suggest" +msgstr "vorschlagen" + #: Tag/Tag.js:23 msgid "tag" msgstr "Stichwort" @@ -2729,7 +2753,7 @@ msgstr "Stichwort" msgid "tags" msgstr "Stichworte" -#: Global/Global.js:1118 +#: Global/Global.js:1120 msgid "tomorrow" msgstr "morgen" @@ -2741,7 +2765,7 @@ msgstr "vertrauenswürdig" msgid "updated // has updated" msgstr "aktualisierte" -#: Site/Site.js:940 +#: Site/Site.js:952 msgid "used" msgstr "benutzt" @@ -2749,25 +2773,25 @@ msgstr "benutzt" msgid "vote" msgstr "Stimme" -#: Global/Global.js:1171 +#: Global/Global.js:1173 msgid "yesterday" msgstr "gestern" -#: User/$User.skin:125 Site/$Site.skin:293 +#: User/$User.skin:125 Site/$Site.skin:309 #, java-format msgid "{0} Comment" msgid_plural "{0} Comments" msgstr[0] "{0} Kommentar" msgstr[1] "{0} Kommentare" -#: User/$User.skin:127 Site/$Site.skin:295 +#: User/$User.skin:127 Site/$Site.skin:311 #, java-format msgid "{0} File" msgid_plural "{0} Files" msgstr[0] "{0} Datei" msgstr[1] "{0} Dateien" -#: User/$User.skin:126 Site/$Site.skin:294 +#: User/$User.skin:126 Site/$Site.skin:310 #, java-format msgid "{0} Image" msgid_plural "{0} Images" @@ -2791,7 +2815,7 @@ msgid_plural "{0} Sites" msgstr[0] "{0} Website" msgstr[1] "{0} Websites" -#: User/$User.skin:124 Site/$Site.skin:292 +#: User/$User.skin:124 Site/$Site.skin:308 #, java-format msgid "{0} Story" msgid_plural "{0} Stories" @@ -2827,29 +2851,29 @@ msgid_plural "{0} characters" msgstr[0] "{0} Zeichen" msgstr[1] "{0} Zeichen" -#: User/$User.skin:174 Story/Story.js:574 Site/$Site.skin:311 +#: User/$User.skin:174 Story/Story.js:574 Site/$Site.skin:327 #, java-format msgid "{0} comment" msgid_plural "{0} comments" msgstr[0] "{0} Kommentar" msgstr[1] "{0} Kommentare" -#: Site/$Site.skin:1305 Site/$Site.skin:1315 Root/$Root.skin:79 -#: Root/Root.js:419 Root/Root.js:428 +#: Site/$Site.skin:1321 Site/$Site.skin:1331 Root/$Root.skin:79 +#: Root/Root.js:430 Root/Root.js:439 #, java-format msgid "{0} day" msgid_plural "{0} days" msgstr[0] "{0} Tag" msgstr[1] "{0} Tage" -#: Global/Global.js:1173 +#: Global/Global.js:1175 #, java-format msgid "{0} day ago" msgid_plural "{0} days ago" msgstr[0] "vor {0} Tag" msgstr[1] "vor {0} Tagen" -#: User/$User.skin:176 Site/$Site.skin:313 +#: User/$User.skin:176 Site/$Site.skin:329 #, java-format msgid "{0} file" msgid_plural "{0} files" @@ -2871,14 +2895,14 @@ msgstr "{0} hat {1} zur Website {2} hinzugefügt:" msgid "{0} has modified {1} at the site {2}:" msgstr "{0} hat {1} auf der Website {2} geändert:" -#: Global/Global.js:1169 +#: Global/Global.js:1171 #, java-format msgid "{0} hour ago" msgid_plural "{0} hours ago" msgstr[0] "vor {0} Stunde" msgstr[1] "vor {0} Stunden" -#: User/$User.skin:175 Site/$Site.skin:312 +#: User/$User.skin:175 Site/$Site.skin:328 #, java-format msgid "{0} image" msgid_plural "{0} images" @@ -2902,14 +2926,14 @@ msgid_plural "{0} mails" msgstr[0] "{0} Nachrichten" msgstr[1] "{0} Nachrichten" -#: Global/Global.js:1167 +#: Global/Global.js:1169 #, java-format msgid "{0} minute ago" msgid_plural "{0} minutes ago" msgstr[0] "vor {0} Minute" msgstr[1] "vor {0} Minuten" -#: Global/Global.js:1177 +#: Global/Global.js:1179 #, java-format msgid "{0} month ago" msgid_plural "{0} months ago" @@ -2926,7 +2950,7 @@ msgstr "{0} von {1} Objekten" msgid "{0} per page" msgstr "{0} pro Seite" -#: User/$User.skin:177 Site/$Site.skin:314 +#: User/$User.skin:177 Site/$Site.skin:330 #, java-format msgid "{0} poll" msgid_plural "{0} polls" @@ -2953,7 +2977,7 @@ msgstr[1] "{0} Websites" msgid "{0} sites sorted by {1} in {2} order." msgstr "Typ: {0} Sortierung: {1} {2}" -#: User/$User.skin:173 Site/$Site.skin:310 +#: User/$User.skin:173 Site/$Site.skin:326 #, java-format msgid "{0} story" msgid_plural "{0} stories" @@ -2972,7 +2996,7 @@ msgid_plural "{0} votes" msgstr[0] "{0} Stimme" msgstr[1] "{0} Stimmen" -#: HopObject/HopObject.js:190 +#: HopObject/HopObject.js:198 #, java-format msgid "{0} was successfully deleted." msgstr "{0} wurde erfolgreich gelöscht." @@ -2982,14 +3006,14 @@ msgstr "{0} wurde erfolgreich gelöscht." msgid "{0} was successfully reset." msgstr "{0} wurde erfolgreich zurückgesetzt." -#: Global/Global.js:1175 +#: Global/Global.js:1177 #, java-format msgid "{0} week ago" msgid_plural "{0} weeks ago" msgstr[0] "vor {0} Woche" msgstr[1] "vor {0} Wochen" -#: Global/Global.js:1179 +#: Global/Global.js:1181 #, java-format msgid "{0} year ago" msgid_plural "{0} years ago" @@ -3018,13 +3042,19 @@ msgstr "{0}% in den letzten 5 Min." msgid "{0}% total" msgstr "{0}% gesamt" -#: User/$User.skin:142 Site/$Site.skin:1269 +#: User/$User.skin:142 Site/$Site.skin:1285 #, java-format msgid "{0}Download the archive{1} or click “Export” to create a new one." msgstr "" "{0}Laden Sie das Archiv herunter{1} oder klicken Sie »Exportieren«, um ein " "neues zu erstellen." +#~ msgid "" +#~ "Edit the rules in the robots.txt skin." +#~ msgstr "" +#~ "Die Regeln können im Skin robots.txt " +#~ "bearbeitet werden." + #~ msgid "Build" #~ msgstr "Gestalt" diff --git a/i18n/messages.de.js b/i18n/messages.de.js index b67b1d33..51778114 100644 --- a/i18n/messages.de.js +++ b/i18n/messages.de.js @@ -125,6 +125,7 @@ global.messages['de'] = { "Edit Poll": "Umfrage bearbeiten", "Edit Story": "Beitrag bearbeiten", "Edit the filter in the site settings.": "Der Filter kann in den Einstellungen bearbeitet werden.", + "Edit the rules in the robots.txt skin.": "Bearbeiten Sie die Regeln im robots.txt-Skin.", "Edit {0}.{1}": "{0}.{1} bearbeiten", "Enabled": "Aktiviert", "Enter one filter {0}pattern{1} per line to be applied on every URL in the referrer and backlink lists.": "Geben Sie ein {0}Filter-Schema{1} pro Zeile ein, das für jede Adresse in den Rückverweis-Listen angewendet werden soll.", @@ -290,6 +291,7 @@ global.messages['de'] = { "Resource type (e.g. Story or Comment)": "Art der Ressource (z.B. Beitrag oder Kommentar)", "Restricted": "Eingeschränkt", "Results": "Ergebnis", + "Robot rules": "Regeln für Robots", "Role": "Rolle", "Running": "Laufende", "Running Polls": "Laufende Umfragen", @@ -529,6 +531,8 @@ global.messages['de'] = { "e-mail": "E-Mail", "e.g. {0}": "z.B. {0}", "enabled": "aktiviert", + "enforce": "erzwingen", + "enforced": "erzwingen", "export": "Exportieren", "featured": "sichtbar", "file": "Datei", @@ -578,6 +582,7 @@ global.messages['de'] = { "soon": "in Kürze", "stories": "Beiträge", "story": "Beitrag", + "suggest": "vorschlagen", "tag": "Stichwort", "tags": "Stichworte", "tomorrow": "morgen", From 1d59aff5a67e77cdcfb67c3caf0f8480afc29ecf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20Sch=C3=A4fer?= Date: Sun, 11 May 2025 01:06:40 +0200 Subject: [PATCH 07/10] Always grant access to robots.txt --- code/Site/Site.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/Site/Site.js b/code/Site/Site.js index d638d1d2..09cdc67d 100644 --- a/code/Site/Site.js +++ b/code/Site/Site.js @@ -1145,9 +1145,10 @@ Site.prototype.enforceRobotsTxt = function() { // Override some URLs to prevent a site from becoming inaccessible even for the owner const overrides = [ this.href('edit'), - this.layout.href(), this.href('main.css'), this.href('main.js'), + this.href('robots.txt'), + this.layout.href(), this.members.href() ]; From c6d86368c589db02797e9e1e5a6a19890a5ac065 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20Sch=C3=A4fer?= Date: Sun, 11 May 2025 01:08:19 +0200 Subject: [PATCH 08/10] Add and translate missing translatable message --- code/HopObject/HopObject.js | 2 +- i18n/antville.pot | 9 +++++++-- i18n/de.po | 14 ++++++++++++-- i18n/messages.de.js | 1 + 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/code/HopObject/HopObject.js b/code/HopObject/HopObject.js index f2a2782a..b9983c5d 100644 --- a/code/HopObject/HopObject.js +++ b/code/HopObject/HopObject.js @@ -147,7 +147,7 @@ HopObject.prototype.onRequest = function() { if (res.handlers.site.enforceRobotsTxt()) { res.status = 403 - res.data.error = "Robots.txt forbids access to this page."; + res.data.error = gettext('The robots.txt file disallows access to this page.', res.handlers.site.href('robots.txt')); root.error_action(); res.stop(); } diff --git a/i18n/antville.pot b/i18n/antville.pot index 076cfc8d..5e30bc20 100644 --- a/i18n/antville.pot +++ b/i18n/antville.pot @@ -22,8 +22,8 @@ msgid "" msgstr "" "Project-Id-Version: Antville-0\n" "Report-Msgid-Bugs-To: mail@antville.org\n" -"POT-Creation-Date: 2025-05-11 00:27+0200\n" -"PO-Revision-Date: 2025-05-11 00:27+0200\n" +"POT-Creation-Date: 2025-05-11 01:00+0200\n" +"PO-Revision-Date: 2025-05-11 01:00+0200\n" "Language-Team: The Antville People \n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" @@ -1773,6 +1773,11 @@ msgstr "" msgid "Thanks, your vote was registered. You can change your mind until the poll is closed." msgstr "" +#: HopObject/HopObject.js:150 +#, java-format +msgid "The robots.txt file disallows access to this page." +msgstr "" + #: User/$User.skin:207 #: Membership/$Membership.skin:107 #: Membership/$Membership.skin:135 diff --git a/i18n/de.po b/i18n/de.po index 4067b2d5..aa8acbbf 100644 --- a/i18n/de.po +++ b/i18n/de.po @@ -18,8 +18,8 @@ msgid "" msgstr "" "Project-Id-Version: Antville-1.5\n" "Report-Msgid-Bugs-To: mail@antville.org\n" -"POT-Creation-Date: 2025-05-11 00:27+0200\n" -"PO-Revision-Date: 2025-05-11 00:29+0200\n" +"POT-Creation-Date: 2025-05-11 01:00+0200\n" +"PO-Revision-Date: 2025-05-11 01:03+0200\n" "Last-Translator: Tobi Schäfer \n" "Language-Team: The Antville People \n" "Language: de\n" @@ -1663,6 +1663,13 @@ msgstr "" "Danke, Ihre Stimme wurde gezählt. Bis die Umfrage beendet ist, können Sie " "Ihre Meinung jederzeit ändern." +#: HopObject/HopObject.js:150 +#, java-format +msgid "The robots.txt file disallows access to this page." +msgstr "" +"Die robots.txt-Datei verbietet den Zugriff auf diese " +"Seite." + #: User/$User.skin:207 Membership/$Membership.skin:107 #: Membership/$Membership.skin:135 Membership/$Membership.skin:124 #: Membership/$Membership.skin:156 Membership/$Membership.skin:146 @@ -3049,6 +3056,9 @@ msgstr "" "{0}Laden Sie das Archiv herunter{1} oder klicken Sie »Exportieren«, um ein " "neues zu erstellen." +#~ msgid "Robots.txt forbids access to this page." +#~ msgstr "Robots.txt verbietet den Zugang zu dieser Seite." + #~ msgid "" #~ "Edit the rules in the robots.txt skin." #~ msgstr "" diff --git a/i18n/messages.de.js b/i18n/messages.de.js index 51778114..b4550753 100644 --- a/i18n/messages.de.js +++ b/i18n/messages.de.js @@ -361,6 +361,7 @@ global.messages['de'] = { "Terms and Conditions": "Nutzungsbedingungen", "Text": "Text", "Thanks, your vote was registered. You can change your mind until the poll is closed.": "Danke, Ihre Stimme wurde gezählt. Bis die Umfrage beendet ist, können Sie Ihre Meinung jederzeit ändern.", + "The robots.txt file disallows access to this page.": "Die robots.txt-Datei verbietet den Zugriff auf diese Seite.", "The Management": "Die Direktion", "The URL endpoint for each of these APIs is located at": "Die Internet-Adresse für jede dieser Schnittstellen lautet", "The account data will be available for download from here within the next days.": "Die Kontodaten stehen demnächst hier zum Download bereit.", From 0b326e71e6224763729ed68e171b1b394256ac4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobi=20Sch=C3=A4fer?= Date: Tue, 27 May 2025 20:26:38 +0200 Subject: [PATCH 09/10] Add compatibility fix for CSS skins with