@@ -27,19 +27,16 @@ or via Yarn:
27
27
``` js
28
28
var robotsParser = require (' robots-parser' );
29
29
30
- var robots = robotsParser (
31
- ' http://www.example.com/robots.txt' ,
32
- [
33
- ' User-agent: *' ,
34
- ' Disallow: /dir/' ,
35
- ' Disallow: /test.html' ,
36
- ' Allow: /dir/test.html' ,
37
- ' Allow: /test.html' ,
38
- ' Crawl-delay: 1' ,
39
- ' Sitemap: http://example.com/sitemap.xml' ,
40
- ' Host: example.com'
41
- ].join (' \n ' )
42
- );
30
+ var robots = robotsParser (' http://www.example.com/robots.txt' , [
31
+ ' User-agent: *' ,
32
+ ' Disallow: /dir/' ,
33
+ ' Disallow: /test.html' ,
34
+ ' Allow: /dir/test.html' ,
35
+ ' Allow: /test.html' ,
36
+ ' Crawl-delay: 1' ,
37
+ ' Sitemap: http://example.com/sitemap.xml' ,
38
+ ' Host: example.com'
39
+ ].join (' \n ' ));
43
40
44
41
robots .isAllowed (' http://www.example.com/test.html' , ' Sams-Bot/1.0' ); // true
45
42
robots .isAllowed (' http://www.example.com/dir/test.html' , ' Sams-Bot/1.0' ); // true
@@ -113,16 +110,13 @@ Returns the preferred host name specified by the `host:` directive or null if th
113
110
For example:
114
111
115
112
``` js
116
- var robots = robotsParser (
117
- ' /robots.txt' ,
118
- [
119
- ' User-agent: *' ,
120
- ' Disallow: /dir/' ,
121
- ' Disallow: /test.html' ,
122
- ' Allow: /dir/test.html' ,
123
- ' Allow: /test.html'
124
- ].join (' \n ' )
125
- );
113
+ var robots = robotsParser (' /robots.txt' , [
114
+ ' User-agent: *' ,
115
+ ' Disallow: /dir/' ,
116
+ ' Disallow: /test.html' ,
117
+ ' Allow: /dir/test.html' ,
118
+ ' Allow: /test.html'
119
+ ].join (' \n ' ));
126
120
127
121
robots .isAllowed (' /test.html' , ' Sams-Bot/1.0' ); // false
128
122
robots .isAllowed (' /dir/test.html' , ' Sams-Bot/1.0' ); // true
0 commit comments