-
Notifications
You must be signed in to change notification settings - Fork 5
/
robots.txt
168 lines (151 loc) · 2.85 KB
/
robots.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
User-agent: *
Disallow: /images/
Disallow: /js/
Disallow: /css/
Disallow: /tags/
Disallow: /series/
Disallow: /posts/
Disallow: /blog/
User-agent: MJ12bot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: BLEXBot
Disallow: /
# Block SISTRIX
User-agent: SISTRIX Crawler
Disallow: /
User-agent: sistrix
Disallow: /
User-agent: 007ac9
Disallow: /
User-agent: 007ac9 Crawler
Disallow: /
# Block Uptime robot
User-agent: UptimeRobot/2.0
Disallow: /
# Block Ezooms Robot
User-agent: Ezooms Robot
Disallow: /
# Block Perl LWP
User-agent: Perl LWP
Disallow: /
# Block netEstate NE Crawler (+http://www.website-datenbank.de/)
User-agent: netEstate NE Crawler (+http://www.website-datenbank.de/)
Disallow: /
# Block WiseGuys Robot
User-agent: WiseGuys Robot
Disallow: /
# Block Turnitin Robot
User-agent: Turnitin Robot
Disallow: /
# Block Heritrix
User-agent: Heritrix
Disallow: /
# Block pricepi
User-agent: pimonster
Disallow: /
User-agent: SurdotlyBot
Disallow: /
User-agent: ZoominfoBot
Disallow: /
User-agent: serpstatbot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /
User-agent: dotbot
Disallow:/
User-agent: rogerbot
Disallow: /
User-agent: Alexibot
Disallow: /
User-agent: SurveyBot
Disallow: /
User-agent: Xenu’s
Disallow: /
User-agent: Xenu’s Link Sleuth 1.1c
Disallow: /
User-agent: NextGenSearchBot
Disallow: /
User-agent: ia_archiver
Disallow: /
User-agent: archive.org_bot
Disallow: /
User-agent: Archive.org Bot
Disallow: /
User-agent: LinkWalker
Disallow: /
User-agent: GigaBlast Spider
Disallow: /
User-agent: ia_archiver-web.archive.org
Disallow: /
User-agent: PicScout
Disallow: /
User-agent: BLEXBot Crawler
Disallow: /
User-agent: TinEye
Disallow: /
User-agent: SEOkicks-Robot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: UptimeRobot/2.0
Disallow: /
User-agent: Ezooms Robot
Disallow: /
User-agent: netEstate NE Crawler (+http://www.website-datenbank.de/)
Disallow: /
User-agent: WiseGuys Robot
Disallow: /
User-agent: Turnitin Robot
Disallow: /
User-agent: pimonster
Disallow: /
User-agent: Pimonster
Disallow: /
User-agent: Pi-Monster
Disallow: /
User-agent: ECCP/1.0 (search@eniro.com)
Disallow: /
User-agent: Psbot
Disallow: /
User-agent: YoudaoBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: NaverBot
User-agent: Yeti
Disallow: /
User-agent: ZBot
Disallow: /
User-agent: Vagabondo
Disallow: /
User-agent: LinkWalker
Disallow: /
User-agent: SimplePie
Disallow: /
User-agent: Wget
Disallow: /
User-agent: Pixray-Seeker
Disallow: /
User-agent: BoardReader
Disallow: /
User-agent: Quantify
Disallow: /
User-agent: Plukkie
Disallow: /
User-agent: Cuam
Disallow: /
User-agent: MegaIndex.ru
Disallow: /
User-agent: megaindex.com
Disallow: /
User-agent: +http://megaindex.com/crawler
Disallow: /
User-agent: MegaIndex.ru/2.0
Disallow: /
User-agent: megaIndex.ru
Disallow: /
Sitemap: https://saintmalik.me/sitemap.xml