protected static class RobotsManager.RobotsData extends Object
Modifier and Type | Field and Description |
---|---|
protected long |
expiration |
protected ArrayList |
records |
Constructor and Description |
---|
RobotsManager.RobotsData(InputStream is,
long expiration,
String hostName,
IVersionActivity activities)
Constructor.
|
Modifier and Type | Method and Description |
---|---|
long |
getExpirationTime()
Get expiration
|
boolean |
isFetchAllowed(String userAgent,
String pathString)
Check if fetch is allowed
|
protected void |
parseRobotsTxt(BufferedReader r,
String hostName,
IVersionActivity activities)
Parse the robots.txt file using a reader.
|
protected long expiration
protected ArrayList records
public RobotsManager.RobotsData(InputStream is, long expiration, String hostName, IVersionActivity activities) throws IOException, ManifoldCFException
IOException
ManifoldCFException
public boolean isFetchAllowed(String userAgent, String pathString)
public long getExpirationTime()
protected void parseRobotsTxt(BufferedReader r, String hostName, IVersionActivity activities) throws IOException, ManifoldCFException
IOException
ManifoldCFException