|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |
java.lang.Objectxsmeral.semnet.crawler.model.CrawlerConfiguration
@XStreamConverter(value=CrawlerConfigurationConverter.class) public class CrawlerConfiguration
Container for HTMLCrawler
configuration.
Contains
RDBLayer
HostDescriptor
s
Field Summary | |
---|---|
static boolean |
DEF_FAKE_REFERRER
|
static int |
DEF_GLOBAL_CRAWL_DELAY_MIN
|
static boolean |
DEF_POLICY_IGNORED
|
Constructor Summary | |
---|---|
CrawlerConfiguration()
|
|
CrawlerConfiguration(Collection<HostDescriptor> hosts,
RDBLayer dbLayer,
int threadsPerHost,
int globalCrawlDelayMinimum,
boolean policyIgnored,
boolean fakeReferrer)
Initializes all fields |
Method Summary | |
---|---|
RDBLayer |
getDBLayer()
The relational DB layer used by the crawler for state persistence (URL storage) |
int |
getGlobalCrawlDelayMinimum()
Minimal crawl delay in milliseconds |
Collection<HostDescriptor> |
getHosts()
Hosts crawled by the crawler |
int |
getThreadsPerHost()
Number of crawling threads per host |
boolean |
isFakeReferrer()
Indication, whether the HTTP Referer header should be set to the base URL of the host |
boolean |
isPolicyIgnored()
Indication of adherence to the Robots Exclusion Protocol |
void |
setDBLayer(RDBLayer dbLayer)
The relational DB layer used by the crawler for state persistence (URL storage) |
void |
setFakeReferrer(boolean fakeReferrer)
Indication, whether the HTTP Referer header should be set to the base URL of the host |
void |
setGlobalCrawlDelayMinimum(int globalCrawlDelayMinimum)
Minimal crawl delay in milliseconds |
void |
setHosts(Collection<HostDescriptor> hosts)
Hosts crawled by the crawler |
void |
setPolicyIgnored(boolean policyIgnored)
Indication of adherence to the Robots Exclusion Protocol |
void |
setThreadsPerHost(int threadsPerHost)
Number of crawling threads per host |
Methods inherited from class java.lang.Object |
---|
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
Field Detail |
---|
public static final int DEF_GLOBAL_CRAWL_DELAY_MIN
public static final boolean DEF_POLICY_IGNORED
public static final boolean DEF_FAKE_REFERRER
Constructor Detail |
---|
public CrawlerConfiguration()
public CrawlerConfiguration(Collection<HostDescriptor> hosts, RDBLayer dbLayer, int threadsPerHost, int globalCrawlDelayMinimum, boolean policyIgnored, boolean fakeReferrer)
Method Detail |
---|
public RDBLayer getDBLayer()
public void setDBLayer(RDBLayer dbLayer)
public boolean isFakeReferrer()
public void setFakeReferrer(boolean fakeReferrer)
public int getGlobalCrawlDelayMinimum()
public void setGlobalCrawlDelayMinimum(int globalCrawlDelayMinimum)
public Collection<HostDescriptor> getHosts()
public void setHosts(Collection<HostDescriptor> hosts)
public boolean isPolicyIgnored()
RobotsPolicy
public void setPolicyIgnored(boolean policyIgnored)
RobotsPolicy
public int getThreadsPerHost()
public void setThreadsPerHost(int threadsPerHost)
|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |