edu.uci.ics.crawler4j.frontier

Class Frontier

java.lang.Object

edu.uci.ics.crawler4j.crawler.Configurable

edu.uci.ics.crawler4j.frontier.Frontier

public class Frontier
extends Configurable

Author:

Yasser Ganjisaffar

Field Summary

Fields

Modifier and Type	Field and Description
protected Counters	counters
protected DocIDServer	docIdServer
protected InProcessPagesDB	inProcessPages
protected boolean	isFinished
protected static org.apache.log4j.Logger	logger
protected Object	mutex
protected long	scheduledPages
protected Object	waitingList
protected WorkQueues	workQueues

Fields inherited from class edu.uci.ics.crawler4j.crawler.Configurable

config

Constructor Summary

Constructors

Constructor and Description
Frontier(com.sleepycat.je.Environment env, CrawlConfig config, DocIDServer docIdServer)

Method Summary

Methods

Modifier and Type	Method and Description
void	close()
void	finish()
void	getNextURLs(int max, List<WebURL> result)
long	getNumberOfAssignedPages()
long	getNumberOfProcessedPages()
long	getQueueLength()
boolean	isFinished()
void	schedule(WebURL url)
void	scheduleAll(List<WebURL> urls)
void	setProcessed(WebURL webURL)
void	sync()

Methods inherited from class edu.uci.ics.crawler4j.crawler.Configurable

getConfig

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

logger

protected static final org.apache.log4j.Logger logger

workQueues

protected WorkQueues workQueues

inProcessPages

protected InProcessPagesDB inProcessPages

mutex

protected final Object mutex

waitingList

protected final Object waitingList

isFinished

protected boolean isFinished

scheduledPages

protected long scheduledPages

docIdServer

protected DocIDServer docIdServer

counters

protected Counters counters

Constructor Detail

Frontier

public Frontier(com.sleepycat.je.Environment env,
        CrawlConfig config,
        DocIDServer docIdServer)

Method Detail

scheduleAll

public void scheduleAll(List<WebURL> urls)

schedule

public void schedule(WebURL url)

getNextURLs

public void getNextURLs(int max,
               List<WebURL> result)

setProcessed

public void setProcessed(WebURL webURL)

getQueueLength

public long getQueueLength()

getNumberOfAssignedPages

public long getNumberOfAssignedPages()

getNumberOfProcessedPages

public long getNumberOfProcessedPages()

sync

public void sync()

isFinished

public boolean isFinished()

close

public void close()

finish

public void finish()