edu.uci.ics.crawler4j.frontier

Class DocIDServer

java.lang.Object

edu.uci.ics.crawler4j.crawler.Configurable

edu.uci.ics.crawler4j.frontier.DocIDServer

public class DocIDServer
extends Configurable

Author:

Yasser Ganjisaffar

Field Summary

Fields

Modifier and Type	Field and Description
protected com.sleepycat.je.Database	docIDsDB
protected int	lastDocID
protected static org.apache.log4j.Logger	logger
protected Object	mutex

Fields inherited from class edu.uci.ics.crawler4j.crawler.Configurable

config

Constructor Summary

Constructors

Constructor and Description
DocIDServer(com.sleepycat.je.Environment env, CrawlConfig config)

Method Summary

Methods

Modifier and Type	Method and Description
void	addUrlAndDocId(String url, int docId)
void	close()
int	getDocCount()
int	getDocId(String url) Returns the docid of an already seen url.
int	getNewDocID(String url)
boolean	isSeenBefore(String url)
void	sync()

Methods inherited from class edu.uci.ics.crawler4j.crawler.Configurable

getConfig

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

logger

protected static final org.apache.log4j.Logger logger

docIDsDB

protected com.sleepycat.je.Database docIDsDB

mutex

protected final Object mutex

lastDocID

protected int lastDocID

Constructor Detail

DocIDServer

public DocIDServer(com.sleepycat.je.Environment env,
           CrawlConfig config)
            throws com.sleepycat.je.DatabaseException

Throws:

com.sleepycat.je.DatabaseException

Method Detail

getDocId

public int getDocId(String url)

Returns the docid of an already seen url.

Parameters:

url - the URL for which the docid is returned.

Returns:

the docid of the url if it is seen before. Otherwise -1 is returned.

getNewDocID

public int getNewDocID(String url)

addUrlAndDocId

public void addUrlAndDocId(String url,
                  int docId)
                    throws Exception

Throws:

Exception

isSeenBefore

public boolean isSeenBefore(String url)

getDocCount

public int getDocCount()

sync

public void sync()

close

public void close()