com.norconex.collector.http.crawler

Class HttpCrawler

java.lang.Object

com.norconex.jef4.job.AbstractResumableJob

com.norconex.collector.core.crawler.AbstractCrawler

com.norconex.collector.http.crawler.HttpCrawler

All Implemented Interfaces:

ICrawler, IJob

public class HttpCrawler
extends AbstractCrawler

The HTTP Crawler.

Author:

Pascal Essiembre

Nested Class Summary

Nested classes/interfaces inherited from class com.norconex.collector.core.crawler.AbstractCrawler

AbstractCrawler.CopyIfNullBeanUtilsBean

Constructor Summary

Constructors

Constructor and Description
HttpCrawler(HttpCrawlerConfig crawlerConfig) Constructor.

Method Summary

Modifier and Type	Method and Description
protected void	beforeFinalizeDocumentProcessing(BaseCrawlData crawlData, ICrawlDataStore store, ImporterDocument doc, ICrawlData cachedData)
protected void	cleanupExecution(JobStatusUpdater statusUpdater, JobSuite suite, ICrawlDataStore refStore)
protected BaseCrawlData	createEmbeddedCrawlData(String embeddedReference, ICrawlData parentCrawlData)
protected void	executeCommitterPipeline(ICrawler crawler, ImporterDocument doc, ICrawlDataStore crawlDataStore, BaseCrawlData crawlData, BaseCrawlData cachedCrawlData)
protected ImporterResponse	executeImporterPipeline(ImporterPipelineContext importerContext)
protected void	executeQueuePipeline(ICrawlData crawlData, ICrawlDataStore crawlDataStore)
HttpCrawlerConfig	getCrawlerConfig()
org.apache.http.client.HttpClient	getHttpClient()
ISitemapResolver	getSitemapResolver()
protected void	initCrawlData(ICrawlData crawlData, ICrawlData cachedCrawlData, ImporterDocument document)
protected void	markReferenceVariationsAsProcessed(BaseCrawlData crawlData, ICrawlDataStore crawlDataStore)
protected void	prepareExecution(JobStatusUpdater statusUpdater, JobSuite suite, ICrawlDataStore crawlDataStore, boolean resume)
void	stop(IJobStatus jobStatus, JobSuite suite)
protected ImporterDocument	wrapDocument(ICrawlData crawlData, ImporterDocument document)

Methods inherited from class com.norconex.collector.core.crawler.AbstractCrawler

createCrawlDataStore, deleteCacheOrphans, execute, fireCrawlerEvent, getBaseDownloadDir, getCrawlerDownloadDir, getCrawlerEventManager, getId, getImporter, getStreamFactory, handleOrphans, isMaxDocuments, isStopped, processNextReference, processReferences, reprocessCacheOrphans, resumeExecution, startExecution

Methods inherited from class com.norconex.jef4.job.AbstractResumableJob

execute

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface com.norconex.jef4.job.IJob

execute

Constructor Detail

HttpCrawler

public HttpCrawler(HttpCrawlerConfig crawlerConfig)

Constructor.

Parameters:

crawlerConfig - HTTP crawler configuration

Method Detail

getCrawlerConfig

public HttpCrawlerConfig getCrawlerConfig()

Specified by:

getCrawlerConfig in interface ICrawler

Overrides:

getCrawlerConfig in class AbstractCrawler

getHttpClient

public org.apache.http.client.HttpClient getHttpClient()

Returns:

the httpClient

getSitemapResolver

public ISitemapResolver getSitemapResolver()

Returns:

the sitemapResolver

stop

public void stop(IJobStatus jobStatus,
                 JobSuite suite)

Specified by:

stop in interface IJob

Overrides:

stop in class AbstractCrawler

prepareExecution

protected void prepareExecution(JobStatusUpdater statusUpdater,
                                JobSuite suite,
                                ICrawlDataStore crawlDataStore,
                                boolean resume)

Specified by:

prepareExecution in class AbstractCrawler

executeQueuePipeline

protected void executeQueuePipeline(ICrawlData crawlData,
                                    ICrawlDataStore crawlDataStore)

Specified by:

executeQueuePipeline in class AbstractCrawler

wrapDocument

protected ImporterDocument wrapDocument(ICrawlData crawlData,
                                        ImporterDocument document)

Specified by:

wrapDocument in class AbstractCrawler

initCrawlData

protected void initCrawlData(ICrawlData crawlData,
                             ICrawlData cachedCrawlData,
                             ImporterDocument document)

Overrides:

initCrawlData in class AbstractCrawler

executeImporterPipeline

protected ImporterResponse executeImporterPipeline(ImporterPipelineContext importerContext)

Specified by:

executeImporterPipeline in class AbstractCrawler

createEmbeddedCrawlData

protected BaseCrawlData createEmbeddedCrawlData(String embeddedReference,
                                                ICrawlData parentCrawlData)

Specified by:

createEmbeddedCrawlData in class AbstractCrawler

executeCommitterPipeline

protected void executeCommitterPipeline(ICrawler crawler,
                                        ImporterDocument doc,
                                        ICrawlDataStore crawlDataStore,
                                        BaseCrawlData crawlData,
                                        BaseCrawlData cachedCrawlData)

Specified by:

executeCommitterPipeline in class AbstractCrawler

beforeFinalizeDocumentProcessing

protected void beforeFinalizeDocumentProcessing(BaseCrawlData crawlData,
                                                ICrawlDataStore store,
                                                ImporterDocument doc,
                                                ICrawlData cachedData)

Overrides:

beforeFinalizeDocumentProcessing in class AbstractCrawler

markReferenceVariationsAsProcessed

protected void markReferenceVariationsAsProcessed(BaseCrawlData crawlData,
                                                  ICrawlDataStore crawlDataStore)

Specified by:

markReferenceVariationsAsProcessed in class AbstractCrawler

cleanupExecution

protected void cleanupExecution(JobStatusUpdater statusUpdater,
                                JobSuite suite,
                                ICrawlDataStore refStore)

Specified by:

cleanupExecution in class AbstractCrawler