org.apache.manifoldcf.crawler.system
Class ManifoldCF

java.lang.Object
  org.apache.manifoldcf.core.system.ManifoldCF
      org.apache.manifoldcf.agents.system.ManifoldCF
          org.apache.manifoldcf.crawler.system.ManifoldCF

public class ManifoldCF
extends ManifoldCF

Nested Class Summary

protected static class

ManifoldCF.InitializationThread

Nested classes/interfaces inherited from class org.apache.manifoldcf.agents.system.ManifoldCF

ManifoldCF.AgentsShutdownHook

Nested classes/interfaces inherited from class org.apache.manifoldcf.core.system.ManifoldCF

ManifoldCF.DatabaseConnectionReleaseThread, ManifoldCF.DatabaseShutdown, ManifoldCF.FileTrack, ManifoldCF.ShutdownThread

Field Summary

static java.lang.String	_rcsid
protected static java.lang.String	API_AUTHORITYCONNECTIONNODE
protected static java.lang.String	API_AUTHORITYCONNECTORNODE
protected static java.lang.String	API_CHECKRESULTNODE
protected static java.lang.String	API_CONNECTIONNAMENODE
protected static java.lang.String	API_JOBIDNODE
protected static java.lang.String	API_JOBNODE
protected static java.lang.String	API_JOBSTATUSNODE
protected static java.lang.String	API_OUTPUTCONNECTIONNODE
protected static java.lang.String	API_OUTPUTCONNECTORNODE
protected static java.lang.String	API_REPOSITORYCONNECTIONNODE
protected static java.lang.String	API_REPOSITORYCONNECTORNODE
protected static DocumentCleanupStufferThread	cleanupStufferThread
protected static java.lang.String	cleanupThreadCountProperty
protected static DocumentCleanupThread[]	cleanupThreads
protected static java.lang.String	CONNECTIONNODE_ACLAUTHORITY
protected static java.lang.String	CONNECTIONNODE_CLASSNAME
protected static java.lang.String	CONNECTIONNODE_CONFIGURATION
protected static java.lang.String	CONNECTIONNODE_DESCRIPTION
protected static java.lang.String	CONNECTIONNODE_ISNEW
protected static java.lang.String	CONNECTIONNODE_MATCH
protected static java.lang.String	CONNECTIONNODE_MATCHDESCRIPTION
protected static java.lang.String	CONNECTIONNODE_MAXCONNECTIONS
protected static java.lang.String	CONNECTIONNODE_NAME
protected static java.lang.String	CONNECTIONNODE_RATE
protected static java.lang.String	CONNECTIONNODE_THROTTLE
protected static java.lang.String	CONNECTORNODE_CLASSNAME
protected static java.lang.String	CONNECTORNODE_DESCRIPTION
protected static boolean	crawlerInitialized
protected static DocumentDeleteStufferThread	deleteStufferThread
protected static java.lang.String	deleteThreadCountProperty
protected static DocumentDeleteThread[]	deleteThreads
protected static DocCleanupResetManager	docCleanupResetManager Cleanup thread pool reset manager
protected static DocDeleteResetManager	docDeleteResetManager Delete thread pool reset manager
protected static ExpireStufferThread	expireStufferThread
protected static java.lang.String	expireThreadCountProperty
protected static ExpireThread[]	expireThreads
protected static FinisherThread	finisherThread
protected static IdleCleanupThread	idleCleanupThread
protected static ManifoldCF.InitializationThread	initializationThread
protected static JobDeleteThread	jobDeleteThread
protected static java.lang.String	JOBNODE_CONNECTIONNAME
protected static java.lang.String	JOBNODE_COUNT
protected static java.lang.String	JOBNODE_DAYOFMONTH
protected static java.lang.String	JOBNODE_DAYOFWEEK
protected static java.lang.String	JOBNODE_DESCRIPTION
protected static java.lang.String	JOBNODE_DOCUMENTSPECIFICATION
protected static java.lang.String	JOBNODE_DURATION
protected static java.lang.String	JOBNODE_ENUMVALUE
protected static java.lang.String	JOBNODE_EXPIRATIONINTERVAL
protected static java.lang.String	JOBNODE_HOPCOUNT
protected static java.lang.String	JOBNODE_HOPCOUNTMODE
protected static java.lang.String	JOBNODE_HOUROFDAY
protected static java.lang.String	JOBNODE_ID
protected static java.lang.String	JOBNODE_LINKTYPE
protected static java.lang.String	JOBNODE_MINUTESOFHOUR
protected static java.lang.String	JOBNODE_MONTHOFYEAR
protected static java.lang.String	JOBNODE_OUTPUTNAME
protected static java.lang.String	JOBNODE_OUTPUTSPECIFICATION
protected static java.lang.String	JOBNODE_PRIORITY
protected static java.lang.String	JOBNODE_RECRAWLINTERVAL
protected static java.lang.String	JOBNODE_RESEEDINTERVAL
protected static java.lang.String	JOBNODE_RUNMODE
protected static java.lang.String	JOBNODE_SCHEDULE
protected static java.lang.String	JOBNODE_STARTMODE
protected static java.lang.String	JOBNODE_TIMEZONE
protected static java.lang.String	JOBNODE_YEAR
protected static JobResetThread	jobResetThread
protected static JobStartThread	jobStartThread
protected static java.lang.String	JOBSTATUSNODE_DOCUMENTSINQUEUE
protected static java.lang.String	JOBSTATUSNODE_DOCUMENTSOUTSTANDING
protected static java.lang.String	JOBSTATUSNODE_DOCUMENTSPROCESSED
protected static java.lang.String	JOBSTATUSNODE_ENDTIME
protected static java.lang.String	JOBSTATUSNODE_ERRORTEXT
protected static java.lang.String	JOBSTATUSNODE_JOBID
protected static java.lang.String	JOBSTATUSNODE_STARTTIME
protected static java.lang.String	JOBSTATUSNODE_STATUS
protected static float	lowWaterFactor
protected static java.lang.String	lowWaterFactorProperty
protected static JobNotificationThread	notificationThread
protected static int	numCleanupThreads
protected static int	numDeleteThreads
protected static int	numExpireThreads
protected static int	numWorkerThreads
protected static SeedingThread	seedingThread
protected static SetPriorityThread	setPriorityThread
protected static StartDeleteThread	startDeleteThread
protected static java.lang.Integer	startupLock This object is used to make sure the initialization sequence is atomic.
protected static StartupThread	startupThread
protected static float	stuffAmtFactor
protected static java.lang.String	stuffAmtFactorProperty
protected static StufferThread	stufferThread
protected static WorkerResetManager	workerResetManager Worker thread pool reset manager
protected static java.lang.String	workerThreadCountProperty
protected static WorkerThread[]	workerThreads
static int	WRITERESULT_CREATED
static int	WRITERESULT_FOUND
static int	WRITERESULT_NOTFOUND

Fields inherited from class org.apache.manifoldcf.agents.system.ManifoldCF

agentsInitialized, API_ERRORNODE, API_SERVICEINTERRUPTIONNODE, runningHash, stopAgentsRun

Fields inherited from class org.apache.manifoldcf.core.system.ManifoldCF

alreadyClosed, applicationName, ATTRIBUTE_NAME, ATTRIBUTE_PATH, ATTRIBUTE_VALUE, cleanupHooks, configSignalCommandProperty, databaseHandleMaxcountProperty, databaseHandleTimeoutProperty, databaseImplementation, dbShutdown, initializeFlagLock, isInitialized, lcfConfigFileProperty, localConfiguration, localProperties, lockManagerImplementation, logConfigFileProperty, maintenanceFileSignalProperty, masterDatabaseName, masterDatabaseNameProperty, masterDatabasePassword, masterDatabasePasswordProperty, masterDatabaseUsername, masterDatabaseUsernameProperty, NODE_LIBDIR, NODE_PROPERTY, propertyFilelastMod, propertyFilePath, resourceLoader, shutdownThread, tracker, workingDirectory

Constructor Summary

ManifoldCF()

Method Summary

static java.lang.String[]	calculateBins(IRepositoryConnector connector, java.lang.String documentIdentifier) Given a connector object and a document identifier, calculate its bins.
static java.lang.String	colonStuff(java.lang.String input) Stuff colons so we can't have conflicts.
static java.lang.String	createConnectionSpecificString(java.lang.String connectionName, java.lang.String simpleString) Create a connection-specific string
static java.lang.String	createGlobalString(java.lang.String simpleString) Create a global string
static java.lang.String	createJobSpecificString(java.lang.Long jobID, java.lang.String simpleString) Create a job-specific string
static java.lang.String	decodeAPIPathElement(java.lang.String startingPathElement) Decode path element.
static void	deinstallSystemTables(IThreadContext threadcontext) Uninstall all the crawler system tables.
static boolean	executeDeleteCommand(IThreadContext tc, Configuration output, java.lang.String path) Execute specified delete command.
static int	executePostCommand(IThreadContext tc, Configuration output, java.lang.String path, Configuration input) Execute specified post command.
static boolean	executeReadCommand(IThreadContext tc, Configuration output, java.lang.String path) Execute specified read command.
static int	executeWriteCommand(IThreadContext tc, Configuration output, java.lang.String path, Configuration input) Execute specified write command.
static void	exportConfiguration(IThreadContext threadContext, java.lang.String exportFilename) Atomically export the crawler configuration
protected static void	formatAuthorityConnection(ConfigurationNode connectionNode, IAuthorityConnection connection) Format an authority connection.
protected static void	formatEnumeratedValues(ConfigurationNode recordNode, java.lang.String childType, EnumeratedValues value)
protected static void	formatJobDescription(ConfigurationNode jobNode, IJobDescription job) Convert a job description into a ConfigurationNode.
protected static void	formatJobStatus(ConfigurationNode jobStatusNode, JobStatus jobStatus) Format a job status.
protected static void	formatOutputConnection(ConfigurationNode connectionNode, IOutputConnection connection) Format an output connection.
protected static void	formatRepositoryConnection(ConfigurationNode connectionNode, IRepositoryConnection connection) Format a repository connection.
static int	getMaxDeleteThreads() Get the maximum number of delete threads.
static int	getMaxExpireThreads() Get the maximum number of expire threads.
static int	getMaxWorkerThreads() Get the maximum number of worker threads.
protected static java.lang.String	hopcountModeMap(int mode)
static void	importConfiguration(IThreadContext threadContext, java.lang.String importFilename) Atomically import a crawler configuration
static void	initializeEnvironment() Initialize environment.
static void	installSystemTables(IThreadContext threadcontext) Install all the crawler system tables.
protected static java.lang.Long	interpretInterval(java.lang.String interval)
static boolean	isOutputConnectionInUse(IThreadContext threadContext, java.lang.String connName) Request permission from agent to delete an output connection.
protected static int	mapToHopcountMode(java.lang.String mode)
protected static int	mapToRunMode(java.lang.String mode)
protected static int	mapToStartMode(java.lang.String startMethod)
static void	noteOutputConnectionChange(IThreadContext threadContext, java.lang.String connectionName) Note the change in configuration of an output connection.
static void	noteOutputConnectorDeregistration(IThreadContext threadContext, java.lang.String[] connectionNames) Note the deregistration of a set of output connections.
static void	noteOutputConnectorRegistration(IThreadContext threadContext, java.lang.String[] connectionNames) Note the registration of a set of output connections.
protected static java.lang.String	presentInterval(java.lang.Long interval)
protected static void	processAuthorityConnection(AuthorityConnection connection, ConfigurationNode connectionNode) Convert input hierarchy into an AuthorityConnection object.
protected static EnumeratedValues	processEnumeratedValues(ConfigurationNode fieldNode)
protected static void	processJobDescription(JobDescription jobDescription, ConfigurationNode jobNode) Convert a node into a job description.
protected static void	processOutputConnection(OutputConnection connection, ConfigurationNode connectionNode) Convert input hierarchy into an OutputConnection object.
protected static void	processRepositoryConnection(RepositoryConnection connection, ConfigurationNode connectionNode) Convert input hierarchy into a RepositoryConnection object.
static java.lang.String	qualifyOutputActivityName(java.lang.String outputActivityName, java.lang.String outputConnectionName) Qualify output activity name.
static void	requeueDocumentsDueToCarrydown(IJobManager jobManager, DocumentDescription[] requeueCandidates, IRepositoryConnector connector, IRepositoryConnection connection, QueueTracker queueTracker, long currentTime) Requeue documents due to carrydown.
protected static java.lang.String	runModeMap(int type)
protected static java.lang.String	startModeMap(int startMethod)
static void	startSystem(IThreadContext threadContext) Start everything.
protected static java.lang.String	statusMap(int status)
static void	stopSystem(IThreadContext threadContext) Stop the system.
static void	writeDocumentPriorities(IThreadContext threadContext, IRepositoryConnectionManager mgr, IJobManager jobManager, DocumentDescription[] descs, java.util.HashMap connectionMap, java.util.HashMap jobDescriptionMap, QueueTracker queueTracker, long currentTime)

Methods inherited from class org.apache.manifoldcf.agents.system.ManifoldCF

createErrorNode, createServiceInterruptionNode, deinstallTables, findConfigurationNode, getRootArgument, installTables, signalOutputConnectionRedo, startAgents, stopAgents

Methods inherited from class org.apache.manifoldcf.core.system.ManifoldCF

addFile, addShutdownHook, addToHash, checkMaintenanceUnderway, checkProperties, cleanUpSystem, createResourceLoader, createSystemDatabase, deleteFile, deleteFolder, deobfuscate, dropSystemDatabase, encrypt, ensureFolder, findClass, getChildDatabaseName, getFileProperty, getHashValue, getIntProperty, getMasterDatabaseName, getMasterDatabasePassword, getMasterDatabaseUsername, getProperty, hash, isFolder, noteConfigurationChange, obfuscate, readByte, readByteArray, readBytes, readDword, readfloat, readLong, readNibble, readSdword, readString, readWord, recursiveDelete, resolvePath, safeFileName, sleep, startHash, writeByte, writeByteArray, writeBytes, writeDword, writefloat, writeLong, writeNibble, writeSdword, writeString, writeWord

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

_rcsid

public static final java.lang.String _rcsid

See Also:

Constant Field Values

crawlerInitialized

protected static boolean crawlerInitialized

initializationThread

protected static ManifoldCF.InitializationThread initializationThread

jobStartThread

protected static JobStartThread jobStartThread

stufferThread

protected static StufferThread stufferThread

finisherThread

protected static FinisherThread finisherThread

notificationThread

protected static JobNotificationThread notificationThread

startupThread

protected static StartupThread startupThread

startDeleteThread

protected static StartDeleteThread startDeleteThread

jobDeleteThread

protected static JobDeleteThread jobDeleteThread

workerThreads

protected static WorkerThread[] workerThreads

expireStufferThread

protected static ExpireStufferThread expireStufferThread

expireThreads

protected static ExpireThread[] expireThreads

deleteStufferThread

protected static DocumentDeleteStufferThread deleteStufferThread

deleteThreads

protected static DocumentDeleteThread[] deleteThreads

cleanupStufferThread

protected static DocumentCleanupStufferThread cleanupStufferThread

cleanupThreads

protected static DocumentCleanupThread[] cleanupThreads

jobResetThread

protected static JobResetThread jobResetThread

seedingThread

protected static SeedingThread seedingThread

idleCleanupThread

protected static IdleCleanupThread idleCleanupThread

setPriorityThread

protected static SetPriorityThread setPriorityThread

workerResetManager

protected static WorkerResetManager workerResetManager

Worker thread pool reset manager

docDeleteResetManager

protected static DocDeleteResetManager docDeleteResetManager

Delete thread pool reset manager

docCleanupResetManager

protected static DocCleanupResetManager docCleanupResetManager

Cleanup thread pool reset manager

numWorkerThreads

protected static int numWorkerThreads

numDeleteThreads

protected static int numDeleteThreads

numCleanupThreads

protected static int numCleanupThreads

numExpireThreads

protected static int numExpireThreads

lowWaterFactor

protected static float lowWaterFactor

stuffAmtFactor

protected static float stuffAmtFactor

workerThreadCountProperty

protected static final java.lang.String workerThreadCountProperty

See Also:

Constant Field Values

deleteThreadCountProperty

protected static final java.lang.String deleteThreadCountProperty

See Also:

Constant Field Values

cleanupThreadCountProperty

protected static final java.lang.String cleanupThreadCountProperty

See Also:

Constant Field Values

expireThreadCountProperty

protected static final java.lang.String expireThreadCountProperty

See Also:

Constant Field Values

lowWaterFactorProperty

protected static final java.lang.String lowWaterFactorProperty

See Also:

Constant Field Values

stuffAmtFactorProperty

protected static final java.lang.String stuffAmtFactorProperty

See Also:

Constant Field Values

startupLock

protected static java.lang.Integer startupLock

This object is used to make sure the initialization sequence is atomic. Shutdown cannot occur until the system is in a known state.

API_JOBNODE

protected static final java.lang.String API_JOBNODE

See Also:

Constant Field Values

API_JOBSTATUSNODE

protected static final java.lang.String API_JOBSTATUSNODE

See Also:

Constant Field Values

API_REPOSITORYCONNECTORNODE

protected static final java.lang.String API_REPOSITORYCONNECTORNODE

See Also:

Constant Field Values

API_OUTPUTCONNECTORNODE

protected static final java.lang.String API_OUTPUTCONNECTORNODE

See Also:

Constant Field Values

API_AUTHORITYCONNECTORNODE

protected static final java.lang.String API_AUTHORITYCONNECTORNODE

See Also:

Constant Field Values

API_REPOSITORYCONNECTIONNODE

protected static final java.lang.String API_REPOSITORYCONNECTIONNODE

See Also:

Constant Field Values

API_OUTPUTCONNECTIONNODE

protected static final java.lang.String API_OUTPUTCONNECTIONNODE

See Also:

Constant Field Values

API_AUTHORITYCONNECTIONNODE

protected static final java.lang.String API_AUTHORITYCONNECTIONNODE

See Also:

Constant Field Values

API_CHECKRESULTNODE

protected static final java.lang.String API_CHECKRESULTNODE

See Also:

Constant Field Values

API_JOBIDNODE

protected static final java.lang.String API_JOBIDNODE

See Also:

Constant Field Values

API_CONNECTIONNAMENODE

protected static final java.lang.String API_CONNECTIONNAMENODE

See Also:

Constant Field Values

CONNECTORNODE_DESCRIPTION

protected static final java.lang.String CONNECTORNODE_DESCRIPTION

See Also:

Constant Field Values

CONNECTORNODE_CLASSNAME

protected static final java.lang.String CONNECTORNODE_CLASSNAME

See Also:

Constant Field Values

WRITERESULT_NOTFOUND

public static final int WRITERESULT_NOTFOUND

See Also:

Constant Field Values

WRITERESULT_FOUND

public static final int WRITERESULT_FOUND

See Also:

Constant Field Values

WRITERESULT_CREATED

public static final int WRITERESULT_CREATED

See Also:

Constant Field Values

JOBNODE_ID

protected static final java.lang.String JOBNODE_ID

See Also:

Constant Field Values

JOBNODE_DESCRIPTION

protected static final java.lang.String JOBNODE_DESCRIPTION

See Also:

Constant Field Values

JOBNODE_CONNECTIONNAME

protected static final java.lang.String JOBNODE_CONNECTIONNAME

See Also:

Constant Field Values

JOBNODE_OUTPUTNAME

protected static final java.lang.String JOBNODE_OUTPUTNAME

See Also:

Constant Field Values

JOBNODE_DOCUMENTSPECIFICATION

protected static final java.lang.String JOBNODE_DOCUMENTSPECIFICATION

See Also:

Constant Field Values

JOBNODE_OUTPUTSPECIFICATION

protected static final java.lang.String JOBNODE_OUTPUTSPECIFICATION

See Also:

Constant Field Values

JOBNODE_STARTMODE

protected static final java.lang.String JOBNODE_STARTMODE

See Also:

Constant Field Values

JOBNODE_RUNMODE

protected static final java.lang.String JOBNODE_RUNMODE

See Also:

Constant Field Values

JOBNODE_HOPCOUNTMODE

protected static final java.lang.String JOBNODE_HOPCOUNTMODE

See Also:

Constant Field Values

JOBNODE_PRIORITY

protected static final java.lang.String JOBNODE_PRIORITY

See Also:

Constant Field Values

JOBNODE_RECRAWLINTERVAL

protected static final java.lang.String JOBNODE_RECRAWLINTERVAL

See Also:

Constant Field Values

JOBNODE_EXPIRATIONINTERVAL

protected static final java.lang.String JOBNODE_EXPIRATIONINTERVAL

See Also:

Constant Field Values

JOBNODE_RESEEDINTERVAL

protected static final java.lang.String JOBNODE_RESEEDINTERVAL

See Also:

Constant Field Values

JOBNODE_HOPCOUNT

protected static final java.lang.String JOBNODE_HOPCOUNT

See Also:

Constant Field Values

JOBNODE_SCHEDULE

protected static final java.lang.String JOBNODE_SCHEDULE

See Also:

Constant Field Values

JOBNODE_LINKTYPE

protected static final java.lang.String JOBNODE_LINKTYPE

See Also:

Constant Field Values

JOBNODE_COUNT

protected static final java.lang.String JOBNODE_COUNT

See Also:

Constant Field Values

JOBNODE_TIMEZONE

protected static final java.lang.String JOBNODE_TIMEZONE

See Also:

Constant Field Values

JOBNODE_DURATION

protected static final java.lang.String JOBNODE_DURATION

See Also:

Constant Field Values

JOBNODE_DAYOFWEEK

protected static final java.lang.String JOBNODE_DAYOFWEEK

See Also:

Constant Field Values

JOBNODE_MONTHOFYEAR

protected static final java.lang.String JOBNODE_MONTHOFYEAR

See Also:

Constant Field Values

JOBNODE_DAYOFMONTH

protected static final java.lang.String JOBNODE_DAYOFMONTH

See Also:

Constant Field Values

JOBNODE_YEAR

protected static final java.lang.String JOBNODE_YEAR

See Also:

Constant Field Values

JOBNODE_HOUROFDAY

protected static final java.lang.String JOBNODE_HOUROFDAY

See Also:

Constant Field Values

JOBNODE_MINUTESOFHOUR

protected static final java.lang.String JOBNODE_MINUTESOFHOUR

See Also:

Constant Field Values

JOBNODE_ENUMVALUE

protected static final java.lang.String JOBNODE_ENUMVALUE

See Also:

Constant Field Values

JOBSTATUSNODE_JOBID

protected static final java.lang.String JOBSTATUSNODE_JOBID

See Also:

Constant Field Values

JOBSTATUSNODE_STATUS

protected static final java.lang.String JOBSTATUSNODE_STATUS

See Also:

Constant Field Values

JOBSTATUSNODE_ERRORTEXT

protected static final java.lang.String JOBSTATUSNODE_ERRORTEXT

See Also:

Constant Field Values

JOBSTATUSNODE_STARTTIME

protected static final java.lang.String JOBSTATUSNODE_STARTTIME

See Also:

Constant Field Values

JOBSTATUSNODE_ENDTIME

protected static final java.lang.String JOBSTATUSNODE_ENDTIME

See Also:

Constant Field Values

JOBSTATUSNODE_DOCUMENTSINQUEUE

protected static final java.lang.String JOBSTATUSNODE_DOCUMENTSINQUEUE

See Also:

Constant Field Values

JOBSTATUSNODE_DOCUMENTSOUTSTANDING

protected static final java.lang.String JOBSTATUSNODE_DOCUMENTSOUTSTANDING

See Also:

Constant Field Values

JOBSTATUSNODE_DOCUMENTSPROCESSED

protected static final java.lang.String JOBSTATUSNODE_DOCUMENTSPROCESSED

See Also:

Constant Field Values

CONNECTIONNODE_ISNEW

protected static final java.lang.String CONNECTIONNODE_ISNEW

See Also:

Constant Field Values

CONNECTIONNODE_NAME

protected static final java.lang.String CONNECTIONNODE_NAME

See Also:

Constant Field Values

CONNECTIONNODE_CLASSNAME

protected static final java.lang.String CONNECTIONNODE_CLASSNAME

See Also:

Constant Field Values

CONNECTIONNODE_MAXCONNECTIONS

protected static final java.lang.String CONNECTIONNODE_MAXCONNECTIONS

See Also:

Constant Field Values

CONNECTIONNODE_DESCRIPTION

protected static final java.lang.String CONNECTIONNODE_DESCRIPTION

See Also:

Constant Field Values

CONNECTIONNODE_CONFIGURATION

protected static final java.lang.String CONNECTIONNODE_CONFIGURATION

See Also:

Constant Field Values

CONNECTIONNODE_ACLAUTHORITY

protected static final java.lang.String CONNECTIONNODE_ACLAUTHORITY

See Also:

Constant Field Values

CONNECTIONNODE_THROTTLE

protected static final java.lang.String CONNECTIONNODE_THROTTLE

See Also:

Constant Field Values

CONNECTIONNODE_MATCH

protected static final java.lang.String CONNECTIONNODE_MATCH

See Also:

Constant Field Values

CONNECTIONNODE_MATCHDESCRIPTION

protected static final java.lang.String CONNECTIONNODE_MATCHDESCRIPTION

See Also:

Constant Field Values

CONNECTIONNODE_RATE

protected static final java.lang.String CONNECTIONNODE_RATE

See Also:

Constant Field Values

Constructor Detail

ManifoldCF

public ManifoldCF()

Method Detail

initializeEnvironment

public static void initializeEnvironment()
                                  throws ManifoldCFException

Initialize environment.

Throws:

ManifoldCFException

installSystemTables

public static void installSystemTables(IThreadContext threadcontext)
                                throws ManifoldCFException

Install all the crawler system tables.

Parameters:

threadcontext - is the thread context.

Throws:

ManifoldCFException

deinstallSystemTables

public static void deinstallSystemTables(IThreadContext threadcontext)
                                  throws ManifoldCFException

Uninstall all the crawler system tables.

Parameters:

threadcontext - is the thread context.

Throws:

ManifoldCFException

startSystem

public static void startSystem(IThreadContext threadContext)
                        throws ManifoldCFException

Start everything.

Throws:

ManifoldCFException

stopSystem

public static void stopSystem(IThreadContext threadContext)
                       throws ManifoldCFException

Stop the system.

Throws:

ManifoldCFException

exportConfiguration

public static void exportConfiguration(IThreadContext threadContext,
                                       java.lang.String exportFilename)
                                throws ManifoldCFException

Atomically export the crawler configuration

Throws:

ManifoldCFException

importConfiguration

public static void importConfiguration(IThreadContext threadContext,
                                       java.lang.String importFilename)
                                throws ManifoldCFException

Atomically import a crawler configuration

Throws:

ManifoldCFException

getMaxWorkerThreads

public static int getMaxWorkerThreads()

Get the maximum number of worker threads.

getMaxDeleteThreads

public static int getMaxDeleteThreads()

Get the maximum number of delete threads.

getMaxExpireThreads

public static int getMaxExpireThreads()

Get the maximum number of expire threads.

requeueDocumentsDueToCarrydown

public static void requeueDocumentsDueToCarrydown(IJobManager jobManager,
                                                  DocumentDescription[] requeueCandidates,
                                                  IRepositoryConnector connector,
                                                  IRepositoryConnection connection,
                                                  QueueTracker queueTracker,
                                                  long currentTime)
                                           throws ManifoldCFException

Requeue documents due to carrydown.

Throws:

ManifoldCFException

colonStuff

public static java.lang.String colonStuff(java.lang.String input)

Stuff colons so we can't have conflicts.

createGlobalString

public static java.lang.String createGlobalString(java.lang.String simpleString)

Create a global string

createConnectionSpecificString

public static java.lang.String createConnectionSpecificString(java.lang.String connectionName,
                                                              java.lang.String simpleString)

Create a connection-specific string

createJobSpecificString

public static java.lang.String createJobSpecificString(java.lang.Long jobID,
                                                       java.lang.String simpleString)

Create a job-specific string

calculateBins

public static java.lang.String[] calculateBins(IRepositoryConnector connector,
                                               java.lang.String documentIdentifier)

Given a connector object and a document identifier, calculate its bins.

writeDocumentPriorities

public static void writeDocumentPriorities(IThreadContext threadContext,
                                           IRepositoryConnectionManager mgr,
                                           IJobManager jobManager,
                                           DocumentDescription[] descs,
                                           java.util.HashMap connectionMap,
                                           java.util.HashMap jobDescriptionMap,
                                           QueueTracker queueTracker,
                                           long currentTime)
                                    throws ManifoldCFException

Throws:

ManifoldCFException

isOutputConnectionInUse

public static boolean isOutputConnectionInUse(IThreadContext threadContext,
                                              java.lang.String connName)
                                       throws ManifoldCFException

Request permission from agent to delete an output connection.

Parameters:

threadContext - is the thread context.

connName - is the name of the output connection.

Returns:

true if the connection is in use, false otherwise.

Throws:

ManifoldCFException

noteOutputConnectorDeregistration

public static void noteOutputConnectorDeregistration(IThreadContext threadContext,
                                                     java.lang.String[] connectionNames)
                                              throws ManifoldCFException

Note the deregistration of a set of output connections.

Parameters:

threadContext - is the thread context.

connectionNames - are the names of the connections being deregistered.

Throws:

ManifoldCFException

noteOutputConnectorRegistration

public static void noteOutputConnectorRegistration(IThreadContext threadContext,
                                                   java.lang.String[] connectionNames)
                                            throws ManifoldCFException

Note the registration of a set of output connections.

Parameters:

threadContext - is the thread context.

connectionNames - are the names of the connections being registered.

Throws:

ManifoldCFException

noteOutputConnectionChange

public static void noteOutputConnectionChange(IThreadContext threadContext,
                                              java.lang.String connectionName)
                                       throws ManifoldCFException

Note the change in configuration of an output connection.

Parameters:

threadContext - is the thread context.

connectionName - is the output connection name.

Throws:

ManifoldCFException

qualifyOutputActivityName

public static java.lang.String qualifyOutputActivityName(java.lang.String outputActivityName,
                                                         java.lang.String outputConnectionName)

Qualify output activity name.

Parameters:

outputActivityName - is the name of the output activity.

outputConnectionName - is the corresponding name of the output connection.

Returns:

the qualified (global) activity name.

decodeAPIPathElement

public static java.lang.String decodeAPIPathElement(java.lang.String startingPathElement)
                                             throws ManifoldCFException

Decode path element. Path elements in the API world cannot have "/" characters, or they become impossible to parse. This method undoes escaping that prevents "/" from appearing.

Throws:

ManifoldCFException

executeReadCommand

public static boolean executeReadCommand(IThreadContext tc,
                                         Configuration output,
                                         java.lang.String path)
                                  throws ManifoldCFException

Execute specified read command.

Parameters:

tc - is the thread context.

output - is the output object, to be filled in.

path - is the object path.

Returns:

true if the resource exists, false otherwise.

Throws:

ManifoldCFException

executePostCommand

public static int executePostCommand(IThreadContext tc,
                                     Configuration output,
                                     java.lang.String path,
                                     Configuration input)
                              throws ManifoldCFException

Execute specified post command.

Parameters:

tc - is the thread context.

output - is the output object, to be filled in.

path - is the object path.

input - is the input object.

Returns:

write result - either "not found", "found", or "created".

Throws:

ManifoldCFException

executeWriteCommand

public static int executeWriteCommand(IThreadContext tc,
                                      Configuration output,
                                      java.lang.String path,
                                      Configuration input)
                               throws ManifoldCFException

Execute specified write command.

Parameters:

tc - is the thread context.

output - is the output object, to be filled in.

path - is the object path.

input - is the input object.

Returns:

write result - either "not found", "found", or "created".

Throws:

ManifoldCFException

executeDeleteCommand

public static boolean executeDeleteCommand(IThreadContext tc,
                                           Configuration output,
                                           java.lang.String path)
                                    throws ManifoldCFException

Execute specified delete command.

Parameters:

tc - is the thread context.

output - is the output object, to be filled in.

path - is the object path.

Returns:

true if the object exists, false otherwise.

Throws:

ManifoldCFException

processJobDescription

protected static void processJobDescription(JobDescription jobDescription,
                                            ConfigurationNode jobNode)
                                     throws ManifoldCFException

Convert a node into a job description.

Parameters:

jobDescription - is the job to be filled in.

jobNode - is the configuration node corresponding to the whole job itself.

Throws:

ManifoldCFException

formatJobDescription

protected static void formatJobDescription(ConfigurationNode jobNode,
                                           IJobDescription job)

Convert a job description into a ConfigurationNode.

Parameters:

jobNode - is the node to be filled in.

job - is the job description.

formatEnumeratedValues

protected static void formatEnumeratedValues(ConfigurationNode recordNode,
                                             java.lang.String childType,
                                             EnumeratedValues value)

processEnumeratedValues

protected static EnumeratedValues processEnumeratedValues(ConfigurationNode fieldNode)
                                                   throws ManifoldCFException

Throws:

ManifoldCFException

presentInterval

protected static java.lang.String presentInterval(java.lang.Long interval)

interpretInterval

protected static java.lang.Long interpretInterval(java.lang.String interval)
                                           throws ManifoldCFException

Throws:

ManifoldCFException

startModeMap

protected static java.lang.String startModeMap(int startMethod)

mapToStartMode

protected static int mapToStartMode(java.lang.String startMethod)
                             throws ManifoldCFException

Throws:

ManifoldCFException

runModeMap

protected static java.lang.String runModeMap(int type)

mapToRunMode

protected static int mapToRunMode(java.lang.String mode)
                           throws ManifoldCFException

Throws:

ManifoldCFException

hopcountModeMap

protected static java.lang.String hopcountModeMap(int mode)

mapToHopcountMode

protected static int mapToHopcountMode(java.lang.String mode)
                                throws ManifoldCFException

Throws:

ManifoldCFException

formatJobStatus

protected static void formatJobStatus(ConfigurationNode jobStatusNode,
                                      JobStatus jobStatus)

Format a job status.

statusMap

protected static java.lang.String statusMap(int status)

processOutputConnection

protected static void processOutputConnection(OutputConnection connection,
                                              ConfigurationNode connectionNode)
                                       throws ManifoldCFException

Convert input hierarchy into an OutputConnection object.

Throws:

ManifoldCFException

formatOutputConnection

protected static void formatOutputConnection(ConfigurationNode connectionNode,
                                             IOutputConnection connection)

Format an output connection.

processAuthorityConnection

protected static void processAuthorityConnection(AuthorityConnection connection,
                                                 ConfigurationNode connectionNode)
                                          throws ManifoldCFException

Convert input hierarchy into an AuthorityConnection object.

Throws:

ManifoldCFException

formatAuthorityConnection

protected static void formatAuthorityConnection(ConfigurationNode connectionNode,
                                                IAuthorityConnection connection)

Format an authority connection.

processRepositoryConnection

protected static void processRepositoryConnection(RepositoryConnection connection,
                                                  ConfigurationNode connectionNode)
                                           throws ManifoldCFException

Convert input hierarchy into a RepositoryConnection object.

Throws:

ManifoldCFException

formatRepositoryConnection

protected static void formatRepositoryConnection(ConfigurationNode connectionNode,
                                                 IRepositoryConnection connection)

Format a repository connection.