//+---------------------------------------------------------------------------
//
// Microsoft Windows
// Copyright (C) Microsoft Corporation, 1991 - 2000.
//
// File: RESMAN.CXX
//
// Contents: Resource Manager
//
// Classes: CResManager
//
// History: 08-Apr-91 BartoszM Created
// 4-Jan-95 BartoszM Separated Filter Manager
// Jan-08-97 mohamedn CFwEventItem and CDmFwEventItem
// 24-Feb-97 SitaramR Push filtering
//
//----------------------------------------------------------------------------
#include <pch.cxx>
#pragma hdrstop
#include <cci.hxx>
#include <xact.hxx>
#include <pstore.hxx>
#include <cifailte.hxx>
#include <ciole.hxx>
#include <regevent.hxx>
#include <ciregkey.hxx>
#include <eventlog.hxx>
#include <cievtmsg.h>
#include <cifrmcom.hxx>
#include <fwevent.hxx>
#include <pidmap.hxx>
#include <identran.hxx>
#include <psavtrak.hxx>
#include "resman.hxx"
#include "ci.hxx"
#include "partn.hxx"
#include "pindex.hxx"
#include "mindex.hxx"
#include "idxids.hxx"
#include "indxact.hxx"
#include "merge.hxx"
#include "mmerglog.hxx"
#include "pendcur.hxx"
#include "fltstate.hxx"
#include "idle.hxx"
#include "notxact.hxx"
#include "lowres.hxx"
const ULONG lowDiskWaterMark = 3 * 512 * 1024; // 1.5 MB
const ULONG highDiskWaterMark = lowDiskWaterMark + 512 * 1024; // 2.0 MB
const ULONG minDiskFreeForMerge = lowDiskWaterMark;
class CRevertBoolValue {
public:
CRevertBoolValue( BOOL & rfValue, BOOL fValue ) :
_rfVal( rfValue ),
_fRevert( TRUE )
{
Win4Assert( rfValue != fValue );
_fPrevVal = _rfVal;
_rfVal = fValue;
}
~CRevertBoolValue()
{
if (_fRevert)
_rfVal = _fPrevVal;
}
void Revert()
{
_rfVal = _fPrevVal;
_fRevert = FALSE;
}
private:
BOOL & _rfVal;
BOOL _fPrevVal;
BOOL _fRevert;
};
CMergeThreadKiller::CMergeThreadKiller( CResManager & resman )
: _resman(resman), _fKill(TRUE)
{
}
CMergeThreadKiller::~CMergeThreadKiller()
{
if ( _fKill )
{
_resman._fStopMerge = TRUE;
_resman._thrMerge.Resume();
_resman.StopMerges();
}
}
//+---------------------------------------------------------------------------
//
// Class: CPendQueueTrans
//
// Purpose: Transaction for flushing the pending queue if there is a
// failure while extracting entries out of the pending queue
// and adding them to the changelog.
//
// History: 9-11-95 srikants Created
//
//----------------------------------------------------------------------------
class CPendQueueTrans : public CTransaction
{
public:
CPendQueueTrans( CPendingQueue & pendQueue ) : _pendQueue( pendQueue )
{
}
~CPendQueueTrans()
{
if ( CTransaction::XActAbort == _status )
{
CLock lock( _pendQueue.GetMutex() );
_pendQueue.LokFlushCompletedEntries();
}
}
private:
CPendingQueue & _pendQueue;
};
//+---------------------------------------------------------------------------
//
// Member: CResManager::CResManager, public
//
// Arguments: [cat] -- catalog
// [xact] -- transaction
//
// History: 08-Apr-91 BartoszM Created
// Jan-07-96 mohamedn CFwEventItem
//
//----------------------------------------------------------------------------
CResManager::CResManager(
PStorage &storage,
CCiFrameworkParams & params,
ICiCDocStore * pDocStore,
CI_STARTUP_INFO const & startupInfo,
IPropertyMapper * pPropMapper,
CTransaction& xact,
XInterface<CIndexNotificationTable> & xIndexNotifTable )
: _sigResman(eSigResman),
_frmwrkParams( params ),
_mergeTime(0),
_storage ( storage ),
_sKeyList(0),
_idxTab( _storage.QueryIndexTable( xact ) ),
_partList ( storage, *_idxTab, _sKeyList, xact, params ),
_fresh ( storage, xact, _partList ),
_partidToMerge ( partidInvalid ),
_fStopMerge(FALSE),
_pMerge(0),
#pragma warning( disable : 4355 ) // this used in base initialization
_thrMerge ( MergeThread, this, TRUE ), // create suspended
_mergeKiller( *this ),
#pragma warning( default : 4355 )
_activeDeletedIndex( _idxTab->GetDeletedIndex() ),
_cQuery(0),
_cFilteredDocuments(0),
_pBackupWorkItem(0),
_isBeingEmptied(FALSE),
_isLowOnDiskSpace(FALSE),
_isDismounted(FALSE),
_isOutOfDate(FALSE),
_isCorrupt(FALSE),
_fFirstTimeUpdatesAreEnabled( TRUE ),
_fPushFiltering( FALSE ),
_fFlushWorkerActive( FALSE ),
_configFlags( startupInfo.startupFlags ),
_xIndexNotifTable( xIndexNotifTable.Acquire() ),
_dwFilteringState( 0 ),
_pFilterAgent( 0 )
{
Win4Assert( 0 != pDocStore );
//
// Look for a resource monitor. If none exists, use the default.
//
_xLowResDefault.Set( new CLowRes(_frmwrkParams) );
SCODE sc = pDocStore->QueryInterface( IID_ICiCResourceMonitor,
_xLowRes.GetQIPointer() );
if ( FAILED(sc) )
{
_xLowResDefault->AddRef();
_xLowRes.Set( _xLowResDefault.GetPointer() );
}
//
//
// The presence or absence of an actual pointer in _xIndexNotifTable,
// indicates whether the client has specified pull filtering or push
// filtering.
//
_fPushFiltering = !_xIndexNotifTable.IsNull();
//
// Initialize the resman in all the partitions.
//
CPartIter iter;
for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
{
CPartition * pPart = iter.LokGet();
pPart->SetResMan( this, FPushFiltering() );
}
if ( _fPushFiltering )
{
//
// In push filtering, an identity translator is used
//
CIdentityNameTranslator *pIdentityTrans = new CIdentityNameTranslator();
sc = pIdentityTrans->QueryInterface( IID_ICiCDocNameToWorkidTranslator,
_translator.GetQIPointer() );
Win4Assert( SUCCEEDED( sc ) );
_translator->Release(); // QI does an AddRef
}
else
{
//
// In pull filtering, the client provides the translator, which may
// be the Ex version.
//
sc = pDocStore->QueryInterface( IID_ICiCDocNameToWorkidTranslatorEx,
_translator2.GetQIPointer() );
if ( SUCCEEDED(sc) )
sc = _translator2->QueryInterface( IID_ICiCDocNameToWorkidTranslator,
_translator.GetQIPointer() );
else
sc = pDocStore->QueryInterface( IID_ICiCDocNameToWorkidTranslator,
_translator.GetQIPointer() );
}
if ( S_OK != sc )
THROW( CException(sc) );
//
// Create a workIdToDocName converter object.
//
_xWorkidToDocName.Set( new CWorkIdToDocName( _translator.GetPointer(), _translator2.GetPointer() ) );
ICiCAdviseStatus * pAdviseStatus;
sc = pDocStore->QueryInterface( IID_ICiCAdviseStatus,
(void **) & pAdviseStatus );
if ( S_OK != sc )
THROW( CException(sc) );
_adviseStatus.Set( pAdviseStatus );
_prfCounter.SetAdviseStatus( pAdviseStatus );
ICiCPropertyStorage * pPropStore;
sc = pDocStore->QueryInterface( IID_ICiCPropertyStorage,
(void **) & pPropStore );
if ( S_OK != sc )
THROW( CException(sc) );
_propStore.Set( pPropStore );
pDocStore->AddRef();
_docStore.Set( pDocStore );
pPropMapper->AddRef();
_mapper.Set( pPropMapper );
ULONG mergeTime = _frmwrkParams.GetMasterMergeTime();
_mergeTime = CMasterMergePolicy::ComputeMidNightMergeTime(mergeTime);
if ( -1 == _mergeTime )
{
CFwEventItem item(EVENTLOG_ERROR_TYPE,
MSG_CI_BAD_SYSTEM_TIME,
0);
item.ReportEvent( _adviseStatus.GetReference() );
THROW( CException( STATUS_INVALID_PARAMETER ) );
}
//
// Restore the information associated with any master merge that
// was stopped when the drive was dismounted.
//
_partList.RestoreMMergeState(*this, _storage);
_partIter.LokInit( _partList );
_isLowOnDiskSpace = LokCheckIfDiskLow( *this,
_storage,
_isLowOnDiskSpace,
_adviseStatus.GetReference() );
_thrMerge.SetPriority ( _frmwrkParams.GetThreadPriorityMerge() );
_mergeKiller.Defuse();
_thrMerge.Resume();
//
// Set the merge progress indicator to 0, and refresh other counters
//
_prfCounter.Update( CI_PERF_MERGE_PROGRESS, 0 );
LokUpdateCounters();
//
// Enable or disables updates/notifications based on disk space available
// and whether the catalog is readonly
//
if ( _isLowOnDiskSpace || _storage.IsReadOnly() )
{
DisableUpdates( partidDefault );
_state.LokSetState( eUpdatesDisabled );
}
else
{
EnableUpdates( partidDefault );
SCODE sc = _docStore->EnableUpdates();
if ( SUCCEEDED( sc ) )
_state.LokSetState( eSteady );
else
_state.LokSetState( eUpdatesToBeEnabled );
}
}
//+-------------------------------------------------------------------------
//
// Member: CResManager::~CResManager, public
//
// Synopsis: Shuts down merge(s)
//
// History: 13-Aug-93 KyleP Created
//
//--------------------------------------------------------------------------
CResManager::~CResManager()
{
_workMan.AbortWorkItems();
LokDeleteFlushWorkItems();
_workMan.WaitForDeath();
StopMerges();
#if CIDBG==1
// ======================================
{
CPriLock lock(_mutex);
Win4Assert( 0 == _pBackupWorkItem );
}
// ======================================
#endif // CIDBG==1
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::MergeThread, public
//
// Synopsis: Entry point for the thread responsible
// for asynchronous merges
//
// History: 05-Mar-92 BartoszM Created
//
//----------------------------------------------------------------------------
DWORD WINAPI CResManager::MergeThread( void* self )
{
if ( !((CResManager*)self)->_fStopMerge )
((CResManager*)self)->DoMerges();
return 0;
}
//==============
// STATE CHANGES
//==============
//+---------------------------------------------------------------------------
//
// Function: Dismount
//
// Synopsis: Dismounts the catalog by stopping any in-progress merge
// and also finishing off any pending writes ( eg. ChangeLog).
//
// History: 6-20-94 srikants Created
//
//----------------------------------------------------------------------------
NTSTATUS CResManager::Dismount()
{
ciDebugOut(( DEB_ITRACE, "*** CI: Initiating DisMount ***\n" ));
NTSTATUS status = STATUS_SUCCESS;
TRY
{
//
// Inform all the partitions to cleanup. We have to abort any
// in-progress merges without having to take a lock. There could
// be another thread (like the filter thread) with the lock doing
// a long operation. In one case we had it trying to create a new
// fresh test and the memory writer could not make any progress
// because the merge was continuing to dirty the pages.
//
// There is no need to take a lock because the partitions are not
// going away and the operation we are about to do is just turning
// on a one-way flag.
//
{
PARTITIONID partid = 1;
CPartition* pPart = _partList.LokGetPartition ( partid );
pPart->PrepareForCleanup();
}
//
// Stop in progress merges.
//
StopMerges();
//
// Dismount each of the partitions.
//
// ================================================
{
CPriLock lock( _mutex );
Win4Assert( !_isDismounted );
_isDismounted = TRUE;
CKeyList * pKeyList = _sKeyList.Acquire();
delete pKeyList;
CPartIter iter;
for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
{
CPartition * pPart = iter.LokGet();
// =======================================================
CChangeTrans xact( *this, pPart );
if ( STATUS_SUCCESS == pPart->LokDismount( xact ) )
{
xact.LokCommit();
}
//=========================================================
}
if ( _pBackupWorkItem )
_pBackupWorkItem->GetSaveProgressTracker().SetAbort();
}
// ================================================
_workMan.WaitForDeath();
}
CATCH( CException, e )
{
status = e.GetErrorCode();
ciDebugOut(( DEB_ERROR,
"ContentIndex Dismount Failed. Error Code 0x%X\n", status ));
}
END_CATCH
ciDebugOut(( DEB_ITRACE, "*** CI: Completed DisMount ***\n" ));
return status;
}
//+---------------------------------------------------------------------------
//
// Function: CResManager::Empty, public
//
// Synopsis: Releases resources associated with the resman object. This
// includes all indexes, the persistent freshlog, the persistant
// changelog, the freshtest, and the master merge log.
//
// History: 15-Aug-94 DwightKr Created
//
//----------------------------------------------------------------------------
void CResManager::Empty()
{
CPriLock lock( _mutex );
_isBeingEmptied = TRUE;
//
// Cancel any connections to the CI Filter service.
//
_pFilterAgent->LokCancel();
//
// Delete everything from the index table here first.
//
_idxTab->LokEmpty();
//
// If anything fails after this point, chkdsk /f or autochk will
// release the disk storage associated with the leaked objects
// since they are no longer part of the persistent index list.
//
//
// For each partition, zombify all indexes.
//
CPartIter iter;
for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
{
CPartition* pPart = iter.LokGet();
//
// Zombify all indexes in this partition, and delete them if their
// ref-count is 0.
//
CIndex ** aIndex = 0;
TRY
{
unsigned cIndex;
aIndex = pPart->LokZombify( cIndex );
ReleaseIndexes( cIndex, aIndex, NULL );
}
CATCH ( CException, e )
{
}
END_CATCH
delete [] aIndex;
//
// Delete the change log associated with this partition. These
// routines do not throw exceptions.
//
WORKID widChangeLog = _partList.GetChangeLogObjectId( pPart->GetId() );
if ( widChangeLog != widInvalid)
_storage.RemoveObject( widChangeLog );
WORKID widMMergeLog;
WORKID widDummy;
pPart->GetMMergeObjectIds( widMMergeLog, widDummy, widDummy );
if ( widMMergeLog != widInvalid)
_storage.RemoveObject( widMMergeLog );
}
_fresh.LokEmpty(); // Release storage associated with fresh test
WORKID widFreshLog = _storage.GetSpecialItObjectId( itFreshLog );
#ifdef KEYLIST_ENABLED
WORKID widKeyList = _storage.GetSpecialItObjectId( itMMKeyList );
#else
WORKID widKeyList = widInvalid;
#endif //
WORKID widPhrLat = _storage.GetSpecialItObjectId( itPhraseLat );
if ( widFreshLog != widInvalid)
_storage.RemoveObject( widFreshLog );
if ( widKeyList != widInvalid)
_storage.RemoveObject( widKeyList );
if ( widPhrLat != widInvalid)
_storage.RemoveObject( widPhrLat );
}
//+---------------------------------------------------------------------------
//
// Function: StopCurrentMerge
//
// Synopsis: Aborts any in-progress merge.
//
// History: 5-04-94 srikants Created
//
//----------------------------------------------------------------------------
NTSTATUS CResManager::StopCurrentMerge()
{
{
CPriLock lock( _mutex );
//
// If we're doing a merge right now then kill it off.
//
if ( 0 != _pMerge )
_pMerge->LokAbort();
}
return STATUS_SUCCESS;
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::DisableUpdates, public
//
// Arguments: [partid] -- partition id
//
// History: 24-Dec-94 Anonymous Created
//
//----------------------------------------------------------------------------
void CResManager::DisableUpdates( PARTITIONID partid )
{
CPriLock lock( _mutex );
CPartition * pPart = _partList.LokGetPartition( partid );
Win4Assert( 0 != pPart );
pPart->LokDisableUpdates();
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::EnableUpdates, public
//
// Arguments: [partid] -- partition id
//
// History: 24-Dec-94 Anonymous Created
//
//----------------------------------------------------------------------------
void CResManager::EnableUpdates( PARTITIONID partid )
{
CPriLock lock( _mutex );
CPartition * pPart = _partList.LokGetPartition( partid );
Win4Assert( 0 != pPart );
pPart->LokEnableUpdates( _fFirstTimeUpdatesAreEnabled );
_fFirstTimeUpdatesAreEnabled = FALSE;
}
//+---------------------------------------------------------------------------
//
// Function: LokCheckIfDiskLow
//
// Synopsis: Checks if we are running low on free disk space. It has
// a "LowWaterMark" and a "HighWaterMark".
//
// If current state is "not full", then it will check to
// see if the free disk space is < the LowWaterMark.
//
// if current state is "full", then it will check to see if
// the free disk space is < the HighWaterMark.
//
// The HighWaterMark is > the LowWaterMark to prevent hysterisis.
//
// Arguments: [resman] -- Resource manager
// [storage] -- Storage object.
// [fCurrent] -- Current state.
// [adviseStatus] -- reference to ICiCAdviseStatus
//
// Returns: TRUE if we are running low on disk space.
// FALSE otherwise.
//
// History: 10-11-94 srikants Created
// Jan-07-96 mohamedn CFwEventItem
//
//----------------------------------------------------------------------------
BOOL LokCheckIfDiskLow( CResManager & resman,
PStorage & storage,
BOOL fIsCurrentFull,
ICiCAdviseStatus & adviseStatus )
{
BOOL fLowOnDisk = fIsCurrentFull;
TRY
{
__int64 diskTotal, diskRemaining;
storage.GetDiskSpace( diskTotal, diskRemaining );
if ( !fIsCurrentFull )
fLowOnDisk = diskRemaining < lowDiskWaterMark;
else
fLowOnDisk = diskRemaining < highDiskWaterMark;
//
// It is okay to read it without mutex as it is only a heuristic.
//
if ( fLowOnDisk && !fIsCurrentFull && !storage.IsReadOnly() )
{
ciDebugOut(( DEB_WARN, "****YOU ARE RUNNING LOW ON DISK SPACE****\n"));
ciDebugOut(( DEB_WARN, "****PLEASE FREE UP SOME SPACE ****\n"));
ULONG mbToFree = highDiskWaterMark/(1024*1024);
ULONG mbIndex;
resman.IndexSize( mbIndex );
mbToFree = max( mbToFree, mbIndex );
mbToFree = min( mbToFree, 50 ); // don't ask for more than 50 MB
CFwEventItem item( EVENTLOG_AUDIT_FAILURE,
MSG_CI_LOW_DISK_SPACE,
2);
item.AddArg(storage.GetVolumeName() );
item.AddArg(mbToFree);
item.ReportEvent(adviseStatus);
}
else if ( fIsCurrentFull && !fLowOnDisk )
{
ciDebugOut(( DEB_WARN, "****DISK SPACE FREED UP ****\n"));
}
}
CATCH( CException,e )
{
ciDebugOut(( DEB_ERROR,
"Error 0x%X while getting disk space info\n",
e.GetErrorCode() ));
}
END_CATCH
return ( fLowOnDisk );
}
//+---------------------------------------------------------------------------
//
// Function: NoFailFreeResources
//
// Synopsis: deletes a big wordlist and schedules refiltering
//
// History: 6-Jan-95 BartoszM Created
//
//----------------------------------------------------------------------------
void CResManager::NoFailFreeResources()
{
ciDebugOut(( DEB_WARN, "Running out of resources. " ));
TRY
{
PARTITIONID partId = 1;
CPriLock lock(_mutex);
CPartition * pPart = LokGetPartition( partId );
//
// Before calling remove word list, we must append any refiled
// documents to the doc queue. This is because, the docqueue
// can handle only one set of refiled documents.
//
{
// =========================================
CChangeTrans xact( *this, pPart );
pPart->LokAppendRefiledDocs( xact );
xact.LokCommit();
// =========================================
}
LokNoFailRemoveWordlist( pPart );
}
CATCH( CException, e )
{
}
END_CATCH
}
//+-------------------------------------------------------------------------
//
// Member: CResManager::IsMemoryLow, private
//
// Returns: TRUE if we're in a low memory situation
//
// History: 10-May-93 AmyA Created from old DoUpdates
// 3-May-96 dlee #if 0'ed it out because the memory
// load # isn't reliable, and we've
// got LOTS of other allocations anyway.
// 05-Nov-97 KyleP New approach
//
//--------------------------------------------------------------------------
BOOL CResManager::IsMemoryLow()
{
SCODE sc = _xLowRes->IsMemoryLow();
if ( E_NOTIMPL == sc )
sc = _xLowResDefault->IsMemoryLow();
return ( S_OK == sc );
}
//+-------------------------------------------------------------------------
//
// Member: CResManager::IsIoHigh, private
//
// Returns: TRUE if the system is performing a 'lot' of I/O
//
// History: 10-Dec-97 KyleP Created
//
// Notes: This call takes time (~ 5 sec) to complete.
//
//--------------------------------------------------------------------------
BOOL CResManager::IsIoHigh()
{
SCODE sc = _xLowRes->IsIoHigh( &_fStopMerge );
if ( E_NOTIMPL == sc )
sc = _xLowResDefault->IsIoHigh( &_fStopMerge );
return ( S_OK == sc );
}
//+-------------------------------------------------------------------------
//
// Member: CResManager::IsBatteryLow, private
//
// Returns: TRUE if the system is running low on battery power.
//
// History: 16-Jul-98 KyleP Created
//
// Notes: By default, even 100% battery (as opposed to A/C) may be
// considered low.
//
//--------------------------------------------------------------------------
BOOL CResManager::IsBatteryLow()
{
SCODE sc = _xLowRes->IsBatteryLow();
if ( E_NOTIMPL == sc )
sc = _xLowResDefault->IsBatteryLow();
return ( S_OK == sc );
}
//+-------------------------------------------------------------------------
//
// Member: CResManager::IsOnBatteryPower, private
//
// Returns: TRUE if the system is running on battery power.
//
// History: 01-Oct-00 dlee Created
//
//--------------------------------------------------------------------------
BOOL CResManager::IsOnBatteryPower()
{
SCODE sc = _xLowRes->IsOnBatteryPower();
if ( E_NOTIMPL == sc )
sc = _xLowResDefault->IsOnBatteryPower();
return ( S_OK == sc );
} //IsOnBatteryPower
//+-------------------------------------------------------------------------
//
// Member: CResManager::SampleUserActivity, private
//
// Returns: - nothing -
//
// History: 31 Jul 98 AlanW Created
//
//--------------------------------------------------------------------------
void CResManager::SampleUserActivity()
{
SCODE sc = _xLowRes->SampleUserActivity();
if ( E_NOTIMPL == sc )
_xLowResDefault->SampleUserActivity();
return;
}
//+-------------------------------------------------------------------------
//
// Member: CResManager::IsUserActive, private
//
// Arguments: [fCheckLongTerm] - TRUE if long-term activity should be checked.
//
// Returns: TRUE if the interactive user is using the keyboard or mouse
//
// History: 28 Jul 98 AlanW Created
//
//--------------------------------------------------------------------------
BOOL CResManager::IsUserActive(BOOL fCheckLongTerm)
{
SCODE sc = _xLowRes->IsUserActive(fCheckLongTerm);
if ( E_NOTIMPL == sc )
sc = _xLowResDefault->IsUserActive(fCheckLongTerm);
return ( S_OK == sc );
}
//======================
// STATE AND STATISTICS
//======================
//+-------------------------------------------------------------------------
//
// Member: CResManager::CountPendingUpdates, public
//
// Arguments: [secCount (output)] -- Count of secondary Q Updates
//
// Returns: Count of pending updates.
//
// History: 14-Dec-92 KyleP Created
//
// Notes: If wordlists are being constructed the count may be low.
//
//--------------------------------------------------------------------------
unsigned CResManager::CountPendingUpdates(unsigned& secCount)
{
CPriLock lock( _mutex );
unsigned count = 0;
CPartIter iter;
secCount = 0;
for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
{
count += iter.LokGet()->LokCountUpdates();
secCount += iter.LokGet()->LokCountSecUpdates();
}
return(count + secCount);
}
//+-------------------------------------------------------------------------
//
// Member: CResManager::UpdateCounters, public
//
// Returns: Updates performance counters.
//
// History: 05-Jan-95 BartoszM Created
//
//--------------------------------------------------------------------------
void CResManager::LokUpdateCounters()
{
if ( !_isDismounted )
{
_prfCounter.Update( CI_PERF_NUM_WORDLIST, _partList.LokWlCount() );
unsigned cSecQDocuments;
unsigned cUnfilteredDocs = CountPendingUpdates( cSecQDocuments );
_prfCounter.Update( CI_PERF_FILES_TO_BE_FILTERED, cUnfilteredDocs );
_prfCounter.Update( CI_PERF_DEFERRED_FILTER_FILES, cSecQDocuments );
_prfCounter.Update( CI_PERF_NUM_UNIQUE_KEY, _sKeyList->MaxKeyIdInUse() );
_prfCounter.Update( CI_PERF_DOCUMENTS_FILTERED, _cFilteredDocuments );
ULONG cDocuments;
GetClientState( cDocuments );
_prfCounter.Update( CI_PERF_NUM_DOCUMENTS, cDocuments );
}
}
//====================
// UPDATES AND QUERIES
//====================
//+---------------------------------------------------------------------------
//
// Member: CResManager::ReserveUpdate, public
//
// Arguments: [wid] -- Used to confirm hint.
//
// Returns: Hint to position of reserved slot.
//
// History: 30-Aug-95 KyleP Created
//
//----------------------------------------------------------------------------
unsigned CResManager::ReserveUpdate( WORKID wid )
{
CPriLock lock ( _pendQueue.GetMutex() );
return _pendQueue.LokPrepare( wid );
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::UpdateDocument, public
//
// Arguments: [iHint] -- Positional hint
// [wid] -- wid to add/delete
// [partid] -- partition containing this wid
// [usn] -- USN associated with change
// [volumeId] -- Volume Id
// [action] -- addition / deletion
//
// History: 08-Apr-91 BartoszM Created
// 08-Oct-93 BartoszM Rewrote to accept single document
// 30-Aug-95 KyleP Use reserved slots
//
//----------------------------------------------------------------------------
SCODE CResManager::UpdateDocument( unsigned iHint,
WORKID wid,
PARTITIONID partid,
USN usn,
VOLUMEID volumeId,
ULONG action )
{
if ( !IsIndexingEnabled() )
return CI_E_FILTERING_DISABLED;
Win4Assert ( partid == 1 );
//
// Try to avoid lock contention
// with filter agent
//
_pFilterAgent->SlowDown();
CPriLock lock ( _mutex );
BOOL fComplete;
{
CPriLock lock2( _pendQueue.GetMutex() );
if ( _isBeingEmptied ) // The content index is empty
{
//
// Just get rid of any pending entries in the queue.
//
if ( !_pendQueue.LokComplete( iHint, wid, usn, volumeId, partid, action ) )
{
while ( _pendQueue.LokRemove( wid, usn, volumeId, partid, action ) )
;
}
return CI_E_SHUTDOWN;
}
//
// The if clause triggers when this document is the only one
// on the queue.
//
fComplete = _pendQueue.LokComplete( iHint, wid, usn, volumeId, partid, action );
}
SCODE sc = S_OK;
if ( fComplete )
{
CPartition* pPart = _partList.LokGetPartition ( partid );
CChangeTrans xact( *this, pPart );
sc = pPart->LokUpdateDocument ( xact, wid, usn, volumeId, action, 1, 0 );
xact.LokCommit();
_pFilterAgent->LokWakeUp();
}
else
{
BOOL fGotOne = FALSE;
CPendQueueTrans pendQueueTrans( _pendQueue );
while ( TRUE )
{
BOOL fGotAnother;
{
CLock lock2( _pendQueue.GetMutex() );
fGotAnother = _pendQueue.LokRemove( wid, usn, volumeId, partid, action );
}
if ( fGotAnother )
{
fGotOne = TRUE;
CPartition* pPart = _partList.LokGetPartition ( partid );
CChangeTrans xact( *this, pPart );
sc = pPart->LokUpdateDocument ( xact, wid, usn, volumeId, action, 1, 0 );
xact.LokCommit();
}
else
break;
}
pendQueueTrans.Commit();
if ( fGotOne )
_pFilterAgent->LokWakeUp();
}
return sc;
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::FlushUpdates
//
// Synopsis: Flushes all update notifications to disk
//
// History: 27-Jun-97 SitaramR Created
//
//----------------------------------------------------------------------------
void CResManager::FlushUpdates()
{
CPriLock lock( _mutex );
CPartition * pPart = _partList.LokGetPartition( partidDefault );
Win4Assert( 0 != pPart );
pPart->LokFlushUpdates();
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::QueryIndexes, public
//
// Arguments: [cPartitions] -- count of partitions
// [aPartID] -- array of partition id's
// [freshTest] -- return arg.: fresh test
// [cInd] -- return arg: count of indexes
// [cPendingUpdates] -- Pending update 'threshold'. If fewer
// pending updates, the pending wids are
// returned in *pcurPending.
// [pcurPending] -- Pending cursors stored here.
// [pFlags] -- return arg: status of indexes
//
// Returns: Array of pointers to indexes
//
// History: 08-Oct-91 BartoszM Created
//
// Notes: Called by Query
// Indexes and fresh test have ref count increased
// Flags may change to indicate status of indexes
//
//----------------------------------------------------------------------------
CIndex** CResManager::QueryIndexes (
unsigned cPartitions,
PARTITIONID aPartID[],
CFreshTest** freshTest,
unsigned& cInd,
ULONG cPendingUpdates,
CCurStack * pcurPending,
ULONG* pFlags )
{
unsigned count = 0;
CPartition* pPart;
CPriLock lock ( _mutex );
if ( _isCorrupt )
THROW( CException( CI_CORRUPT_DATABASE ) );
if ( _isBeingEmptied ) // Content index is empty or corrupt
{
cInd = 0;
cPartitions = 0;
return 0;
}
for ( unsigned i = 0; i < cPartitions; i++ )
{
pPart = _partList.LokGetPartition( aPartID [i] );
count += pPart->LokIndexCount();
}
XArray<CIndex *> apIndex( count );
#if CIDBG || DBG
unsigned j = 0;
#endif
for ( i = 0; i < cPartitions; i++ )
{
pPart = _partList.LokGetPartition(aPartID [i]);
#if CIDBG || DBG
j +=
#endif // CIDBG || DBG
pPart->LokGetIndexes ( &apIndex[i] );
if ( LokIsScanNeeded() && !_storage.IsReadOnly() )
*pFlags |= CI_NOT_UP_TO_DATE;
//
// If we're looking for pending updates, get them. Otherwise just
// set the out-of-date flag.
//
unsigned cPending = pPart->LokCountUpdates();
cPending += pPart->LokCountSecUpdates();
if ( cPending != 0 )
{
ULONG fFlags = CI_NOT_UP_TO_DATE; // assume non-success
if ( cPendingUpdates > 0 )
{
Win4Assert( 0 != pcurPending );
if ( cPending <= cPendingUpdates )
{
XArray<WORKID> pWid( cPending );
BOOL fSucceeded = pPart->LokGetPendingUpdates(
pWid.GetPointer(),
cPending );
if ( fSucceeded )
{
Win4Assert( cPending > 0 );
XPtr<CPendingCursor> xCursor( new CPendingCursor( pWid, cPending ) );
pcurPending->Push( xCursor.GetPointer() );
xCursor.Acquire();
fFlags = 0;
}
}
}
*pFlags |= fFlags;
}
}
//
// Also check to see if we are actually in the process of filtering some
// documents.
//
if ( _docList.Count() > 0 )
{
if ( _docList.Count() <= cPendingUpdates )
{
Win4Assert( 0 != pcurPending );
XArray<WORKID> pWid( _docList.Count() );
ciDebugOut((DEB_FILTERWIDS,
"CResManager::QueryIndexes - pending documents: %d %x\n",
_docList.Count(), pWid.GetPointer()));
_docList.LokGetWids( pWid );
XPtr<CPendingCursor> xCursor( new CPendingCursor( pWid, _docList.Count() ) );
pcurPending->Push( xCursor.GetPointer() );
xCursor.Acquire();
}
else
*pFlags |= CI_NOT_UP_TO_DATE;
}
//
// Also check documents pending notifcation that may be complete but
// out-of-order.
//
{
CPriLock lock( _pendQueue.GetMutex() );
unsigned cwidPending = _pendQueue.LokCountCompleted();
if ( cwidPending > 0 )
{
if ( cwidPending <= cPendingUpdates )
{
Win4Assert( 0 != pcurPending );
XArray<WORKID> pWid( cwidPending );
_pendQueue.LokGetCompleted( pWid.GetPointer() );
XPtr<CPendingCursor> xCursor( new CPendingCursor( pWid, cwidPending ) );
pcurPending->Push( xCursor.GetPointer() );
xCursor.Acquire();
}
else
*pFlags |= CI_NOT_UP_TO_DATE;
}
}
//
// Finally, check for pending scans.
//
if ( _isOutOfDate )
*pFlags |= CI_NOT_UP_TO_DATE;
Win4Assert ( j == count );
*freshTest = _fresh.LokGetFreshTest();
cInd = count;
return apIndex.Acquire();
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::ReleaseIndexes, public
//
// Synopsis: Decrements ref counts, deletes if indexes
// marked to be deleted.
//
// Arguments: [cInd] -- count of indexes
// [apIndex] -- array of indexes
// [freshTest] -- fresh test
//
// History: 08-Oct-91 BartoszM Created
//
// Notes: Takes ResMan lock
//
//----------------------------------------------------------------------------
void CResManager::ReleaseIndexes ( unsigned cInd, CIndex** apIndex,
CFreshTest* pFreshTest )
{
ciDebugOut (( DEB_ITRACE, "Release Indexes\n" ));
CPriLock lock ( _mutex );
for ( unsigned i = 0; i < cInd; i++ )
{
if ( apIndex[i] != 0 )
LokReleaseIndex ( apIndex[i] );
}
if ( pFreshTest)
_fresh.LokReleaseFreshTest (pFreshTest);
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::BackupContentIndexData
//
// Synopsis: Private method called by the merge thread to backup the
// content index data. If successful, it sets the status of
// the backup work item to indicate success. Otherwise, it
// must be considered to have failed.
//
// History: 3-18-97 srikants Created
//
//----------------------------------------------------------------------------
void CResManager::BackupContentIndexData()
{
Win4Assert( 0 != _pBackupWorkItem );
ciDebugOut(( DEB_WARN, "Starting Backup of Ci Data\n" ));
//
// Create the backup object.
//
CPartition *pPartition =_partList.LokGetPartition ( partidDefault );
CBackupCiPersData backup( *_pBackupWorkItem,
*this,
*pPartition );
// =========================================
{
CPriLock lock(_mutex);
backup.LokGrabResources();
}
// =========================================
backup.BackupIndexes();
// =========================================
{
CPriLock lock(_mutex);
backup.LokBackupMetaInfo();
ciDebugOut(( DEB_WARN, "Completed Backup of Ci Data\n" ));
_pBackupWorkItem->LokSetStatus( STATUS_SUCCESS );
}
// =========================================
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::LokUpdateBackupMergeProgress
//
// Synopsis: Updates the backup progress during the merge.
//
// History: 3-20-97 srikants Created
//
//----------------------------------------------------------------------------
void CResManager::LokUpdateBackupMergeProgress()
{
Win4Assert( 0 != _pBackupWorkItem );
CCiFrmPerfCounter counter( _adviseStatus.GetPointer(),
CI_PERF_MERGE_PROGRESS );
DWORD dwMergeProgress = counter.GetCurrValue();
//
// Assuming Merge is half of the work, we will assume half
// progress. So make the denominator 200 instead of 100.
//
if ( _pMerge )
{
const cShadowMergePart = 5;
if ( _pBackupWorkItem->IsFullSave() )
{
if ( _pMerge->GetMergeType() != mtMaster )
{
//
// The shadow merge preceding a master is approx. 5%
// say.
//
dwMergeProgress = cShadowMergePart;
}
else
{
dwMergeProgress = min(dwMergeProgress + cShadowMergePart, 100);
}
}
_pBackupWorkItem->LokUpdateMergeProgress( (ULONG) dwMergeProgress, 100 );
}
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::BackupCIData
//
// Synopsis: External method to backup content index data. It creates
// a backup work item and waits for the backup to complete.
//
// Arguments: [storage] - Destination storage
// [fFull] - [in/out] Set to TRUE if a full merge is
// needed on input; On output will be set to TRUE if a full
// merge was done.
// [progressTracker] - Progress tracker.
//
// History: 3-18-97 srikants Created
//
//----------------------------------------------------------------------------
SCODE CResManager::BackupCIData( PStorage & storage,
BOOL & fFull,
XInterface<ICiEnumWorkids> & xEnumWorkids,
PSaveProgressTracker & progressTracker )
{
if ( !IsIndexMigrationEnabled() || !IsIndexingEnabled() )
return CI_E_INVALID_STATE;
SCODE sc = S_OK;
Win4Assert( fFull );
TRY
{
// ===================================================
{
CLock lock(_mutex);
if ( _isDismounted )
THROW( CException( CI_E_SHUTDOWN ) );
// There can be only one operation going on at a time.
if ( 0 != _pBackupWorkItem )
THROW( CException( CI_E_INVALID_STATE ) );
_pBackupWorkItem = new CBackupCiWorkItem( storage,
fFull,
progressTracker );
_eventMerge.Set();
}
// ===================================================
ciDebugOut(( DEB_WARN, "Waiting for backup to complete\n" )) ;
//
// Wait for the work-item to be completed.
//
while ( TRUE )
{
DWORD const dwWaitTime = 1 * 60 * 1000; // 1 minute in millisecs
_pBackupWorkItem->WaitForCompletion( dwWaitTime );
//============================================================
{
CPriLock lock(_mutex);
if ( _pBackupWorkItem->LokIsDone() )
break;
//
// If there is a merge going on, we must update the progress.
//
if ( _pBackupWorkItem->LokIsDoingMerge() )
LokUpdateBackupMergeProgress();
_pBackupWorkItem->LokReset();
}
//============================================================
} // of while
sc= _pBackupWorkItem->GetStatus();
ciDebugOut(( DEB_WARN, "Backup completed with code 0x%X\n", sc ));
if ( S_OK == sc )
{
fFull = _pBackupWorkItem->IsFullSave();
xEnumWorkids.Set( _pBackupWorkItem->AcquireWorkIdEnum() );
}
// ====================================
{
CLock lock(_mutex);
delete _pBackupWorkItem;
_pBackupWorkItem = 0;
}
// ====================================
}
CATCH( CException,e )
{
sc = HRESULT_FROM_WIN32( e.GetErrorCode() );
Win4Assert( !"How Did we Come Here" );
}
END_CATCH
return sc;
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::CiState, public
//
// Arguments: [state] -- internal state of the CI
//
// History: 01-Nov-95 DwightKr Created
//
//
//----------------------------------------------------------------------------
#define setState( field, value ) \
if ( state.cbStruct >= ( offsetof( CIF_STATE, field) + \
sizeof( state.field ) ) ) \
{ \
state.field = ( value ); \
}
#define roundup(a, b) ((a%b) ? (a/b + 1) : (a/b))
NTSTATUS CResManager::CiState(CIF_STATE & state)
{
CPriLock lock( _mutex );
if ( _isDismounted )
return CI_E_SHUTDOWN;
if ( _isCorrupt )
return CI_CORRUPT_DATABASE;
setState( cWordList, _partList.LokWlCount() );
setState( cPersistentIndex, _partList.LokIndexCount() - state.cWordList );
// Get the perf counter for the # of queries executed
Win4Assert( !_adviseStatus.IsNull() );
long cQ;
_adviseStatus->GetPerfCounterValue( CI_PERF_TOTAL_QUERIES, &cQ );
setState( cQueries, cQ );
unsigned cSecQDocuments;
ULONG cUnfilteredDocs = CountPendingUpdates(cSecQDocuments) + _docList.Count();
setState( cDocuments, cUnfilteredDocs );
setState( cSecQDocuments, cSecQDocuments );
setState( cFreshTest, LokGetFreshCount() );
CCiFrmPerfCounter counter( _adviseStatus.GetPointer(),
CI_PERF_MERGE_PROGRESS );
setState( dwMergeProgress, (ULONG) counter.GetCurrValue() );
setState( cFilteredDocuments, _cFilteredDocuments );
unsigned size = roundup(_partList.LokIndexSize(), ((1024*1024)/CI_PAGE_SIZE));
setState( dwIndexSize, size);
setState( cUniqueKeys, _sKeyList->MaxKeyIdInUse() );
state.cbStruct = min( state.cbStruct, sizeof(state) );
DWORD eCiState = _dwFilteringState;
//
// Set each of the state bits independently.
//
//
// Are we in a shadow or master merge?
//
if ( 0 != _pMerge )
{
switch (_pMerge->GetMergeType() )
{
case mtShadow:
eCiState |= CIF_STATE_SHADOW_MERGE;
break;
case mtAnnealing:
eCiState |= CIF_STATE_ANNEALING_MERGE;
break;
case mtIncrBackup:
eCiState |= CIF_STATE_INDEX_MIGRATION_MERGE;
break;
default:
eCiState |= CIF_STATE_MASTER_MERGE;
break;
}
}
else
{
CPartition *pPartition = _partList.LokGetPartition ( partidDefault );
if ( pPartition->InMasterMerge() )
eCiState |= CIF_STATE_MASTER_MERGE_PAUSED;
}
if ( LokIsScanNeeded() )
eCiState |= CIF_STATE_CONTENT_SCAN_REQUIRED;
setState( eState, (CIF_STATE_FLAGS) eCiState );
return STATUS_SUCCESS;
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::IndexSize
//
// Synopsis: Computes the size of the indexes in all partitions
//
// Arguments: [mbIndex] - On output, will have the size of all indexes
// in MBs.
//
// History: 4-15-96 srikants Created
//
//----------------------------------------------------------------------------
void CResManager::IndexSize( ULONG & mbIndex )
{
CLock lock(_mutex);
mbIndex = _partList.LokIndexSize() / ((1024*1024)/CI_PAGE_SIZE);
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::LokCheckWordlistQuotas, public
//
// Synopsis: Determine if we've reached wordlist capacity.
//
// Returns: TRUE if we have as much wordlist data in memory as
// parameters allow.
//
// History: 14-Jan-1999 KyleP Created
//
//----------------------------------------------------------------------------
BOOL CResManager::LokCheckWordlistQuotas()
{
CPartIter iter;
for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList) )
{
CPartition* pPart = iter.LokGet();
if ( pPart->LokCheckWordlistMerge() )
return TRUE;
}
return FALSE;
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::MarkCorruptIndex
//
// Synopsis: Marks the index as corrupt, disables updates in all partitions
// and wakes up the merge thread to notify the framework client
// about the corruption.
//
// History: 1-30-97 srikants Created
//
//----------------------------------------------------------------------------
NTSTATUS CResManager::MarkCorruptIndex()
{
CPriLock lock(_mutex);
//
// Disable updates in all partitions
//
CPartIter iter;
for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
{
CPartition * pPart = iter.LokGet();
pPart->LokDisableUpdates();
}
if ( _state.FLokCorruptionNotified() )
return S_OK;
_isCorrupt = TRUE;
StopCurrentMerge();
_eventMerge.Set();
return S_OK;
}
//=====================
// KEYINDEX
//=====================
#ifdef KEYLIST_ENABLED
//+---------------------------------------------------------------------------
//
// Member: CResManager::_AddRefKeyList, private
//
// Synopsis: Adds to refcount on the keylist
//
// History: 07-Jul-94 dlee Created
//
//----------------------------------------------------------------------------
CKeyList * CResManager::_AddRefKeyList()
{
CPriLock lock( _mutex );
_sKeyList->Reference();
return _sKeyList.GetPointer();
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::_ReleaseKeyList, private
//
// Synopsis: Release a refcount on the keylist
//
// History: 07-Jul-94 dlee Created
//
//----------------------------------------------------------------------------
void CResManager::_ReleaseKeyList()
{
CPriLock lock( _mutex );
_sKeyList->Release();
if ( !_sKeyList->InUse() && _sKeyList->IsZombie() )
{
ciDebugOut(( DEB_ITRACE,
"Keylist %x unreferenced zombie. Deleting\n",
_sKeyList->GetId() ));
CKeyList * pKeyList = _sKeyList.Acquire();
pKeyList->Remove();
delete pKeyList;
}
}
#endif // KEYLIST_ENABLED
//+---------------------------------------------------------------------------
//
// Member: CResManager::KeyToId, public
//
// Synopsis: Maps from a key to an id.
//
// Arguments: [pkey] -- pointer to the key to be mapped to ULONG
//
// Returns: key id - a ULONG
//
// History: 03-Nov-93 w-Patg Created.
// 17-Feb-94 KyleP Initial version
//
//----------------------------------------------------------------------------
ULONG CResManager::KeyToId( CKey const * pkey )
{
#ifdef KEYLIST_ENABLED
CKeyList * pKeyList = _AddRefKeyList();
ULONG kid = pKeyList->KeyToId( pkey );
_ReleaseKeyList();
return( kid );
#else
return widInvalid;
#endif // KEYLIST_ENABLED
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::IdToKey, public
//
// Synopsis: Maps from an id to a key.
//
// Arguments: [ulKid] -- key id to be mapped to a key
// [rkey] -- reference to the returned key
//
// Returns: void
//
// History: 03-Nov-93 w-Patg Created.
// 17-Feb-94 KyleP Initial version
//
//----------------------------------------------------------------------------
void CResManager::IdToKey( ULONG ulKid, CKey & rkey )
{
#ifdef KEYLIST_ENABLED
CKeyList * pKeyList = _AddRefKeyList();
pKeyList->IdToKey( ulKid, rkey );
_ReleaseKeyList();
#else
rkey.FillMin();
#endif // KEYLIST_ENABLED
}
//+-------------------------------------------------------------------------
//
// Member: CResManager::ComputeRelevantWords, public
//
// Synopsis: Computes and returns relevant words for a set of wids
//
// Arguments: [cRows] -- # of rows to compute and in pwid array
// [cRW] -- # of rw per wid
// [pwid] -- array of wids to work over
// [partid] -- partition
//
// History: 10-May-94 v-dlee Created
//
//--------------------------------------------------------------------------
CRWStore * CResManager::ComputeRelevantWords(ULONG cRows,ULONG cRW,
WORKID *pwids,
PARTITIONID partid)
{
CRWStore *prws = 0;
#ifdef KEYLIST_ENABLED
CPriLock lock( _mutex );
CPartition *pPart = LokGetPartition(partid);
if (pPart != 0)
{
CPersIndex *pIndex= pPart->GetCurrentMasterIndex();
if (pIndex != 0)
{
prws = pIndex->ComputeRelevantWords(cRows,cRW,pwids,
_sKeyList.GetPointer() );
}
}
#endif // KEYLIST_ENABLED
return prws;
} //ComputeRelevantWords
//+-------------------------------------------------------------------------
//
// Member: CRWStore::RetrieveRelevantWords, public
//
// Synopsis: Retrieves relevant words already computed
//
// Arguments: [fAcquire] -- TRUE if ownership is transferred.
// [partid] -- partition
//
// History: 10-May-94 v-dlee Created
//
//--------------------------------------------------------------------------
CRWStore * CResManager::RetrieveRelevantWords(BOOL fAcquire,
PARTITIONID partid)
{
CRWStore *prws = 0;
#ifdef KEYLIST_ENABLED
CPriLock lock( _mutex );
CPartition *pPart = LokGetPartition(partid);
if (pPart != 0)
{
prws = pPart->RetrieveRelevantWords(fAcquire);
}
#endif // KEYLIST_ENABLED
return prws;
} //RetrieveRelevantWords
//================
// PRIVATE METHODS
//================
//+-------------------------------------------------------------------------
//
// Member: CResManager::CompactResources
//
// History: 05-Jan-95 BartoszM Created
//
//--------------------------------------------------------------------------
void CResManager::CompactResources()
{
CPriLock lock( _mutex );
CPartIter iter;
for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
{
iter.LokGet()->LokCompact();
}
}
//=================================//
// MERGES //
//=================================//
//+---------------------------------------------------------------------------
//
// Member: CResManager::DoMerges, private
//
// History: 08-Apr-91 BartoszM Created
// 25-Feb-92 BartoszM Rewrote to use thread
// 30-Jun-97 KrishnaN Calling docstore.FlushPropertyStore
// before a shadow merge.
// 03-Mar-98 KitmanH Don't merge if _storage is read-only
//
// Notes: Entry point for captive threads
//
//----------------------------------------------------------------------------
void CResManager::DoMerges()
{
if ( _storage.IsReadOnly() )
return;
CCiFrmPerfCounter pPerIndexCounter( _adviseStatus.GetPointer(), CI_PERF_NUM_PERSISTENT_INDEXES );
CCiFrmPerfCounter pIndexSizeCounter( _adviseStatus.GetPointer(), CI_PERF_INDEX_SIZE );
CCiFrmPerfCounter pPendingCounter( _adviseStatus.GetPointer(), CI_PERF_FILES_TO_BE_FILTERED );
CCiFrmPerfCounter pNumKeysCounter( _adviseStatus.GetPointer(), CI_PERF_NUM_UNIQUE_KEY);
CCiFrmPerfCounter pWordListCounter(_adviseStatus.GetPointer(), CI_PERF_NUM_WORDLIST);
CCiFrmPerfCounter pDeferredFilesCounter( _adviseStatus.GetPointer(), CI_PERF_DEFERRED_FILTER_FILES );
XPtr<CIdleTime> xIdle( new CIdleTime() );
//
// Allow mount to complete before checking for merges. O/W we will
// start a merge before the mount/startup is complete and it might
// prevent a system from coming up.
//
const lowDiskPollTime = 1; // 1 minute
DWORD dwWaitTime = _isLowOnDiskSpace ? lowDiskPollTime : _frmwrkParams.GetMaxMergeInterval();
_eventMerge.Wait( dwWaitTime * 1000 * 60 );
BOOL fAnnealing = FALSE;
while ( !_fStopMerge )
{
BOOL fCorrupt = FALSE;
ciDebugOut (( DEB_ITRACE, "\t|Merge Wakeup!\n" ));
//
// Book keeping chores.
//
BOOL fBackupCiData = FALSE;
BOOL fBackupStarted = FALSE;
TRY
{
// ===============================================
CPriLock lock(_mutex);
if ( _isCorrupt )
{
if ( !_state.FLokCorruptionNotified() )
LokNotifyCorruptionToClient();
}
else
{
//
// Use low disk condition plus the current state to decide if
// updates should be enabled or not
//
_isLowOnDiskSpace = LokCheckIfDiskLow( *this,
_storage,
_isLowOnDiskSpace,
_adviseStatus.GetReference() );
if ( _isLowOnDiskSpace )
{
if ( _state.LokGetState() != eUpdatesToBeDisabled
&& _state.LokGetState() != eUpdatesDisabled )
{
_state.LokSetState( eUpdatesToBeDisabled );
_state.LokSetUpdateType( eIncremental );
}
}
else
{
if ( _state.LokGetState() == eUpdatesDisabled )
_state.LokSetState( eUpdatesToBeEnabled );
}
if ( _state.LokGetState() == eUpdatesToBeDisabled )
LokNotifyDisableUpdatesToClient();
else if ( _state.LokGetState() == eUpdatesToBeEnabled )
LokNotifyEnableUpdatesToClient();
}
fBackupCiData = 0 != _pBackupWorkItem;
// ===============================================
}
CATCH( CException, e )
{
// ignore and try again the next time through the loop.
}
END_CATCH
TRY
{
if ( !_isCorrupt && _partidToMerge != partidInvalid )
{
ciDebugOut (( DEB_ITRACE, "Forced merge in %ld\n",
_partidToMerge ));
switch ( _mtForceMerge )
{
case CI_MASTER_MERGE:
MasterMerge ( _partidToMerge );
break;
case CI_SHADOW_MERGE:
Merge( mtShadow, _partidToMerge );
break;
case CI_ANNEALING_MERGE:
Merge( mtAnnealing, _partidToMerge );
break;
case CI_ANY_MERGE:
break; // Do nothing. Let normal mechanism work it out.
default:
Win4Assert( !"Invalid ForceMerge type" );
}
}
if ( !_isLowOnDiskSpace && !_isCorrupt )
{
//
// Refile the documents from the secondary queue to the primary
// queue in pull filtering.
//
// ======================= lock ======================
if ( !_fPushFiltering )
{
CLock lock(_mutex);
LokRefileSecQueueDocs();
}
// ======================= unlock ======================
if ( fBackupCiData )
{
BOOL fIsMasterPresent = FALSE;
BOOL fIsMMergeInProgress = FALSE;
//================================
{
CPriLock lock(_mutex);
_pBackupWorkItem->LokSetDoingMerge();
fIsMasterPresent = LokIsMasterIndexPresent( partidDefault );
fIsMMergeInProgress = LokIsMasterMergeInProgress( partidDefault );
}
//================================
if ( _pBackupWorkItem->IsFullSave() ||
fIsMMergeInProgress ||
!fIsMasterPresent )
{
_pBackupWorkItem->SetDoingFullSave();
MasterMerge( partidDefault );
}
else
{
Merge( mtIncrBackup, partidDefault );
}
fBackupStarted = TRUE;
BackupContentIndexData();
}
else
{
if ( fAnnealing )
{
fAnnealing = FALSE;
CPartIdStack partitionIds;
CheckAndDoMerge( mtAnnealing, partitionIds );
while ( partitionIds.Count() > 0 )
Merge( mtAnnealing, partitionIds.Pop() );
}
{
CPartIdStack partitionIds;
CheckAndDoMerge( mtShadow, partitionIds );
while ( partitionIds.Count() > 0 )
{
// Call FlushPropertyStore on DocStore to give it
// a chance to persist changes before a shadow merge.
_docStore->FlushPropertyStore();
Merge( mtShadow, partitionIds.Pop() );
}
}
{
if ( CheckDeleteMerge( _partidToMerge != partidInvalid ) )
{
// Call FlushPropertyStore on DocStore to give it
// a chance to persist changes before a shadow merge.
_docStore->FlushPropertyStore();
Merge( mtDeletes, 1 );
}
}
BOOL fIsMasterPresent = FALSE;
BOOL fIsMMergeInProgress = FALSE;
//================================
{
CPriLock lock(_mutex);
fIsMasterPresent = LokIsMasterIndexPresent( partidDefault );
fIsMMergeInProgress = LokIsMasterMergeInProgress( partidDefault );
}
//================================
{
CPartIdStack partitionIds;
CheckAndDoMerge( mtMaster, partitionIds );
while ( partitionIds.Count() > 0 )
MasterMerge( partitionIds.Pop() );
}
}
}
}
CATCH ( CException, e )
{
ciDebugOut (( DEB_ERROR,
"ResMan::DoMerges -- merge failed with 0x%x\n",
e.GetErrorCode() ));
if ( IsDiskFull(e.GetErrorCode()) )
{
ciDebugOut(( DEB_ERROR, "***** DISK IS FULL *****\n" ));
CPriLock lock(_mutex);
_isLowOnDiskSpace = LokCheckIfDiskLow( *this,
_storage,
_isLowOnDiskSpace,
_adviseStatus.GetReference() );
}
else if ( IsCorrupt( e.GetErrorCode() ) )
{
Win4Assert( "!Merge failed, but not for low disk. Why?" );
fCorrupt = TRUE;
}
//
// We have to prevent a master merge from being restarted
// immediately. The reason for failure could be something like
// out of disk space (typical reason) or log full. In either
// case we will only be exacerbating the situation if we
// restart immediately.
//
_eventMerge.Reset();
}
END_CATCH
// ==========================================================
{
CPriLock lock( _mutex );
Win4Assert( _partList.LokIndexCount() >= _partList.LokWlCount() );
pPerIndexCounter.Update(_partList.LokIndexCount()-_partList.LokWlCount());
pIndexSizeCounter.Update(_partList.LokIndexSize() / ((1024*1024)/CI_PAGE_SIZE) );
pNumKeysCounter.Update(_sKeyList->MaxKeyIdInUse());
pWordListCounter.Update(_partList.LokWlCount());
_partidToMerge = partidInvalid;
if ( _fStopMerge )
{
ciDebugOut(( DEB_ITRACE, "Stopping Merge\n" ));
break;
}
if ( !_isCorrupt && ( _state.LokGetState() == eSteady ) && ( 0 != _pFilterAgent ) )
_pFilterAgent->LokWakeUp();
_eventMerge.Reset(); // sleep
if ( fBackupStarted || _isCorrupt )
{
//
// Set that backup to be complete even if it failed. Once
// Backup starts, it either succeeds or fails but in either
// case the operation is considered done.
//
if ( 0 != _pBackupWorkItem )
{
if ( _isCorrupt )
_pBackupWorkItem->LokSetStatus( CI_CORRUPT_DATABASE );
_pBackupWorkItem->LokSetDone();
}
}
}
// ==========================================================
BOOL fSteadyState = TRUE;
if ( fCorrupt )
{
CPriLock lock(_mutex);
_isCorrupt = TRUE;
fSteadyState = _state.LokGetState() == eSteady;
}
unsigned cSecQCount;
unsigned cPending = CountPendingUpdates( cSecQCount );
//
// Wait for either the merge event to be signaled or the deadman timeout.
//
dwWaitTime = ( ( _isLowOnDiskSpace ) ||
( !fSteadyState ) ||
( 0 != cPending ) ) ?
lowDiskPollTime : _frmwrkParams.GetMaxMergeInterval();
NTSTATUS Status = _eventMerge.Wait( dwWaitTime * 60 * 1000 );
if ( !_isCorrupt )
{
unsigned PercentIdle = xIdle->PercentIdle();
//
// Are we idle enough to consider an annealing merge?
//
if ( STATUS_TIMEOUT == Status && PercentIdle >= _frmwrkParams.GetMinMergeIdleTime() )
{
ciDebugOut(( DEB_ITRACE, "Idle time for period: %u percent\n", PercentIdle ));
fAnnealing = TRUE;
}
}
if ( _fStopMerge )
{
ciDebugOut(( DEB_ITRACE, "Stopping Merge\n" ));
break;
}
cPending = CountPendingUpdates( cSecQCount );
pPendingCounter.Update(cPending);
pDeferredFilesCounter.Update(cSecQCount);
ciDebugOut (( DEB_ITRACE | DEB_PENDING,
"%d updates pending\n", cPending ));
}
// ===========================================
{
CPriLock lock2(_mutex);
if ( _pBackupWorkItem )
{
ciDebugOut(( DEB_WARN, "Forcing the backup to be done\n" ));
_pBackupWorkItem->LokSetDone();
}
}
// ===========================================
}
//+-------------------------------------------------------------------------
//
// Member: CResManager::StopMerges, private
//
// Synopsis: Aborts merge-in-progress (if any) and sets merge flag.
//
// History: 13-Aug-93 KyleP Created
//
//--------------------------------------------------------------------------
void CResManager::StopMerges()
{
{
CPriLock lock( _mutex );
_fStopMerge = TRUE;
_eventMerge.Set(); // wake up
//
// If we're doing a merge right now then kill it off.
//
if ( 0 != _pMerge )
{
_pMerge->LokAbort();
}
}
//
// Wait for merge thread to die.
//
_thrMerge.WaitForDeath();
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::CheckAndDoMerge, private
//
// History: 08-Apr-91 BartoszM Created
//
// Notes: Called by captive thread.
//
//----------------------------------------------------------------------------
void CResManager::CheckAndDoMerge( MergeType mt, CPartIdStack & partitionIds)
{
if ( !IsBatteryLow() )
{
CPriLock lock( _mutex );
ciDebugOut (( DEB_ITRACE, "check merge\n" ));
CPartIter iter;
for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
{
CPartition* pPart = iter.LokGet();
if ( mt == mtMaster )
{
__int64 shadowIndexSize = pPart->LokIndexSize();
CPersIndex * pMasterIndex = pPart->GetCurrentMasterIndex();
if ( pMasterIndex )
shadowIndexSize -= pMasterIndex->Size();
shadowIndexSize *= CI_PAGE_SIZE;
CMasterMergePolicy mergePolicy( _storage, shadowIndexSize,
LokGetFreshCount(),
_mergeTime,
_frmwrkParams,
_adviseStatus.GetReference() );
if ( mergePolicy.IsTimeToMasterMerge() ||
( mergePolicy.IsMidNightMergeTime() &&
!IsOnBatteryPower() ) )
partitionIds.Push( pPart->GetId() );
}
else
{
if ( pPart->LokCheckMerge(mt) ||
( IsMemoryLow() && pPart->LokCheckLowMemoryMerge() ) )
{
#if CIDBG == 1
if ( !pPart->LokCheckMerge(mt) )
{
ciDebugOut(( DEB_WARN, "Merge due to low memory. %u wordlists, %u shadow.\n",
pPart->WordListCount(), pPart->LokIndexCount() ));
}
#endif
partitionIds.Push( pPart->GetId() );
}
}
}
}
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::CheckDeleteMerge, private
//
// Synopsis: Merge check for delete merge
//
// Arguments: [fForce] -- If TRUE, even 1 pending delete will force merge.
//
// Returns: TRUE if merge should be performed.
//
// History: 12-Jun-97 KyleP Created
//
// Notes: Called by captive thread.
//
//----------------------------------------------------------------------------
BOOL CResManager::CheckDeleteMerge( BOOL fForce )
{
CPriLock lock( _mutex );
ciDebugOut (( DEB_ITRACE, "check delete merge\n" ));
//
// A 'delete merge' occurs when enough documents have been deleted, and no
// documents have been added or modified (which would cause a shadow merge).
//
return ( ( _fresh.LokDeleteCount() > _frmwrkParams.GetMaxFreshDeletes() ) ||
( fForce && _fresh.LokDeleteCount() > 0 ) );
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::Merge, private
//
// Synopsis: Perform a merge in partition
//
// Arguments: [partid] -- partition id
// [mt] -- Merge type
//
// Signals: CExcetpion
//
// History: 08-Apr-91 BartoszM Created
// 13-Nov-91 BartoszM Rewrote using CMerge
//
//----------------------------------------------------------------------------
void CResManager::Merge( MergeType mt, PARTITIONID partid )
{
ciDebugOut (( DEB_ITRACE, "%s", (mtDeletes == mt) ? "Delete merging: Have a quick stretch" :
"Shadow merging: Get a cup of coffee\n" ));
CMerge Merge( *this, partid, mt );
//
// Perform resource acquisition under storage transaction. If
// aborted all persistent storage will be deleted by the storage
// transaction.
//
{
CPriLock lock ( _mutex );
Win4Assert( _pMerge == 0 );
if (_fStopMerge)
{
ciDebugOut(( DEB_ITRACE, "Stopping Merge\n" ));
return;
}
Merge.LokGrabResources(); // may throw an exception
//
// Optimization: Don't merge zero indexes.
//
if ( 0 != Merge.LokCountOld() )
{
_pMerge = &Merge;
}
else if ( mtDeletes != mt )
{
ciDebugOut(( DEB_ITRACE, "Shadow merge of zero indexes. Cancelled.\n" ));
return;
}
}
XPtr<CNotificationTransaction> xNotifyTrans;
// Perform the merge under simple transaction
// No lock held.
{
TRY
{
//
// Perform merge, if necessary.
//
if ( 0 != Merge.LokCountOld() )
{
CCiFrmPerfCounter counter( _adviseStatus.GetPointer(), CI_PERF_MERGE_PROGRESS );
Merge.Do( counter );
}
//
// NotifTrans should be outside resman lock and the resman lock
// is acquired in notifTran's destructor
//
xNotifyTrans.Set( new CNotificationTransaction( this,
_xIndexNotifTable.GetPointer() ) );
}
CATCH ( CException, e )
{
CPriLock lock ( _mutex );
Merge.LokRollBack();
_pMerge = 0;
// Really bad errors indicate the index is corrupt.
SCODE scE = e.GetErrorCode();
if ( STATUS_INTEGER_DIVIDE_BY_ZERO == scE ||
STATUS_ACCESS_VIOLATION == scE ||
STATUS_IN_PAGE_ERROR == scE )
{
ciDebugOut(( DEB_ERROR,
"Corrupt index, caught 0x%x\n", scE ));
_storage.ReportCorruptComponent( L"ShadowMerge" );
THROW( CException( CI_CORRUPT_DATABASE ) );
}
RETHROW();
}
END_CATCH
}
// nb: no failures allowed until CMergeTrans is constructed!
CDiskIndex* pIndexNew = 0;
{ //=================== begin notification transaction =========================
{
CLock lock( _mutex );
XPtr<CFreshTest> xFreshTestAtMerge;
{
_pMerge = 0;
CMergeTrans Xact( *this, _storage, Merge );
unsigned cIndOld = Merge.LokCountOld();
INDEXID* aIidOld = Merge.LokGetIidList();
pIndexNew = Merge.LokGetNewIndex();
//
// Write the new fresh test persistently to disk.
//
CPersFresh newFreshLog( _storage, _partList );
CShadowMergeSwapInfo swapInfo;
swapInfo._widNewFreshLog = _fresh.LokUpdate( Merge,
Xact,
newFreshLog,
pIndexNew ? pIndexNew->GetId() : iidInvalid,
cIndOld,
aIidOld,
xFreshTestAtMerge );
swapInfo._widOldFreshLog =
_storage.GetSpecialItObjectId( itFreshLog );
swapInfo._cIndexOld = cIndOld;
swapInfo._aIidOld = aIidOld;
_partList.LokSwapIndexes( Xact,
partid,
pIndexNew,
swapInfo );
//
// No failures allowed after this point on until the transaction
// is committed.
//
Merge.ReleaseNewIndex();
ciDebugOut (( DEB_ITRACE, "done merging\n" ));
Xact.LokCommit();
#if CIDBG == 1
if ( pIndexNew )
pIndexNew->Reference();
#endif
//==========================================
}
ciFAILTEST( STATUS_NO_MEMORY );
CPartition *pPartition =_partList.LokGetPartition ( partid );
//
// Delete the wids in the change log that have made it to
// the new shadow index created in this merge. The algorithm
// needs the latest fresh test, fresh test snapshoted at the
// start of shadow merge and the doc list. If the latest fresh
// test is the same as the fresh test at merge, then as an
// optimization, the fresh test at merge is null. So, in the
// optimized case, we use the latest fresh test as the fresh
// test at merge.
//
CFreshTest *pFreshTestLatest = _fresh.LokGetFreshTest();
CFreshTestLock freshTestLock( pFreshTestLatest );
CFreshTest *pFreshTestAtMerge;
if ( xFreshTestAtMerge.IsNull() )
pFreshTestAtMerge = pFreshTestLatest;
else
pFreshTestAtMerge = xFreshTestAtMerge.GetPointer();
//==========================================
CChangeTrans xact( *this, pPartition );
pPartition->LokDeleteWIDsInPersistentIndexes( xact,
*pFreshTestLatest,
*pFreshTestAtMerge,
_docList,
xNotifyTrans.GetReference() );
xact.Commit();
}
xNotifyTrans.Free();
} //=================== end notification transaction =========================
#if CIDBG == 1
if ( pIndexNew )
{
pIndexNew->VerifyContents();
{
CPriLock lock ( _mutex );
LokReleaseIndex( pIndexNew );
}
}
#endif
_storage.CheckPoint();
} //Merge
//+---------------------------------------------------------------------------
//
// Member: CResManager::LogMMergeStartFailure
//
// Synopsis: Writes an eventlog message to indicate a MasterMerge could
// not be started.
//
// Arguments: [fRestart] - Indicates if this is a restarted master merge.
// [storage] - Storage reference
// [error] - Error code.
// [adviseStatus] - reference to ICiCAdviseStatus
//
// History: 5-27-96 srikants Created
// Jan-07-96 mohamedn CDmFwEventItem
//
//----------------------------------------------------------------------------
void CResManager::LogMMergeStartFailure( BOOL fRestart,
PStorage & storage,
ULONG error,
ICiCAdviseStatus & adviseStatus)
{
TRY
{
DWORD dwMsgCode = fRestart ?
MSG_CI_MASTER_MERGE_CANT_RESTART :
MSG_CI_MASTER_MERGE_CANT_START;
CDmFwEventItem Item( EVENTLOG_AUDIT_FAILURE,
dwMsgCode,
2,
sizeof(error),
(void *)&error );
Item.AddArg( storage.GetVolumeName() );
Item.AddArg( error );
Item.ReportEvent( adviseStatus );
}
CATCH( CException, e )
{
ciDebugOut(( DEB_ERROR, "Error 0x%X in LogMMergeStartFailure\n",
e.GetErrorCode() ));
}
END_CATCH
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::LogMMergePaused
//
// Synopsis: Writes an eventlog message that master merge was paused.
//
// Arguments: [storage] - Storage
// [error] - Error which caused the merge to be paused.
// [adviseStatus] - Advise status to use for notifying about the
// event.
//
// History: 1-24-97 srikants Added header
//
//----------------------------------------------------------------------------
void CResManager::LogMMergePaused( PStorage & storage,
ULONG error,
ICiCAdviseStatus &adviseStatus)
{
if ( IsCorrupt( error ) )
{
// don't log "paused" when there is corruption
return;
}
TRY
{
CDmFwEventItem Item( EVENTLOG_INFORMATION_TYPE,
MSG_CI_MASTER_MERGE_ABORTED,
2,
sizeof(error),
(void *)&error );
Item.AddArg( storage.GetVolumeName() );
Item.AddArg( error );
Item.ReportEvent( adviseStatus );
}
CATCH( CException, e )
{
ciDebugOut(( DEB_ERROR, "Error 0x%X in LogMMergePaused\n",
e.GetErrorCode() ));
}
END_CATCH
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::MasterMerge, private
//
// Synopsis: Perform a MasterMerge in partition
//
// Arguments: [partid] -- partition id
//
// Signals: CException
//
// History: 08-Apr-91 BartoszM Created
// 13-Nov-91 BartoszM Rewrote using CMerge
// 17-Feb-94 KyleP Added keylist merge
// Jan-07-96 mohamedn CDmFwEventItem
//
//----------------------------------------------------------------------------
void CResManager::MasterMerge ( PARTITIONID partid )
{
ciDebugOut (( DEB_ITRACE, "\n===MASTER MERGE===\n" ));
//
// Force a shadow merge if we are not restarting a master merge. This
// will force all word lists into persistent indexes.
//
BOOL fMasterMergePaused = FALSE;
//
// Before starting the master merge, we have to swap the deleted index
// iid. However, if there is a failure before we commit the beginning
// of the master merge, we have to roll back this change. The
// CDeletedIIDTrans object does this.
//
CDeletedIIDTrans delIIDTrans( *this );
INDEXID iidDelPreMerge = iidInvalid;
{
CPriLock lock ( _mutex );
CPartition *pPartition = _partList.LokGetPartition ( partid );
fMasterMergePaused = pPartition->InMasterMerge();
if ( !fMasterMergePaused )
{
if ( 0 == LokGetFreshCount() )
{
//
// There is no benefit in doing a master merge. Just return
//
return;
}
iidDelPreMerge = _activeDeletedIndex.GetIndex();
delIIDTrans.LokLogNewDeletedIid( iidDelPreMerge,
_activeDeletedIndex.GetNewIndex() );
_activeDeletedIndex.SwapIndex();
}
}
if ( !fMasterMergePaused )
{
//
// First do a delete merge to finish merging all outstanding
// word-lists into a persistent index.
//
// A delete merge will force an optimized null merge if there are
// outstanding deletes in the fresh test which need to be logged.
// This is an important case for Push filtering. Since it doesn't
// hurt in the other model, just keep the code identical.
//
Merge( mtDeletes, partid );
}
else
{
//
// This is a restarted master merge. The _activeDeletedIndex contains
// the "Post" master merge deleted index. The "New" one will be same
// as the "Prev" one.
//
iidDelPreMerge = _activeDeletedIndex.GetNewIndex();
CDmFwEventItem Item( EVENTLOG_INFORMATION_TYPE,
MSG_CI_MASTER_MERGE_RESTARTED,
1 );
Item.AddArg( _storage.GetVolumeName() );
Item.ReportEvent( _adviseStatus.GetReference() );
}
CMasterMerge Merge( *this, partid );
TRY
{
CPriLock lock ( _mutex );
Win4Assert( _pMerge == 0 );
if ( _fStopMerge )
{
ciDebugOut(( DEB_ITRACE, "Stopping Merge\n" ));
return;
}
ciFAILTEST( STATUS_NO_MEMORY );
CPartition *pPartition =_partList.LokGetPartition ( partid );
if ( !pPartition->InMasterMerge() )
{
Win4Assert( delIIDTrans.IsTransLogged() );
Merge.LokGrabResources( delIIDTrans ); // may throw an exception
if ( 0 == Merge.LokCountOld() )
{
ciDebugOut(( DEB_ITRACE, "Master merge of 0 indexes cancelled\n"));
return;
}
}
else
{
Merge.LokLoadRestartResources();
}
//
// The deletionIIDTrans must be either committed or not logged at
// all.
//
Win4Assert( !delIIDTrans.IsRollBackTrans() );
_pMerge = &Merge;
}
CATCH( CException, e )
{
ciDebugOut(( DEB_ERROR, "Error 0x%X while starting a MasterMerge\n",
e.GetErrorCode() ));
LogMMergeStartFailure( fMasterMergePaused,
_storage,
e.GetErrorCode(),
_adviseStatus.GetReference() );
RETHROW();
}
END_CATCH
// Perform the merge under simple transaction
// No lock held.
{
TRY
{
// Perform merge
CCiFrmPerfCounter counter( _adviseStatus.GetPointer(), CI_PERF_MERGE_PROGRESS );
Merge.Do( counter );
}
CATCH ( CException, e )
{
CPriLock lock ( _mutex );
Merge.LokRollBack();
_pMerge = 0;
LogMMergePaused( _storage,
e.GetErrorCode(),
_adviseStatus.GetReference() );
RETHROW();
}
END_CATCH
}
CMasterMergeIndex* pMMergeIndex = 0;
{
CPriLock lock ( _mutex );
{
_pMerge = 0;
unsigned cIndOld = Merge.LokCountOld();
INDEXID* aIidOld = Merge.LokGetIidList();
pMMergeIndex = Merge.LokGetMasterMergeIndex();
CMergeTrans MergeTrans( *this, _storage, Merge );
ciFAILTEST( STATUS_NO_MEMORY );
//==========================================
CMasterMergeTrans xact( *this,
_storage,
MergeTrans,
pMMergeIndex,
Merge );
CPartition *pPart = LokGetPartition(partid);
#ifdef KEYLIST_ENABLED
pPart->SetRelevantWords(pMMergeIndex->AcquireRelevantWords());
#endif // KEYLIST_ENABLED
//
// Remove old indexes from fresh list after master merge
//
CPersFresh newFreshLog( _storage, _partList );
CMasterMergeSwapInfo SwapInfo;
SwapInfo._widNewFreshLog = _fresh.LokRemoveIndexes (
MergeTrans,
newFreshLog,
cIndOld, aIidOld, iidDelPreMerge );
SwapInfo._widOldFreshLog = _storage.GetSpecialItObjectId( itFreshLog );
SwapInfo._cIndexOld = cIndOld;
SwapInfo._aIidOld = aIidOld;
SwapInfo._partid = partid;
CKeyList const * pOldKeyList = _sKeyList.GetPointer();
CKeyList const * pNewKeyList = Merge.LokGetNewKeyList();
_partList.LokSwapIndexes( MergeTrans,
partid,
pMMergeIndex,
SwapInfo,
pOldKeyList,
pNewKeyList );
//
// After this point, we must not resume the master merge even
// if there is a soft failure.
//
Merge.ReleaseNewIndex();
#ifdef KEYLIST_ENABLED
//
// Get rid of the current KeyList and replace it with the
// new KeyList
//
_storage.SetSpecialItObjectId( itMMKeyList, widInvalid );
CKeyList * pKeyList = _sKeyList.Acquire();
Win4Assert( 0 != pKeyList );
_sKeyList.Set( Merge.LokGetNewKeyList() );
Merge.LokReleaseNewKeyList();
pKeyList->Zombify();
if ( !pKeyList->InUse() )
{
pKeyList->Remove();
delete pKeyList;
}
#else
CKeyList * pKeyList = _sKeyList.Acquire();
Win4Assert( 0 != pKeyList );
_sKeyList.Set( Merge.LokGetNewKeyList() );
Merge.LokReleaseNewKeyList();
delete pKeyList;
#endif // KEYLIST_ENABLED
#if CIDBG == 1
pMMergeIndex->Reference();
#endif
xact.LokCommit();
//==========================================
ciDebugOut (( DEB_ITRACE, "done merging\n" ));
}
//
// There is no need to compact the change log here because the
// participants in the master merge are persistent indexes and
// there will be no wids to be deleted from the change log when
// persistent indexes are merged into another persistent index.
//
}
#if CIDBG == 1
pMMergeIndex->VerifyContents();
{
CPriLock lock ( _mutex );
LokReleaseIndex( pMMergeIndex );
}
#endif
_storage.CheckPoint();
CDmFwEventItem Item( EVENTLOG_INFORMATION_TYPE,
MSG_CI_MASTER_MERGE_COMPLETED,
1 );
Item.AddArg( _storage.GetVolumeName() );
Item.ReportEvent( _adviseStatus.GetReference() );
}
//+---------------------------------------------------------------------------
//
// Class: CReleaseMMergeIndex
//
// Purpose: Class to acquire the ownernship of the target and current
// master indexes during the destruction of a mastermerge index
// and release the target and current indexes.
//
// History: 9-29-94 srikants Created
//
//----------------------------------------------------------------------------
class CReleaseMMergeIndex
{
public:
CReleaseMMergeIndex( CResManager & resman, CMasterMergeIndex * mmIndex );
~CReleaseMMergeIndex();
private:
CResManager & _resman;
CMasterMergeIndex * _pmmIndex;
CPersIndex * _pTargetMaster;
CPersIndex * _pCurrentMaster;
};
CReleaseMMergeIndex::CReleaseMMergeIndex( CResManager & resman,
CMasterMergeIndex * mmIndex ) :
_resman(resman),
_pmmIndex(mmIndex),
_pTargetMaster(0),
_pCurrentMaster(0)
{
Win4Assert( _pmmIndex );
Win4Assert( _pmmIndex->IsZombie() && !_pmmIndex->InUse() );
_pmmIndex->AcquireCurrentAndTarget( &_pCurrentMaster, &_pTargetMaster );
Win4Assert( 0 == _pCurrentMaster || _pCurrentMaster->IsZombie() );
if ( 0 != _pCurrentMaster )
{
_pCurrentMaster->Reference();
}
Win4Assert( 0 != _pTargetMaster );
_pTargetMaster->Reference();
}
CReleaseMMergeIndex::~CReleaseMMergeIndex()
{
delete _pmmIndex;
if ( 0 != _pCurrentMaster )
_resman.LokReleaseIndex( _pCurrentMaster );
_resman.LokReleaseIndex( _pTargetMaster );
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::LokReleaseIndex, private
//
// Synopsis: Decrements ref counts, deletes if index
// marked to be deleted.
//
// Arguments: [pIndex] -- index to be released
//
// History: 08-Oct-91 BartoszM Created
//
// Notes: ResMan LOCKED
//
//----------------------------------------------------------------------------
void CResManager::LokReleaseIndex ( CIndex* pIndex )
{
pIndex->Release();
if ( pIndex->IsZombie() && !pIndex->InUse() )
{
CIndexId iid = pIndex->GetId();
//
// If it is not a master index, it has to be removed from the
// Partition object. A master index would have been already removed
// when merge completed.
//
// If we are in the Empty path, then we want to force all indexes
// to be deleted if their ref-count is 0.
//
if ( !pIndex->IsMaster() || _isBeingEmptied )
{
if ( !_isDismounted )
{
TRY
{
_partList.LokRemoveIndex ( iid );
}
CATCH( CException, e )
{
ciDebugOut(( DEB_ERROR,
"Index 0x%08x could not be deleted from index table\n",
iid ));
}
END_CATCH
TRY
{
// Must impersonate as system since query threads
// impersonating as a query user can't delete the
// files sometimes deleted when an index is released
// after a merge. Impersonation can fail, which will
// leak the file on disk.
CImpersonateSystem impersonate;
if ( iid.IsPersistent() )
pIndex->Remove();
}
CATCH( CException, e )
{
ciDebugOut(( DEB_ERROR,
"Index 0x%08x could not be deleted from disk, %#x\n",
iid, e.GetErrorCode() ));
}
END_CATCH
}
else
{
//
// Zombie indexes are cleaned up after reboot.
//
//
// After a dismount, we should not have any dirty streams
// and releasing a persistent zombie index will update the
// index table on disk. That is not allowed. This situation
// happens only when queries have not been released and
// shutdown is happening.
//
//
ciDebugOut(( DEB_WARN,
"Index with iid 0x%X being released AFTER CI dismount\n",
iid ));
}
delete pIndex;
}
else
{
//
// This must be a master merge index as no other master index
// can be a zombie.
//
Win4Assert( pIndex->IsMasterMergeIndex() );
CReleaseMMergeIndex mmIndexRel( *this,
(CMasterMergeIndex *)pIndex );
}
}
}
//+---------------------------------------------------------------------------
//
// Function: LokReplaceMasterIndex
//
// Synopsis: Replaces the CMasterMergeIndex with the CPersIndex incarnation
// of the target master index in the index list. Also prepares
// the CMasterMergeIndex for removal from the in-memory index
// list by zombifying it.
//
// Arguments: [pMMergeIndex] -- The target index in the CMasterMergeIndex
// form.
// [pNewMaster] -- The CPersIndex form of the target index.
//
// History: 6-30-94 srikants Created
//
//----------------------------------------------------------------------------
void CResManager::LokReplaceMasterIndex ( CMasterMergeIndex* pMMergeIndex )
{
Win4Assert( pMMergeIndex->IsMaster() && pMMergeIndex->IsZombie() );
CPersIndex * pNewMaster = pMMergeIndex->GetTargetMaster();
Win4Assert( pNewMaster->IsMaster() && !pNewMaster->IsZombie() );
//
// Delete the master merge index from the partition.
//
CIndexId iid = pMMergeIndex->GetId();
CPartition * pPart = _partList.LokGetPartition( iid.PartId() );
pPart->LokRemoveIndex ( iid );
//
// Check if there is a query in progress. If not, we can
// go ahead and delete it. If a query is in progress, the
// indexsnapshot of the query will delete it.
//
if ( !pMMergeIndex->InUse() )
{
Win4Assert( !pNewMaster->InUse() );
pMMergeIndex->ReleaseTargetAndCurrent();
delete pMMergeIndex;
}
//
// Add the new master index to the partition object. This will
// now be the only master accessible to new queries.
//
pPart->AddIndex( pNewMaster );
}
//
//+---------------------------------------------------------------------------
//
// Function: RollBackDeletedIIDChange
//
// Synopsis: Rolls back the changed made to the deleted iid during the
// beginning of a master merge.
//
// Arguments: [xact] - The transaction object which is rolling back the
// change to the deleted iid.
//
// History: 7-17-95 srikants Created
//
// Notes: Consider the following sequence of events:
//
// 1. Pre-MasterMerge preparations : Change Deleleted IID to iidDeleted2 from
// iidDeleted1.
//
// 2. Start a shadow merge.
//
// 3. Commit the beginning of a new MasterMerge
//
// 4. Complete the master merge .....
//
// If there is a failure between steps 2 and 3, we just revert the deleted
// IID back to iidDeleted2. Even if some entries were created in the fresh
// list between steps 2 & 3 of the form, WID->iidDeleted2, it is okay. They
// will get deleted from the fresh list on the second MasterMerge.
//
//----------------------------------------------------------------------------
void CResManager::RollBackDeletedIIDChange( CDeletedIIDTrans & xact )
{
CPriLock lock(_mutex);
Win4Assert( _activeDeletedIndex.GetIndex() == xact.GetNewDelIID() );
Win4Assert( _activeDeletedIndex.GetNewIndex() == xact.GetOldDelIID() );
_activeDeletedIndex.SwapIndex();
}
//========================
// FILTER RELATED METHODS
//========================
//+---------------------------------------------------------------------------
//
// Function: LokGetFilterDocs
//
// Synopsis: Gets documents from the change log for filtering. _docList
// will be filled with the documents.
//
// Arguments: [cMaxDocs] -- [in] Maximum number of docs to retrieve
// [cDocs] -- [out] Number of documents retrieved
// [state] -- [in/out] State of the document retrieval
// progress.
//
// Returns: Number of documents in the docBuffer.
//
// History: 6-16-94 srikants Separated for making kernel mode call
// Asynchronous.
//
// Notes: This function has two goals:
//
// 1. Select documents from different partitions in a round
// robin fashion.
//
//----------------------------------------------------------------------------
BOOL CResManager::LokGetFilterDocs ( ULONG cMaxDocs,
ULONG & cDocs,
CGetFilterDocsState & state )
{
ciDebugOut(( DEB_ITRACE, "# CResManager::LokGetFilterDocs.\n" ));
Win4Assert( state.GetTriesLeft() > 0 && state.GetTriesLeft() <=2 );
cDocs = 0;
Win4Assert ( !_isBeingEmptied );
unsigned maxDocs = min( cMaxDocs, CI_MAX_DOCS_IN_WORDLIST);
if ( _partIter.LokAtEnd() )
return FALSE; // There are no valid partitions
// we are sure that the partition is valid
CPartition* pPart = _partIter.LokGet();
Win4Assert ( pPart != 0 );
//
// Don't bother looking for more documents to filter if there
// are too many wordlists in memory -- wait until they are
// merged.
//
if ( pPart->WordListCount() < _frmwrkParams.GetMaxWordlists() )
{
CChangeTrans xact( *this, pPart );
pPart->LokQueryPendingUpdates ( xact, maxDocs, _docList );
xact.LokCommit();
}
else
{
ciDebugOut(( DEB_ITRACE,
"Too many wordlists. Waking merge\n" ));
_eventMerge.Set();
}
_docList.SetPartId ( pPart->GetId() );
//==================================
//
// If there is a failure after this point, we have to re-add
// the documents to the change log.
//
cDocs = _docList.Count();
for (unsigned i = 0; i < cDocs; i++)
{
if (_docList.Status(i) != DELETED && _docList.Status(i) != WL_NULL)
break;
}
BOOL fRetry = FALSE;
if ( i == cDocs && i != 0 )
{
CIndexTrans xact( *this );
_fresh.LokDeleteDocuments ( xact, _docList, _activeDeletedIndex.GetIndex() );
xact.LokCommit();
cDocs = 0;
_docList.LokClear();
state.Reset();
//
// If we are doing a bunch of deletions in a "delnode" case,
// we should see if the number of changed documents has exceeded
// the max fresh test count and wake up the merge thread. Note
// that we are not starting a merge here - the merge thread will
// deal with the policy issues.
//
if ( LokGetFreshCount() > _frmwrkParams.GetMaxFreshCount() ||
CheckDeleteMerge( FALSE ) )
{
_eventMerge.Set();
}
LokUpdateCounters();
}
_partIter.LokAdvance( _partList );
if ( _partIter.LokAtEnd() )
{
_partIter.LokInit( _partList );
state.DecrementTries();
fRetry = (0 == cDocs) && (state.GetTriesLeft() > 0);
}
else
{
fRetry = ( 0 != cDocs );
}
Win4Assert( cDocs == _docList.Count() );
return fRetry;
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::FillDocBuffer, public
//
// Synopsis: Fills buffer with strings for paths and properties of docs
// to be filtered
//
// Arguments: [docBuffer] -- (in, out) buffer to be filled with document
// names.
// [cb] -- (in, out) count of bytes in docBuffer
// [cDocs] -- (out) count of # documents to filter
// [state] -- [in/out] State of the document retrieval
// progress.
//
// History: 08-Apr-1991 BartoszM Created
// 24-Nov-1992 KyleP Retry pending documents
// 22-Mar-1993 AmyA Split into several functions
// 22-Jan-1997 SrikantS Changed doc. names to bytes for frame
// work.
// 18-May-2000 KitmanH Changed to push retried docs to filter
// one by one
//
//----------------------------------------------------------------------------
void CResManager::FillDocBuffer( BYTE * docBuffer,
ULONG & cb, ULONG & cDocs,
CGetFilterDocsState & state )
{
Win4Assert( docBuffer != 0 );
// =========================================
{
CPriLock lock(_mutex);
_docList.LokSortOnWid();
}
// ==========================================
unsigned cDocuments = _docList.Count();
//
// Format of the data
// 4 Bytes remaining files count, 2 Bytes number of files. Following
// that is (2 Bytes of Name Length in bytes, Name)*
//
if ( _fPushFiltering )
{
//
// We can't take an exception while growing the aborted wids
// array while doing exception processing in the filter manager.
// So, pre-allocate the array now to insure we won't run out of
// room later. Now is a good time to die; later isn't.
//
CPriLock lock( _mutex );
_aAbortedWids.LokReserve( cDocuments );
//
// In push filtering there are no refiles, and hence the buffer
// cannot overflow. Check that the buffer can fit 16 documents.
// A document name is the serialized form of the wid, i.e. it's
// 4 bytes long. The buffer size is 4K.
//
Win4Assert( cb > 4 + 2 + 16 * (2 + 4) );
}
const cbLenField = sizeof(ULONG);
Win4Assert( cb >= cbLenField );
Win4Assert( sizeof(ULONG) == 4 );
if ( cb < cbLenField )
THROW( CException( STATUS_INVALID_PARAMETER ) );
//
// Fill the number of documents left in the changelog in the first
// 4 bytes of the buffer.
//
// =========================================================
{
CPriLock lock( _mutex );
ULONG partid = 1;
CPartition * pPart = _partList.LokGetPartition( partid );
ULONG cDocsLeft = pPart->LokCountUpdates();
RtlCopyMemory( docBuffer, &cDocsLeft, sizeof(ULONG) );
}
// =========================================================
RtlZeroMemory( docBuffer + cbLenField, sizeof USHORT );
const cbUSHORT = sizeof USHORT;
BYTE * pCount = docBuffer + cbLenField;
*pCount = 0;
BYTE * pBuf = pCount + cbUSHORT;
BYTE * pEnd = docBuffer + cb;
cDocs = 0;
// copy path/property pairs into buffer until we run out of either
// buffer space or pairs. We stop filling the buffer if we encounter a
// document that has a retry count > 1, we will fill the docBuffer with
// these docs one at a time
ciDebugOut((DEB_FILTERWIDS | DEB_ITRACE,
"CResManager::FillDocBuffer..."));
unsigned cbExtraSpaceNeeded = 0;
for (unsigned cSrcDocs = 0; cSrcDocs < cDocuments; cSrcDocs++)
{
//
// Leave the first 2 bytes for the length field
//
unsigned cbPath = (unsigned)(pEnd - pBuf);
if ( cbPath <= cbUSHORT )
{
// We would definitely need more space than a USHORT, but
// we don't know how much right now. But since the filter
// increases space by a PAGE, I guess we should do fine.
cbExtraSpaceNeeded = cbUSHORT - cbPath + 1;
break;
}
if (_docList.Status(cSrcDocs) == DELETED)
{
cbPath = 0;
ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, " %dX",
_docList.Wid(cSrcDocs) ));
}
else
{
if ( _docList.Retries(cSrcDocs) > 1 )
{
ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME,
"************FillDocBuffer: Retry count is %d. Wid is %d\n",
_docList.Retries(cSrcDocs),
_docList.Wid(cSrcDocs) ));
ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, "cDocs is %d\n", cDocs ));
if ( cDocs > 0 )
{
//
// There is at least one document in the docBuffer. Stop
// now so we can file the doc(s) being retried one by one.
//
break;
}
else
{
//
// Add the currenct doc. Stop if the current add is successful
//
//
// Make room for the length field at the beginning.
//
cbPath -= cbUSHORT;
ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, " %d",
_docList.Wid(cSrcDocs) ));
unsigned cbPathIn = cbPath;
//
// If this is not the first try, then attempt to get a more
// accurate path. We may be suffering from NTBUG: 270566
//
// changes cbPath.
if ( !WorkidToDocName( _docList.Wid(cSrcDocs),
pBuf+cbUSHORT,
cbPath,
TRUE ) )
{
if ( cbPathIn < cbPath)
{
cbExtraSpaceNeeded = cbPath - cbPathIn;
}
break;
}
// cbPath = 0 if object has been deleted (not added to docBuffer)
if ( cbPath == 0 )
{
_docList.SetStatus( cSrcDocs, DELETED );
ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, "X" ));
}
else
{
ClearNonStoragePropertiesForWid( _docList.Wid(cSrcDocs) );
cSrcDocs++;
cDocs++;
USHORT uscb = (USHORT) cbPath;
RtlCopyMemory( pBuf, &uscb, cbUSHORT );
pBuf += (cbPath+cbUSHORT);
break;
}
}
}
else // not retries
{
ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME,
"FillDocBuffer: Not retry for wid %d\n",
_docList.Wid(cSrcDocs) ));
//
// Make room for the length field at the beginning.
//
cbPath -= cbUSHORT;
ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, " %d",
_docList.Wid(cSrcDocs) ));
unsigned cbPathIn = cbPath;
// changes cbPath.
if ( !WorkidToDocName( _docList.Wid(cSrcDocs),
pBuf+cbUSHORT,
cbPath,
FALSE ) )
{
if ( cbPathIn < cbPath)
{
cbExtraSpaceNeeded = cbPath - cbPathIn;
}
break;
}
// cbPath = 0 if object has been deleted
if ( cbPath == 0 )
{
_docList.SetStatus( cSrcDocs, DELETED );
ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, "X" ));
}
else
{
cDocs++;
ClearNonStoragePropertiesForWid( _docList.Wid(cSrcDocs) );
}
}
}
USHORT uscb = (USHORT) cbPath;
RtlCopyMemory( pBuf, &uscb, cbUSHORT );
pBuf += (cbPath+cbUSHORT);
}
ciDebugOut (( DEB_FILTERWIDS | DEB_NOCOMPNAME, "\n" ));
//
// Fill in the count of the number of documents.
//
USHORT usFileCount = (USHORT) cSrcDocs;
RtlCopyMemory( pCount, &usFileCount, cbUSHORT );
if ( cSrcDocs < cDocuments )
{
if ( 0 == cSrcDocs && cbExtraSpaceNeeded > 0 )
{
// We need a larger buffer
cb += cbExtraSpaceNeeded;
}
//
// No refiling in push filtering
//
Win4Assert( !_fPushFiltering );
ciDebugOut (( DEB_ITRACE, "Putting back documents\n" ));
CDocList docListExtra;
docListExtra.SetPartId( _docList.PartId() );
for ( unsigned i = 0, j = cSrcDocs; j < cDocuments; i++, j++ )
{
ciDebugOut(( DEB_ITRACE | DEB_NOCOMPNAME, " %d", _docList.Wid(j) ));
ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, "Refiling Wid %d\n", _docList.Wid(j) ));
docListExtra.Set( i,
_docList.Wid(j),
_docList.Usn(j),
_docList.VolumeId(j) );
}
ciDebugOut (( DEB_ITRACE | DEB_NOCOMPNAME, "\n" ));
CPriLock lock2(_mutex);
docListExtra.LokSetCount(i);
_docList.LokSetCount( cSrcDocs );
ReFileCommit ( docListExtra );
}
//
// If we were unable to convert any of the wids to paths, then all of
// them have been deleted.
//
if ( 0 == cDocs )
{
ciDebugOut (( DEB_ITRACE, "All docs were deleted\n" ));
CPriLock lock3(_mutex);
_docList.LokClear();
state.Reset();
}
else
{
cb = (ULONG)(pBuf - docBuffer);
}
}
//+---------------------------------------------------------------------------
//
// Function: NoFailReFile
//
// Synopsis: re-adds the documents to the change log witout committing
//
// History: 6-16-94 srikants Separated from FilterReady for making
// kernel mode call Asynchronous.
//
//----------------------------------------------------------------------------
BOOL CResManager::NoFailReFile ( CDocList& docList )
{
//
// No refiling in push filtering
//
Win4Assert( !_fPushFiltering );
CPriLock lock ( _mutex );
if ( _isBeingEmptied )
return FALSE;
LokNoFailReFileChanges( docList );
docList.LokClear();
return TRUE;
}
//+---------------------------------------------------------------------------
//
// Function: ReFileCommit
//
// Synopsis: re-adds the documents to the change log and commits
// them to the change list.
//
// History: 6-16-94 srikants Separated from FilterReady for making
// kernel mode call Asynchronous.
//
//----------------------------------------------------------------------------
BOOL CResManager::ReFileCommit ( CDocList& docList )
{
//
// No refiling in push filtering
//
Win4Assert( !_fPushFiltering );
CPriLock lock ( _mutex );
if ( _isBeingEmptied )
return FALSE;
LokNoFailReFileChanges( docList );
docList.LokClear();
//
// The docqueue can handle only one set of refiled documents.
// we MUST complete processing of the
// refiled documents.
//
PARTITIONID partId = 1;
CPartition * pPart = LokGetPartition( partId );
{
// =========================================
CChangeTrans xact( *this, pPart );
pPart->LokAppendRefiledDocs( xact );
xact.LokCommit();
// =========================================
}
return TRUE;
}
//+---------------------------------------------------------------------------
//
// Function: LokNoFailReFileChanges
//
// Synopsis: Refiles the updates with the change log.
//
// Arguments: [docList] -- contains the documents to be added back to the
// change log.
//
// History: 5-16-94 srikants Modified to deal with failures during
// append of docs to change log.
//
// Notes: This CANNOT throw
//
//----------------------------------------------------------------------------
void CResManager::LokNoFailReFileChanges( CDocList& docList )
{
//
// No refiling in push filtering
//
Win4Assert( !_fPushFiltering );
Win4Assert ( docList.PartId() == 1 );
CPartition* pPart = _partList.LokGetPartition ( docList.PartId() );
pPart->LokRefileDocs( docList );
_pFilterAgent->LokWakeUp ();
}
//+-------------------------------------------------------------------------
//
// Member: CResManager::LokReFileDocument, public
//
// Returns: Put a document back on the quueue to be refiltered
//
// History: 05-Jan-95 BartoszM Created
//
//--------------------------------------------------------------------------
void CResManager::LokReFileDocument ( PARTITIONID partid,
WORKID wid,
USN usn,
VOLUMEID volumeId,
ULONG retries,
ULONG secQRetries )
{
//
// No refiling in push filtering
//
Win4Assert( !_fPushFiltering );
CPartition* pPart = _partList.LokGetPartition(partid);
//==========================================
CChangeTrans xact( *this, pPart );
//
// A usn of 0 is used for all refiled documents
//
pPart->LokUpdateDocument (xact, wid, 0, volumeId, CI_UPDATE_OBJ, retries, secQRetries);
xact.LokCommit();
//==========================================
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::LokAddToSecQueue
//
// Synopsis: Adds the given document to the secondary change log of the
// partition.
//
// Arguments: [partid] - PartitionId
// [wid] - WorkId of the document
// [volumeId] - Volume id
// [cSecQRetries] - Sec Q Retry Count
//
// History: 5-09-96 srikants Created
//
//----------------------------------------------------------------------------
void CResManager::LokAddToSecQueue( PARTITIONID partid,
WORKID wid,
VOLUMEID volumeId,
ULONG cSecQRetries )
{
//
// No refiling in push filtering
//
Win4Assert( !_fPushFiltering );
CPartition* pPart = _partList.LokGetPartition(partid);
//==========================================
CChangeTrans xact( *this, pPart );
pPart->LokAddToSecQueue (xact, wid, volumeId, cSecQRetries);
xact.LokCommit();
//==========================================
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::LokRefileSecQueueDocs
//
// Synopsis: Transfers the documents from the secondary queue to the primary
// change queue.
//
// History: 5-09-96 srikants Created
//
//----------------------------------------------------------------------------
void CResManager::LokRefileSecQueueDocs()
{
//
// No refiling in push filtering
//
Win4Assert( !_fPushFiltering );
CPartIter iter;
for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
{
CPartition * pPart = iter.LokGet();
//==========================================
CChangeTrans xact( *this, pPart );
pPart->LokRefileSecQueue( xact );
xact.LokCommit();
//==========================================
}
}
//+-------------------------------------------------------------------------
//
// Member: CResManager::LokTransferWordlist, public
//
// Returns: Transfer a wordlist from the filter manager
// to the content index
//
// History: 05-Jan-95 BartoszM Created
//
//--------------------------------------------------------------------------
BOOL CResManager::LokTransferWordlist ( PWordList& pWordList )
{
BOOL fSuccess = TRUE;
PARTITIONID partid = _docList.PartId();
CPartition* pPart = _partList.LokGetPartition(partid);
INDEXID iid = pWordList->GetId();
TRY
{
//==========================================
CIndexTrans xact( *this );
//
// LokDone can fail while trying to add unfinished docs to the
// change log. In that case we must disable further USN updates.
// We should commit the change transaction immediately after the
// LokDone succeeds.
//
CChangeTrans changeXact( *this, pPart );
pPart->LokDone ( changeXact, iid, _docList );
ciFAILTEST( STATUS_NO_MEMORY );
changeXact.LokCommit();
_fresh.LokAddIndex ( xact, iid, _activeDeletedIndex.GetIndex(),
_docList, *pWordList );
ciDebugOut (( DEB_ITRACE, "CFilterManager::FilterDone "
"Success: transfering wordlist\n" ));
pWordList->Done();
//
// This may be an empty wordlist: all documents were deleted or in error.
// If so, just get rid of the wordlist.
//
if ( pWordList->IsEmpty() )
{
#if CIDBG == 1
unsigned cShouldHaveData = 0;
unsigned cDocuments = _docList.Count();
for ( unsigned iDoc = 0; iDoc < cDocuments; iDoc++ )
{
switch ( _docList.Status(iDoc) )
{
case TOO_MANY_RETRIES:
case SUCCESS:
case TOO_MANY_BLOCKS_TO_FILTER:
cShouldHaveData++;
break;
default:
break;
}
}
if ( cShouldHaveData != 0 )
{
for ( unsigned iDoc = 0; iDoc < cDocuments; iDoc++ )
{
ciDebugOut(( DEB_ERROR, "WID: 0x%x STATUS: %d\n",
_docList.Wid(iDoc), _docList.Status(iDoc) ));
}
}
Win4Assert( cShouldHaveData == 0 );
#endif
pWordList.Delete();
}
else
pWordList.Transfer ( pPart );
xact.LokCommit();
}
CATCH ( CException, e )
{
ciDebugOut(( DEB_ERROR,
"Exception 0x%x caught commiting wordlist.\n"
"delete wordlist and clear doclist\n",
e.GetErrorCode() ));
fSuccess = FALSE;
}
END_CATCH
_eventMerge.Set(); // wake up
return fSuccess;
}
//+---------------------------------------------------------------------------
//
// Function: LokNoFailRemoveWordlist
//
// Synopsis: Removes the biggest wordlist from memory and adds the
// documents back to the change log.
//
// Requires: _docList.Count() == 0 when this method is called. This
// relies on the fact that there are no other outstanding
// wordlists being constructed.
//
// History: 10-20-94 srikants Added the header section and modified
// to deal with the change in refiling
// documents.
//
//----------------------------------------------------------------------------
void CResManager::LokNoFailRemoveWordlist( CPartition * pPart )
{
//
// In push filtering, we don't delete completed wordlist to simplify
// the logic of aborting the notification transaction on the client
//
if ( _isBeingEmptied || _fPushFiltering )
return;
//
// Look for biggest wordlist
//
unsigned cInd;
CIndex ** apIndex = pPart->LokQueryMergeIndexes( cInd, mtWordlist );
XPtrST<CIndex *> xIndex( apIndex );
unsigned size = 0;
CWordList * pWordList = 0;
for ( unsigned i = 0; i < cInd; i++ )
{
apIndex[i]->Release();
if ( apIndex[i]->Size() > size )
{
Win4Assert( apIndex[i]->IsWordlist() );
pWordList = (CWordList *)apIndex[i];
size = pWordList->Size();
}
}
if ( pWordList )
{
//
// Reschedule contents for later filtering.
//
INDEXID iid = pWordList->GetId();
ciDebugOut(( DEB_ITRACE,
"Removing wordlist %x to reduce resource consumption.\n",
iid ));
CDocList _docList;
_docList.SetPartId ( pPart->GetId() );
pWordList->GetDocuments( _docList );
#if CIDBG == 1
for ( unsigned cWid = 0; cWid < _docList.Count(); cWid++ )
{
ciDebugOut(( DEB_ITRACE, "Reschedule filtering of wid %d\n",
_docList.Wid( cWid ) ));
}
#endif // CIDBG == 1
LokNoFailReFileChanges ( _docList );
//
// Remove it
//
ciDebugOut(( DEB_WARN, "Deleting wordlist with USN (%x, %x).\n",
lltoHighPart(pWordList->Usn()), (ULONG) pWordList->Usn() ));
pPart->LokRemoveIndex( iid );
if ( !pWordList->InUse() )
{
_partList.LokRemoveIndex ( iid );
delete pWordList;
}
else
{
pWordList->Zombify();
}
}
}
//+-------------------------------------------------------------------------
//
// Member: CResManager::LokMakeWordlistId, public
//
// Returns: Makes unique wordlist id
//
// History: 05-Jan-95 BartoszM Created
//
//--------------------------------------------------------------------------
INDEXID CResManager::LokMakeWordlistId (PARTITIONID partid)
{
CPartition* pPart = _partList.LokGetPartition(partid);
if ( pPart == 0 )
return iidInvalid;
// Get unique index id
INDEXID iid = pPart->LokMakeWlstId ();
ciDebugOut((DEB_FILTERWIDS,
"CResManager::LokMakeWorkListId - new wordlist iid: %x\n", iid));
return iid;
}
//+-------------------------------------------------------------------------
//
// Member: CResManager::ReportFilterFailure, public
//
// Synopsis: Write to event log about unsuccessful filtering
//
// Arguments: [wid] -- workid of the file that couldn't be filtered
//
// History: 05-Jan-95 BartoszM Created
//
//--------------------------------------------------------------------------
void CResManager::ReportFilterFailure (WORKID wid)
{
PROPVARIANT var;
var.vt = VT_UI4;
var.ulVal = wid;
_adviseStatus->NotifyStatus( CI_NOTIFY_FILTERING_FAILURE,
1,
&var );
}
//=========
// SCANNING
//=========
//+---------------------------------------------------------------------------
//
// Member: CResManager::SetUpdateLost
//
// Synopsis: Sets that an update got lost and wakes up the merge thread
// to notify the client about it.
//
// Arguments: [partId] - Partition id
//
// History: 1-24-97 srikants Created
//
//----------------------------------------------------------------------------
void CResManager::SetUpdatesLost( PARTITIONID partId )
{
CPriLock lock(_mutex);
if ( _state.LokGetState() != eUpdatesToBeDisabled )
{
_state.LokSetState( eUpdatesToBeDisabled );
_state.LokSetUpdateType( eIncremental );
StopCurrentMerge();
_eventMerge.Set();
}
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::LokCleanupForFullCiScan, public
//
// History: 10-Nov-94 DwightKr Created
//
// Notes: This routine performs the disk I/O and other operations
// which does the cleanup after a corruption is detected.
//
// This routine is similar to the Empty() method, except
// that we do not delete the change log object or the
// freshlog object, and the index list must contain
// the change log and fresh log after the cleanup is complete.
//
//----------------------------------------------------------------------------
void CResManager::LokCleanupForFullCiScan()
{
if ( 0 != _pMerge ) // Stop in progress merges.
_pMerge->LokAbort();
//
// Delete everything from the index table here first.
//
_idxTab->LokEmpty();
//
// If anything fails after this point, chkdsk /f or autochk will
// release the disk storage associated with the leaked objects
// since they are no longer part of the persistent index list.
//
//
// For each partition, zombify all indexes, and re-add the change
// log to the index table on disk.
//
CPartIter iter;
for ( iter.LokInit(_partList);
!iter.LokAtEnd();
iter.LokAdvance(_partList))
{
CPartition* pPart = iter.LokGet();
Win4Assert( pPart != NULL );
pPart->LokDisableUpdates();
//
// Zombify all indexes in this partition, and delete them if
// their ref-count is 0.
//
unsigned cIndex;
CIndex ** aIndex = pPart->LokZombify( cIndex );
ReleaseIndexes( cIndex, aIndex, NULL );
delete [] aIndex;
WORKID widMMergeLog;
WORKID widDummy;
pPart->GetMMergeObjectIds( widMMergeLog, widDummy, widDummy );
if ( widMMergeLog != widInvalid)
_storage.RemoveObject( widMMergeLog );
//
// Empty the fresh test and the fresh log, and add it to the
// index table on disk.
//
pPart->LokEmpty(); // Release change list/log storage
WORKID oidChangeLog = pPart->GetChangeLogObjectId();
Win4Assert( oidChangeLog != widInvalid );
_partList.LokAddIt( oidChangeLog, itChangeLog, pPart->GetId() );
}
//
// Empty the fresh test, fresh log, and add the fresh log to
// the index table on disk.
//
_fresh.LokInit(); // Release fresh test/log storage
WORKID oidFreshLog = _storage.GetSpecialItObjectId(itFreshLog);
Win4Assert( oidFreshLog != widInvalid );
_partList.LokAddIt( oidFreshLog, itFreshLog, partidFresh1 );
#ifdef KEYLIST_ENABLED
WORKID widKeyList = _storage.GetSpecialItObjectId( itMMKeyList );
#else
WORKID widKeyList = widInvalid;
#endif // KEYLIST_ENABLED
WORKID widPhrLat = _storage.GetSpecialItObjectId( itPhraseLat );
if ( widKeyList != widInvalid)
{
_storage.RemoveObject( widKeyList );
_storage.SetSpecialItObjectId( itMMKeyList, widInvalid );
}
if ( widPhrLat != widInvalid)
{
_storage.RemoveObject( widPhrLat );
_storage.SetSpecialItObjectId( itPhraseLat, widInvalid );
}
_isBeingEmptied = FALSE; // Allow new queries
}
// ===========================================================
// Content Index Frame Work Interfacing.
// ===========================================================
//+---------------------------------------------------------------------------
//
// Member: CResManager::WorkidToDocName
//
// Synopsis: Converts the given workid to path and fill it in the buffer
// given.
//
// Arguments: [wid] -- Workid to convert.
// [pbBuf] -- Destination buffer;
// [cb] -- On input the total number of bytes that can be
// put. On output, the actual number of bytes copied.
// For a deleted file, the cb will be 0.
// [fAccurate] -- TRUE --> Attempt to get a more accurate
// workid-to-path translation.
//
// Returns: TRUE if successfully converted; FALSE means the buffer was
// not big enough.
//
// History: 12-04-96 srikants Created
//
// Notes: WorkidToDocName is called outside the resman lock. It is a
// potentially long operation and taking resman lock is not
// advisable. Also, creation of a CWorkidToDocName object once
// for each wid->docName conversion is expensive. So, we create
// one in resman constructor and protect it with a separate
// lock.
//
//----------------------------------------------------------------------------
BOOL CResManager::WorkidToDocName( WORKID wid,
BYTE * pbBuf,
unsigned & cb,
BOOL fAccurate )
{
CLock lock(_workidToDocNameMutex);
ULONG cbDocName;
BYTE const * pbDocName = _xWorkidToDocName->GetDocName( wid, cbDocName, fAccurate );
BOOL fStatus = TRUE;
if ( pbDocName )
{
ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME,
"WorkidToDocName: wid is %d and DocName is %ws\n",
wid,
pbDocName ));
if ( cbDocName > cb )
fStatus = FALSE;
else
RtlCopyMemory( pbBuf, pbDocName, cbDocName );
cb = cbDocName;
}
else
{
cb = 0;
}
return fStatus;
} // WorkidToDocName
//+---------------------------------------------------------------------------
//
// Member: CResManager::GetClientState
//
// Synopsis: Gets the client specific status information like total
// number of documents.
//
// Arguments: [cDocuments] - [out] Total # of documents
//
// History: 12-04-96 srikants Created
//
//----------------------------------------------------------------------------
void CResManager::GetClientState( ULONG & cDocuments )
{
CI_CLIENT_STATUS status;
SCODE sc = _docStore->GetClientStatus( &status );
if ( S_OK == sc )
{
cDocuments = status.cDocuments;
}
else
{
cDocuments = 0;
}
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::LokNotifyCorruptionToClient
//
// Synopsis: Informs the client that the catalog is corrupt
//
// History: 12-04-96 srikants Created
//
//----------------------------------------------------------------------------
void CResManager::LokNotifyCorruptionToClient()
{
Win4Assert( _isCorrupt );
DisableUpdates( partidDefault );
SCODE sc = _docStore->DisableUpdates( FALSE, CI_CORRUPT_INDEX );
if ( SUCCEEDED( sc ) )
_state.LokSetCorruptionNotified();
//
// Disabled following assert so that the general NT 5 user is not
// bothered by this assert.
//
//Win4Assert( ! "We must empty the Content Index" );
// Empty();
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::LokNotifyDisableUpdatesToClient
//
// Synopsis: Informs the client that updates should be disabled
//
// History: 07-05-97 SitaramR Created
//
//----------------------------------------------------------------------------
void CResManager::LokNotifyDisableUpdatesToClient()
{
Win4Assert( _state.LokGetState() == eUpdatesToBeDisabled );
DisableUpdates( partidDefault );
BOOL fIncremental = _state.LokGetUpdateType() == eIncremental;
SCODE sc = _docStore->DisableUpdates( fIncremental, CI_LOST_UPDATE );
if ( SUCCEEDED(sc) )
{
ciDebugOut(( DEB_WARN, "Notified disabled updates to DocStore\n" ));
_state.LokSetState( eUpdatesDisabled );
}
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::LokNotifyEnableUpdatesToClient
//
// Synopsis: Informs the client that updates should be enabled
//
// History: 07-05-97 SitaramR Created
//
//----------------------------------------------------------------------------
void CResManager::LokNotifyEnableUpdatesToClient()
{
Win4Assert( _state.LokGetState() == eUpdatesToBeEnabled );
EnableUpdates( partidDefault );
SCODE sc = _docStore->EnableUpdates();
if ( SUCCEEDED(sc) )
{
ciDebugOut(( DEB_WARN, "Notified enable updates to DocStore\n" ));
_state.LokSetState( eSteady );
}
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::StoreValue
//
// Synopsis: Stores the given value in the document store
//
// Arguments: [wid] - WorkId of the document
// [ps] - The FULLPROPSPEC of the property
// [var] - Value of the property
// [fInSchema] - Returns TRUE if the property is in the schema
//
// Returns: SCODE result
//
// History: 12-18-96 srikants Created
//
//----------------------------------------------------------------------------
SCODE CResManager::StoreValue( WORKID wid,
CFullPropSpec const & ps,
CStorageVariant const & var,
BOOL & fInSchema )
{
FULLPROPSPEC const * fps = ps.CastToStruct();
PROPVARIANT const * pVar = ConvertToPropVariant( &var );
// Default to TRUE since the return code isn't checked!
fInSchema = TRUE;
SCODE sc = _propStore->StoreProperty( wid, fps, pVar );
if ( CI_E_PROPERTY_NOT_CACHED == sc )
fInSchema = FALSE;
else if ( FAILED( sc ) )
return sc;
return S_OK;
}
BOOL CResManager::StoreSecurity( WORKID wid,
PSECURITY_DESCRIPTOR pSD,
ULONG cbSD )
{
return S_OK == _docStore->StoreSecurity( wid, (BYTE *) pSD, cbSD );
}
//+---------------------------------------------------------------------------
//
// Member: CResManager::FPSToPROPID, public
//
// Synopsis: Converts FULLPROPSPEC property to PROPID
//
// Arguments: [fps] -- FULLPROPSPEC representation of property
// [pid] -- PROPID written here on success
//
// Returns: S_OK on success
//
// History: 29-Dec-1997 KyleP Created.
//
//----------------------------------------------------------------------------
SCODE CResManager::FPSToPROPID( CFullPropSpec const & fps, PROPID & pid )
{
return _mapper->PropertyToPropid( fps.CastToStruct(), TRUE, &pid );
}
//
// Flush notify support
//
//+---------------------------------------------------------------------------
//
// Member: CResManager::NotifyFlush
//
// Synopsis: Notifies the client that changes got flushed and gives the
// flush information.
//
// Arguments: [ft] - FileTime of the last successful flush.
// [cEntries] - Number of entries in the aInfo
// [aInfo] - Array of USN information for the flush.
//
// History: 1-27-97 srikants Created
//
// Note: We should not call directly into framework client because
// it violates resman->client lock hierarchy. We should not call
// into client with resman lock held.
//
//----------------------------------------------------------------------------
void CResManager::NotifyFlush( FILETIME const & ft,
ULONG cEntries,
USN_FLUSH_INFO ** ppInfo )
{
if ( !_fPushFiltering )
{
//
// Notification of flushes are sent in pull filtering only.
// For push filtering, the ICiCIndexNotificationStatus
// interface is used to notify clients.
//
CPriLock lock(_mutex);
CChangesFlushNotifyItem * pItem =
new CChangesFlushNotifyItem( ft, cEntries, ppInfo );
_flushList.Queue( pItem );
if (! _fFlushWorkerActive)
{
CFwFlushNotify * pWorkItem = new CFwFlushNotify( _workMan, *this );
_workMan.AddToWorkList( pWorkItem );
pWorkItem->AddToWorkQueue();
pWorkItem->Release();
}
}
} //NotifyFlush
//+---------------------------------------------------------------------------
//
// Member: CResManager::ProcessFlushNotifies
//
// Synopsis: Processes the queued flush notifies in a FIFO order. This
// method is called only by the asynchronous worker thread
// responsible for notifying the client about flushes.
//
// History: 1-27-97 srikants Created
//
//----------------------------------------------------------------------------
void CResManager::ProcessFlushNotifies()
{
//
// Only one worker thread allowed to operate on the list at a time.
//
CLock lock (_mtxChangeFlush );
CRevertBoolValue RevertFlushActive( _fFlushWorkerActive, TRUE );
while ( TRUE )
{
CChangesFlushNotifyItem * pItem = 0;
//
// Remove the first flush item from the list under resman lock.
//
//===========================================
{
CPriLock lock(_mutex);
pItem = _flushList.GetFirst();
if ( pItem )
_flushList.Pop();
if (_flushList.IsEmpty())
RevertFlushActive.Revert();
}
//===========================================
if ( pItem )
{
SCODE sc = _docStore->CheckPointChangesFlushed(
pItem->GetFlushTime(),
pItem->GetCount(),
pItem->GetFlushInfo() );
if ( FAILED(sc) )
{
//
// We don't have to requeue this item. The client's restart
// work will just increase.
//
ciDebugOut(( DEB_ERROR,
"CheckPointChangesFlushed failed. Error (0x%X)\n",
sc ));
}
pItem->Close();
delete pItem;
}
else
{
break;
}
} // While (TRUE)
} //ProcessFlushNotifies
//+---------------------------------------------------------------------------
//
// Member: CResManager::_LokDeleteFlushWorkItems
//
// Synopsis: Deletes the flush notifies in the queue. This is called
// during shutdown (dismount) to clean up any asynchronous
// worker threads referring to resman.
//
// History: 1-27-97 srikants Created
//
//----------------------------------------------------------------------------
void CResManager::LokDeleteFlushWorkItems()
{
for ( CChangesFlushNotifyItem * pItem = _flushList.Pop();
0 != pItem;
pItem = _flushList.Pop() )
{
pItem->Close();
delete pItem;
}
} //LokDeleteFlushWorkItems
//+---------------------------------------------------------------------------
//
// Member: CResManager::NoFailAbortWidsInDocList
//
// Synopsis: Used in push filtering to abort all wids in the current
// doclist
//
// History: 24-Feb-97 SitaramR Created
//
//----------------------------------------------------------------------------
void CResManager::NoFailAbortWidsInDocList()
{
CPriLock lock ( _mutex );
if ( _isBeingEmptied )
return ;
for ( unsigned i=0; i<_docList.Count(); i++ )
{
WORKID wid = _docList.Wid(i);
USN usn = _docList.Usn(i);
//
// Abort the client transaction
//
_xIndexNotifTable->AbortWid( wid, usn );
//
// Keep track of aborted wids for use during
// changlog shrink from front.
//
_aAbortedWids.NoFailLokAddWid( wid, usn );
}
_docList.LokClear();
} //NoFailAbortWidsInDocList
//+---------------------------------------------------------------------------
//
// Member: CResManager::NoFailAbortWid
//
// Synopsis: Used in push filtering to abort a wid
//
// Arguments: [wid] -- Workid
// [usn] -- Usn
//
// History: 24-Feb-97 SitaramR Created
//
//----------------------------------------------------------------------------
void CResManager::NoFailAbortWid( WORKID wid, USN usn )
{
CPriLock lock ( _mutex );
if ( _isBeingEmptied )
return ;
//
// Abort the client transaction
//
_xIndexNotifTable->AbortWid( wid, usn );
//
// Keep track of aborted wid for use during
// changlog shrink from front.
//
_aAbortedWids.NoFailLokAddWid( wid, usn );
} //NoFailAbortWid
//+---------------------------------------------------------------------------
//
// Member: CResManager::ClearNonStoragePropertiesForWid
//
// Synopsis: Clear non-storage properties from the property storage for wid
//
// Arguments: [wid] -- Workid
//
// History: 10-Oct-2000 KitmanH Created
//
//----------------------------------------------------------------------------
SCODE CResManager::ClearNonStoragePropertiesForWid( WORKID wid )
{
return _propStore->ClearNonStoragePropertiesForWid( wid );
}
//====================
// MASTER MERGE POLICY
//====================
//+---------------------------------------------------------------------------
//
// Member: CMasterMergePolicy::IsTimeToMasterMerge, public
//
// Synopsis: Determines if the system should start a master merge
// immediately, and not wait until the master merge time.
//
// History: 4-Aug-94 DwightKr Created
//
// Notes: If new tests are added simply add them here.
//
//----------------------------------------------------------------------------
BOOL CMasterMergePolicy::IsTimeToMasterMerge() const
{
return IsMaxShadowIndexSize() ||
IsMaxFreshListCount() ||
IsMinDiskFreeForceMerge();
}
//+---------------------------------------------------------------------------
//
// Member: CMasterMergePolicy::IsMinDiskFreeForceMerge, private
//
// Synopsis: If the amount of free disk space is lower than a threshold,
// then we should master merge, since it is likely that disk
// space will be released.
//
// History: 4-Aug-94 DwightKr Created
// Jan-07-96 mohamedn CDmFwEventItem
//
//----------------------------------------------------------------------------
BOOL CMasterMergePolicy::IsMinDiskFreeForceMerge() const
{
//
// If we've exceeded the MIN_DISKFREE_FORCE_MERGE space used on
// the system, and a master merge has the potential to make some
// useful progress, then it's time to force a master merge. The
// values of MIN_DISKFREE_FORCE_MERGE and MAX_SHADOW_FREESPACE
// are expressed as a percentage.
//
__int64 diskTotal, diskRemaining;
_storage.GetDiskSpace( diskTotal, diskRemaining );
if ( diskRemaining < minDiskFreeForMerge )
return FALSE;
if (diskRemaining*100/diskTotal < _frmwrkParams.GetMinDiskFreeForceMerge() &&
_shadowIndexSize*100/diskRemaining > _frmwrkParams.GetMaxShadowFreeForceMerge() )
{
CDmFwEventItem item( EVENTLOG_INFORMATION_TYPE,
MSG_CI_MASTER_MERGE_STARTED_TOTAL_DISK_SPACE_USED,
2);
item.AddArg( _storage.GetVolumeName() );
item.AddArg( _frmwrkParams.GetMinDiskFreeForceMerge() );
item.ReportEvent(_adviseStatus);
ciDebugOut(( DEB_ITRACE | DEB_PENDING,
"Master merge, reason: Low on disk space\n" ));
return TRUE;
}
return FALSE;
}
//+---------------------------------------------------------------------------
//
// Member: CMasterMergePolicy::IsMaxFreshListCount, private
//
// Synopsis: If the number of entries in the fresh list is great than
// a threshold, we should master merge, and free up memory
// occupied by the fresh hash.
//
// History: 4-Aug-94 DwightKr Created
// Jan-07-96 mohamedn CDmFwEventItem
//
//----------------------------------------------------------------------------
BOOL CMasterMergePolicy::IsMaxFreshListCount() const
{
if ( _freshCount > _frmwrkParams.GetMaxFreshCount() )
{
CDmFwEventItem item( EVENTLOG_INFORMATION_TYPE,
MSG_CI_MASTER_MERGE_STARTED_FRESH_LIST_COUNT,
2);
item.AddArg( _storage.GetVolumeName() );
item.AddArg( _frmwrkParams.GetMaxFreshCount() );
item.ReportEvent(_adviseStatus);
ciDebugOut(( DEB_ITRACE | DEB_PENDING,
"Master merge, reason: More than %d entries in freshhash\n",
_frmwrkParams.GetMaxFreshCount() ));
return TRUE;
}
return FALSE;
}
//+---------------------------------------------------------------------------
//
// Member: CMasterMergePolicy::IsMaxShadowIndexSize, private
//
// Synopsis: If the combined size of all shadow indexes is greater than
// a threahold, we should master merge, and free up disk space.
// The content index should not take up more than a given
// percentage of disk space.
//
// History: 4-Aug-94 DwightKr Created
// Jan-07-96 mohamedn CDmFwEventItem
//
//----------------------------------------------------------------------------
BOOL CMasterMergePolicy::IsMaxShadowIndexSize() const
{
__int64 diskTotal, diskRemaining;
_storage.GetDiskSpace( diskTotal, diskRemaining );
if ( diskRemaining < minDiskFreeForMerge )
return FALSE;
if ( _shadowIndexSize*100/diskTotal > _frmwrkParams.GetMaxShadowIndexSize() )
{
CDmFwEventItem item( EVENTLOG_INFORMATION_TYPE,
MSG_CI_MASTER_MERGE_STARTED_SHADOW_INDEX_SIZE,
2);
item.AddArg( _storage.GetVolumeName() );
item.AddArg( _frmwrkParams.GetMaxShadowIndexSize() );
item.ReportEvent(_adviseStatus);
ciDebugOut(( DEB_ITRACE | DEB_PENDING,
"Master merge, reason: Combined shadow indexes > %d%% of disk\n",
_frmwrkParams.GetMaxShadowIndexSize() ));
return TRUE;
}
return FALSE;
}
//+---------------------------------------------------------------------------
//
// Method: CMasterMergePolicy::ComputeMidNightMergeTime
//
// Synopsis: Function to compute the time (in hundreds of nanoseconds
// from midnight Dec 31, 1969) for the next master merge based on
// the value given from registry.
//
// Arguments: [mergeTime] - The time read from the registry control params.
// It's an offset in minutes after midnight.
//
// Returns: LONG - time for next merge
//
// History: 19-Feb-1996 srikants Moved the code from svcfilt.cxx
// 17-Nov-1999 KyleP Stop using CRT
//
// Notes: If the time is before January 1, 1970 it will return -1.
//
//----------------------------------------------------------------------------
ULONGLONG CMasterMergePolicy::ComputeMidNightMergeTime( LONG mergeTime )
{
Win4Assert( mergeTime < 24 * 60 );
mergeTime *= 60;
//
// What time is it here?
//
SYSTEMTIME stLocal;
GetLocalTime( &stLocal );
LONG lNow = stLocal.wHour*60*60 + stLocal.wMinute*60 + stLocal.wSecond;
//
// Have we passed the daily merge time?
//
LONGLONG llHNSTilMerge; // Hundred-Nano-Second-Til-Merge
LONGLONG const llHNSOneDay = 24*60*60*10000000i64;
if ( lNow < mergeTime )
llHNSTilMerge = mergeTime * 10000000i64;
else
llHNSTilMerge = mergeTime * 10000000i64 + llHNSOneDay;
//
// Create the *local* filetime for the next merge
//
stLocal.wHour = 0;
stLocal.wMinute = 0;
stLocal.wSecond = 0;
stLocal.wMilliseconds = 1;
FILETIME ftLocal;
if ( !SystemTimeToFileTime( &stLocal, &ftLocal ) )
return -1;
((ULARGE_INTEGER *)&ftLocal)->QuadPart += llHNSTilMerge;
//
// Now, adjust to UTC and return
//
FILETIME ft;
if ( !LocalFileTimeToFileTime( &ftLocal, &ft ) )
return -1;
#if CIDBG == 1
FILETIME ft2;
GetSystemTimeAsFileTime( &ft2 );
Win4Assert( ((ULARGE_INTEGER *)&ft2)->QuadPart < ((ULARGE_INTEGER *)&ft)->QuadPart );
#endif // CIDBG
return ((ULARGE_INTEGER *)&ft)->QuadPart;
} //ComputeMidNightMergeTime
//+---------------------------------------------------------------------------
//
// Member: CMasterMergePolicy::IsMidNightMergeTime, public
//
// Synopsis: Determines if it's been 24 hours since the last merge time
//
// History: 4-Aug-1994 DwightKr Created
// 17-Nov-1999 KyleP Stop using CRT
//
//----------------------------------------------------------------------------
BOOL CMasterMergePolicy::IsMidNightMergeTime()
{
FILETIME ft;
GetSystemTimeAsFileTime( &ft );
if ( ((ULARGE_INTEGER *)&ft)->QuadPart > _mergeTime )
{
ULONG mergeTime = _frmwrkParams.GetMasterMergeTime();
ULONGLONG oldMergeTime = _mergeTime;
_mergeTime = ComputeMidNightMergeTime( mergeTime );
//
// If it's been more than 30 minutes since the master merge time, don't
// do it. This can happen if we were doing a master merge as part of
// indexing documents or if you turn on a machine in the morning and it
// was off at the master merge time.
//
LONGLONG const llHNSHalfHour = 30*60*10000000i64;
if ( ((ULARGE_INTEGER *)&ft)->QuadPart > ( oldMergeTime + llHNSHalfHour ) )
return FALSE;
return TRUE;
}
return FALSE;
} //IsMidNightMergeTime