_SuffixArrayLanguageModel.h

Go to the documentation of this file.

// Revision $Rev: 3824 $
// Last Modified $LastChangedDate: 2007-07-07 00:51:51 -0400 (Sat, 07 Jul 2007) $

#if ! defined (__HEADER_SUFFIXARRAY_LANGUAGE_MODEL_INCLUDED__)
#define __HEADER_SUFFIXARRAY_LANGUAGE_MODEL_INCLUDED__


#include "_SuffixArraySearchApplicationBase.h"
#include "salm_shared.h"

typedef unsigned int LMState;


typedef struct s_cachedLmInfo{
        int nextState;
        double logProb;
}S_CachedLmInfo;

typedef struct s_NgramLocationInCorpus{
        TextLenType posInCorpus;
        unsigned char len;
}S_NgramLocationInCorpus;

typedef struct s_lmStateInfo{
        S_NgramLocationInCorpus locationInCorpus;
        map<IndexType, S_CachedLmInfo> cachedNextWordExtension; //cached information of this LMState extended by the next word
}S_LMStateInfo;

struct lt_ngramLocationInCorpus
{
  bool operator()(S_NgramLocationInCorpus a, S_NgramLocationInCorpus b) const{
                if(a.posInCorpus<b.posInCorpus){
                        return true;
                }

                if(a.posInCorpus>b.posInCorpus){
                        return false;
                }

                if(a.len<b.len){
                        return true;
                }

                return false;   
        }
};


class C_SuffixArrayLanguageModel : public C_SuffixArraySearchApplicationBase
{

public:
        IndexType returnVocId(C_String aWord);  

        LMState beginOfSentenceState();
        
        double logProb(LMState lmState, IndexType nextWord, LMState & nextState);
        
        double logProb(LMState lmState, vector<IndexType> nextPhrase, LMState & nextState);     
        
        double logProbEnd(LMState lmState);     

        void setParam_interpolationStrategy(char interpolationStrategy);


        C_SuffixArrayLanguageModel(const char * cfgFileName);
        C_SuffixArrayLanguageModel();
        ~C_SuffixArrayLanguageModel();


private:

        void calcNgramMatchingInfoTokenFreqOnlyExtendingCurrentMatch(TextLenType currentMatchStart, unsigned char currentMatchLen, IndexType nextWord, double *freqTable, TextLenType &updatedMatchingStart, unsigned char &updatedMatchingLen);

        //Log prob calculation
        double logProbFromFreq(TextLenType currentMatchStart, unsigned char currentMatchLen, IndexType nextWord, TextLenType &updatedMatchingStart, unsigned char &updatedMatchingLen);
        double calcLogProb(double *freq);
        double calcLogProb_equalWeightedInterpolation(double *freq);
        double calcLogProb_ibmHeuristicInterpolation(double *freq);
        double calcLogProb_maxProbInterpolation(double * freq);

        char interpolationStrategy;
        int maxN;
        IndexType vocIdForSentStart;
        IndexType vocIdForSentEnd;
        IndexType vocIdForCorpusEnd;

        void constructDiscountingMap();
        double *discountingMap;
        double discountFreq(int n, unsigned int observedFreq);
        bool applyDiscounting;
        int maxFreqForDiscounting;
        S_nGramScanningInfoElement * nGramScanningList; 
        

        void resetLmStates();
        void initialLmState();  
        
        //caching lm prob for each sentence     
        vector<S_LMStateInfo> allLMStates;
        map<S_NgramLocationInCorpus, int, lt_ngramLocationInCorpus> ngramLocation2LmStateId;



};

#endif

---