/*------------------------------------------------------------------------------ * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team * * Distributable under the terms of either the Apache License (Version 2.0) or * the GNU Lesser General Public License, as specified in the COPYING file. ------------------------------------------------------------------------------*/ #include "CLucene/StdHeader.h" #include "Analyzers.h" #include "CLucene/util/StringBuffer.h" CL_NS_USE(util) CL_NS_DEF(analysis) CharTokenizer::CharTokenizer(Reader* in) : Tokenizer(in), offset(0), bufferIndex(0), dataLen(0), ioBuffer(NULL) { buffer[0]=0; } TCHAR CharTokenizer::normalize(const TCHAR c) const { return c; } bool CharTokenizer::next(Token* token){ int32_t length = 0; int32_t start = offset; while (true) { TCHAR c; offset++; if (bufferIndex >= dataLen) { dataLen = input->read(ioBuffer, LUCENE_IO_BUFFER_SIZE); if (dataLen == -1) dataLen = 0; bufferIndex = 0; } if (dataLen <= 0 ) { if (length > 0) break; else return false; }else c = ioBuffer[bufferIndex++]; if (isTokenChar(c)) { // if it's a token TCHAR if (length == 0) // start of token start = offset-1; buffer[length++] = normalize(c); // buffer it, normalized if (length == LUCENE_MAX_WORD_LEN) // buffer overflow! break; } else if (length > 0) // at non-Letter w/ chars break; // return 'em } buffer[length]=0; token->set( buffer, start, start+length); return true; } bool LetterTokenizer::isTokenChar(const TCHAR c) const { return _istalpha(c)!=0; } TCHAR LowerCaseTokenizer::normalize(const TCHAR chr) const { return _totlower(chr); } bool WhitespaceTokenizer::isTokenChar(const TCHAR c) const{ return _istspace(c)==0; //(return true if NOT a space) } TokenStream* WhitespaceAnalyzer::tokenStream(const TCHAR* fieldName, Reader* reader) { return _CLNEW WhitespaceTokenizer(reader); } TokenStream* SimpleAnalyzer::tokenStream(const TCHAR* fieldName, Reader* reader) { return _CLNEW LowerCaseTokenizer(reader); } bool LowerCaseFilter::next(Token* t){ if (!input->next(t)) return false; stringCaseFold( t->_termText ); return true; } StopFilter::StopFilter(TokenStream* in, bool deleteTokenStream, const TCHAR** stopWords): TokenFilter(in, deleteTokenStream), table(_CLNEW CLSetList(false)) { fillStopTable( table,stopWords ); } void StopFilter::fillStopTable(CLSetList* stopTable, const TCHAR** stopWords) { for (int32_t i = 0; stopWords[i]!=NULL; i++) stopTable->insert(stopWords[i]); } bool StopFilter::next(Token* token) { // return the first non-stop word found while (input->next(token)){ if (table->find(token->_termText)==table->end()){ return true; } } // reached EOS -- return nothing return false; } StopAnalyzer::StopAnalyzer():stopTable(false) { StopFilter::fillStopTable(&stopTable,ENGLISH_STOP_WORDS); } StopAnalyzer::~StopAnalyzer() { } StopAnalyzer::StopAnalyzer( const TCHAR** stopWords) { StopFilter::fillStopTable(&stopTable,stopWords); } TokenStream* StopAnalyzer::tokenStream(const TCHAR* fieldName, Reader* reader) { return _CLNEW StopFilter(_CLNEW LowerCaseTokenizer(reader),true, &stopTable); } const TCHAR* StopAnalyzer::ENGLISH_STOP_WORDS[] = { _T("a"), _T("an"), _T("and"), _T("are"), _T("as"), _T("at"), _T("be"), _T("but"), _T("by"), _T("for"), _T("if"), _T("in"), _T("into"), _T("is"), _T("it"), _T("no"), _T("not"), _T("of"), _T("on"), _T("or"), _T("s"), _T("such"), _T("t"), _T("that"), _T("the"), _T("their"), _T("then"), _T("there"), _T("these"), _T("they"), _T("this"), _T("to"), _T("was"), _T("will"), _T("with"), NULL }; PerFieldAnalyzerWrapper::PerFieldAnalyzerWrapper(Analyzer* defaultAnalyzer): analyzerMap(true,true) { this->defaultAnalyzer = defaultAnalyzer; } PerFieldAnalyzerWrapper::~PerFieldAnalyzerWrapper(){ analyzerMap.clear(); _CLDELETE(defaultAnalyzer); } void PerFieldAnalyzerWrapper::addAnalyzer(const TCHAR* fieldName, Analyzer* analyzer) { analyzerMap.put(STRDUP_TtoT(fieldName), analyzer); } TokenStream* PerFieldAnalyzerWrapper::tokenStream(const TCHAR* fieldName, Reader* reader) { Analyzer* analyzer = (fieldName==NULL?defaultAnalyzer:analyzerMap.get(fieldName)); if (analyzer == NULL) { analyzer = defaultAnalyzer; } return analyzer->tokenStream(fieldName, reader); } bool ISOLatin1AccentFilter::next(Token* token){ if ( input->next(token) ){ int32_t l = token->termTextLength(); const TCHAR* chars = token->termText(); bool doProcess = false; for (int32_t i = 0; i < l; ++i) { #ifdef _UCS2 if ( chars[i] >= 0xC0 && chars[i] <= 0x178 ) { #else if ( (chars[i] >= 0xC0 && chars[i] <= 0xFF) || chars[i] < 0 ) { #endif doProcess = true; break; } } if ( !doProcess ) { return true; } StringBuffer output(l*2); for (int32_t j = 0; j < l; j++) { #ifdef _UCS2 TCHAR c = chars[j]; #else unsigned char c = chars[j]; #endif switch (c) { case 0xC0 : // À case 0xC1 : // Á case 0xC2 : // Â case 0xC3 : // Ã case 0xC4 : // Ä case 0xC5 : // Å output.appendChar('A'); break; case 0xC6 : // Æ output.append(_T("AE")); break; case 0xC7 : // Ç output.appendChar('C'); break; case 0xC8 : // È case 0xC9 : // É case 0xCA : // Ê case 0xCB : // Ë output.appendChar('E'); break; case 0xCC : // Ì case 0xCD : // Í case 0xCE : // Î case 0xCF : // Ï output.appendChar('I'); break; case 0xD0 : // Ð output.appendChar('D'); break; case 0xD1 : // Ñ output.appendChar('N'); break; case 0xD2 : // Ò case 0xD3 : // Ó case 0xD4 : // Ô case 0xD5 : // Õ case 0xD6 : // Ö case 0xD8 : // Ø output.appendChar('O'); break; case 0xDE : // Þ output.append(_T("TH")); break; case 0xD9 : // Ù case 0xDA : // Ú case 0xDB : // Û case 0xDC : // Ü output.appendChar('U'); break; case 0xDD : // Ý output.appendChar('Y'); break; case 0xE0 : // à case 0xE1 : // á case 0xE2 : // â case 0xE3 : // ã case 0xE4 : // ä case 0xE5 : // å output.appendChar('a'); break; case 0xE6 : // æ output.append(_T("ae")); break; case 0xE7 : // ç output.appendChar('c'); break; case 0xE8 : // è case 0xE9 : // é case 0xEA : // ê case 0xEB : // ë output.appendChar('e'); break; case 0xEC : // ì case 0xED : // í case 0xEE : // î case 0xEF : // ï output.appendChar('i'); break; case 0xF0 : // ð output.appendChar('d'); break; case 0xF1 : // ñ output.appendChar('n'); break; case 0xF2 : // ò case 0xF3 : // ó case 0xF4 : // ô case 0xF5 : // õ case 0xF6 : // ö case 0xF8 : // ø output.appendChar('o'); break; case 0xDF : // ß output.append(_T("ss")); break; case 0xFE : // þ output.append(_T("th")); break; case 0xF9 : // ù case 0xFA : // ú case 0xFB : // û case 0xFC : // ü output.appendChar('u'); break; case 0xFD : // ý case 0xFF : // ÿ output.appendChar('y'); break; #ifdef _UCS2 case 0x152 : // Œ output.append(_T("OE")); break; case 0x153 : // œ output.append(_T("oe")); break; case 0x178 : // Ÿ output.appendChar('Y'); break; #endif default : output.appendChar(c); break; } } token->setText(output.getBuffer()); return true; } return false; } TokenStream* KeywordAnalyzer::tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader){ return _CLNEW KeywordTokenizer(reader); } KeywordTokenizer::KeywordTokenizer(CL_NS(util)::Reader* input, int bufferSize): Tokenizer(input) { this->done = false; if ( bufferSize < 0 ) this->bufferSize = DEFAULT_BUFFER_SIZE; } KeywordTokenizer::~KeywordTokenizer(){ } bool KeywordTokenizer::next(Token* token){ if (!done) { done = true; int32_t rd; const TCHAR* buffer=0; while (true) { rd = input->read(buffer, bufferSize); if (rd == -1) break; token->growBuffer(token->_termTextLen +rd+1); int32_t cp = rd; if ( token->_termTextLen + cp > token->bufferLength() ) cp = token->bufferLength() - token->_termTextLen; _tcsncpy(token->_termText+token->_termTextLen,buffer,cp); token->_termTextLen+=rd; } token->_termText[token->_termTextLen]=0; token->set(token->_termText,0,token->_termTextLen); return true; } return false; } LengthFilter::LengthFilter(TokenStream* in, int _min, int _max): TokenFilter(in) { this->_min = _min; this->_max = _max; } bool LengthFilter::next(Token* token) { // return the first non-stop word found while ( input->next(token) ) { size_t len = token->termTextLength(); if (len >= _min && len <= _max) return true; // note: else we ignore it but should we index each part of it? } // reached EOS -- return null return false; } CL_NS_END