|
/*
*/
using System;
using System.Collections;
using System.Text.RegularExpressions;
namespace ServiceRanking
{
/// <summary>
/// Summary description for Tokeniser.
/// Partition string off into subwords
/// </summary>
public class Tokeniser
{
public static string[] ArrayListToArray(ArrayList arraylist)
{
return (string[])arraylist.ToArray (typeof(string));
}
void Normalize_Casing(ref string input)
{
for(int i=0; i < input.Length; i++)
{
if (Char.IsSeparator(input[i]))
input=input.Replace(input[i].ToString() , " ") ;
}
int idx=1;
while (idx < input.Length - 2)
{
++idx;
if (
(Char.IsUpper(input[idx])
&& Char.IsLower(input[idx + 1]))
&&
(!Char.IsWhiteSpace(input[idx - 1]) && !Char.IsSeparator(input[idx - 1]) )
)
{
input=input.Insert(idx, " ") ;
++idx;
}
}
}
public string[] Partition(string input)
{
Regex r=new Regex("([ \\t{}():;.,!?_/ \n])");
Normalize_Casing(ref input);
input=input.ToLower() ;
String [] tokens=r.Split(input);
ArrayList filter=new ArrayList() ;
for (int i=0; i < tokens.Length ; i++)
{
MatchCollection mc=r.Matches(tokens[i]);
if (mc.Count <= 0 && tokens[i].Trim().Length > 0
&& !StopWordsHandler.IsStopword (tokens[i]) )
filter.Add(tokens[i]) ;
}
StemmerInterface istem = new PorterStemmer();
tokens=new string[filter.Count] ;
for (int i = 0; i < filter.Count; i++)
{
string stem=istem.stemTerm ((string)filter[i]);
tokens[i] = stem;
}
return tokens;
}
public Tokeniser()
{
StopWordsHandler stop = new StopWordsHandler();
}
}
}
|
By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.
If a file you wish to view isn't highlighted, and is a text file (not binary), please
let us know and we'll add colourisation support for it.
I'm still alive...but temporarily moved to work on mobile & web stuffs(j2me/brew/php/flash...something not M$). things have just been very busy, and probably will continue...so don't have chance to maintain & respond. Hope will have time to try to write again, because many ideas with WPF &silver light are waiting. wish me luck
FYI:
- MESHSimPack project(c# library for measuring similarity among concepts of the MESH ontology):
http://sourceforge.net/projects/meshsimpack.