标题:
聚类(8)
[打印本页]
作者:
look_w
时间:
2019-4-17 19:05
标题:
聚类(8)
Tokeniser.cs
/*
Tokenization
Author: Thanh Ngoc Dao -
Thanh.dao@gmx.net
Copyright (c) 2005 by Thanh Ngoc Dao.
*/
using System;
using System.Collections;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using WawaSoft.Search.Common;
namespace WawaSoft.Search.Common
{
/// <summary>
/// Summary description for Tokeniser.
/// Partition string into SUBwords
/// </summary>
internal class Tokeniser : ITokeniser
{
/// <summary>
/// 以空白字符进行简单分词,并忽略大小写,
/// 实际情况中可以用其它中文分词算法
/// </summary>
/// <param name="input"></param>
/// <returns></returns>
public IList<string> Partition(string input)
{
Regex r=new Regex("([ \\t{}():;. \n])");
input=input.ToLower() ;
String [] tokens=r.Split(input);
List<string> filter=new List<string>() ;
for (int i=0; i < tokens.Length ; i++)
{
MatchCollection mc=r.Matches(tokens
);
if (mc.Count <= 0 && tokens
.Trim().Length > 0
&& !StopWordsHandler.IsStopword (tokens
) )
filter.Add(tokens
) ;
}
return filter.ToArray();
}
public Tokeniser()
{
}
}
}
欢迎光临 电子技术论坛_中国专业的电子工程师学习交流社区-中电网技术论坛 (http://bbs.eccn.com/)
Powered by Discuz! 7.0.0