CKeditor关键词统计工具无法识别非英文字符及变音符号问题

我找到一款适用于CKEditor的文本关键词统计排序代码，可用于撰写文章时的SEO关键词建议。但该代码无法识别öäåÖÄÅ等非英文字符，以及é、ñ这类变音符号。尝试修改去重音函数未解决问题，希望移除去重音功能，让特殊字符能正常被统计。

原HTML代码 #

<!-- Textarea -->
<div class="form-group">
<label class="col-md-2 control-label" for="editor1">HTML </label>
<div class="col-md-10">                     
<textarea class="form-control" id="editor1" name="editor1"><p>text example with ahöäåra</p></textarea>
</div>
</div>
<!-- KW density result -->
<div class="form-group">
<label class="col-md-2 control-label" for="editor1">Words Repeat</label>
<div class="col-md-10">                     
<div id="KWdensity" ></div>
</div>
</div> 

原JavaScript代码 #

$(document).ready(function () {
//----------------------------------------------------------------------
// Editor init
//----------------------------------------------------------------------
CKEDITOR.replace( 'editor1' );

//----------------------------------------------------------------------
// KW init
//----------------------------------------------------------------------
$(initKW);

//----------------------------------------------------------------------
// Editor key intercept
//----------------------------------------------------------------------
CKEDITOR.instances.editor1.on('contentDom', function() {
CKEDITOR.instances.editor1.document.on('keyup', function(event) { $(initKW); });
});    
function removeDiacritics (str) {

var defaultDiacriticsRemovalMap = [
{'base':'A', 'letters':'AⒶＡÀÁÂẦẤẪẨÃĀĂẰẮẴẲȦǠÄǞẢÅǺǍȀȂẠẬẶḀĄȺⱯ'}
];        
var diacriticsMap = {};
for (var i=0; i < defaultDiacriticsRemovalMap.length; i++){
var letters = defaultDiacriticsRemovalMap[i].letters;
for (var j=0; j < letters.length ; j++){
diacriticsMap[letters[j]] = defaultDiacriticsRemovalMap[i].base;
}}    
return str.replace(/[^\u0000-\u007E]/g, function(a){ 
return diacriticsMap[a] || a; 
});}  
function KeyDensityShow(srctext, MaxKeyOut, keylenMin) {
var Output;
// Split text on non word characters
var words = srctext.toLowerCase().split(/\W+/)
var positions = new Array()
var word_counts = new Array()
try {
for (var i=0; i<words.length; i++) {
var word = words[i]
if (!word || word.length < keylenMin) {
continue
}
if (!positions.hasOwnProperty(word)) {
positions[word] = word_counts.length;
word_counts.push([word, 1]);
} else {
word_counts[positions[word]][1]++;
}}
// Put most frequent words at the beginning.
word_counts.sort(function (a, b) {return b[1] - a[1]})
// Return the first MaxKeyOut items
return word_counts.slice(0, MaxKeyOut)
}        
catch(err) {
return "";
}}
function removeStopWords(input) {
var stopwords = ['test'];
var filtered  = input.split( /\b/ ).filter( function( v ){
return stopwords.indexOf( v ) == -1;
});
stopwords.forEach(function(item) {
var reg = new RegExp('\\W'+item +'\\W','gmi');
input = input.replace(reg, " ");
});
return input.toString();
}
function initKW() {
$('#KWdensity').html('');
var TextGrab = CKEDITOR.instances['editor1'].getData();
TextGrab = $(TextGrab).text(); // html to text
TextGrab = removeDiacritics(TextGrab);
TextGrab = removeStopWords(TextGrab);            
TextGrab = TextGrab.replace(/\r?\n|\r/gm," ").trim(); // remove line breaks   
TextGrab = TextGrab.replace(/\s\s+/g, " ").trim(); // remove double spaces
TextGrab = TextGrab.replace(/[^a-zA-Z ]+/g, "").trim(); // only letters and space
if (TextGrab != "") 
{    
var keyCSV = KeyDensityShow(TextGrab, 11, 3);        
var KeysArr = keyCSV.toString().split(',');
var item, items = '';
for (var i = 0; i < KeysArr.length; i++) {
item = '';
item = item + '<b>' + KeysArr[i] + "</b></button>&nbsp;";
i++;
item = '<button class="btn btn-default btn-xs" type="button"><span class="badge">' + KeysArr[i] + "</span>&nbsp;" + item;    
items = items + item; 
} 
$('#KWdensity').html(items);
}
}
});

修改方案 #

要让特殊变音字符正常被统计，需做以下4处修改：

1. 删除去重音函数：直接移除removeDiacritics函数，不再处理字符的变音标记。
2. 移除去重音调用：在initKW函数中，删掉TextGrab = removeDiacritics(TextGrab);这一行。
3. 修改字符过滤正则：将TextGrab = TextGrab.replace(/[^a-zA-Z ]+/g, "").trim();中的正则替换为/[^\p{L} ]+/gu，该正则会保留所有Unicode字母（包括öäå、é、ñ等带变音的字符）和空格。
4. 修改单词分割正则：在KeyDensityShow函数中，把var words = srctext.toLowerCase().split(/\W+/)改为var words = srctext.toLowerCase().split(/[^\p{L}]+/u)，确保带变音的字符能被正确识别为单词的一部分。

修改后的JavaScript代码 #

$(document).ready(function () {
//----------------------------------------------------------------------
// Editor init
//----------------------------------------------------------------------
CKEDITOR.replace( 'editor1' );

//----------------------------------------------------------------------
// KW init
//----------------------------------------------------------------------
$(initKW);

//----------------------------------------------------------------------
// Editor key intercept
//----------------------------------------------------------------------
CKEDITOR.instances.editor1.on('contentDom', function() {
CKEDITOR.instances.editor1.document.on('keyup', function(event) { $(initKW); });
});  

function KeyDensityShow(srctext, MaxKeyOut, keylenMin) {
var Output;
// Split text on non letter characters (supports Unicode letters)
var words = srctext.toLowerCase().split(/[^\p{L}]+/u)
var positions = new Array()
var word_counts = new Array()
try {
for (var i=0; i<words.length; i++) {
var word = words[i]
if (!word || word.length < keylenMin) {
continue
}
if (!positions.hasOwnProperty(word)) {
positions[word] = word_counts.length;
word_counts.push([word, 1]);
} else {
word_counts[positions[word]][1]++;
}}
// Put most frequent words at the beginning.
word_counts.sort(function (a, b) {return b[1] - a[1]})
// Return the first MaxKeyOut items
return word_counts.slice(0, MaxKeyOut)
}        
catch(err) {
return "";
}}
function removeStopWords(input) {
var stopwords = ['test'];
var filtered  = input.split( /\b/ ).filter( function( v ){
return stopwords.indexOf( v ) == -1;
});
stopwords.forEach(function(item) {
var reg = new RegExp('\\W'+item +'\\W','gmi');
input = input.replace(reg, " ");
});
return input.toString();
}
function initKW() {
$('#KWdensity').html('');
var TextGrab = CKEDITOR.instances['editor1'].getData();
TextGrab = $(TextGrab).text(); // html to text
TextGrab = removeStopWords(TextGrab);            
TextGrab = TextGrab.replace(/\r?\n|\r/gm," ").trim(); // remove line breaks   
TextGrab = TextGrab.replace(/\s\s+/g, " ").trim(); // remove double spaces
// Keep all Unicode letters and spaces
TextGrab = TextGrab.replace(/[^\p{L} ]+/gu, "").trim(); 
if (TextGrab != "") 
{    
var keyCSV = KeyDensityShow(TextGrab, 11, 3);        
var KeysArr = keyCSV.toString().split(',');
var item, items = '';
for (var i = 0; i < KeysArr.length; i++) {
item = '';
item = item + '<b>' + KeysArr[i] + "</b></button>&nbsp;";
i++;
item = '<button class="btn btn-default btn-xs" type="button"><span class="badge">' + KeysArr[i] + "</span>&nbsp;" + item;    
items = items + item; 
} 
$('#KWdensity').html(items);
}
}
});

内容的提问来源于stack exchange，提问作者newspapper.eu