【文本挖掘】模拟退火英文分词

 1 from random import randint
 2 
 3 
 4 def segment(text,segs):
 5     words=[]
 6     last=0
 7     for i in range(len(segs)):
 8         if segs[i]==‘1‘:
 9             words.append(text[last:i+1])
10             last=i+1
11     words.append(text[last:])
12     return words
13 
14 def evaluate(text,segs):
15     words=segment(text, segs)
16     text_size=len(words)
17     lexicon_size=len(‘‘.join(list(set(words))))
18     return text_size+lexicon_size
19 
20 def filp(segs,pos):
21     return segs[:pos]+str(1-int(segs[pos]))+segs[pos+1:]
22 
23 def filp_n(segs,n):
24     for i in range(n):
25         segs=filp(segs,randint(0,len(segs)-1))
26     return segs
27 
28 def anneal(text,segs,iterations,cooling_rate):
29     temperature=float(len(segs))
30     while temperature>0.5:
31         best_segs,best=segs,evaluate(text, segs)
32         for i in range(iterations):
33             guess=filp_n(segs,int(round(temperature)))
34             score=evaluate(text,guess)
35             if score<best:
36                 best,best_segs=score,guess
37         score,segs=best,best_segs
38         temperature=temperature/cooling_rate
39         print  temperature,evaluate(text,segs),segment(text,segs)
40     print ‘----------END--------------------‘
41     return segs