
#判断字符ch是否为汉字
def isKannji(ch):
    by = bytes(ch,encoding='gbk')
    #小于两个字节是英文字符
    if len(by) < 2:
        return False
    #排除各种符号
    if (by[0] >= 0xa1 and by[0] <= 0xa9):
        return False
    # if ch in ['，','；','。','、','：','！','【','】','■','‘','’','●','□','（','）']:
    #     return False

    return True

ci = open('Ci.txt')
line = True
count = dict()
count2 = dict()

word1sum = 0
word2sum = 0

k =0

while line:

    try:
        line = ci.readline()
    except:
        pass

    ch1 = ''
    ch2 = ''
    if len(line) < 6:
        continue

    for i in line:

        if not isKannji(i):
            ch1 = ''
            ch2 = ''
            continue
        word1sum += 1

        if i not in count:
            count[i] = 1
        else:
            count[i] = count[i] + 1
        
        ch1 = ch2
        ch2 = i
        if ch1 == '' or ch2 == '':
            continue
        
        word2sum += 1
        str2 = ch1 + ch2#词语

        if str2 not in count2:
            count2[str2] = 1
        else:
            count2[str2] += 1 
        
    
count= sorted(count.items(), key=lambda d:d[1], reverse = True)  
count2= sorted(count2.items(), key=lambda d:d[1], reverse = True)  


out = open('1word.csv','w')

for ch in count:
    out.write(ch[0]+','+ str(ch[1])+','+str(float(ch[1]) / word1sum)+'\n')

ci.close()

out.close()


out2 = open('2word.csv','w')

for ch in count2:
    out2.write(ch[0]+','+ str(ch[1])+','+str(float(ch[1])/word2sum)+'\n')

ci.close()
out2.close()