bekommen Alphabete nach dem Anwenden von Satz Tokenizer von Nltk anstelle von Sätzen in Python 3.5.1

Question

import codecs, os 
import re 
import string 
import mysql 
import mysql.connector 
y_ = "" 

'''Searching and reading text files from a folder.''' 
for root, dirs, files in os.walk("/Users/ultaman/Documents/PAN dataset/Pan  Plagiarism dataset 2010/pan-plagiarism-corpus-2010/source-documents/test1"): 
for file in files: 
    if file.endswith(".txt"): 
     x_ = codecs.open(os.path.join(root,file),"r", "utf-8-sig") 
     for lines in x_.readlines(): 
      y_ = y_ + lines 
'''Tokenizing the senteces of the text file.''' 
from nltk.tokenize import sent_tokenize 
raw_docs = sent_tokenize(y_) 

tokenized_docs = [sent_tokenize(y_) for sent in raw_docs] 

'''Removing punctuation marks.''' 

regex = re.compile('[%s]' % re.escape(string.punctuation)) 

tokenized_docs_no_punctuation = '' 

for review in tokenized_docs: 

new_review = '' 
for token in review: 
    new_token = regex.sub(u'', token) 
    if not new_token == u'': 
     new_review+= new_token 

tokenized_docs_no_punctuation += (new_review) 
print(tokenized_docs_no_punctuation) 

'''Connecting and inserting tokenized documents without punctuation in database field.''' 
def connect(): 
    for i in range(len(tokenized_docs_no_punctuation)): 
     conn = mysql.connector.connect(user = 'root', password = '', unix_socket = "/tmp/mysql.sock", database = 'test') 
     cursor = conn.cursor() 
     cursor.execute("""INSERT INTO splitted_sentences(sentence_id, splitted_sentences) VALUES(%s, %s)""",(cursor.lastrowid,(tokenized_docs_no_punctuation[i]))) 
     conn.commit() 
     conn.close() 
if __name__ == '__main__': 
    connect() 


After writing the above code, The result is like 

      2 | S  | N                                                                            | 
|   3 | S  | o                                                                           | 

| 4 | S | | | 5 | S | d | | 6 | S | o | | 7 | S | du | | 8 | S | b | | 9 | S | t | | 10 | S | | | 11 | S | m | | 12 | S | y | | 13 | S |
| 14 | S | d
in der Datenbank.

It should be like: 
    1 | S  | No doubt, my dear friend. 
    2 | S  | no doubt.                                         

Answer 1

nw = [] 
for review in tokenized_docs[0]: 
    new_review = '' 
    for token in review: 
     new_token = regex.sub(u'', token) 
     if not new_token == u'': 
     new_review += new_token 
nw.append(new_review) 
'''Inserting into database''' 
def connect(): 
    for j in nw: 
     conn = mysql.connector.connect(user = 'root', password = '', unix_socket = "/tmp/mysql.sock", database = 'Thesis') 
     cursor = conn.cursor() 
     cursor.execute("""INSERT INTO splitted_sentences(sentence_id, splitted_sentences) VALUES(%s, %s)""",(cursor.lastrowid,j)) 
     conn.commit() 
     conn.close() 
if __name__ == '__main__': 
    connect() 

Answer 2

Ich schlage vor, die folgenden Änderungen vorzunehmen (verwenden Sie, was Sie möchten). Aber das ist es, was ich benutzt habe, um deinen Code zum Laufen zu bringen. Ihr Problem ist, dass review in for review in tokenized_docs: bereits eine Zeichenfolge ist. Also, das macht token in for token in review: Zeichen. Deshalb ist diese zu beheben versuchte ich -

tokenized_docs = ['"No doubt, my dear friend, no doubt; but in the meanwhile suppose we talk of this annuity.', 'Shall we say one thousand francs a year."', '"What!"', 'asked Bonelle, looking at him very fixedly.', '"My dear friend, I mistook; I meant two thousand francs per annum," hurriedly rejoined Ramin.', 'Monsieur Bonelle closed his eyes, and appeared to fall into a gentle slumber.', 'The mercer coughed;\nthe sick man never moved.', '"Monsieur Bonelle."'] 

'''Removing punctuation marks.''' 

regex = re.compile('[%s]' % re.escape(string.punctuation)) 

tokenized_docs_no_punctuation = [] 
for review in tokenized_docs: 
    new_token = regex.sub(u'', review) 
    if not new_token == u'': 
     tokenized_docs_no_punctuation.append(new_token) 

print(tokenized_docs_no_punctuation) 

und bekam diese -

['No doubt my dear friend no doubt but in the meanwhile suppose we talk of this annuity', 'Shall we say one thousand francs a year', 'What', 'asked Bonelle looking at him very fixedly', 'My dear friend I mistook I meant two thousand francs per annum hurriedly rejoined Ramin', 'Monsieur Bonelle closed his eyes and appeared to fall into a gentle slumber', 'The mercer coughed\nthe sick man never moved', 'Monsieur Bonelle'] 

Das endgültige Format der Ausgabe ist bis zu Ihnen. Ich bevorzuge Listen. Aber Sie könnten dies auch in eine Zeichenfolge verketten.