You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
29 lines
1.0 KiB
29 lines
1.0 KiB
|
|
import re as clear
|
|
|
|
class process():
|
|
def master_clean_text(text):
|
|
#clean up all the html tags
|
|
text = clear.sub(r'<.*?>','',text)
|
|
#remove the unwanted punctation chars
|
|
|
|
text = clear.sub(r"\\","",text)
|
|
text = clear.sub(r"\'","",text)
|
|
text = clear.sub(r"\"","",text)
|
|
|
|
# coversion to lowercase to remove complexity
|
|
text = text.strip().lower()
|
|
|
|
#removing unwanted expressions
|
|
|
|
unwanted = '!"\'#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n'
|
|
|
|
convert = dict((c," ") for c in unwanted)
|
|
|
|
# str.maketrans() --->> creates a one to one mapping of a character to its translation/replacement.
|
|
mapping_trans = str.maketrans(convert)
|
|
|
|
text = text.translate(mapping_trans)
|
|
|
|
return text
|
|
#master_clean_text("<a> Say youre scrapping a text from you'r website !! WEll it might be swap CASE or unevened you wanna remove all the punctation's into separate WOrd !!!!</a>").split()
|