A simple implementation
class NaiveFilter(): '''Filter Messages from keywords very simple filter implementation >>> f = NaiveFilter() >>> f.add("sexy") >>> f.filter("hello sexy baby") hello **** baby ''' def __init__(self): self.keywords = set([]) def parse(self, path): for keyword in open(path): self.keywords.add(keyword.strip().decode('utf-8').lower()) def filter(self, message, repl="*"): message = str(message).lower() for kw in self.keywords: message = message.replace(kw, repl) return message
The strip() function deletes some nearby spaces, decodes them in utf-8, and then converts them to lowercase.
The parse() function opens a file, extracts keywords from it, and stores them in the keyword set.
The filter() function is a filter function in which messages are converted to lowercase and keywords are replaced.
class BSFilter: '''Filter Messages from keywords Use Back Sorted Mapping to reduce replacement times >>> f = BSFilter() >>> f.add("sexy") >>> f.filter("hello sexy baby") hello **** baby ''' def __init__(self): self.keywords = [] self.kwsets = set([]) self.bsdict = defaultdict(set) self.pat_en = re.compile(r'^[0-9a-zA-Z]+$') # english phrase or not def add(self, keyword): if not isinstance(keyword, str): keyword = keyword.decode('utf-8') keyword = keyword.lower() if keyword not in self.kwsets: self.keywords.append(keyword) self.kwsets.add(keyword) index = len(self.keywords) - 1 for word in keyword.split(): if self.pat_en.search(word): self.bsdict[word].add(index) else: for char in word: self.bsdict[char].add(index) def parse(self, path): with open(path, "r") as f: for keyword in f: self.add(keyword.strip()) def filter(self, message, repl="*"): if not isinstance(message, str): message = message.decode('utf-8') message = message.lower() for word in message.split(): if self.pat_en.search(word): for index in self.bsdict[word]: message = message.replace(self.keywords[index], repl) else: for char in word: for index in self.bsdict[char]: message = message.replace(self.keywords[index], repl) return message
In the implementation example above, the search search is optimized, and for English words, the dictionary search is directly indexed by words. For other language patterns, we use character-by-character search to find a matching pattern.
BFS: Width First Search.
class DFAFilter(): '''Filter Messages from keywords Use DFA to keep algorithm perform constantly >>> f = DFAFilter() >>> f.add("sexy") >>> f.filter("hello sexy baby") hello **** baby ''' def __init__(self): self.keyword_chains = {} self.delimit = '\x00' def add(self, keyword): if not isinstance(keyword, str): keyword = keyword.decode('utf-8') keyword = keyword.lower() chars = keyword.strip() if not chars: return level = self.keyword_chains for i in range(len(chars)): if chars[i] in level: level = level[chars[i]] else: if not isinstance(level, dict): break for j in range(i, len(chars)): level[chars[j]] = {} last_level, last_char = level, chars[j] level = level[chars[j]] last_level[last_char] = {self.delimit: 0} break if i == len(chars) - 1: level[self.delimit] = 0 def parse(self, path): with open(path,encoding='UTF-8') as f: for keyword in f: self.add(keyword.strip()) def filter(self, message, repl="*"): if not isinstance(message, str): message = message.decode('utf-8') message = message.lower() ret = [] start = 0 while start < len(message): level = self.keyword_chains step_ins = 0 for char in message[start:]: if char in level: step_ins += 1 if self.delimit not in level[char]: level = level[char] else: ret.append(repl * step_ins) start += step_ins - 1 break else: ret.append(message[start]) break else: ret.append(message[start]) start += 1 return ''.join(ret)
DFA is Deterministic Finite Automaton, that is, deterministic finite automaton.
Nested dictionaries are used to achieve this.