论文降重算法代码详解与实现

import jieba from synonyms import nearby import random class SynonymReplacer: def __init__(self): self.similarity_threshold = 0.7 def replace_synonyms(self, text, replacement_rate=0.3): """同义词替换降重算法""" words = list(jieba.cut(text)) result_words = [] for word in words: if len(word.strip()) > 1 and random.random() < replacement_rate: try: syns, scores = nearby(word) if syns and scores[0] > self.similarity_threshold: # 选择相似度最高的同义词进行替换 new_word = syns[0] result_words.append(new_word) else: result_words.append(word) except: result_words.append(word) else: result_words.append(word) return ''.join(result_words) # 使用示例 replacer = SynonymReplacer() original_text = "人工智能技术在医疗诊断中发挥重要作用" rewritten_text = replacer.replace_synonyms(original_text) print(f"原文: {original_text}") print(f"改写: {rewritten_text}")

import re import nltk from nltk.tokenize import sent_tokenize from transformers import T5ForConditionalGeneration, T5Tokenizer class SentenceRestructurer: def __init__(self): self.model_name = "t5-base" self.tokenizer = T5Tokenizer.from_pretrained(self.model_name) self.model = T5ForConditionalGeneration.from_pretrained(self.model_name) def restructure_sentences(self, text): """句子结构重组算法""" sentences = sent_tokenize(text) restructured_sentences = [] for sentence in sentences: # 使用T5模型进行句式转换 input_text = f"paraphrase: {sentence}" inputs = self.tokenizer.encode(input_text, return_tensors="pt", max_length=512, truncation=True) outputs = self.model.generate(inputs, max_length=512, num_beams=4, temperature=0.7, early_stopping=True) paraphrased = self.tokenizer.decode(outputs[0], skip_special_tokens=True) restructured_sentences.append(paraphrased) return ' '.join(restructured_sentences) def change_voice(self, sentence): """主动被动语态转换""" # 简单的语态检测与转换规则 passive_pattern = r'被(\w+)(.*?)[了，。]' active_pattern = r'(\w+)(.*?)了(\w+)' if re.search(passive_pattern, sentence): # 被动转主动 match = re.search(passive_pattern, sentence) agent = match.group(1) action_desc = match.group(2) new_sentence = f"{agent}{action_desc}主动完成了" return new_sentence elif re.search(active_pattern, sentence): # 主动转被动 match = re.search(active_pattern, sentence) subject = match.group(1) action_desc = match.group(2) obj = match.group(3) new_sentence = f"{obj}被{subject}{action_desc}完成" return new_sentence return sentence # 使用示例 restructurer = SentenceRestructurer() text = "机器学习算法被广泛应用于数据分析领域。研究人员开发了新方法来提高准确率。" result = restructurer.restructure_sentences(text) print(f"原文: {text}") print(f"重组后: {result}")

import torch from transformers import GPT2LMHeadModel, GPT2Tokenizer from sklearn.metrics.pairwise import cosine_similarity import numpy as np class SemanticParaphraser: def __init__(self): self.tokenizer = GPT2Tokenizer.from_pretrained('gpt2-medium') self.model = GPT2LMHeadModel.from_pretrained('gpt2-medium') self.tokenizer.pad_token = self.tokenizer.eos_token def semantic_paraphrase(self, text, num_return_sequences=3): """基于语义的文本改写""" prompt = f"Rewrite the following text with different words but same meaning:\n{text}\n\nRewritten:" inputs = self.tokenizer.encode(prompt, return_tensors='pt', max_length=400, truncation=True) # 生成多个候选改写 outputs = self.model.generate( inputs, max_length=500, num_return_sequences=num_return_sequences, temperature=0.8, do_sample=True, pad_token_id=self.tokenizer.eos_token_id, top_k=50, top_p=0.95 ) paraphrases = [] for output in outputs: paraphrase = self.tokenizer.decode(output, skip_special_tokens=True) # 提取改写部分 rewritten_part = paraphrase.split("Rewritten:")[-1].strip() paraphrases.append(rewritten_part) return paraphrases def calculate_semantic_similarity(self, original, paraphrase): """计算语义相似度确保改写质量""" # 这里可以使用Sentence-BERT等模型计算语义相似度 # 简化版本使用词向量平均 orig_embedding = self._get_text_embedding(original) para_embedding = self._get_text_embedding(paraphrase) similarity = cosine_similarity([orig_embedding], [para_embedding])[0][0] return similarity def _get_text_embedding(self, text): """获取文本嵌入向量（简化实现）""" inputs = self.tokenizer.encode(text, return_tensors='pt', truncation=True) with torch.no_grad(): outputs = self.model(inputs, output_hidden_states=True) embedding = outputs.hidden_states[-1].mean(dim=1).numpy() return embedding.flatten() # 使用示例 paraphraser = SemanticParaphraser() original = "深度学习神经网络在计算机视觉领域取得了突破性进展。" paraphrases = paraphraser.semantic_paraphrase(original) print("原文:", original) for i, para in enumerate(paraphrases, 1): print(f"改写{i}: {para}")

class HybridParaphraser: def __init__(self): self.synonym_replacer = SynonymReplacer() self.structure_restructurer = SentenceRestructurer() self.semantic_paraphraser = SemanticParaphraser() def hybrid_paraphrase(self, text, strategy='balanced'): """混合降重策略""" if strategy == 'conservative': # 保守策略：主要使用同义词替换 return self.synonym_replacer.replace_synonyms(text, 0.2) elif strategy == 'aggressive': # 激进策略：多算法组合使用 step1 = self.synonym_replacer.replace_synonyms(text, 0.4) step2 = self.structure_restructurer.restructure_sentences(step1) paraphrases = self.semantic_paraphraser.semantic_paraphrase(step2, 1) return paraphrases[0] if paraphrases else step2 else: # balanced # 平衡策略：适度组合各种方法 step1 = self.synonym_replacer.replace_synonyms(text, 0.3) step2 = self.structure_restructurer.change_voice(step1) return step2 # 策略使用示例 hybrid = HybridParaphraser() text = "神经网络模型在图像识别任务中表现出色。研究者们不断探索新的架构来提高性能。" print("保守策略:", hybrid.hybrid_paraphrase(text, 'conservative')) print("平衡策略:", hybrid.hybrid_paraphrase(text, 'balanced')) print("激进策略:", hybrid.hybrid_paraphrase(text, 'aggressive'))