Merge pull request #139 from CNlaojing/main

修复生成目录时报错问题,新增部分逻辑,优化生成流程
This commit is contained in:
IdleCloud
2025-03-19 22:13:25 +08:00
committed by GitHub
12 changed files with 774 additions and 261 deletions
+1 -1
View File
@@ -11,7 +11,7 @@ CONSISTENCY_PROMPT = """\
- 角色状态(可能包含重要信息):
{character_state}
- 全局摘要:
- 前文摘要:
{global_summary}
- 已记录的未解决冲突或剧情要点:
+32 -11
View File
@@ -210,20 +210,41 @@ class SiliconFlowEmbeddingAdapter(BaseEmbeddingAdapter):
def embed_documents(self, texts: List[str]) -> List[List[float]]:
embeddings = []
for text in texts:
self.payload["input"] = text
response = requests.post(self.url, json=self.payload, headers=self.headers)
result = response.json()
# 从返回数据中提取第一个 embedding
emb = result.get("data", [{}])[0].get("embedding", [])
embeddings.append(emb)
try:
self.payload["input"] = text
response = requests.post(self.url, json=self.payload, headers=self.headers)
response.raise_for_status()
result = response.json()
if not result or "data" not in result or not result["data"]:
logging.error(f"Invalid response format from SiliconFlow API: {result}")
embeddings.append([])
continue
emb = result["data"][0].get("embedding", [])
embeddings.append(emb)
except requests.exceptions.RequestException as e:
logging.error(f"SiliconFlow API request failed: {str(e)}")
embeddings.append([])
except (KeyError, IndexError, ValueError, TypeError) as e:
logging.error(f"Error parsing SiliconFlow API response: {str(e)}")
embeddings.append([])
return embeddings
def embed_query(self, query: str) -> List[float]:
self.payload["input"] = query
# print('SiliconFlowEmbeddingAdapter发送',self.payload)
response = requests.post(self.url, json=self.payload, headers=self.headers)
result = response.json()
return result.get("data", [{}])[0].get("embedding", [])
try:
self.payload["input"] = query
response = requests.post(self.url, json=self.payload, headers=self.headers)
response.raise_for_status()
result = response.json()
if not result or "data" not in result or not result["data"]:
logging.error(f"Invalid response format from SiliconFlow API: {result}")
return []
return result["data"][0].get("embedding", [])
except requests.exceptions.RequestException as e:
logging.error(f"SiliconFlow API request failed: {str(e)}")
return []
except (KeyError, IndexError, ValueError, TypeError) as e:
logging.error(f"Error parsing SiliconFlow API response: {str(e)}")
return []
def create_embedding_adapter(
interface_format: str,
+2 -3
View File
@@ -3,8 +3,7 @@
import logging
from typing import Optional
from langchain_openai import ChatOpenAI, AzureChatOpenAI
from google import genai
from google.genai import types
import google.generativeai as genai
from azure.ai.inference import ChatCompletionsClient
from azure.core.credentials import AzureKeyCredential
from azure.ai.inference.models import SystemMessage, UserMessage
@@ -112,7 +111,7 @@ class GeminiAdapter(BaseLLMAdapter):
response = self._client.models.generate_content(
model = self.model_name,
contents = prompt,
config = types.GenerateContentConfig(
config = genai.types.GenerateContentConfig(
max_output_tokens=self.max_tokens,
temperature=self.temperature,
),
+7 -1
View File
@@ -1,7 +1,13 @@
#novel_generator/__init__.py
from .architecture import Novel_architecture_generate
from .blueprint import Chapter_blueprint_generate
from .chapter import generate_chapter_draft, get_last_n_chapters_text
from .chapter import (
get_last_n_chapters_text,
summarize_recent_chapters,
get_filtered_knowledge_context,
build_chapter_prompt,
generate_chapter_draft
)
from .finalization import finalize_chapter, enrich_chapter_text
from .knowledge import import_knowledge_file
from .vectorstore_utils import clear_vector_store
+7 -3
View File
@@ -48,6 +48,7 @@ def Chapter_blueprint_generate(
llm_model: str,
filepath: str,
number_of_chapters: int,
user_guidance: str = "", # 新增参数
temperature: float = 0.7,
max_tokens: int = 4096,
timeout: int = 600
@@ -106,7 +107,8 @@ def Chapter_blueprint_generate(
chapter_list=limited_blueprint,
number_of_chapters=number_of_chapters,
n=current_start,
m=current_end
m=current_end,
user_guidance=user_guidance # 新增参数
)
logging.info(f"Generating chapters [{current_start}..{current_end}] in a chunk...")
chunk_result = invoke_with_cleaning(llm_adapter, chunk_prompt)
@@ -126,7 +128,8 @@ def Chapter_blueprint_generate(
if chunk_size >= number_of_chapters:
prompt = chapter_blueprint_prompt.format(
novel_architecture=architecture_text,
number_of_chapters=number_of_chapters
number_of_chapters=number_of_chapters,
user_guidance=user_guidance # 新增参数
)
blueprint_text = invoke_with_cleaning(llm_adapter, prompt)
if not blueprint_text.strip():
@@ -149,7 +152,8 @@ def Chapter_blueprint_generate(
chapter_list=limited_blueprint,
number_of_chapters=number_of_chapters,
n=current_start,
m=current_end
m=current_end,
user_guidance=user_guidance # 新增参数
)
logging.info(f"Generating chapters [{current_start}..{current_end}] in a chunk...")
chunk_result = invoke_with_cleaning(llm_adapter, chunk_prompt)
+384 -86
View File
@@ -1,16 +1,27 @@
# novel_generator/chapter.py
# -*- coding: utf-8 -*-
"""
章节草稿生成及获取历史章节文本、短期摘要等
章节草稿生成及获取历史章节文本、当前章节摘要等
"""
import os
import json
import logging
import re # 添加re模块导入
from llm_adapters import create_llm_adapter
from prompt_definitions import first_chapter_draft_prompt, next_chapter_draft_prompt, summarize_recent_chapters_prompt
from prompt_definitions import (
first_chapter_draft_prompt,
next_chapter_draft_prompt,
summarize_recent_chapters_prompt,
knowledge_filter_prompt,
knowledge_search_prompt
)
from chapter_directory_parser import get_chapter_info_from_blueprint
from novel_generator.common import invoke_with_cleaning
from utils import read_file, clear_file_content, save_string_to_txt
from novel_generator.vectorstore_utils import get_relevant_context_from_vector_store
from novel_generator.vectorstore_utils import (
get_relevant_context_from_vector_store,
load_vector_store # 添加导入
)
def get_last_n_chapters_text(chapters_dir: str, current_chapter_num: int, n: int = 3) -> list:
"""
@@ -35,37 +46,228 @@ def summarize_recent_chapters(
temperature: float,
max_tokens: int,
chapters_text_list: list,
novel_number: int, # 新增参数
chapter_info: dict, # 新增参数
next_chapter_info: dict, # 新增参数
timeout: int = 600
) -> tuple:
) -> str: # 修改返回值类型为 str,不再是 tuple
"""
生成 (short_summary, next_chapter_keywords)
如果解析失败,则返回 (合并文本, "")
根据前三章内容生成当前章节的精准摘要。
如果解析失败,则返回空字符串。
"""
combined_text = "\n".join(chapters_text_list).strip()
if not combined_text:
return ("", "")
llm_adapter = create_llm_adapter(
interface_format=interface_format,
base_url=base_url,
model_name=model_name,
api_key=api_key,
temperature=temperature,
max_tokens=max_tokens,
timeout=timeout
try:
combined_text = "\n".join(chapters_text_list).strip()
if not combined_text:
return ""
# 限制组合文本长度
max_combined_length = 4000
if len(combined_text) > max_combined_length:
combined_text = combined_text[-max_combined_length:]
llm_adapter = create_llm_adapter(
interface_format=interface_format,
base_url=base_url,
model_name=model_name,
api_key=api_key,
temperature=temperature,
max_tokens=max_tokens,
timeout=timeout
)
# 确保所有参数都有默认值
chapter_info = chapter_info or {}
next_chapter_info = next_chapter_info or {}
prompt = summarize_recent_chapters_prompt.format(
combined_text=combined_text,
novel_number=novel_number,
chapter_title=chapter_info.get("chapter_title", "未命名"),
chapter_role=chapter_info.get("chapter_role", "常规章节"),
chapter_purpose=chapter_info.get("chapter_purpose", "内容推进"),
suspense_level=chapter_info.get("suspense_level", "中等"),
foreshadowing=chapter_info.get("foreshadowing", ""),
plot_twist_level=chapter_info.get("plot_twist_level", "★☆☆☆☆"),
chapter_summary=chapter_info.get("chapter_summary", ""),
next_chapter_number=novel_number + 1,
next_chapter_title=next_chapter_info.get("chapter_title", "(未命名)"),
next_chapter_role=next_chapter_info.get("chapter_role", "过渡章节"),
next_chapter_purpose=next_chapter_info.get("chapter_purpose", "承上启下"),
next_chapter_summary=next_chapter_info.get("chapter_summary", "衔接过渡内容"),
next_chapter_suspense_level=next_chapter_info.get("suspense_level", "中等"),
next_chapter_foreshadowing=next_chapter_info.get("foreshadowing", "无特殊伏笔"),
next_chapter_plot_twist_level=next_chapter_info.get("plot_twist_level", "★☆☆☆☆")
)
response_text = invoke_with_cleaning(llm_adapter, prompt)
summary = extract_summary_from_response(response_text)
if not summary:
logging.warning("Failed to extract summary, using full response")
return response_text[:2000] # 限制长度
return summary[:2000] # 限制摘要长度
except Exception as e:
logging.error(f"Error in summarize_recent_chapters: {str(e)}")
return ""
def extract_summary_from_response(response_text: str) -> str:
"""从响应文本中提取摘要部分"""
if not response_text:
return ""
# 查找摘要标记
summary_markers = [
"当前章节摘要:",
"章节摘要:",
"摘要:",
"本章摘要:"
]
for marker in summary_markers:
if (marker in response_text):
parts = response_text.split(marker, 1)
if len(parts) > 1:
return parts[1].strip()
return response_text.strip()
def format_chapter_info(chapter_info: dict) -> str:
"""将章节信息字典格式化为文本"""
template = """
章节编号:第{number}
章节标题:《{title}
章节定位:{role}
核心作用:{purpose}
主要人物:{characters}
关键道具:{items}
场景地点:{location}
伏笔设计:{foreshadow}
悬念密度:{suspense}
转折程度:{twist}
章节简述:{summary}
"""
return template.format(
number=chapter_info.get('chapter_number', '未知'),
title=chapter_info.get('chapter_title', '未知'),
role=chapter_info.get('chapter_role', '未知'),
purpose=chapter_info.get('chapter_purpose', '未知'),
characters=chapter_info.get('characters_involved', '未指定'),
items=chapter_info.get('key_items', '未指定'),
location=chapter_info.get('scene_location', '未指定'),
foreshadow=chapter_info.get('foreshadowing', ''),
suspense=chapter_info.get('suspense_level', '一般'),
twist=chapter_info.get('plot_twist_level', '★☆☆☆☆'),
summary=chapter_info.get('chapter_summary', '未提供')
)
prompt = summarize_recent_chapters_prompt.format(combined_text=combined_text)
response_text = invoke_with_cleaning(llm_adapter, prompt)
short_summary = ""
next_chapter_keywords = ""
for line in response_text.splitlines():
line = line.strip()
if line.startswith("短期摘要:"):
short_summary = line.replace("短期摘要:", "").strip()
elif line.startswith("下一章关键字:"):
next_chapter_keywords = line.replace("下一章关键字:", "").strip()
if not short_summary and not next_chapter_keywords:
short_summary = response_text
return (short_summary, next_chapter_keywords)
def parse_search_keywords(response_text: str) -> list:
"""解析新版关键词格式(示例输入:'科技公司·数据泄露\n地下实验室·基因编辑'"""
return [
line.strip().replace('·', ' ')
for line in response_text.strip().split('\n')
if '·' in line
][:5] # 最多取5组
def apply_content_rules(texts: list, novel_number: int) -> list:
"""应用内容处理规则"""
processed = []
for text in texts:
if re.search(r'第[\d]+章', text) or re.search(r'chapter_[\d]+', text):
chap_nums = list(map(int, re.findall(r'\d+', text)))
recent_chap = max(chap_nums) if chap_nums else 0
time_distance = novel_number - recent_chap
if time_distance <= 2:
processed.append(f"[SKIP] 跳过近章内容:{text[:120]}...")
elif 3 <= time_distance <= 5:
processed.append(f"[MOD40%] {text}(需修改≥40%")
else:
processed.append(f"[OK] {text}(可引用核心)")
else:
processed.append(f"[PRIOR] {text}(优先使用)")
return processed
def apply_knowledge_rules(contexts: list, chapter_num: int) -> list:
"""应用知识库使用规则"""
processed = []
for text in contexts:
# 检测历史章节内容
if "" in text and "" in text:
# 提取章节号判断时间远近
chap_nums = [int(s) for s in text.split() if s.isdigit()]
recent_chap = max(chap_nums) if chap_nums else 0
time_distance = chapter_num - recent_chap
# 相似度处理规则
if time_distance <= 3: # 近三章内容
processed.append(f"[历史章节限制] 跳过近期内容: {text[:50]}...")
continue
# 允许引用但需要转换
processed.append(f"[历史参考] {text} (需进行30%以上改写)")
else:
# 第三方知识优先处理
processed.append(f"[外部知识] {text}")
return processed
def get_filtered_knowledge_context(
api_key: str,
base_url: str,
model_name: str,
interface_format: str,
embedding_adapter,
filepath: str,
chapter_info: dict,
retrieved_texts: list,
max_tokens: int = 2048,
timeout: int = 600
) -> str:
"""优化后的知识过滤处理"""
if not retrieved_texts:
return "(无相关知识库内容)"
try:
processed_texts = apply_knowledge_rules(retrieved_texts, chapter_info.get('chapter_number', 0))
llm_adapter = create_llm_adapter(
interface_format=interface_format,
base_url=base_url,
model_name=model_name,
api_key=api_key,
temperature=0.3,
max_tokens=max_tokens,
timeout=timeout
)
# 限制检索文本长度并格式化
formatted_texts = []
max_text_length = 600
for i, text in enumerate(processed_texts, 1):
if len(text) > max_text_length:
text = text[:max_text_length] + "..."
formatted_texts.append(f"[预处理结果{i}]\n{text}")
# 使用格式化函数处理章节信息
formatted_chapter_info = (
f"当前章节定位:{chapter_info.get('chapter_role', '')}\n"
f"核心目标:{chapter_info.get('chapter_purpose', '')}\n"
f"关键要素:{chapter_info.get('characters_involved', '')} | "
f"{chapter_info.get('key_items', '')} | "
f"{chapter_info.get('scene_location', '')}"
)
prompt = knowledge_filter_prompt.format(
chapter_info=formatted_chapter_info,
retrieved_texts="\n\n".join(formatted_texts) if formatted_texts else "(无检索结果)"
)
filtered_content = invoke_with_cleaning(llm_adapter, prompt)
return filtered_content if filtered_content else "(知识内容过滤失败)"
except Exception as e:
logging.error(f"Error in knowledge filtering: {str(e)}")
return "(内容过滤过程出错)"
def build_chapter_prompt(
api_key: str,
@@ -90,8 +292,13 @@ def build_chapter_prompt(
timeout: int = 600
) -> str:
"""
构造当前章节的请求提示词,新增对下一章节元数据的引用
构造当前章节的请求提示词(完整实现版)
修改重点:
1. 优化知识库检索流程
2. 新增内容重复检测机制
3. 集成提示词应用规则
"""
# 读取基础文件
arch_file = os.path.join(filepath, "Novel_architecture.txt")
novel_architecture_text = read_file(arch_file)
directory_file = os.path.join(filepath, "Novel_directory.txt")
@@ -101,7 +308,7 @@ def build_chapter_prompt(
character_state_file = os.path.join(filepath, "character_state.txt")
character_state_text = read_file(character_state_file)
# 获取当前章节信息
# 获取章节信息
chapter_info = get_chapter_info_from_blueprint(blueprint_text, novel_number)
chapter_title = chapter_info["chapter_title"]
chapter_role = chapter_info["chapter_role"]
@@ -111,7 +318,7 @@ def build_chapter_prompt(
plot_twist_level = chapter_info["plot_twist_level"]
chapter_summary = chapter_info["chapter_summary"]
# 新增:获取下一章节信息
# 获取下一章节信息
next_chapter_number = novel_number + 1
next_chapter_info = get_chapter_info_from_blueprint(blueprint_text, next_chapter_number)
next_chapter_title = next_chapter_info.get("chapter_title", "(未命名)")
@@ -122,11 +329,13 @@ def build_chapter_prompt(
next_chapter_twist = next_chapter_info.get("plot_twist_level", "★☆☆☆☆")
next_chapter_summary = next_chapter_info.get("chapter_summary", "衔接过渡内容")
# 创建章节目录
chapters_dir = os.path.join(filepath, "chapters")
os.makedirs(chapters_dir, exist_ok=True)
# 第一章特殊处理
if novel_number == 1:
prompt_text = first_chapter_draft_prompt.format(
return first_chapter_draft_prompt.format(
novel_number=novel_number,
word_number=word_number,
chapter_title=chapter_title,
@@ -143,74 +352,163 @@ def build_chapter_prompt(
user_guidance=user_guidance,
novel_setting=novel_architecture_text
)
else:
recent_3_texts = get_last_n_chapters_text(chapters_dir, novel_number, n=3)
short_summary, next_chapter_keywords = summarize_recent_chapters(
# 获取前文内容和摘要
recent_texts = get_last_n_chapters_text(chapters_dir, novel_number, n=3)
try:
logging.info("Attempting to generate summary")
short_summary = summarize_recent_chapters(
interface_format=interface_format,
api_key=api_key,
base_url=base_url,
model_name=model_name,
temperature=temperature,
max_tokens=max_tokens,
chapters_text_list=recent_3_texts,
chapters_text_list=recent_texts,
novel_number=novel_number,
chapter_info=chapter_info,
next_chapter_info=next_chapter_info,
timeout=timeout
)
previous_chapter_excerpt = ""
for text_block in reversed(recent_3_texts):
if text_block.strip():
if len(text_block) > 1500:
previous_chapter_excerpt = text_block[-1500:]
else:
previous_chapter_excerpt = text_block
break
from embedding_adapters import create_embedding_adapter # 避免循环依赖
logging.info("Summary generated successfully")
except Exception as e:
logging.error(f"Error in summarize_recent_chapters: {str(e)}")
short_summary = "(摘要生成失败)"
# 获取前一章结尾
previous_excerpt = ""
for text in reversed(recent_texts):
if text.strip():
previous_excerpt = text[-800:] if len(text) > 800 else text
break
# 知识库检索和处理
try:
# 生成检索关键词
llm_adapter = create_llm_adapter(
interface_format=interface_format,
base_url=base_url,
model_name=model_name,
api_key=api_key,
temperature=0.3,
max_tokens=max_tokens,
timeout=timeout
)
search_prompt = knowledge_search_prompt.format(
chapter_number=novel_number,
chapter_title=chapter_title,
characters_involved=characters_involved,
key_items=key_items,
scene_location=scene_location,
chapter_role=chapter_role,
chapter_purpose=chapter_purpose,
foreshadowing=foreshadowing,
short_summary=short_summary,
user_guidance=user_guidance,
time_constraint=time_constraint
)
search_response = invoke_with_cleaning(llm_adapter, search_prompt)
keyword_groups = parse_search_keywords(search_response)
# 执行向量检索
all_contexts = []
from embedding_adapters import create_embedding_adapter
embedding_adapter = create_embedding_adapter(
embedding_interface_format,
embedding_api_key,
embedding_url,
embedding_model_name
)
retrieval_query = short_summary + " " + next_chapter_keywords
relevant_context = get_relevant_context_from_vector_store(
embedding_adapter=embedding_adapter,
query=retrieval_query,
filepath=filepath,
k=embedding_retrieval_k
)
if not relevant_context.strip():
relevant_context = "(无检索到的上下文)"
prompt_text = next_chapter_draft_prompt.format(
novel_number=novel_number,
word_number=word_number,
chapter_title=chapter_title,
chapter_role=chapter_role,
chapter_purpose=chapter_purpose,
suspense_level=suspense_level,
foreshadowing=foreshadowing,
plot_twist_level=plot_twist_level,
chapter_summary=chapter_summary,
characters_involved=characters_involved,
key_items=key_items,
scene_location=scene_location,
time_constraint=time_constraint,
user_guidance=user_guidance,
novel_setting=novel_architecture_text,
global_summary=global_summary_text,
character_state=character_state_text,
context_excerpt=relevant_context,
previous_chapter_excerpt=previous_chapter_excerpt,
# 新增下一章节参数
next_chapter_number=next_chapter_number,
next_chapter_title=next_chapter_title,
next_chapter_role=next_chapter_role,
next_chapter_purpose=next_chapter_purpose,
next_chapter_suspense_level=next_chapter_suspense,
next_chapter_foreshadowing=next_chapter_foreshadow,
next_chapter_plot_twist_level=next_chapter_twist,
next_chapter_summary=next_chapter_summary
store = load_vector_store(embedding_adapter, filepath)
if store:
collection_size = store._collection.count()
actual_k = min(embedding_retrieval_k, max(1, collection_size))
for group in keyword_groups:
context = get_relevant_context_from_vector_store(
embedding_adapter=embedding_adapter,
query=group,
filepath=filepath,
k=actual_k
)
if context:
if any(kw in group.lower() for kw in ["技法", "手法", "模板"]):
all_contexts.append(f"[TECHNIQUE] {context}")
elif any(kw in group.lower() for kw in ["设定", "技术", "世界观"]):
all_contexts.append(f"[SETTING] {context}")
else:
all_contexts.append(f"[GENERAL] {context}")
# 应用内容规则
processed_contexts = apply_content_rules(all_contexts, novel_number)
# 执行知识过滤
chapter_info_for_filter = {
"chapter_number": novel_number,
"chapter_title": chapter_title,
"chapter_role": chapter_role,
"chapter_purpose": chapter_purpose,
"characters_involved": characters_involved,
"key_items": key_items,
"scene_location": scene_location,
"foreshadowing": foreshadowing, # 修复拼写错误
"suspense_level": suspense_level,
"plot_twist_level": plot_twist_level,
"chapter_summary": chapter_summary,
"time_constraint": time_constraint
}
filtered_context = get_filtered_knowledge_context(
api_key=api_key,
base_url=base_url,
model_name=model_name,
interface_format=interface_format,
embedding_adapter=embedding_adapter,
filepath=filepath,
chapter_info=chapter_info_for_filter,
retrieved_texts=processed_contexts,
max_tokens=max_tokens,
timeout=timeout
)
return prompt_text
except Exception as e:
logging.error(f"知识处理流程异常:{str(e)}")
filtered_context = "(知识库处理失败)"
# 返回最终提示词
return next_chapter_draft_prompt.format(
user_guidance=user_guidance if user_guidance else "无特殊指导",
global_summary=global_summary_text,
previous_chapter_excerpt=previous_excerpt,
character_state=character_state_text,
short_summary=short_summary,
novel_number=novel_number,
chapter_title=chapter_title,
chapter_role=chapter_role,
chapter_purpose=chapter_purpose,
suspense_level=suspense_level,
foreshadowing=foreshadowing,
plot_twist_level=plot_twist_level,
chapter_summary=chapter_summary,
word_number=word_number,
characters_involved=characters_involved,
key_items=key_items,
scene_location=scene_location,
time_constraint=time_constraint,
next_chapter_number=next_chapter_number,
next_chapter_title=next_chapter_title,
next_chapter_role=next_chapter_role,
next_chapter_purpose=next_chapter_purpose,
next_chapter_suspense_level=next_chapter_suspense,
next_chapter_foreshadowing=next_chapter_foreshadow,
next_chapter_plot_twist_level=next_chapter_twist,
next_chapter_summary=next_chapter_summary,
filtered_context=filtered_context
)
def generate_chapter_draft(
api_key: str,
+1 -1
View File
@@ -29,7 +29,7 @@ def finalize_chapter(
timeout: int = 600
):
"""
对指定章节做最终处理:更新全局摘要、更新角色状态、插入向量库等。
对指定章节做最终处理:更新前文摘要、更新角色状态、插入向量库等。
默认无需再做扩写操作,若有需要可在外部调用 enrich_chapter_text 处理后再定稿。
"""
chapters_dir = os.path.join(filepath, "chapters")
+23 -29
View File
@@ -8,47 +8,41 @@ import logging
import re
import traceback
import nltk
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
import warnings
from utils import read_file
from novel_generator.vectorstore_utils import load_vector_store, init_vector_store
from langchain.docstore.document import Document
# 禁用特定的Torch警告
warnings.filterwarnings('ignore', message='.*Torch was not compiled with flash attention.*')
os.environ["TOKENIZERS_PARALLELISM"] = "false"
def advanced_split_content(content: str, similarity_threshold: float = 0.7, max_length: int = 500) -> list:
"""使用基本分段策略"""
nltk.download('punkt', quiet=True)
nltk.download('punkt_tab', quiet=True)
sentences = nltk.sent_tokenize(content)
if not sentences:
return []
model = SentenceTransformer('paraphrase-MiniLM-L6-v2')
embeddings = model.encode(sentences)
merged_paragraphs = []
current_sentences = [sentences[0]]
current_embedding = embeddings[0]
for i in range(1, len(sentences)):
sim = cosine_similarity([current_embedding], [embeddings[i]])[0][0]
if sim >= similarity_threshold:
current_sentences.append(sentences[i])
current_embedding = (current_embedding + embeddings[i]) / 2.0
else:
merged_paragraphs.append(" ".join(current_sentences))
current_sentences = [sentences[i]]
current_embedding = embeddings[i]
if current_sentences:
merged_paragraphs.append(" ".join(current_sentences))
final_segments = []
for para in merged_paragraphs:
if len(para) > max_length:
sub_segments = []
start_idx = 0
while start_idx < len(para):
end_idx = min(start_idx + max_length, len(para))
segment = para[start_idx:end_idx].strip()
sub_segments.append(segment)
start_idx = end_idx
final_segments.extend(sub_segments)
current_segment = []
current_length = 0
for sentence in sentences:
sentence_length = len(sentence)
if current_length + sentence_length > max_length:
if current_segment:
final_segments.append(" ".join(current_segment))
current_segment = [sentence]
current_length = sentence_length
else:
final_segments.append(para)
current_segment.append(sentence)
current_length += sentence_length
if current_segment:
final_segments.append(" ".join(current_segment))
return final_segments
def import_knowledge_file(
+44 -28
View File
@@ -7,10 +7,19 @@ import os
import logging
import traceback
import nltk
import numpy as np
import re
import ssl
import requests
import warnings
from langchain_chroma import Chroma
# 禁用特定的Torch警告
warnings.filterwarnings('ignore', message='.*Torch was not compiled with flash attention.*')
os.environ["TOKENIZERS_PARALLELISM"] = "false" # 禁用tokenizer并行警告
from chromadb.config import Settings
from langchain.docstore.document import Document
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
from .common import call_with_retry
@@ -131,8 +140,8 @@ def split_by_length(text: str, max_length: int = 500):
def split_text_for_vectorstore(chapter_text: str, max_length: int = 500, similarity_threshold: float = 0.7):
"""
对新的章节文本进行分段后再用于存入向量库。
先句子切分 -> 语义相似度合并 -> 再按 max_length 切分
对新的章节文本进行分段后,再用于存入向量库。
使用 embedding 进行文本相似度计算
"""
if not chapter_text.strip():
return []
@@ -143,33 +152,24 @@ def split_text_for_vectorstore(chapter_text: str, max_length: int = 500, similar
if not sentences:
return []
model = SentenceTransformer('paraphrase-MiniLM-L6-v2')
embeddings = model.encode(sentences)
merged_paragraphs = []
current_sentences = [sentences[0]]
current_embedding = embeddings[0]
for i in range(1, len(sentences)):
sim = cosine_similarity([current_embedding], [embeddings[i]])[0][0]
if sim >= similarity_threshold:
current_sentences.append(sentences[i])
current_embedding = (current_embedding + embeddings[i]) / 2.0
else:
merged_paragraphs.append(" ".join(current_sentences))
current_sentences = [sentences[i]]
current_embedding = embeddings[i]
if current_sentences:
merged_paragraphs.append(" ".join(current_sentences))
# 直接按长度分段,不做相似度合并
final_segments = []
for para in merged_paragraphs:
if len(para) > max_length:
sub_segments = split_by_length(para, max_length=max_length)
final_segments.extend(sub_segments)
current_segment = []
current_length = 0
for sentence in sentences:
sentence_length = len(sentence)
if current_length + sentence_length > max_length:
if current_segment:
final_segments.append(" ".join(current_segment))
current_segment = [sentence]
current_length = sentence_length
else:
final_segments.append(para)
current_segment.append(sentence)
current_length += sentence_length
if current_segment:
final_segments.append(" ".join(current_segment))
return final_segments
@@ -226,3 +226,19 @@ def get_relevant_context_from_vector_store(embedding_adapter, query: str, filepa
logging.warning(f"Similarity search failed: {e}")
traceback.print_exc()
return ""
def _get_sentence_transformer(model_name: str = 'paraphrase-MiniLM-L6-v2'):
"""获取sentence transformer模型,处理SSL问题"""
try:
# 设置torch环境变量
os.environ["TORCH_ALLOW_TF32_CUBLAS_OVERRIDE"] = "0"
os.environ["TORCH_CUDNN_V8_API_ENABLED"] = "0"
# 禁用SSL验证
ssl._create_default_https_context = ssl._create_unverified_context
# ...existing code...
except Exception as e:
logging.error(f"Failed to load sentence transformer model: {e}")
traceback.print_exc()
return None
+225 -85
View File
@@ -2,21 +2,156 @@
# -*- coding: utf-8 -*-
"""
集中存放所有提示词 (Prompt),整合雪花写作法、角色弧光理论、悬念三要素模型等
并包含新增加的短期摘要/下一章关键字提炼提示词,以及章节正文写作提示词。
并包含新增加的前三章摘要/下一章关键字提炼提示词,以及章节正文写作提示词。
"""
# =============== 摘要与下一章关键字提炼 ===============
# =============== 生成草稿提示词当前章节摘要、知识库提炼 ===============
# 当前章节摘要生成提示词
summarize_recent_chapters_prompt = """\
你是一名资深长篇小说编辑,请分析以下合并文本(可能包含最近几章内容)
作为一名专业的小说编辑和知识管理专家,正在基于已完成的前三章内容和本章信息生成当前章节的精准摘要。请严格遵循以下工作流程
前三章内容:
{combined_text}
当前章节信息:
{novel_number}章《{chapter_title}》:
├── 本章定位:{chapter_role}
├── 核心作用:{chapter_purpose}
├── 悬念密度:{suspense_level}
├── 伏笔操作:{foreshadowing}
├── 认知颠覆:{plot_twist_level}
└── 本章简述:{chapter_summary}
下一章信息:
{next_chapter_number}章《{next_chapter_title}》:
├── 本章定位:{next_chapter_role}
├── 核心作用:{next_chapter_purpose}
├── 悬念密度:{next_chapter_suspense_level}
├── 伏笔操作:{next_chapter_foreshadowing}
├── 认知颠覆:{next_chapter_plot_twist_level}
└── 本章简述:{next_chapter_summary}
**上下文分析阶段**
1. 回顾前三章核心内容:
- 第一章核心要素:[章节标题]→[核心冲突/理论]→[关键人物/概念]
- 第二章发展路径:[已建立的人物关系]→[技术/情节进展]→[遗留伏笔]
- 第三章转折点:[新出现的变量]→[世界观扩展]→[待解决问题]
2. 提取延续性要素:
- 必继承要素:列出前3章中必须延续的3个核心设定
- 可调整要素:识别2个允许适度变化的辅助设定
**当前章节摘要生成规则**
1. 内容架构:
- 继承权重:70%内容需与前3章形成逻辑递进
- 创新空间:30%内容可引入新要素,但需标注创新类型(如:技术突破/人物黑化)
2. 结构控制:
- 采用"承继→发展→铺垫"三段式结构
- 每段含1个前文呼应点+1个新进展
3. 预警机制:
- 若检测到与前3章设定冲突,用[!]标记并说明
- 对开放式发展路径,提供2种合理演化方向
现在请你基于目前故事的进展,完成以下两件事:
1) 用最多200字,写一个简洁明了的「当前情节短期摘要」
2) 提炼「下一章」的关键字(例如关键物品、重要人物、地点、事件、情节等),可以用逗号分隔或条目列出。
用最多800字,写一个简洁明了的「当前章节摘要」
请按如下格式输出(不需要额外解释):
短期摘要: <这里写短期摘要>
下一章关键字: <这里写下一章关键字>
当前章节摘要: <这里写当前章节摘要>
"""
# 知识库相关性检索提示词
knowledge_search_prompt = """\
请基于以下当前写作需求,生成合适的知识库检索关键词:
章节元数据:
- 准备创作:第{chapter_number}
- 章节主题:{chapter_title}
- 核心人物:{characters_involved}
- 关键道具:{key_items}
- 场景位置:{scene_location}
写作目标:
- 本章定位:{chapter_role}
- 核心作用:{chapter_purpose}
- 伏笔操作:{foreshadowing}
当前摘要:
{short_summary}
- 用户指导:
{user_guidance}
- 核心人物(可能未指定){characters_involved}
- 关键道具(可能未指定){key_items}
- 空间坐标(可能未指定){scene_location}
- 时间压力(可能未指定){time_constraint}
生成规则:
1.关键词组合逻辑:
-类型1[实体]+[属性](如"量子计算机 故障日志"
-类型2[事件]+[后果](如"实验室爆炸 辐射泄漏"
-类型3[地点]+[特征](如"地下城 氧气循环系统"
2.优先级:
-首选用户指导中明确提及的术语
-次选当前章节涉及的核心道具/地点
-最后补充可能关联的扩展概念
3.过滤机制:
-排除抽象程度高于"中级"的概念
-排除与前3章重复率超60%的词汇
请生成3-5组检索词,按优先级降序排列。
格式:每组用"·"连接2-3个关键词,每组占一行
示例:
科技公司·数据泄露
地下实验室·基因编辑·禁忌实验
"""
# 知识库内容过滤提示词
knowledge_filter_prompt = """\
对知识库内容进行三级过滤:
待过滤内容:
{retrieved_texts}
当前叙事需求:
{chapter_info}
过滤流程:
冲突检测:
删除与已有摘要重复度>40%的内容
标记存在世界观矛盾的内容(使用▲前缀)
价值评估:
关键价值点(❗标记):
· 提供新的角色关系可能性
· 包含可转化的隐喻素材
· 存在至少2个可延伸的细节锚点
次级价值点(·标记):
· 补充环境细节
· 提供技术/流程描述
结构重组:
"情节燃料/人物维度/世界碎片/叙事技法"分类
为每个分类添加适用场景提示(如"可用于XX类型伏笔"
输出格式:
[分类名称]→[适用场景]
❗/· [内容片段] (▲冲突提示)
...
示例:
[情节燃料]→可用于时间压力类悬念
❗ 地下氧气系统剩余23%储量(可制造生存危机)
▲ 与第三章提到的"永久生态循环系统"存在设定冲突
"""
# =============== 1. 核心种子设定(雪花第1层)===================
@@ -219,22 +354,22 @@ chunked_chapter_blueprint_prompt = """\
仅给出最终文本,不要解释任何内容。
"""
# =============== 6. 全局摘要更新 ===================
# =============== 6. 前文摘要更新 ===================
summary_prompt = """\
以下是新完成的章节文本:
{chapter_text}
这是当前的全局摘要(可为空):
这是当前的前文摘要(可为空):
{global_summary}
请根据本章新增内容,更新全局摘要。
请根据本章新增内容,更新前文摘要。
要求:
- 保留既有重要信息,同时融入新剧情要点
- 以简洁、连贯的语言描述全书进展
- 客观描绘,不展开联想或解释
- 字数控制在2000字以内
- 字数控制在2000字以内
仅返回全局摘要文本,不要解释任何内容。
仅返回前文摘要文本,不要解释任何内容。
"""
# =============== 7. 角色状态更新 ===================
@@ -243,7 +378,7 @@ create_character_state_prompt = """\
请生成一个角色状态文档,内容格式:
例:
李员外
张三
├──物品:
│ ├──青衫:一件破损的青色长袍,带有暗红色的污渍
│ └──寒铁长剑:一柄断裂的铁剑,剑身上刻有古老的符文
@@ -254,11 +389,11 @@ create_character_state_prompt = """\
│ ├──身体状态: 身材挺拔,穿着华丽的铠甲,面色冷峻
│ └──心理状态: 目前的心态比较平静,但内心隐藏着对柳溪镇未来掌控的野心和不安
├──主要角色间关系网
│ ├──林婉儿:李员外从小就与她有关联,对她的成长一直保持关注
│ └──苏明远:两人之间有着复杂的过去,最近因一场冲突而让对方感到威胁
│ ├──李四:张三从小就与她有关联,对她的成长一直保持关注
│ └──王二:两人之间有着复杂的过去,最近因一场冲突而让对方感到威胁
├──触发或加深的事件
│ ├──村庄内突然出现不明符号:这个不明符号似乎在暗示柳溪镇即将发生重大事件
│ └──林婉儿被刺穿皮肤:这次事件让两人意识到对方的强大实力,促使他们迅速离开队伍
│ └──李四被刺穿皮肤:这次事件让两人意识到对方的强大实力,促使他们迅速离开队伍
角色名:
├──物品:
@@ -274,8 +409,8 @@ create_character_state_prompt = """\
│ └──心理状态:描述
├──主要角色间关系网
│ ├──角色B:描述
│ └──角色C:描述
│ ├──李四:描述
│ └──王二:描述
│ ...
├──触发或加深的事件
│ ├──事件1:描述
@@ -298,7 +433,7 @@ update_character_state_prompt = """\
请更新主要角色状态,内容格式:
例:
李员外
张三
├──物品:
│ ├──青衫:一件破损的青色长袍,带有暗红色的污渍
│ └──寒铁长剑:一柄断裂的铁剑,剑身上刻有古老的符文
@@ -309,11 +444,11 @@ update_character_state_prompt = """\
│ ├──身体状态: 身材挺拔,穿着华丽的铠甲,面色冷峻
│ └──心理状态: 目前的心态比较平静,但内心隐藏着对柳溪镇未来掌控的野心和不安
├──主要角色间关系网
│ ├──林婉儿:李员外从小就与她有关联,对她的成长一直保持关注
│ └──苏明远:两人之间有着复杂的过去,最近因一场冲突而让对方感到威胁
│ ├──李四:张三从小就与她有关联,对她的成长一直保持关注
│ └──王二:两人之间有着复杂的过去,最近因一场冲突而让对方感到威胁
├──触发或加深的事件
│ ├──村庄内突然出现不明符号:这个不明符号似乎在暗示柳溪镇即将发生重大事件
│ └──林婉儿被刺穿皮肤:这次事件让两人意识到对方的强大实力,促使他们迅速离开队伍
│ └──李四被刺穿皮肤:这次事件让两人意识到对方的强大实力,促使他们迅速离开队伍
角色名:
├──物品:
@@ -329,8 +464,8 @@ update_character_state_prompt = """\
│ └──心理状态:描述
├──主要角色间关系网
│ ├──角色B:描述
│ └──角色C:描述
│ ├──李四:描述
│ └──王二:描述
│ ...
├──触发或加深的事件
│ ├──事件1:描述
@@ -371,11 +506,10 @@ first_chapter_draft_prompt = """\
- 小说设定:
{novel_setting}
完成第 {novel_number} 章的正文,字数要求{word_number}字,至少设计下方2个或以上具有动态张力的场景:
完成第 {novel_number} 章的正文,字数要求{word_number}字,至少设计下方2个或以上具有动态张力的场景:
1. 对话场景:
- 潜台词冲突(表面谈论A,实际博弈B)
- 权力关系变化(通过非对称对话长度体现)
- 至少1处双关语暗示未来危机
2. 动作场景:
- 环境交互细节(至少3个感官描写)
@@ -391,9 +525,6 @@ first_chapter_draft_prompt = """\
- 空间透视变化(宏观→微观→异常焦点)
- 非常规感官组合(如"听见阳光的重量"
- 动态环境反映心理(环境与人物心理对应)
- 隐藏线索植入(环境暗示未来事件)
文末设置一个"钩链转折":结尾时回收旧悬念/创造新悬念/抛出新危机/颠覆某个认知/神转折等。
格式要求:
- 仅返回章节正文文本;
@@ -406,80 +537,89 @@ first_chapter_draft_prompt = """\
# 8.2 后续章节草稿提示
next_chapter_draft_prompt = """\
参考文档:
- 小说设定
{novel_setting}
└── 前文摘要
{global_summary}
- 全局摘要
{global_summary}
└── 前章结尾段
{previous_chapter_excerpt}
- 角色状态
{character_state}
└── 用户指导
{user_guidance}
本地知识库检索到的片段
{context_excerpt}
└── 角色状态
{character_state}
即将创作:第 {novel_number} 章《{chapter_title}
本章定位:{chapter_role}
核心作用:{chapter_purpose}
悬念密度:{suspense_level}
伏笔操作:{foreshadowing}
认知颠覆:{plot_twist_level}
本章简述:{chapter_summary}
└── 当前章节摘要:
{short_summary}
下一章节介绍
{next_chapter_number} 章《{next_chapter_title}
本章定位:{next_chapter_role}
核心作用:{next_chapter_purpose}
悬念密度:{next_chapter_suspense_level}
伏笔操作:{next_chapter_foreshadowing}
认知颠覆:{next_chapter_plot_twist_level}
本章简述:{next_chapter_summary}
参考下一章内容简介,避免情节脱节或冲突。
当前章节信息
{novel_number}章《{chapter_title}
├── 章节定位:{chapter_role}
├── 核心作用:{chapter_purpose}
├── 悬念密度:{suspense_level}
├── 伏笔设计:{foreshadowing}
├── 转折程度:{plot_twist_level}
├── 章节简述:{chapter_summary}
├── 字数要求:{word_number}
├── 核心人物:{characters_involved}
├── 关键道具:{key_items}
├── 场景地点:{scene_location}
└── 时间压力:{time_constraint}
可用元素:
- 核心人物(可能未指定){characters_involved}
- 关键道具(可能未指定){key_items}
- 空间坐标(可能未指定){scene_location}
- 时间压力(可能未指定){time_constraint}
下一章节目录
{next_chapter_number}章《{next_chapter_title}》:
├── 章节定位:{next_chapter_role}
├── 核心作用:{next_chapter_purpose}
├── 悬念密度:{next_chapter_suspense_level}
├── 伏笔设计:{next_chapter_foreshadowing}
├── 转折程度:{next_chapter_plot_twist_level}
└── 章节简述:{next_chapter_summary}
前章结尾段:
{previous_chapter_excerpt}
知识库参考:(按优先级应用)
{filtered_context}
依据前章结尾剧情,开始完成第 {novel_number} 章的正文,字数要求{word_number}字,确保与前章结尾衔接流畅,
🎯 知识库应用规则:
1. 内容分级:
- 写作技法类(优先):
▸ 场景构建模板
▸ 对话写作技巧
▸ 悬念营造手法
- 设定资料类(选择性):
▸ 独特世界观元素
▸ 未使用过的技术细节
- 禁忌项类(必须规避):
▸ 已在前文出现过的特定情节
▸ 重复的人物关系发展
本章至少设计下方2个或以上具有动态张力的场景
1. 对话场景:
- 潜台词冲突(表面谈论A,实际博弈B)
- 权力关系变化(通过非对称对话长度体现
- 至少1处双关语暗示未来危机
2. 使用限制
● 禁止直接复制已有章节的情节模式
● 历史章节内容仅允许:
→ 参照叙事节奏(不超过20%相似度
→ 延续必要的人物反应模式(需改编30%以上)
● 第三方写作知识优先用于:
→ 增强场景表现力(占知识应用的60%以上)
→ 创新悬念设计(至少1处新技巧)
2. 动作场景
- 环境交互细节(至少3个感官描写)
- 节奏控制(短句加速+比喻减速)
- 动作揭示人物隐藏特质
3. 冲突检测
⚠️ 若检测到与历史章节重复:
- 相似度>40%:必须重构叙事角度
- 相似度20-40%:替换至少3个关键要素
- 相似度<20%:允许保留核心概念但改变表现形式
3. 心理场景:
- 认知失调的具体表现(行为矛盾)
- 隐喻系统的运用(连接世界观符号)
- 决策前的价值天平描写
4. 环境场景:
- 空间透视变化(宏观→微观→异常焦点)
- 非常规感官组合(如"听见阳光的重量"
- 动态环境反映心理(环境与人物心理对应)
- 隐藏线索植入(环境暗示未来事件)
文末设置一个"钩链转折":结尾时回收旧悬念/创造新悬念/抛出新危机/颠覆某个认知/神转折等。
依据前面所有设定,开始完成第 {novel_number} 章的正文,字数要求{word_number}字,
内容生成严格遵循:
-用户指导
-当前章节摘要
-当前章节信息
-无逻辑漏洞,
确保章节内容与前文摘要、前章结尾段衔接流畅、下一章目录保证上下文完整性,
格式要求:
- 仅返回章节正文文本;
- 不使用分章节小标题;
- 不要使用markdown格式。
额外指导(可能未指定){user_guidance}
"""
Character_Import_Prompt = """\
根据以下文本内容,分析出所有角色及其属性信息,严格按照以下格式要求:
BIN
View File
Binary file not shown.
+48 -13
View File
@@ -77,7 +77,7 @@ def generate_chapter_blueprint_ui(self):
return
def task():
if not messagebox.askyesno("确认", "确定要生成章节草稿吗?"):
if not messagebox.askyesno("确认", "确定要生成章节目录吗?"):
self.enable_button_safe(self.btn_generate_chapter)
return
self.disable_button_safe(self.btn_generate_directory)
@@ -90,6 +90,7 @@ def generate_chapter_blueprint_ui(self):
temperature = self.temperature_var.get()
max_tokens = self.max_tokens_var.get()
timeout_val = self.safe_get_int(self.timeout_var, 600)
user_guidance = self.user_guide_text.get("0.0", "end").strip() # 新增获取用户指导
self.safe_log("开始生成章节蓝图...")
Chapter_blueprint_generate(
@@ -101,7 +102,8 @@ def generate_chapter_blueprint_ui(self):
filepath=filepath,
temperature=temperature,
max_tokens=max_tokens,
timeout=timeout_val
timeout=timeout_val,
user_guidance=user_guidance # 新增参数
)
self.safe_log("✅ 章节蓝图生成完成。请在 'Chapter Blueprint' 标签页查看或编辑。")
except Exception:
@@ -371,7 +373,7 @@ def finalize_chapter_ui(self):
max_tokens=max_tokens,
timeout=timeout_val
)
self.safe_log(f"✅ 第{chap_num}章定稿完成(已更新全局摘要、角色状态、向量库)。")
self.safe_log(f"✅ 第{chap_num}章定稿完成(已更新前文摘要、角色状态、向量库)。")
final_text = read_file(chapter_file)
self.master.after(0, lambda: self.show_chapter_in_textbox(final_text))
@@ -443,20 +445,53 @@ def import_knowledge_handler(self):
emb_format = self.embedding_interface_format_var.get().strip()
emb_model = self.embedding_model_name_var.get().strip()
self.safe_log(f"开始导入知识库文件: {selected_file}")
import_knowledge_file(
embedding_api_key=emb_api_key,
embedding_url=emb_url,
embedding_interface_format=emb_format,
embedding_model_name=emb_model,
file_path=selected_file,
filepath=self.filepath_var.get().strip()
)
self.safe_log("✅ 知识库文件导入完成。")
# 尝试不同编码读取文件
content = None
encodings = ['utf-8', 'gbk', 'gb2312', 'ansi']
for encoding in encodings:
try:
with open(selected_file, 'r', encoding=encoding) as f:
content = f.read()
break
except UnicodeDecodeError:
continue
except Exception as e:
self.safe_log(f"读取文件时发生错误: {str(e)}")
raise
if content is None:
raise Exception("无法以任何已知编码格式读取文件")
# 创建临时UTF-8文件
import tempfile
import os
with tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', delete=False, suffix='.txt') as temp:
temp.write(content)
temp_path = temp.name
try:
self.safe_log(f"开始导入知识库文件: {selected_file}")
import_knowledge_file(
embedding_api_key=emb_api_key,
embedding_url=emb_url,
embedding_interface_format=emb_format,
embedding_model_name=emb_model,
file_path=temp_path,
filepath=self.filepath_var.get().strip()
)
self.safe_log("✅ 知识库文件导入完成。")
finally:
# 清理临时文件
try:
os.unlink(temp_path)
except:
pass
except Exception:
self.handle_exception("导入知识库时出错")
finally:
self.enable_button_safe(self.btn_import_knowledge)
try:
thread = threading.Thread(target=task, daemon=True)
thread.start()