Direction9/database/verify_download.py

#!/usr/bin/env python
"""
验证下载的数据集
"""

from database.corpus import CorpusLoader

def main():
    print("=" * 50)
    print("验证数据集")
    print("=" * 50)
    print()

    loader = CorpusLoader("database/corpus")
    domains = loader.list_available_domains()

    if not domains:
        print("未找到任何语料数据！")
        print("请先运行：python database/download_50_samples.py")
        return

    print(f"可用领域：{domains}")
    print()

    for domain in domains:
        texts = loader.load_texts(domain, limit=5)
        stats = loader.get_corpus_stats(domain)

        print(f"[{domain}]")
        print(f"  语料总数：{stats['num_texts']} 条")
        print(f"  平均长度：{stats['avg_chars']:.1f} 字符")
        print(f"  前 3 条预览:")
        for i, text in enumerate(texts[:3]):
            preview = text[:60] + "..." if len(text) > 60 else text
            print(f"    [{i}] {preview}")
        print()

    print("=" * 50)
    print("验证完成!")
    print("=" * 50)


if __name__ == "__main__":
    main()