ai-ml-domain-llm/pyproject.toml at main · Bansnetsajak007/ai-ml-domain-llm · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
[project]
name = "dataset-for-llm"
version = "0.1.0"
description = "Dataset collection, validation and cleaning pipeline for LLM training data"
readme = "README.md"
requires-python = ">=3.11"

dependencies = [
    # Web & API
    "flask",
    "openai>=1.0.0",
    "python-dotenv",
    "fastmcp",
    "google-genai",
    # Database
    "pymongo",
    "postgrest",
    # Data Collection - Z-Library
    "playwright",
    "numpy",
    # Data Collection - arXiv
    "aiohttp>=3.8.0",
    "aiofiles>=23.0.0",
    "certifi",
    # OCR Pipeline
    "pymupdf",
    # >=1.0 for the use_ocr kwarg (see ocr_pipeline/pipeline_md_latex.py)
    "pymupdf4llm>=1.0",
    "pytesseract",
    "pdf2image",
    "ebooklib",
    "beautifulsoup4",
    "python-docx",
    "tqdm",
    # Utils
    "python-levenshtein",
]

[tool.uv]
# This repo is a collection of scripts/agents, not an installable package.
package = false