commit f697ab193f83e1f005bf331d0a8f972e23b27d5a Author: Tanishq Dubey Date: Mon Sep 23 18:20:00 2024 -0400 Initial Commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..80abc65 --- /dev/null +++ b/.gitignore @@ -0,0 +1,176 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Virtualenv +# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ +.Python +[Bb]in +[Ss]hare +[Ii]nclude +[Ll]ib +[Ll]ib64 +[Ll]ocal +[Ss]cripts +pyvenv.cfg +.venv +pip-selfcheck.json diff --git a/main.py b/main.py new file mode 100644 index 0000000..061080d --- /dev/null +++ b/main.py @@ -0,0 +1,22 @@ +from vllm import LLM +from vllm.sampling_params import SamplingParams + +model_name = "mistralai/Pixtral-12B-2409" + +sampling_params = SamplingParams(max_tokens=8192) + +llm = LLM(model=model_name, tokenizer_mode="mistral") + +prompt = "Describe this image in one sentence." +image_url = "https://picsum.photos/id/237/200/300" + +messages = [ + { + "role": "user", + "content": [{"type": "text", "text": prompt}, {"type": "image_url", "image_url": {"url": image_url}}] + }, +] + +outputs = vllm_model.model.chat(messages, sampling_params=sampling_params) + +print(outputs[0].outputs[0].text) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..642aa96 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,119 @@ +accelerate==0.33.0 +aiohappyeyeballs==2.4.0 +aiohttp==3.10.5 +aiosignal==1.3.1 +annotated-types==0.7.0 +anyio==4.6.0 +attrs==24.2.0 +certifi==2024.7.4 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +datasets==3.0.0 +dill==0.3.8 +diskcache==5.6.3 +distro==1.9.0 +duckduckgo_search==6.2.6 +einops==0.8.0 +fastapi==0.115.0 +filelock==3.15.4 +frozenlist==1.4.1 +fsspec==2024.6.1 +gguf==0.9.1 +h11==0.14.0 +httpcore==1.0.5 +httptools==0.6.1 +httpx==0.27.2 +huggingface-hub==0.24.5 +idna==3.7 +importlib_metadata==8.5.0 +inquirerpy==0.3.4 +interegular==0.3.3 +Jinja2==3.1.4 +jiter==0.5.0 +jsonschema==4.23.0 +jsonschema-specifications==2023.12.1 +lark==1.2.2 +llvmlite==0.43.0 +lm-format-enforcer==0.10.6 +MarkupSafe==2.1.5 +mistral_common==1.4.3 +mpmath==1.3.0 +msgpack==1.1.0 +msgspec==0.18.6 +multidict==6.1.0 +multiprocess==0.70.16 +nest-asyncio==1.6.0 +networkx==3.3 +numba==0.60.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-ml-py==12.560.30 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.6.20 +nvidia-nvtx-cu12==12.1.105 +openai==1.47.1 +outlines==0.0.46 +packaging==24.1 +pandas==2.2.3 +partial-json-parser==0.2.1.1.post4 +pfzy==0.3.4 +pillow==10.4.0 +primp==0.5.5 +prometheus-fastapi-instrumentator==7.0.0 +prometheus_client==0.21.0 +prompt_toolkit==3.0.47 +protobuf==5.28.2 +psutil==6.0.0 +py-cpuinfo==9.0.0 +pyairports==2.1.1 +pyarrow==17.0.0 +pycountry==24.6.1 +pydantic==2.9.2 +pydantic_core==2.23.4 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 +pytz==2024.2 +PyYAML==6.0.2 +pyzmq==26.2.0 +ray==2.36.1 +referencing==0.35.1 +regex==2024.7.24 +requests==2.32.3 +rpds-py==0.20.0 +safetensors==0.4.4 +sentencepiece==0.2.0 +setuptools==72.1.0 +six==1.16.0 +sniffio==1.3.1 +starlette==0.38.6 +sympy==1.13.2 +tiktoken==0.7.0 +tokenizers==0.19.1 +torch==2.4.0 +torchvision==0.19.0 +tqdm==4.66.5 +transformers==4.44.0 +triton==3.0.0 +typing_extensions==4.12.2 +tzdata==2024.2 +urllib3==2.2.2 +uvicorn==0.30.6 +uvloop==0.20.0 +vllm==0.6.1.post2 +vllm-flash-attn==2.6.1 +watchfiles==0.24.0 +wcwidth==0.2.13 +websockets==13.1 +xformers==0.0.27.post2 +xxhash==3.5.0 +yarl==1.12.0 +zipp==3.20.2