{"id":28442,"library":"vllm-tpu","title":"vLLM TPU","description":"vLLM TPU is a variant of vLLM that runs on Google Cloud TPUs (v5e/v5p). It provides a high-throughput and memory-efficient inference and serving engine for large language models, leveraging TPU-specific optimizations like Pallas kernels for attention and quantization. The current version is 0.19.0, following the main vLLM release cadence (monthly).","status":"active","version":"0.19.0","language":"python","source_language":"en","source_url":"https://github.com/vllm-project/vllm","tags":["vllm","tpu","inference","llm"],"install":[{"cmd":"pip install vllm-tpu","lang":"bash","label":"PyPI"}],"dependencies":[{"reason":"Required for tensor operations on TPU (via torch_xla).","package":"torch","optional":false},{"reason":"Required for TPU runtime integration.","package":"torch_xla","optional":false},{"reason":"Required for Pallas kernels and TPU-specific ops.","package":"jax","optional":false}],"imports":[{"wrong":"from vllm import LLM","symbol":"LLM","correct":"from vllm import LLM"}],"quickstart":{"code":"import os\nos.environ['VLLM_TPU'] = '1'  # Optional: explicitly enable TPU backend\nfrom vllm import LLM, SamplingParams\n\nprompts = [\"Hello, my name is\", \"The capital of France is\"]\nsampling_params = SamplingParams(temperature=0.8, top_p=0.95)\n\nllm = LLM(model=\"Qwen/Qwen2.5-1.5B\", max_num_seqs=8)\noutputs = llm.generate(prompts, sampling_params)\nfor output in outputs:\n    print(output.outputs[0].text)","lang":"python","description":"Basic inference with a small model on TPU. Assumes a TPU VM (v5e/v5p) with torch_xla installed."},"warnings":[{"fix":"Verify model compatibility before use; refer to the vLLM TPU docs.","message":"vLLM TPU is experimental and does not support all features of the main vLLM (e.g., tensor parallelism, quantization). Check the official docs for supported model architectures and features.","severity":"gotcha","affected_versions":"<0.20"},{"fix":"Provision a TPU VM and install the TPU runtime: https://cloud.google.com/tpu/docs/users-guide-tpu-vm","message":"You must run on a TPU VM (v5e/v5p) with torch_xla installed. Installing vllm-tpu on CPU/GPU will fail.","severity":"gotcha","affected_versions":"all"},{"fix":"Uninstall vllm first: pip uninstall vllm; then pip install vllm-tpu.","message":"As of v0.19.0, the vllm-tpu package is a separate PyPI package from vllm. Mixing installations may cause conflicts.","severity":"breaking","affected_versions":">=0.19.0"}],"env_vars":null,"search_vec":"'0.19.0':51 'attent':44 'cadenc':57 'cloud':14 'current':48 'effici':26 'engin':30 'follow':52 'googl':13 'high':21 'high-throughput':20 'infer':27,61 'kernel':42 'languag':33 'larg':32 'leverag':35 'like':40 'llm':62 'main':54 'memori':25 'memory-effici':24 'model':34 'month':58 'optim':39 'palla':41 'provid':18 'quantiz':46 'releas':56 'run':11 'serv':29 'specif':38 'throughput':22 'tpu':2,4,37,60 'tpu-specif':36 'tpus':15 'v5e/v5p':16 'variant':7 'version':49 'vllm':1,3,9,55,59","created_at":"2026-05-09T05:55:32.275663+00:00","updated_at":"2026-05-09T05:55:32.275663+00:00","problems":{"verify_error":"Traceback (most recent call last):\n  File \"<string>\", line 1, in <module>\n  File \"<frozen importlib._bootstrap>\", line 1412, in _handle_fromlist\n  File \"/tmp/tmp2bbu5e8y/venv/lib/python3.12/site-packages/vllm/__init__.py\", line 70, in __getattr__\n    module = import_module(module_name, __package__)\n"},"ecosystem":"pypi","meta_description":null,"install_score":null,"quickstart_score":null,"quickstart_tag":null,"pypi_latest":null,"cli_name":"vllm","cli_version":"","type":"library","homepage":null,"github":"https://github.com/vllm-project/vllm","docs":"https://docs.vllm.ai/en/latest/","changelog":null,"pypi":"https://pypi.org/project/vllm-tpu/","npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"categories":["llm-agents","ai-ml","gcp"],"base_url":null,"auth_type":null,"provenance":{"verified_status":"import_fail","verified_at":"2026-07-03","last_verified":"2026-07-03","next_check":"2026-07-10","install_tag":null}}