{"id":28402,"library":"torchscale","title":"torchscale","description":"torchscale is a PyTorch library for building large-scale Transformer models, providing components like Multi-head Attention (MHA), Long Short-Term Memory (LSTM), and other scalable architectures. As of version 0.3.0, it supports Python >=3.8 and is maintained by Microsoft. Releases are infrequent.","status":"active","version":"0.3.0","language":"python","source_language":"en","source_url":"https://github.com/microsoft/torchscale","tags":["transformer","scale","pytorch","microsoft","deep-learning"],"install":[{"cmd":"pip install torchscale","lang":"bash","label":"PyPI install"}],"dependencies":[{"reason":"Core dependency for all tensor operations and models.","package":"torch","optional":false},{"reason":"Used for tensor rearrangement in attention modules.","package":"einops","optional":true},{"reason":"Optional for loading Hugging Face model weights.","package":"transformers","optional":true}],"imports":[{"wrong":"from torchscale.component import MHA","symbol":"MHA","correct":"from torchscale.component import MHA"}],"quickstart":{"code":"import torch\nfrom torchscale.component import MHA\nfrom torchscale.model import IncrementalDecoder\n\n# Example: Multi-head attention\nmha = MHA(embed_dim=512, num_heads=8)\nx = torch.randn(4, 10, 512)\noutput = mha(x, x, x)\nprint(output.shape)\n\n# Example: Decoder\ndecoder = IncrementalDecoder(\n    vocab_size=1000,\n    embed_dim=512,\n    num_heads=8,\n    num_layers=6,\n)\ntokens = torch.randint(0, 1000, (4, 20))\nlogits = decoder(tokens)\nprint(logits.shape)","lang":"python","description":"Initialize MHA and IncrementalDecoder with random input."},"warnings":[{"fix":"Replace `LongShortTerm` with `IncrementalDecoder` or `TemporalDecoder` depending on use case.","message":"The `torchscale.model.LongShortTerm` class is deprecated in 0.3.0; use `IncrementalDecoder` or `TemporalDecoder` instead.","severity":"deprecated","affected_versions":">=0.3.0"},{"fix":"Remove `kdim` and `vdim` from MHA constructor and ensure all dimensions match `embed_dim`.","message":"In version 0.3.0, the `MHA` class no longer accepts `kdim` and `vdim` arguments; use `embed_dim` for all.","severity":"breaking","affected_versions":">=0.3.0"},{"fix":"Ensure input tensors have shape (batch, sequence, features) or use .transpose() if needed.","message":"torchscale components expect batch-first tensors (batch, seq, dim), not sequence-first. Incorrect ordering may cause shape mismatches.","severity":"gotcha","affected_versions":"all"}],"env_vars":null,"search_vec":"'0.3.0':35 '3.8':39 'architectur':31 'attent':20 'build':8 'compon':15 'deep':53 'deep-learn':52 'head':19 'infrequ':47 'larg':10 'large-scal':9 'learn':54 'librari':6 'like':16 'long':22 'lstm':27 'maintain':42 'memori':26 'mha':21 'microsoft':44,51 'model':13 'multi':18 'multi-head':17 'provid':14 'python':38 'pytorch':5,50 'releas':45 'scalabl':30 'scale':11,49 'short':24 'short-term':23 'support':37 'term':25 'torchscal':1,2 'transform':12,48 'version':34","created_at":"2026-05-09T05:55:19.734215+00:00","updated_at":"2026-05-09T05:55:19.734215+00:00","problems":{"verify_error":"Traceback (most recent call last):\n  File \"<string>\", line 1, in <module>\nImportError: cannot import name 'MHA' from 'torchscale.component' (/tmp/tmpfjfc35xm/venv/lib/python3.12/site-packages/torchscale/component/__init__.py)"},"ecosystem":"pypi","meta_description":null,"install_score":null,"quickstart_score":null,"quickstart_tag":null,"pypi_latest":null,"cli_name":"","cli_version":null,"type":"library","homepage":null,"github":"https://github.com/microsoft/torchscale","docs":null,"changelog":null,"pypi":"https://pypi.org/project/torchscale/","npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"categories":["ai-ml","llm-agents"],"base_url":null,"auth_type":null,"provenance":{"verified_status":"import_fail","verified_at":"2026-07-03","last_verified":"2026-07-03","next_check":"2026-07-10","install_tag":null}}