{"id":46302,"library":"sagemaker-huggingface-inference-toolkit","title":"SageMaker Hugging Face Inference Toolkit","description":"Open source library for serving Hugging Face models on Amazon SageMaker. It wraps the Hugging Face Transformers pipeline and provides built-in handlers for inference. Latest version 2.6.0 supports PyTorch 2.0+ and TensorFlow 2.x. Released quarterly.","status":"active","version":"2.6.0","language":"python","source_language":"en","source_url":"https://github.com/aws/sagemaker-huggingface-inference-toolkit","tags":["sagemaker","huggingface","inference","deployment","aws"],"install":[{"cmd":"pip install sagemaker-huggingface-inference-toolkit","lang":"bash","label":"Install from PyPI"}],"dependencies":[{"reason":"Core dependency for model loading and inference.","package":"transformers","optional":false},{"reason":"Required for PyTorch models.","package":"torch","optional":true},{"reason":"Required for TensorFlow models.","package":"tensorflow","optional":true},{"reason":"For deployment and endpoint interaction.","package":"sagemaker","optional":false}],"imports":[{"wrong":"from sagemaker.huggingface import HuggingFaceModel","symbol":"HuggingFaceModel","correct":"from sagemaker_huggingface_inference_toolkit import HuggingFaceModel"}],"quickstart":{"code":"from sagemaker.huggingface import HuggingFaceModel\nimport os\n\nhub = {\n    'HF_MODEL_ID':'distilbert-base-uncased-distilled-squad',\n    'HF_TASK':'question-answering'\n}\n\nhuggingface_model = HuggingFaceModel(\n    env=hub,\n    role='SageMakerRole',\n    transformers_version='4.26',\n    pytorch_version='1.13',\n    py_version='py39',\n)\n\npredictor = huggingface_model.deploy(\n    initial_instance_count=1,\n    instance_type='ml.m5.large'\n)\n\ndata = {\n    'question': 'What is the capital of France?',\n    'context': 'France is a country in Europe. Its capital is Paris.'\n}\n\nresponse = predictor.predict(data)\nprint(response)","lang":"python","description":"Deploy a Hugging Face model on SageMaker using the inference toolkit."},"warnings":[{"fix":"Update imports to `from sagemaker.huggingface import HuggingFaceModel`.","message":"In version 2.0+, the default SageMaker SDK integration changed. Use `HuggingFaceModel` from `sagemaker.huggingface` instead of `sagemaker.model.Model`.","severity":"breaking","affected_versions":">=2.0"},{"fix":"Refer to Hugging Face pipeline tasks: https://huggingface.co/docs/transformers/main_classes/pipelines","message":"The `HF_TASK` environment variable must match a valid Hugging Face pipeline task. Common mistakes: using 'text-classification' instead of 'text-classification' (correct) or misspelling.","severity":"gotcha","affected_versions":"all"},{"fix":"Ensure the inference script is named `inference.py` and included in the model tar.gz file.","message":"When using a custom inference script, the entry point must be named `inference.py` and placed in the same directory as the model artifacts.","severity":"gotcha","affected_versions":"all"}],"env_vars":null,"search_vec":"'2':40 '2.0':37 '2.6.0':34 'amazon':15 'aw':48 'built':27 'built-in':26 'deploy':47 'face':3,12,21 'handler':29 'hug':2,11,20 'huggingfac':45 'infer':4,31,46 'latest':32 'librari':8 'model':13 'open':6 'pipelin':23 'provid':25 'pytorch':36 'quarter':43 'releas':42 'sagemak':1,16,44 'serv':10 'sourc':7 'support':35 'tensorflow':39 'toolkit':5 'transform':22 'version':33 'wrap':18 'x':41","created_at":"2026-06-07T12:59:16.496613+00:00","updated_at":"2026-06-07T12:59:16.496613+00:00","problems":{"verify_error":"Traceback (most recent call last):\n  File \"<string>\", line 1, in <module>\nImportError: cannot import name 'HuggingFaceModel' from 'sagemaker_huggingface_inference_toolkit' (/tmp/tmpkoookqu7/venv/lib/python3.12/site-packages/sagemaker_huggingface_inference_toolkit/__init__.py)"},"ecosystem":"pypi","meta_description":null,"install_score":null,"quickstart_score":null,"quickstart_tag":null,"pypi_latest":null,"cli_name":null,"cli_version":null,"type":"library","homepage":"https://github.com/aws/sagemaker-huggingface-inference-toolkit","github":"https://github.com/aws/sagemaker-huggingface-inference-toolkit","docs":null,"changelog":null,"pypi":null,"npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"categories":["ai-ml","devops"],"base_url":null,"auth_type":null,"provenance":{"verified_status":"import_fail","verified_at":"2026-07-03","last_verified":"2026-07-03","next_check":"2026-07-10","install_tag":null}}