Transformers

Installation

You need to install transformers and torch to use this in blendsql.

TransformersLLM

Bases: ConstrainedModel

Class for Transformers local Model.

Parameters:

Name	Type	Description	Default
model_name_or_path	str	Name of the model on HuggingFace, or the path to a local model	required
caching	bool	Bool determining whether we access the model's cache	True

Examples:

from blendsql.models import TransformersLLM
model = TransformersLLM("Qwen/Qwen1.5-0.5B")

Source code in blendsql/models/constrained/guidance.py

class TransformersLLM(ConstrainedModel):
    """Class for Transformers local Model.

    Args:
        model_name_or_path: Name of the model on HuggingFace, or the path to a local model
        caching: Bool determining whether we access the model's cache

    Examples:
        ```python
        from blendsql.models import TransformersLLM
        model = TransformersLLM("Qwen/Qwen1.5-0.5B")
        ```
    """

    def __init__(
        self,
        model_name_or_path: str,
        config: Optional[dict] = None,
        caching: bool = True,
        **kwargs,
    ):
        import transformers

        transformers.logging.set_verbosity_error()
        if config is None:
            config = {}

        super().__init__(
            model_name_or_path=model_name_or_path,
            requires_config=False,
            tokenizer=transformers.AutoTokenizer.from_pretrained(model_name_or_path),
            config=DEFAULT_KWARGS | config,
            caching=caching,
            **kwargs,
        )

    def _load_model(self) -> ModelObj:
        # https://huggingface.co/blog/how-to-generate
        # from guidance.models._transformers import Transformers
        from guidance.models import Transformers

        lm = Transformers(
            self.model_name_or_path,
            echo=False,
            device_map=self.config.pop("device_map", None),
            **self.config,
        )
        # Try to infer if we're in chat mode
        if lm.engine.tokenizer._orig_tokenizer.chat_template is None:
            logger.debug(
                Fore.YELLOW
                + "chat_template not found in tokenizer config.\nBlendSQL currently only works with chat models"
                + Fore.RESET
            )
        return lm