Skip to content

Transformers

Installation

You need to install transformers and torch to use this in blendsql.

TransformersLLM

Bases: ConstrainedModel

Class for Transformers local Model.

Parameters:

Name Type Description Default
model_name_or_path str

Name of the model on HuggingFace, or the path to a local model

 required
caching bool

Bool determining whether we access the model's cache

 True

Examples:

from blendsql.models import TransformersLLM
model = TransformersLLM("Qwen/Qwen1.5-0.5B")
Source code in blendsql/models/constrained/guidance.py 
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
class TransformersLLM(ConstrainedModel):
    """Class for Transformers local Model.

    Args:
        model_name_or_path: Name of the model on HuggingFace, or the path to a local model
        caching: Bool determining whether we access the model's cache

    Examples:
        ```python
        from blendsql.models import TransformersLLM
        model = TransformersLLM("Qwen/Qwen1.5-0.5B")
        ```
    """

    def __init__(
        self,
        model_name_or_path: str,
        config: Optional[dict] = None,
        caching: bool = True,
        **kwargs,
    ):
        import transformers

        transformers.logging.set_verbosity_error()
        if config is None:
            config = {}

        super().__init__(
            model_name_or_path=model_name_or_path,
            requires_config=False,
            tokenizer=transformers.AutoTokenizer.from_pretrained(model_name_or_path),
            config=DEFAULT_KWARGS | config,
            caching=caching,
            **kwargs,
        )

    @cached_property
    def model_obj(self) -> ModelObj:
        """Allows for lazy loading of underlying model weights."""
        return self._load_model()

    def _load_model(self) -> ModelObj:
        # https://huggingface.co/blog/how-to-generate
        # from guidance.models._transformers import Transformers
        from ._guidance._transformers import Transformers

        if "chat_template" not in self.config:
            self.config["chat_template"] = infer_chat_template(self.model_name_or_path)

        lm = Transformers(
            self.model_name_or_path,
            echo=False,
            device_map=self.config.pop("device_map", None),
            **self.config,
        )
        # Try to infer if we're in chat mode
        if lm.engine.tokenizer._orig_tokenizer.chat_template is None:
            logger.debug(
                Fore.YELLOW
                + "chat_template not found in tokenizer config.\nBlendSQL currently only works with chat models"
                + Fore.RESET
            )
        return lm