FM / pipelines

/usr/local/lib/python3.6/site-packages/transformers/pipelines/

/usr/local/lib/python3.6/site-packages/transformers/pipelines

Name	Size	Mode	Actions
__pycache__/	-	0755	rm
audio_classification.py	5840	0644	edit dl rm
audio_utils.py	7784	0644	edit dl rm
automatic_speech_recognition.py	18992	0644	edit dl rm
base.py	43958	0644	edit dl rm
conversational.py	13466	0644	edit dl rm
feature_extraction.py	3581	0644	edit dl rm
fill_mask.py	9870	0644	edit dl rm
image_classification.py	4325	0644	edit dl rm
image_segmentation.py	8045	0644	edit dl rm
object_detection.py	5304	0644	edit dl rm
pt_utils.py	11513	0644	edit dl rm
question_answering.py	26000	0644	edit dl rm
table_question_answering.py	18110	0644	edit dl rm
text2text_generation.py	15123	0644	edit dl rm
text_classification.py	7290	0644	edit dl rm
text_generation.py	12494	0644	edit dl rm
token_classification.py	20838	0644	edit dl rm
zero_shot_classification.py	10807	0644	edit dl rm
zero_shot_image_classification.py	5088	0644	edit dl rm
__init__.py	30359	0644	edit dl rm

Edit: /usr/local/lib/python3.6/site-packages/transformers/pipelines/feature_extraction.py (3581B)

from typing import Dict

from .base import GenericTensor, Pipeline

# Can't use @add_end_docstrings(PIPELINE_INIT_ARGS) here because this one does not accept `binary_output`
class FeatureExtractionPipeline(Pipeline):
    """
    Feature extraction pipeline using no model head. This pipeline extracts the hidden states from the base
    transformer, which can be used as features in downstream tasks.

This feature extraction pipeline can currently be loaded from [`pipeline`] using the task identifier:
    `"feature-extraction"`.

All models may be used for this pipeline. See a list of all models, including community-contributed models on
    [huggingface.co/models](https://huggingface.co/models).

Arguments:
        model ([`PreTrainedModel`] or [`TFPreTrainedModel`]):
            The model that will be used by the pipeline to make predictions. This needs to be a model inheriting from
            [`PreTrainedModel`] for PyTorch and [`TFPreTrainedModel`] for TensorFlow.
        tokenizer ([`PreTrainedTokenizer`]):
            The tokenizer that will be used by the pipeline to encode data for the model. This object inherits from
            [`PreTrainedTokenizer`].
        modelcard (`str` or [`ModelCard`], *optional*):
            Model card attributed to the model for this pipeline.
        framework (`str`, *optional*):
            The framework to use, either `"pt"` for PyTorch or `"tf"` for TensorFlow. The specified framework must be
            installed.

If no framework is specified, will default to the one currently installed. If no framework is specified and
            both frameworks are installed, will default to the framework of the `model`, or to PyTorch if no model is
            provided.
        task (`str`, defaults to `""`):
            A task-identifier for the pipeline.
        args_parser ([`~pipelines.ArgumentHandler`], *optional*):
            Reference to the object in charge of parsing supplied pipeline parameters.
        device (`int`, *optional*, defaults to -1):
            Device ordinal for CPU/GPU supports. Setting this to -1 will leverage CPU, a positive will run the model on
            the associated CUDA device id.
    """

def _sanitize_parameters(self, truncation=None, **kwargs):
        preprocess_params = {}
        if truncation is not None:
            preprocess_params["truncation"] = truncation
        return preprocess_params, {}, {}

def preprocess(self, inputs, truncation=None) -> Dict[str, GenericTensor]:
        return_tensors = self.framework
        if truncation is None:
            kwargs = {}
        else:
            kwargs = {"truncation": truncation}
        model_inputs = self.tokenizer(inputs, return_tensors=return_tensors, **kwargs)
        return model_inputs

def _forward(self, model_inputs):
        model_outputs = self.model(**model_inputs)
        return model_outputs

def postprocess(self, model_outputs):
        # [0] is the first available tensor, logits or last_hidden_state.
        if self.framework == "pt":
            return model_outputs[0].tolist()
        elif self.framework == "tf":
            return model_outputs[0].numpy().tolist()

def __call__(self, *args, **kwargs):
        """
        Extract the features of the input(s).

Args:
            args (`str` or `List[str]`): One or several texts (or one list of texts) to get the features of.

Return:
            A nested list of `float`: The features computed by the model.
        """
        return super().__call__(*args, **kwargs)

from typing import Dict from .base import GenericTensor, Pipeline # Can't use @add_end_docstrings(PIPELINE_INIT_ARGS) here because this one does not accept `binary_output` class FeatureExtractionPipeline(Pipeline): """ Feature extraction pipeline using no model head. This pipeline extracts the hidden states from the base transformer, which can be used as features in downstream tasks. This feature extraction pipeline can currently be loaded from [`pipeline`] using the task identifier: `"feature-extraction"`. All models may be used for this pipeline. See a list of all models, including community-contributed models on [huggingface.co/models](https://huggingface.co/models). Arguments: model ([`PreTrainedModel`] or [`TFPreTrainedModel`]): The model that will be used by the pipeline to make predictions. This needs to be a model inheriting from [`PreTrainedModel`] for PyTorch and [`TFPreTrainedModel`] for TensorFlow. tokenizer ([`PreTrainedTokenizer`]): The tokenizer that will be used by the pipeline to encode data for the model. This object inherits from [`PreTrainedTokenizer`]. modelcard (`str` or [`ModelCard`], *optional*): Model card attributed to the model for this pipeline. framework (`str`, *optional*): The framework to use, either `"pt"` for PyTorch or `"tf"` for TensorFlow. The specified framework must be installed. If no framework is specified, will default to the one currently installed. If no framework is specified and both frameworks are installed, will default to the framework of the `model`, or to PyTorch if no model is provided. task (`str`, defaults to `""`): A task-identifier for the pipeline. args_parser ([`~pipelines.ArgumentHandler`], *optional*): Reference to the object in charge of parsing supplied pipeline parameters. device (`int`, *optional*, defaults to -1): Device ordinal for CPU/GPU supports. Setting this to -1 will leverage CPU, a positive will run the model on the associated CUDA device id. """ def _sanitize_parameters(self, truncation=None, **kwargs): preprocess_params = {} if truncation is not None: preprocess_params["truncation"] = truncation return preprocess_params, {}, {} def preprocess(self, inputs, truncation=None) -> Dict[str, GenericTensor]: return_tensors = self.framework if truncation is None: kwargs = {} else: kwargs = {"truncation": truncation} model_inputs = self.tokenizer(inputs, return_tensors=return_tensors, **kwargs) return model_inputs def _forward(self, model_inputs): model_outputs = self.model(**model_inputs) return model_outputs def postprocess(self, model_outputs): # [0] is the first available tensor, logits or last_hidden_state. if self.framework == "pt": return model_outputs[0].tolist() elif self.framework == "tf": return model_outputs[0].numpy().tolist() def __call__(self, *args, **kwargs): """ Extract the features of the input(s). Args: args (`str` or `List[str]`): One or several texts (or one list of texts) to get the features of. Return: A nested list of `float`: The features computed by the model. """ return super().__call__(*args, **kwargs)