From c0e8d78f3de85daae5dc18cd90967092c7d2c557 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 8 Sep 2026 17:25:02 -0500 Subject: [PATCH 01/10] Rewrite `cuml.feature_extraction.text` - Reorganize submodule into a single file - Update all estimators to properly follow cuml idioms. This includes type reflection, input validation, etc... - Rewrite logic using modern `cudf` features - Update docstrings. --- .../cuml/cuml/feature_extraction/__init__.py | 5 +- .../cuml/feature_extraction/_stop_words.py | 69 - python/cuml/cuml/feature_extraction/_tfidf.py | 289 ---- .../feature_extraction/_tfidf_vectorizer.py | 284 ---- .../cuml/feature_extraction/_vectorizers.py | 1058 ------------ python/cuml/cuml/feature_extraction/text.py | 1419 ++++++++++++++++- .../tests/test_text_feature_extraction.py | 216 +-- 7 files changed, 1476 insertions(+), 1864 deletions(-) delete mode 100644 python/cuml/cuml/feature_extraction/_stop_words.py delete mode 100644 python/cuml/cuml/feature_extraction/_tfidf.py delete mode 100644 python/cuml/cuml/feature_extraction/_tfidf_vectorizer.py delete mode 100644 python/cuml/cuml/feature_extraction/_vectorizers.py diff --git a/python/cuml/cuml/feature_extraction/__init__.py b/python/cuml/cuml/feature_extraction/__init__.py index 3841c12e1d..b3decd8aa5 100644 --- a/python/cuml/cuml/feature_extraction/__init__.py +++ b/python/cuml/cuml/feature_extraction/__init__.py @@ -1,8 +1,7 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # - from . import text -__all__ = ["text"] +__all__ = ("text",) diff --git a/python/cuml/cuml/feature_extraction/_stop_words.py b/python/cuml/cuml/feature_extraction/_stop_words.py deleted file mode 100644 index 0f73e58a58..0000000000 --- a/python/cuml/cuml/feature_extraction/_stop_words.py +++ /dev/null @@ -1,69 +0,0 @@ -# SPDX-FileCopyrightText: Olivier Grisel -# SPDX-FileCopyrightText: Mathieu Blondel -# SPDX-FileCopyrightText: Lars Buitinck -# SPDX-FileCopyrightText: Robert Layton -# SPDX-FileCopyrightText: Jochen Wersdörfer -# SPDX-FileCopyrightText: Roman Sinayev -# SPDX-FileCopyrightText: Copyright (c) 2018-2025, NVIDIA CORPORATION. -# SPDX-License-Identifier: BSD-3-Clause - -# Portion of this code are derived from the scikit-learn feature_extraction -# package, which has the following license: -# -# -*- coding: utf-8 -*- -# Authors: Olivier Grisel -# Mathieu Blondel -# Lars Buitinck -# Robert Layton -# Jochen Wersdörfer -# Roman Sinayev -# -# License: BSD 3 clause -# - -# This list of English stop words is taken from the "Glasgow Information -# Retrieval Group". The original list can be found at -# http://ir.dcs.gla.ac.uk/resources/linguistic_utils/stop_words -ENGLISH_STOP_WORDS = frozenset([ - "about", "above", "across", "after", "afterwards", "again", "against", - "all", "almost", "alone", "along", "already", "also", "although", "always", - "am", "among", "amongst", "amoungst", "amount", "an", "and", "another", - "any", "anyhow", "anyone", "anything", "anyway", "anywhere", "are", - "around", "as", "at", "back", "be", "became", "because", "become", - "becomes", "becoming", "been", "before", "beforehand", "behind", "being", - "below", "beside", "besides", "between", "beyond", "bill", "both", - "bottom", "but", "by", "call", "can", "cannot", "cant", "co", "con", - "could", "couldnt", "cry", "de", "describe", "detail", "do", "done", - "down", "due", "during", "each", "eg", "eight", "either", "eleven", "else", - "elsewhere", "empty", "enough", "etc", "even", "ever", "every", "everyone", - "everything", "everywhere", "except", "few", "fifteen", "fifty", "fill", - "find", "fire", "first", "five", "for", "former", "formerly", "forty", - "found", "four", "from", "front", "full", "further", "get", "give", "go", - "had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter", - "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his", - "how", "however", "hundred", "ie", "if", "in", "inc", "indeed", - "interest", "into", "is", "it", "its", "itself", "keep", "last", "latter", - "latterly", "least", "less", "ltd", "made", "many", "may", "me", - "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly", - "move", "much", "must", "my", "myself", "name", "namely", "neither", - "never", "nevertheless", "next", "nine", "no", "nobody", "none", "noone", - "nor", "not", "nothing", "now", "nowhere", "of", "off", "often", "on", - "once", "one", "only", "onto", "or", "other", "others", "otherwise", "our", - "ours", "ourselves", "out", "over", "own", "part", "per", "perhaps", - "please", "put", "rather", "re", "same", "see", "seem", "seemed", - "seeming", "seems", "serious", "several", "she", "should", "show", "side", - "since", "sincere", "six", "sixty", "so", "some", "somehow", "someone", - "something", "sometime", "sometimes", "somewhere", "still", "such", - "system", "take", "ten", "than", "that", "the", "their", "them", - "themselves", "then", "thence", "there", "thereafter", "thereby", - "therefore", "therein", "thereupon", "these", "they", "thick", "thin", - "third", "this", "those", "though", "three", "through", "throughout", - "thru", "thus", "to", "together", "too", "top", "toward", "towards", - "twelve", "twenty", "two", "un", "under", "until", "up", "upon", "us", - "very", "via", "was", "we", "well", "were", "what", "whatever", "when", - "whence", "whenever", "where", "whereafter", "whereas", "whereby", - "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", - "who", "whoever", "whole", "whom", "whose", "why", "will", "with", - "within", "without", "would", "yet", "you", "your", "yours", "yourself", - "yourselves", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", - "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"]) diff --git a/python/cuml/cuml/feature_extraction/_tfidf.py b/python/cuml/cuml/feature_extraction/_tfidf.py deleted file mode 100644 index 8775f11376..0000000000 --- a/python/cuml/cuml/feature_extraction/_tfidf.py +++ /dev/null @@ -1,289 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -import cupy as cp -import cupyx -from sklearn.exceptions import NotFittedError - -import cuml.internals -from cuml.common.sparse import ( - csr_diag_mul, - csr_row_normalize_l1, - csr_row_normalize_l2, -) -from cuml.internals.base import Base - - -def _sparse_document_frequency(X): - """Count the number of non-zero values for each feature in sparse X.""" - if cupyx.scipy.sparse.isspmatrix_csr(X): - return cp.bincount(X.indices, minlength=X.shape[1]) - else: - return cp.diff(X.indptr) - - -def _get_dtype(X): - """ - Returns the valid dtype for tf-idf transformer - """ - import numpy as np - - FLOAT_DTYPES = (np.float64, np.float32, np.float16) - - dtype = X.dtype if X.dtype in FLOAT_DTYPES else cp.float32 - return dtype - - -class TfidfTransformer(Base): - """ - Transform a count matrix to a normalized tf or tf-idf representation - Tf means term-frequency while tf-idf means term-frequency times inverse - document-frequency. This is a common term weighting scheme in information - retrieval, that has also found good use in document classification. - The goal of using tf-idf instead of the raw frequencies of occurrence of a - token in a given document is to scale down the impact of tokens that occur - very frequently in a given corpus and that are hence empirically less - informative than features that occur in a small fraction of the training - corpus. - The formula that is used to compute the tf-idf for a term t of a document d - in a document set is tf-idf(t, d) = tf(t, d) * idf(t), and the idf is - computed as idf(t) = log [ n / df(t) ] + 1 (if ``smooth_idf=False``), where - n is the total number of documents in the document set and df(t) is the - document frequency of t; the document frequency is the number of documents - in the document set that contain the term t. The effect of adding "1" to - the idf in the equation above is that terms with zero idf, i.e., terms - that occur in all documents in a training set, will not be entirely - ignored. - (Note that the idf formula above differs from the standard textbook - notation that defines the idf as - idf(t) = log [ n / (df(t) + 1) ]). - If ``smooth_idf=True`` (the default), the constant "1" is added to the - numerator and denominator of the idf as if an extra document was seen - containing every term in the collection exactly once, which prevents - zero divisions: idf(t) = log [ (1 + n) / (1 + df(t)) ] + 1. - Furthermore, the formulas used to compute tf and idf depend - on parameter settings that correspond to the SMART notation used in IR - as follows: - Tf is "n" (natural) by default, "l" (logarithmic) when - ``sublinear_tf=True``. - Idf is "t" when use_idf is given, "n" (none) otherwise. - Normalization is "c" (cosine) when ``norm='l2'``, "n" (none) - when ``norm=None``. - - Parameters - ---------- - - norm : {'l1', 'l2'}, default='l2' - Each output row will have unit norm, either: - * 'l2': Sum of squares of vector elements is 1. The cosine similarity - between two vectors is their dot product when l2 norm has been - applied. - * 'l1': Sum of absolute values of vector elements is 1. - use_idf : bool, default=True - Enable inverse-document-frequency reweighting. - smooth_idf : bool, default=True - Smooth idf weights by adding one to document frequencies, as if an - extra document was seen containing every term in the collection - exactly once. Prevents zero divisions. - sublinear_tf : bool, default=False - Apply sublinear tf scaling, i.e. replace tf with 1 + log(tf). - verbose : int or boolean, default=False - Sets logging level. It must be one of `cuml.common.logger.level_*`. - See :ref:`verbosity-levels` for more info. - output_type : {None, 'input', 'cupy', 'numpy', 'cudf', 'pandas'}, default=None - Return results and set estimator attributes to the indicated output - type. If None, the output type set at the module level - (`cuml.global_settings.output_type`) will be used. See - :ref:`output-data-type-configuration` for more info. - - Attributes - ---------- - idf_ : array of shape (n_features) - The inverse document frequency (IDF) vector; only defined - if ``use_idf`` is True. - - """ - - def __init__( - self, - *, - norm="l2", - use_idf=True, - smooth_idf=True, - sublinear_tf=False, - verbose=False, - output_type=None, - ): - super().__init__(verbose=verbose, output_type=output_type) - self.norm = norm - self.use_idf = use_idf - self.smooth_idf = smooth_idf - self.sublinear_tf = sublinear_tf - - def _set_doc_stats(self, X): - """ - We set the following document level statistics here: - n_samples - n_features - df(document frequency) - """ - # Should not have a cost if already sparse - output_dtype = _get_dtype(X) - X = self._convert_to_csr(X, output_dtype) - n_samples, n_features = X.shape - df = _sparse_document_frequency(X) - df = df.astype(output_dtype, copy=False) - self.__df = df - self.__n_samples = n_samples - self.__n_features = n_features - - return - - def _set_idf_diag(self): - """ - Sets idf_diagonal sparse array - """ - # perform idf smoothing if required - df = self.__df + int(self.smooth_idf) - n_samples = self.__n_samples + int(self.smooth_idf) - - # log+1 instead of log makes sure terms with zero idf don't get - # suppressed entirely. - idf = cp.log(n_samples / df) + 1 - self._idf_diag = cupyx.scipy.sparse.dia_matrix( - (idf, 0), - shape=(self.__n_features, self.__n_features), - dtype=df.dtype, - ) - # Free up memory occupied by below - del self.__df - - @cuml.internals.mlfunc(convert_output=False) - def fit(self, X, y=None) -> "TfidfTransformer": - """Learn the idf vector (global term weights). - - Parameters - ---------- - X : array-like of shape n_samples, n_features - A matrix of term/token counts. - """ - output_dtype = _get_dtype(X) - X = self._convert_to_csr(X, output_dtype) - if self.use_idf: - self._set_doc_stats(X) - self._set_idf_diag() - - return self - - @cuml.internals.mlfunc(convert_output=False) - def transform(self, X, copy=True): - """Transform a count matrix to a tf or tf-idf representation - - Parameters - ---------- - X : array-like of (n_samples, n_features) - A matrix of term/token counts - copy : bool, default=True - Whether to copy X and operate on the copy or perform in-place - operations. - - Returns - ------- - vectors : array-like of shape (n_samples, n_features) - """ - if copy: - X = X.copy() - - dtype = _get_dtype(X) - - X = self._convert_to_csr(X, dtype) - if X.dtype != dtype: - X = X.astype(dtype) - - n_samples, n_features = X.shape - - if self.sublinear_tf: - cp.log(X.data, X.data) - X.data += 1 - - if self.use_idf: - self._check_is_idf_fitted() - - expected_n_features = self._idf_diag.shape[0] - if n_features != expected_n_features: - raise ValueError( - "Input has n_features=%d while the model" - " has been trained with n_features=%d" - % (n_features, expected_n_features) - ) - - csr_diag_mul(X, self._idf_diag, inplace=True) - - if self.norm: - if self.norm == "l1": - csr_row_normalize_l1(X, inplace=True) - elif self.norm == "l2": - csr_row_normalize_l2(X, inplace=True) - - return X - - @cuml.internals.mlfunc(convert_output=False) - def fit_transform(self, X, y=None, copy=True): - """ - Fit TfidfTransformer to X, then transform X. - Equivalent to fit(X).transform(X). - - Parameters - ---------- - X : array-like of (n_samples, n_features) - A matrix of term/token counts - copy : bool, default=True - Whether to copy X and operate on the copy or perform in-place - operations. - Returns - ------- - vectors : array-like of shape (n_samples, n_features) - """ - return self.fit(X).transform(X, copy=copy) - - def _check_is_idf_fitted(self): - if not hasattr(self, "idf_"): - msg = ( - "This TfidfTransformer instance is not fitted or the " - "value of use_idf is not consistent between " - ".fit() and .transform()." - ) - raise NotFittedError(msg) - - def _convert_to_csr(self, X, dtype): - """Convert array to CSR format if it not sparse nor CSR.""" - if not cupyx.scipy.sparse.isspmatrix_csr(X): - if not cupyx.scipy.sparse.issparse(X): - X = cupyx.scipy.sparse.csr_matrix(X.astype(dtype)) - else: - X = X.tocsr() - return X - - @property - def idf_(self): - # if _idf_diag is not set, this will raise an attribute error, - # which means hasattr(self, "idf_") is False - return self._idf_diag.data - - @idf_.setter - def idf_(self, value): - value = cp.asarray(value, dtype=cp.float32) - n_features = value.shape[0] - self._idf_diag = cupyx.scipy.sparse.dia_matrix( - (value, 0), shape=(n_features, n_features), dtype=cp.float32 - ) - - @classmethod - def _get_param_names(cls): - return super()._get_param_names() + [ - "norm", - "use_idf", - "smooth_idf", - "sublinear_tf", - ] diff --git a/python/cuml/cuml/feature_extraction/_tfidf_vectorizer.py b/python/cuml/cuml/feature_extraction/_tfidf_vectorizer.py deleted file mode 100644 index 9452ad07dc..0000000000 --- a/python/cuml/cuml/feature_extraction/_tfidf_vectorizer.py +++ /dev/null @@ -1,284 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Portions of this code are derived from the scikit-learn feature_extraction -# package, which has the following license: -# -# -*- coding: utf-8 -*- -# Authors: Olivier Grisel -# Mathieu Blondel -# Lars Buitinck -# Robert Layton -# Jochen Wersdörfer -# Roman Sinayev -# -# License: BSD 3 clause -# - -import cupy as cp - -from cuml.feature_extraction._tfidf import TfidfTransformer -from cuml.feature_extraction._vectorizers import CountVectorizer - - -class TfidfVectorizer(CountVectorizer): - """Convert a collection of raw documents to a matrix of TF-IDF features. - - Equivalent to :class:`CountVectorizer` followed by - :class:`TfidfTransformer`. - - Parameters - ---------- - lowercase : boolean, True by default - Convert all characters to lowercase before tokenizing. - preprocessor : callable or None (default) - Override the preprocessing (string transformation) stage while - preserving the tokenizing and n-grams generation steps. - stop_words : string {'english'}, list, or None (default) - If 'english', a built-in stop word list for English is used. - If a list, that list is assumed to contain stop words, all of which - will be removed from the input documents. - If None, no stop words will be used. max_df can be set to a value - to automatically detect and filter stop words based on intra corpus - document frequency of terms. - ngram_range : tuple (min_n, max_n), default=(1, 1) - The lower and upper boundary of the range of n-values for different - word n-grams or char n-grams to be extracted. All values of n such - such that min_n <= n <= max_n will be used. For example an - ``ngram_range`` of ``(1, 1)`` means only unigrams, ``(1, 2)`` means - unigrams and bigrams, and ``(2, 2)`` means only bigrams. - analyzer : string, {'word', 'char', 'char_wb'}, default='word' - Whether the feature should be made of word n-gram or character - n-grams. - Option 'char_wb' creates character n-grams only from text inside - word boundaries; n-grams at the edges of words are padded with space. - max_df : float in range [0.0, 1.0] or int, default=1.0 - When building the vocabulary ignore terms that have a document - frequency strictly higher than the given threshold (corpus-specific - stop words). - If float, the parameter represents a proportion of documents, integer - absolute counts. - This parameter is ignored if vocabulary is not None. - min_df : float in range [0.0, 1.0] or int, default=1 - When building the vocabulary ignore terms that have a document - frequency strictly lower than the given threshold. This value is also - called cut-off in the literature. - If float, the parameter represents a proportion of documents, integer - absolute counts. - This parameter is ignored if vocabulary is not None. - max_features : int or None, default=None - If not None, build a vocabulary that only consider the top - max_features ordered by term frequency across the corpus. - This parameter is ignored if vocabulary is not None. - vocabulary : cudf.Series, optional - If not given, a vocabulary is determined from the input documents. - binary : boolean, default=False - If True, all non zero counts are set to 1. This is useful for discrete - probabilistic models that model binary events rather than integer - counts. - dtype : type, optional - Type of the matrix returned by fit_transform() or transform(). - delimiter : str, whitespace by default - String used as a replacement for stop words if stop_words is not None. - Typically the delimiting character between words is a good choice. - norm : {'l1', 'l2'}, default='l2' - Each output row will have unit norm, either: - * 'l2': Sum of squares of vector elements is 1. The cosine similarity - between two vectors is their dot product when l2 norm has been - applied. - * 'l1': Sum of absolute values of vector elements is 1. - - use_idf : bool, default=True - Enable inverse-document-frequency reweighting. - smooth_idf : bool, default=True - Smooth idf weights by adding one to document frequencies, as if an - extra document was seen containing every term in the collection - exactly once. Prevents zero divisions. - sublinear_tf : bool, default=False - Apply sublinear tf scaling, i.e. replace tf with 1 + log(tf). - - Attributes - ---------- - idf_ : array of shape (n_features) - The inverse document frequency (IDF) vector; only defined - if `use_idf` is True. - vocabulary_ : cudf.Series[str] - Array mapping from feature integer indices to feature name. - stop_words_ : cudf.Series[str] - Terms that were ignored because they either: - - occurred in too many documents (`max_df`) - - occurred in too few documents (`min_df`) - - were cut off by feature selection (`max_features`). - - This is only available if no vocabulary was given. - - Notes - ----- - The ``stop_words_`` attribute can get large and increase the model size - when pickling. This attribute is provided only for introspection and can - be safely removed using delattr or set to None before pickling. - - This class is largely based on scikit-learn 0.23.1's TfIdfVectorizer code, - which is provided under the BSD-3 license. - """ - - def __init__( - self, - input=None, - encoding=None, - decode_error=None, - strip_accents=None, - lowercase=True, - preprocessor=None, - tokenizer=None, - stop_words=None, - token_pattern=None, - ngram_range=(1, 1), - analyzer="word", - max_df=1.0, - min_df=1, - max_features=None, - vocabulary=None, - binary=False, - dtype=cp.float32, - delimiter=" ", - norm="l2", - use_idf=True, - smooth_idf=True, - sublinear_tf=False, - ): - super().__init__( - input=input, - encoding=encoding, - decode_error=decode_error, - strip_accents=strip_accents, - lowercase=lowercase, - preprocessor=preprocessor, - tokenizer=tokenizer, - analyzer=analyzer, - stop_words=stop_words, - token_pattern=token_pattern, - ngram_range=ngram_range, - max_df=max_df, - min_df=min_df, - max_features=max_features, - vocabulary=vocabulary, - binary=binary, - dtype=dtype, - delimiter=delimiter, - ) - - self._tfidf = TfidfTransformer( - norm=norm, - use_idf=use_idf, - smooth_idf=smooth_idf, - sublinear_tf=sublinear_tf, - ) - - # Broadcast the TF-IDF parameters to the underlying transformer instance - # for easy grid search and repr - - @property - def norm(self): - return self._tfidf.norm - - @norm.setter - def norm(self, value): - self._tfidf.norm = value - - @property - def use_idf(self): - return self._tfidf.use_idf - - @use_idf.setter - def use_idf(self, value): - self._tfidf.use_idf = value - - @property - def smooth_idf(self): - return self._tfidf.smooth_idf - - @smooth_idf.setter - def smooth_idf(self, value): - self._tfidf.smooth_idf = value - - @property - def sublinear_tf(self): - return self._tfidf.sublinear_tf - - @sublinear_tf.setter - def sublinear_tf(self, value): - self._tfidf.sublinear_tf = value - - @property - def idf_(self): - return self._tfidf.idf_ - - @idf_.setter - def idf_(self, value): - if hasattr(self, "vocabulary_"): - if len(self.vocabulary_) != len(value): - raise ValueError( - "idf length = %d must be equal " - "to vocabulary size = %d" - % (len(value), len(self.vocabulary)) - ) - self._tfidf.idf_ = value - - def fit(self, raw_documents): - """Learn vocabulary and idf from training set. - - Parameters - ---------- - raw_documents : cudf.Series or pd.Series - A Series of string documents - - Returns - ------- - self : object - Fitted vectorizer. - """ - X = super().fit_transform(raw_documents) - self._tfidf.fit(X) - return self - - def fit_transform(self, raw_documents, y=None): - """Learn vocabulary and idf, return document-term matrix. - This is equivalent to fit followed by transform, but more efficiently - implemented. - - Parameters - ---------- - raw_documents : cudf.Series or pd.Series - A Series of string documents - y : None - Ignored. - - Returns - ------- - X : cupy csr array of shape (n_samples, n_features) - Tf-idf-weighted document-term matrix. - """ - X = super().fit_transform(raw_documents) - self._tfidf.fit(X) - # X is already a transformed view of raw_documents so - # we set copy to False - return self._tfidf.transform(X, copy=False) - - def transform(self, raw_documents): - """Transform documents to document-term matrix. - Uses the vocabulary and document frequencies (df) learned by fit (or - fit_transform). - - Parameters - ---------- - raw_documents : cudf.Series or pd.Series - A Series of string documents - - Returns - ------- - X : cupy csr array of shape (n_samples, n_features) - Tf-idf-weighted document-term matrix. - """ - X = super().transform(raw_documents) - return self._tfidf.transform(X, copy=False) diff --git a/python/cuml/cuml/feature_extraction/_vectorizers.py b/python/cuml/cuml/feature_extraction/_vectorizers.py deleted file mode 100644 index 50b26f705a..0000000000 --- a/python/cuml/cuml/feature_extraction/_vectorizers.py +++ /dev/null @@ -1,1058 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -import numbers -from functools import partial - -import cudf -import cupy as cp -import cupyx.scipy.sparse as cp_sp -import numpy as np -import pandas as pd -from cudf import Series -from sklearn.exceptions import NotFittedError - -import cuml.internals.logger as logger -from cuml.common.sparse import csr_row_normalize_l1, csr_row_normalize_l2 -from cuml.feature_extraction._stop_words import ENGLISH_STOP_WORDS -from cuml.internals.mixins import DeprecatedGetFeatureNamesMixin - -CUPY_SPARSE_DTYPES = [cp.float32, cp.float64, cp.complex64, cp.complex128] - - -def min_signed_type(n): - for int_dtype in (np.int8, np.int16, np.int32, np.int64): - dtype = np.dtype(int_dtype) - if (dtype.itemsize * 8) >= 8: - if np.iinfo(int_dtype).min <= n <= np.iinfo(int_dtype).max: - return dtype - # resort to using `int64` and let numpy raise appropriate exception: - return np.int64(n).dtype - - -def _insert_zeros(ary, zero_indices): - """ - Create a new array of len(ary + zero_indices) where zero_indices - indicates indexes of 0s in the new array. Ary is used to fill the rest. - - Examples - -------- - _insert_zeros([1, 2, 3], [1, 3]) => [1, 0, 2, 0, 3] - """ - if len(zero_indices) == 0: - return ary.values - - new_ary = cp.zeros((len(ary) + len(zero_indices)), dtype=cp.int32) - - # getting mask of non-zeros - data_mask = ~cp.in1d( - cp.arange(0, len(new_ary), dtype=cp.int32), zero_indices - ) - - new_ary[data_mask] = ary - return new_ary - - -def _create_csr_matrix_from_count_df( - count_df, empty_doc_ids, n_doc, n_features, dtype=np.float32 -): - """ - Create a sparse matrix from the count of tokens by document - - Parameters - ---------- - count_df = cudf.DataFrame({'count':..., 'doc_id':.., 'token':.. }) - sorted by doc_id and token - empty_doc_ids = cupy array containing doc_ids with no tokens - n_doc: Total number of documents - n_features: Number of features - dtype: Output dtype - """ - data = count_df["count"].values - indices = count_df["token"].values - - doc_token_counts = count_df["doc_id"].value_counts().reset_index() - del count_df - - doc_token_counts = doc_token_counts.rename( - {"count": "token_counts"}, axis=1 - ).sort_values(by="doc_id") - - token_counts = _insert_zeros( - doc_token_counts["token_counts"], empty_doc_ids - ) - indptr = token_counts.cumsum() - indptr = cp.pad(indptr, (1, 0), "constant") - - return cp_sp.csr_matrix( - arg1=(data, indices, indptr), dtype=dtype, shape=(n_doc, n_features) - ) - - -def _preprocess( - doc, - lower=False, - remove_non_alphanumeric=False, - delimiter=" ", - keep_underscore_char=True, - remove_single_token_len=True, -): - """ - Chain together an optional series of text preprocessing steps to - apply to a document. - - Parameters - ---------- - doc: cudf.Series[str] or pd.Series[str] - The string to preprocess - lower: bool - Whether to use str.lower to lowercase all of the text - remove_non_alphanumeric: bool - Whether or not to remove non-alphanumeric characters. - keep_underscore_char: bool - Whether or not to keep the underscore character - - Returns - ------- - doc: cudf.Series[str] - preprocessed string - """ - if isinstance(doc, pd.Series): - doc = Series(doc) - if lower: - doc = doc.str.lower() - if remove_non_alphanumeric: - if keep_underscore_char: - # why: sklearn by default keeps `_` char along with alphanumerics - # currently we dont have a easy way of removing - # all chars but `_` - # in cudf.Series[str] below works around it - temp_string = "cumlSt" - doc = doc.str.replace("_", temp_string, regex=False) - doc = doc.str.filter_alphanum(" ", keep=True) - doc = doc.str.replace(temp_string, "_", regex=False) - else: - doc = doc.str.filter_alphanum(" ", keep=True) - - # sklearn by default removes tokens of - # length 1, if its remove alphanumerics - if remove_single_token_len: - doc = doc.str.filter_tokens(2) - - return doc - - -class _VectorizerMixin: - """ - Provides common code for text vectorizers (tokenization logic). - """ - - def _remove_stop_words(self, doc): - """ - Remove stop words only if needed. - """ - if self.analyzer == "word" and self.stop_words is not None: - stop_words = Series(self._get_stop_words()) - doc = doc.str.replace_tokens( - stop_words, - replacements=self.delimiter, - delimiter=self.delimiter, - ) - return doc - - def build_preprocessor(self): - """ - Return a function to preprocess the text before tokenization. - - If analyzer == 'word' and stop_words is not None, stop words are - removed from the input documents after preprocessing. - - Returns - ------- - preprocessor: callable - A function to preprocess the text before tokenization. - """ - if self.preprocessor is not None: - preprocess = self.preprocessor - else: - remove_non_alpha = self.analyzer == "word" - preprocess = partial( - _preprocess, - lower=self.lowercase, - remove_non_alphanumeric=remove_non_alpha, - delimiter=self.delimiter, - ) - return lambda doc: self._remove_stop_words(preprocess(doc)) - - def _get_stop_words(self): - """ - Build or fetch the effective stop words list. - - Returns - ------- - stop_words: list or None - A list of stop words. - """ - if self.stop_words == "english": - return list(ENGLISH_STOP_WORDS) - elif isinstance(self.stop_words, str): - raise ValueError("not a built-in stop list: %s" % self.stop_words) - elif self.stop_words is None: - return None - else: # assume it's a collection - return list(self.stop_words) - - def get_char_ngrams(self, ngram_size, str_series, doc_id_sr): - """ - Handles ngram generation for characters analyzers. - - When analyzer is 'char_wb', we generate ngrams within word boundaries, - meaning we need to first tokenize and pad each token with a delimiter. - """ - if self.analyzer == "char_wb" and ngram_size != 1: - token_count = str_series.str.token_count(delimiter=self.delimiter) - tokens = str_series.str.tokenize(self.delimiter) - del str_series - - # tokens keeps the original per-document index (repeated per - # token); reset both to a plain range first so the two str.cat() - # calls below align positionally instead of by that index. - tokens = tokens.reset_index(drop=True) - padding = ( - Series(self.delimiter) - .repeat(len(tokens)) - .reset_index(drop=True) - ) - tokens = tokens.str.cat(padding) - tokens = padding.str.cat(tokens) - - ngram_sr = tokens.str.character_ngrams(n=ngram_size) - - doc_id_df = cudf.DataFrame( - { - "doc_id": doc_id_sr.repeat(token_count).reset_index( - drop=True - ), - # formula to count ngrams given number of letters per token: - "ngram_count": tokens.str.len() - (ngram_size - 1), - } - ) - del tokens - ngram_count = doc_id_df.groupby("doc_id", sort=True).sum()[ - "ngram_count" - ] - # A document that tokenizes to zero tokens (e.g. an empty string) - # never appears in the groupby above, so its doc_id is silently - # missing from ngram_count's index instead of being present with - # a count of 0. Reindex onto token_count's full per-document - # index so the two stay aligned for get_ngrams' later - # ngram_count[not_empty_docs] boolean filter. - ngram_count = ngram_count.reindex(token_count.index, fill_value=0) - return ngram_sr, ngram_count, token_count - - if ngram_size == 1: - token_count = str_series.str.len() - ngram_sr = str_series.str.character_tokenize() - del str_series - elif self.analyzer == "char": - token_count = str_series.str.len() - ngram_sr = str_series.str.character_ngrams(n=ngram_size) - del str_series - - ngram_count = token_count - (ngram_size - 1) - - return ngram_sr, ngram_count, token_count - - def get_ngrams(self, str_series, ngram_size, doc_id_sr): - """ - This returns the ngrams for the string series - - Parameters - ---------- - str_series : (cudf.Series) - String series to tokenize - ngram_size : int - Gram level to get (1 for unigram, 2 for bigram etc) - doc_id_sr : cudf.Series - Int series containing documents ids - """ - - if self.analyzer == "word": - token_count_sr = str_series.str.token_count(self.delimiter) - ngram_sr = str_series.str.ngrams_tokenize( - n=ngram_size, separator=" ", delimiter=self.delimiter - ) - # formula to count ngrams given number of tokens x per doc: x-(n-1) - ngram_count = token_count_sr - (ngram_size - 1) - else: - ngram_sr, ngram_count, token_count_sr = self.get_char_ngrams( - ngram_size, str_series, doc_id_sr - ) - - not_empty_docs = token_count_sr > 0 - doc_id_sr = doc_id_sr[not_empty_docs] - ngram_count = ngram_count[not_empty_docs] - - doc_id_sr = doc_id_sr.repeat(ngram_count).reset_index(drop=True) - tokenized_df = cudf.DataFrame() - tokenized_df["doc_id"] = doc_id_sr - - ngram_sr = ngram_sr.reset_index(drop=True) - tokenized_df["token"] = ngram_sr - return tokenized_df - - def _create_tokenized_df(self, docs): - """ - Creates a tokenized DataFrame from a string Series. - Each row describes the token string and the corresponding document id. - """ - min_n, max_n = self.ngram_range - - doc_id = cp.arange(start=0, stop=len(docs), dtype=cp.int32) - doc_id = Series(doc_id) - - tokenized_df_ls = [ - self.get_ngrams(docs, n, doc_id) for n in range(min_n, max_n + 1) - ] - del docs - tokenized_df = cudf.concat(tokenized_df_ls) - tokenized_df = tokenized_df.reset_index(drop=True) - - return tokenized_df - - def _compute_empty_doc_ids(self, count_df, n_doc): - """ - Compute empty docs ids using the remaining docs, given the total number - of documents. - """ - remaining_docs = count_df["doc_id"].unique() - dtype = min_signed_type(n_doc) - doc_ids = cudf.DataFrame( - data={"all_ids": cp.arange(0, n_doc, dtype=dtype)}, dtype=dtype - ) - - empty_docs = doc_ids - doc_ids.iloc[remaining_docs] - empty_ids = empty_docs[empty_docs["all_ids"].isnull()].index.values - return empty_ids - - def _validate_params(self): - """ - Check validity of ngram_range parameter - """ - min_n, max_m = self.ngram_range - msg = "" - if min_n < 1: - msg += "lower boundary must be >= 1. " - if min_n > max_m: - msg += "lower boundary larger than the upper boundary. " - if msg != "": - msg = f"Invalid value for ngram_range={self.ngram_range} {msg}" - raise ValueError(msg) - - if hasattr(self, "n_features"): - if not isinstance(self.n_features, numbers.Integral): - raise TypeError( - f"n_features must be integral, got {self.n_features}\ - ({type(self.n_features)})." - ) - - def _warn_for_unused_params(self): - if self.analyzer != "word" and self.stop_words is not None: - logger.warn( - "The parameter 'stop_words' will not be used" - " since 'analyzer' != 'word'" - ) - - def _check_sklearn_params(self, analyzer, sklearn_params): - if callable(analyzer): - raise ValueError( - "cuML does not support callable analyzer," - " please refer to the cuML documentation for" - " more information." - ) - - for key, vals in sklearn_params.items(): - if vals is not None: - raise TypeError( - "The Scikit-learn variable", - key, - " is not supported in cuML," - " please read the cuML documentation for" - " more information.", - ) - - -def _document_frequency(X): - """ - Count the number of non-zero values for each feature in X. - """ - doc_freq = X[["token", "doc_id"]].groupby(["token"], sort=True).count() - return doc_freq["doc_id"].values - - -def _term_frequency(X): - """ - Count the number of occurrences of each term in X. - """ - term_freq = X[["token", "count"]].groupby(["token"], sort=True).sum() - return term_freq["count"].values - - -class CountVectorizer(DeprecatedGetFeatureNamesMixin, _VectorizerMixin): - """ - Convert a collection of text documents to a matrix of token counts - - If you do not provide an a-priori dictionary then the number of features - will be equal to the vocabulary size found by analyzing the data. - - Parameters - ---------- - lowercase : boolean, True by default - Convert all characters to lowercase before tokenizing. - preprocessor : callable or None (default) - Override the preprocessing (string transformation) stage while - preserving the tokenizing and n-grams generation steps. - stop_words : string {'english'}, list, or None (default) - If 'english', a built-in stop word list for English is used. - If a list, that list is assumed to contain stop words, all of which - will be removed from the input documents. - If None, no stop words will be used. max_df can be set to a value - to automatically detect and filter stop words based on intra corpus - document frequency of terms. - ngram_range : tuple (min_n, max_n), default=(1, 1) - The lower and upper boundary of the range of n-values for different - word n-grams or char n-grams to be extracted. All values of n such - such that min_n <= n <= max_n will be used. For example an - ``ngram_range`` of ``(1, 1)`` means only unigrams, ``(1, 2)`` means - unigrams and bigrams, and ``(2, 2)`` means only bigrams. - analyzer : string, {'word', 'char', 'char_wb'} - Whether the feature should be made of word n-gram or character - n-grams. - Option 'char_wb' creates character n-grams only from text inside - word boundaries; n-grams at the edges of words are padded with space. - max_df : float in range [0.0, 1.0] or int, default=1.0 - When building the vocabulary ignore terms that have a document - frequency strictly higher than the given threshold (corpus-specific - stop words). - If float, the parameter represents a proportion of documents, integer - absolute counts. - This parameter is ignored if vocabulary is not None. - min_df : float in range [0.0, 1.0] or int, default=1 - When building the vocabulary ignore terms that have a document - frequency strictly lower than the given threshold. This value is also - called cut-off in the literature. - If float, the parameter represents a proportion of documents, integer - absolute counts. - This parameter is ignored if vocabulary is not None. - max_features : int or None, default=None - If not None, build a vocabulary that only consider the top - max_features ordered by term frequency across the corpus. - This parameter is ignored if vocabulary is not None. - vocabulary : cudf.Series, optional - If not given, a vocabulary is determined from the input documents. - binary : boolean, default=False - If True, all non zero counts are set to 1. This is useful for discrete - probabilistic models that model binary events rather than integer - counts. - dtype : type, optional - Type of the matrix returned by fit_transform() or transform(). - delimiter : str, whitespace by default - String used as a replacement for stop words if stop_words is not None. - Typically the delimiting character between words is a good choice. - - Attributes - ---------- - vocabulary_ : cudf.Series[str] - Array mapping from feature integer indices to feature name. - stop_words_ : cudf.Series[str] - Terms that were ignored because they either: - - occurred in too many documents (`max_df`) - - occurred in too few documents (`min_df`) - - were cut off by feature selection (`max_features`). - - This is only available if no vocabulary was given. - - """ - - def __init__( - self, - input=None, - encoding=None, - decode_error=None, - strip_accents=None, - lowercase=True, - preprocessor=None, - tokenizer=None, - stop_words=None, - token_pattern=None, - ngram_range=(1, 1), - analyzer="word", - max_df=1.0, - min_df=1, - max_features=None, - vocabulary=None, - binary=False, - dtype=cp.float32, - delimiter=" ", - ): - self.preprocessor = preprocessor - self.analyzer = analyzer - self.lowercase = lowercase - self.stop_words = stop_words - self.max_df = max_df - self.min_df = min_df - if max_df < 0 or min_df < 0: - raise ValueError("negative value for max_df or min_df") - self.max_features = max_features - if max_features is not None: - if not isinstance(max_features, int) or max_features <= 0: - raise ValueError( - "max_features=%r, neither a positive integer nor None" - % max_features - ) - self.ngram_range = ngram_range - self.vocabulary = vocabulary - self.binary = binary - self.dtype = dtype - self.delimiter = delimiter - if dtype not in CUPY_SPARSE_DTYPES: - msg = f"Expected dtype in {CUPY_SPARSE_DTYPES}, got {dtype}" - raise ValueError(msg) - - sklearn_params = { - "input": input, - "encoding": encoding, - "decode_error": decode_error, - "strip_accents": strip_accents, - "tokenizer": tokenizer, - "token_pattern": token_pattern, - } - self._check_sklearn_params(analyzer, sklearn_params) - - def _count_vocab(self, tokenized_df): - """ - Count occurrences of tokens in each document. - """ - # Transform string tokens into token indexes from 0 to len(vocab) - # The indexes are based on lexicographical ordering. - tokenized_df["token"] = tokenized_df["token"].astype("category") - tokenized_df["token"] = ( - tokenized_df["token"] - .cat.set_categories(self.vocabulary_) - .cat.codes - ) - - # Count of each token in each document - count_df = ( - tokenized_df[["doc_id", "token"]] - .groupby(["doc_id", "token"], sort=True) - .size() - .reset_index() - .rename({0: "count"}, axis=1) - ) - - return count_df - - def _filter_and_renumber(self, df, keep_values, column): - """ - Filter dataframe to keep only values from column matching - keep_values. - """ - df[column] = ( - df[column] - .astype("category") - .cat.set_categories(keep_values) - .cat.codes - ) - df = df.dropna(subset=column) - return df - - def _limit_features(self, count_df, vocab, high, low, limit): - """ - Remove too rare or too common features. - - Prune features that are non zero in more samples than high or less - documents than low, modifying the vocabulary, and restricting it to - at most the limit most frequent. - - Sets `self.vocabulary_` and `self.stop_words_` with the new values. - """ - if high is None and low is None and limit is None: - self.stop_words_ = None - return count_df - - document_frequency = _document_frequency(count_df) - - mask = cp.ones(len(document_frequency), dtype=bool) - if high is not None: - mask &= document_frequency <= high - if low is not None: - mask &= document_frequency >= low - if limit is not None and mask.sum() > limit: - term_frequency = _term_frequency(count_df) - mask_inds = (-term_frequency[mask]).argsort()[:limit] - new_mask = cp.zeros(len(document_frequency), dtype=bool) - new_mask[cp.where(mask)[0][mask_inds]] = True - mask = new_mask - - keep_idx = cp.where(mask)[0].astype(cp.int32) - keep_num = keep_idx.shape[0] - - if keep_num == 0: - raise ValueError( - "After pruning, no terms remain. Try a lower" - " min_df or a higher max_df." - ) - - if len(vocab) - keep_num != 0: - count_df = self._filter_and_renumber(count_df, keep_idx, "token") - - self.stop_words_ = vocab[~mask].reset_index(drop=True) - self.vocabulary_ = vocab[mask].reset_index(drop=True) - - return count_df - - def _preprocess(self, raw_documents): - preprocess = self.build_preprocessor() - return preprocess(raw_documents) - - def fit(self, raw_documents, y=None): - """ - Build a vocabulary of all tokens in the raw documents. - - Parameters - ---------- - - raw_documents : cudf.Series or pd.Series - A Series of string documents - y : None - Ignored. - - Returns - ------- - self - - """ - self.fit_transform(raw_documents) - return self - - def fit_transform(self, raw_documents, y=None): - """ - Build the vocabulary and return document-term matrix. - - Equivalent to ``self.fit(X).transform(X)`` but preprocess `X` only - once. - - Parameters - ---------- - raw_documents : cudf.Series or pd.Series - A Series of string documents - y : None - Ignored. - - Returns - ------- - X : cupy csr array of shape (n_samples, n_features) - Document-term matrix. - """ - self._warn_for_unused_params() - self._validate_params() - - self._fixed_vocabulary = self.vocabulary is not None - - docs = self._preprocess(raw_documents) - n_doc = len(docs) - - tokenized_df = self._create_tokenized_df(docs) - - if self._fixed_vocabulary: - self.vocabulary_ = self.vocabulary - else: - self.vocabulary_ = ( - tokenized_df["token"].drop_duplicates().sort_values() - ) - - count_df = self._count_vocab(tokenized_df) - - if not self._fixed_vocabulary: - max_doc_count = ( - self.max_df - if isinstance(self.max_df, numbers.Integral) - else self.max_df * n_doc - ) - min_doc_count = ( - self.min_df - if isinstance(self.min_df, numbers.Integral) - else self.min_df * n_doc - ) - if max_doc_count < min_doc_count: - raise ValueError( - "max_df corresponds to < documents than min_df" - ) - count_df = self._limit_features( - count_df, - self.vocabulary_, - max_doc_count, - min_doc_count, - self.max_features, - ) - - empty_doc_ids = self._compute_empty_doc_ids(count_df, n_doc) - - X = _create_csr_matrix_from_count_df( - count_df, - empty_doc_ids, - n_doc, - len(self.vocabulary_), - dtype=self.dtype, - ) - if self.binary: - X.data.fill(1) - return X - - def transform(self, raw_documents): - """ - Transform documents to document-term matrix. - - Extract token counts out of raw text documents using the vocabulary - fitted with fit or the one provided to the constructor. - - Parameters - ---------- - raw_documents : cudf.Series or pd.Series - A Series of string documents - - Returns - ------- - X : cupy csr array of shape (n_samples, n_features) - Document-term matrix. - """ - if not hasattr(self, "vocabulary_"): - if self.vocabulary is not None: - self.vocabulary_ = self.vocabulary - else: - raise NotFittedError() - - docs = self._preprocess(raw_documents) - n_doc = len(docs) - tokenized_df = self._create_tokenized_df(docs) - count_df = self._count_vocab(tokenized_df) - empty_doc_ids = self._compute_empty_doc_ids(count_df, n_doc) - X = _create_csr_matrix_from_count_df( - count_df, - empty_doc_ids, - n_doc, - len(self.vocabulary_), - dtype=self.dtype, - ) - if self.binary: - X.data.fill(1) - return X - - def inverse_transform(self, X): - """ - Return terms per document with nonzero entries in X. - - Parameters - ---------- - X : array-like of shape (n_samples, n_features) - Document-term matrix. - - Returns - ------- - X_inv : list of cudf.Series of shape (n_samples,) - List of Series of terms. - """ - vocab = Series(self.vocabulary_) - return [vocab[X[i, :].indices] for i in range(X.shape[0])] - - def get_feature_names_out(self, input_features=None): - """Get output feature names for transformation. - - Parameters - ---------- - input_features : array-like of str or None, default=None - Not used, present here for API consistency by convention. - - Returns - ------- - feature_names_out : numpy.ndarray of str objects. - Transformed feature names. - """ - # TODO: use `check_is_fitted` once this class subclasses from `Base` - return self.vocabulary_.to_numpy(dtype=object) - - -class HashingVectorizer(_VectorizerMixin): - """ - Convert a collection of text documents to a matrix of token occurrences - - It turns a collection of text documents into a cupyx.scipy.sparse matrix - holding token occurrence counts (or binary occurrence information), - possibly normalized as token frequencies if norm='l1' or projected on the - euclidean unit sphere if norm='l2'. - - This text vectorizer implementation uses the hashing trick to find the - token string name to feature integer index mapping. - - This strategy has several advantages: - - - it is very low memory scalable to large datasets as there is no need to - store a vocabulary dictionary in memory which is even more important - as GPU's that are often memory constrained - - it is fast to pickle and un-pickle as it holds no state besides the - constructor parameters - - it can be used in a streaming (partial fit) or parallel pipeline as - there is no state computed during fit. - - There are also a couple of cons (vs using a CountVectorizer with an - in-memory vocabulary): - - - there is no way to compute the inverse transform (from feature indices - to string feature names) which can be a problem when trying to - introspect which features are most important to a model. - - there can be collisions: distinct tokens can be mapped to the same - feature index. However in practice this is rarely an issue if n_features - is large enough (e.g. 2 ** 18 for text classification problems). - - no IDF weighting as this would render the transformer stateful. - - The hash function employed is the signed 32-bit version of Murmurhash3. - - Parameters - ---------- - lowercase : bool, default=True - Convert all characters to lowercase before tokenizing. - preprocessor : callable or None (default) - Override the preprocessing (string transformation) stage while - preserving the tokenizing and n-grams generation steps. - stop_words : string {'english'}, list, default=None - If 'english', a built-in stop word list for English is used. - There are several known issues with 'english' and you should - consider an alternative. - If a list, that list is assumed to contain stop words, all of which - will be removed from the resulting tokens. - Only applies if ``analyzer == 'word'``. - ngram_range : tuple (min_n, max_n), default=(1, 1) - The lower and upper boundary of the range of n-values for different - word n-grams or char n-grams to be extracted. All values of n such - such that min_n <= n <= max_n will be used. For example an - ``ngram_range`` of ``(1, 1)`` means only unigrams, ``(1, 2)`` means - unigrams and bigrams, and ``(2, 2)`` means only bigrams. - analyzer : string, {'word', 'char', 'char_wb'} - Whether the feature should be made of word n-gram or character - n-grams. - Option 'char_wb' creates character n-grams only from text inside - word boundaries; n-grams at the edges of words are padded with space. - n_features : int, default=(2 ** 20) - The number of features (columns) in the output matrices. Small numbers - of features are likely to cause hash collisions, but large numbers - will cause larger coefficient dimensions in linear learners. - binary : bool, default=False. - If True, all non zero counts are set to 1. This is useful for discrete - probabilistic models that model binary events rather than integer - counts. - norm : {'l1', 'l2'}, default='l2' - Norm used to normalize term vectors. None for no normalization. - alternate_sign : bool, default=True - When True, an alternating sign is added to the features as to - approximately conserve the inner product in the hashed space even for - small n_features. This approach is similar to sparse random projection. - dtype : type, optional - Type of the matrix returned by fit_transform() or transform(). - delimiter : str, whitespace by default - String used as a replacement for stop words if `stop_words` is not - None. Typically the delimiting character between words is a good - choice. - - Examples - -------- - - .. code-block:: python - - >>> from cuml.feature_extraction.text import HashingVectorizer - >>> import pandas as pd - >>> corpus = [ - ... 'This is the first document.', - ... 'This document is the second document.', - ... 'And this is the third one.', - ... 'Is this the first document?', - ... ] - >>> vectorizer = HashingVectorizer(n_features=2**4) - >>> X = vectorizer.fit_transform(pd.Series(corpus)) - >>> print(X.shape) - (4, 16) - - See Also - -------- - CountVectorizer, TfidfVectorizer - """ - - def __init__( - self, - input=None, - encoding=None, - decode_error=None, - strip_accents=None, - lowercase=True, - preprocessor=None, - tokenizer=None, - stop_words=None, - token_pattern=None, - ngram_range=(1, 1), - analyzer="word", - n_features=(2**20), - binary=False, - norm="l2", - alternate_sign=True, - dtype=cp.float32, - delimiter=" ", - ): - self.preprocessor = preprocessor - self.analyzer = analyzer - self.lowercase = lowercase - self.stop_words = stop_words - self.n_features = n_features - self.ngram_range = ngram_range - self.binary = binary - self.norm = norm - self.alternate_sign = alternate_sign - self.dtype = dtype - self.delimiter = delimiter - - if dtype not in CUPY_SPARSE_DTYPES: - msg = f"Expected dtype in {CUPY_SPARSE_DTYPES}, got {dtype}" - raise ValueError(msg) - - if self.norm not in ("l1", "l2", None): - raise ValueError(f"{self.norm} is not a supported norm") - - sklearn_params = { - "input": input, - "encoding": encoding, - "decode_error": decode_error, - "strip_accents": strip_accents, - "tokenizer": tokenizer, - "token_pattern": token_pattern, - } - self._check_sklearn_params(analyzer, sklearn_params) - - def partial_fit(self, X, y=None): - """ - Does nothing: This transformer is stateless - This method is just there to mark the fact that this transformer - can work in a streaming setup. - - Parameters - ---------- - X : cudf.Series(A Series of string documents). - """ - return self - - def fit(self, X, y=None): - """ - This method only checks the input type and the model parameter. - It does not do anything meaningful as this transformer is stateless - - Parameters - ---------- - X : cudf.Series or pd.Series - A Series of string documents - """ - self._warn_for_unused_params() - self._validate_params() - return self - - def _preprocess(self, raw_documents): - preprocess = self.build_preprocessor() - return preprocess(raw_documents) - - def _count_hash(self, tokenized_df): - """ - Count occurrences of tokens in each document. - """ - # Transform string tokens into token indexes from 0 to n_features - tokenized_df["token"] = tokenized_df["token"].hash_values() - if self.alternate_sign: - # below logic is equivalent to: value *= ((h >= 0) * 2) - 1 - tokenized_df["value"] = (tokenized_df["token"] >= 0).astype( - cp.int8 - ) * 2 - 1 - tokenized_df["token"] = ( - tokenized_df["token"].abs() % self.n_features - ) - count_ser = tokenized_df.groupby( - ["doc_id", "token"], sort=True - ).value.sum() - count_ser.name = "count" - else: - tokenized_df["token"] = ( - tokenized_df["token"].abs() % self.n_features - ) - count_ser = tokenized_df.groupby( - ["doc_id", "token"], sort=True - ).size() - count_ser.name = "count" - - count_df = count_ser.reset_index(drop=False) - del count_ser, tokenized_df - return count_df - - def fit_transform(self, X, y=None): - """ - Transform a sequence of documents to a document-term matrix. - - Parameters - ---------- - X : iterable over raw text documents, length = n_samples - Samples. Each sample must be a text document (either bytes or - unicode strings, file name or file object depending on the - constructor argument) which will be tokenized and hashed. - y : any - Ignored. This parameter exists only for compatibility with - sklearn.pipeline.Pipeline. - - Returns - ------- - X : sparse CuPy CSR matrix of shape (n_samples, n_features) - Document-term matrix. - """ - return self.fit(X, y).transform(X) - - def transform(self, raw_documents): - """ - Transform documents to document-term matrix. - - Extract token counts out of raw text documents using the vocabulary - fitted with fit or the one provided to the constructor. - - Parameters - ---------- - raw_documents : cudf.Series or pd.Series - A Series of string documents - - Returns - ------- - X : sparse CuPy CSR matrix of shape (n_samples, n_features) - Document-term matrix. - """ - docs = self._preprocess(raw_documents) - del raw_documents - n_doc = len(docs) - tokenized_df = self._create_tokenized_df(docs) - del docs - count_df = self._count_hash(tokenized_df) - del tokenized_df - empty_doc_ids = self._compute_empty_doc_ids(count_df, n_doc) - X = _create_csr_matrix_from_count_df( - count_df, empty_doc_ids, n_doc, self.n_features, dtype=self.dtype - ) - - if self.binary: - X.data.fill(1) - if self.norm: - if self.norm == "l1": - csr_row_normalize_l1(X, inplace=True) - elif self.norm == "l2": - csr_row_normalize_l2(X, inplace=True) - - return X diff --git a/python/cuml/cuml/feature_extraction/text.py b/python/cuml/cuml/feature_extraction/text.py index f46e55bfec..91545b701c 100644 --- a/python/cuml/cuml/feature_extraction/text.py +++ b/python/cuml/cuml/feature_extraction/text.py @@ -1,27 +1,1408 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# +import warnings +from collections.abc import Mapping +from numbers import Integral +import cudf +import cupy as cp +import cupyx.scipy.sparse as cp_sp +from sklearn.base import OneToOneFeatureMixin -""" -The following imports are needed so that we can import those classes -from cuml.feature_extraction.text just like scikit-learn. Do not remove. -""" - -from cuml.feature_extraction._tfidf import ( # noqa # pylint: disable=unused-import - TfidfTransformer, -) -from cuml.feature_extraction._tfidf_vectorizer import ( # noqa # pylint: disable=unused-import - TfidfVectorizer, -) -from cuml.feature_extraction._vectorizers import ( # noqa # pylint: disable=unused-import - CountVectorizer, - HashingVectorizer, +from cuml.common.sparse import csr_row_normalize_l1, csr_row_normalize_l2 +from cuml.internals.base import Base +from cuml.internals.mixins import DeprecatedGetFeatureNamesMixin +from cuml.internals.outputs import ReflectedAttr, mlfunc +from cuml.internals.validation import ( + check_array, + check_cudf, + check_inputs, + check_is_fitted, ) -__all__ = [ - "CountVectorizer", +__all__ = ( "HashingVectorizer", + "CountVectorizer", "TfidfTransformer", "TfidfVectorizer", -] +) + + +def _check_vocabulary(vocab): + """Validate and normalizer a user-provided vocabulary. + + Parameters + ---------- + vocab : Iterable[str], Mapping[str, int], cudf.Series + The user provided vocabulary. See the docstring for `CountVectorizer` + for more info. + + Returns + ------- + vocabulary_ : cudf.Series + The validated and normalized vocabulary. + """ + if isinstance(vocab, set): + vocab = sorted(vocab) + + if isinstance(vocab, Mapping): + vocab = cudf.Series(vocab.keys(), index=vocab.values()) + if not vocab.index.is_unique: + raise ValueError("Vocabulary contains repeated indices.") + vocab.sort_index(inplace=True) + missing = cudf.RangeIndex(len(vocab)).difference(vocab.index) + if len(missing): + raise ValueError( + f"Vocabulary of size {len(vocab)} doesn't contain index " + f"{missing[0]}" + ) + vocab = vocab.reset_index(drop=True) + else: + vocab = cudf.Series(vocab) + duplicates = vocab[vocab.duplicated()] + if len(duplicates): + raise ValueError( + f"Duplicate term in vocabulary: {duplicates.iloc[0]}" + ) + + if not len(vocab): + raise ValueError("empty vocabulary passed to fit") + + return vocab + + +def _check_oneof(estimator, name, options): + """A helper for validating estimator parameters are within a certain set of + options""" + value = getattr(estimator, name) + if value not in options: + raise ValueError( + f"Expected `{name}` to be one of {options!r}, got {value!r}" + ) + + +class _BaseVectorizer(Base): + """A base class for all vectorizers""" + + def __init__( + self, + *, + lowercase=True, + preprocessor=None, + stop_words=None, + ngram_range=(1, 1), + analyzer="word", + delimiter=" ", + binary=False, + dtype=cp.float32, + verbose=False, + output_type=None, + ): + super().__init__(verbose=verbose, output_type=output_type) + self.lowercase = lowercase + self.preprocessor = preprocessor + self.stop_words = stop_words + self.ngram_range = ngram_range + self.analyzer = analyzer + self.delimiter = delimiter + self.binary = binary + self.dtype = dtype + + def _get_param_names(self): + return [ + "lowercase", + "preprocessor", + "stop_words", + "ngram_range", + "analyzer", + "delimiter", + "binary", + "dtype", + *super()._get_param_names(), + ] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.string = True + tags.input_tags.one_d_array = True + tags.input_tags.two_d_array = False + return tags + + def _check_params(self): + min_n, max_n = self.ngram_range + if min_n < 1: + raise ValueError( + f"Invalid value for ngram_range={self.ngram_range} " + "lower boundary must be >= 1." + ) + elif min_n > max_n: + raise ValueError( + f"Invalid value for ngram_range={self.ngram_range} " + "lower boundary larger than the upper boundary." + ) + _check_oneof(self, "analyzer", ["word", "char", "char_wb"]) + + if self.analyzer != "word" and self.stop_words is not None: + warnings.warn( + "The parameter 'stop_words' will not be used" + " since 'analyzer' != 'word'" + ) + + def _get_stop_words(self): + """Validate and normalize the specified `stop_words`""" + if self.stop_words == "english": + from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS + + return cudf.Series(ENGLISH_STOP_WORDS) + elif isinstance(self.stop_words, str): + raise ValueError("not a built-in stop list: %s" % self.stop_words) + return cudf.Series(self.stop_words) + + def _preprocess(self, X): + """Preprocess `X`. + + The default preprocessor lowercases all inputs. + + Additionally, if `analyzer="word"`, it: + - Removes all non-alphanumeric characters (excluding "_") + - Normalizes whitespace to " " + - Removes any single character tokens + - Removes any tokens specified by `stop_words`. + + The whole step may be overridden by specifying `preprocessor` + explicitly. + """ + if self.preprocessor is not None: + return self.preprocessor(X) + + if self.lowercase: + X = X.str.lower() + + if self.analyzer == "word": + # XXX: a filler string to take the place of _ temporarily + # since `filter_alphanum` strips `_` but sklearn keeps `_`. + # We can use `X` in the common case of lowercase normalization + # since no uppercase letters will remain. Otherwise pick + # an unlikely key of unicode characters" + flag = "X" if self.lowercase else "cuᵐl" + X = ( + X.str.replace("_", flag, regex=False) + .str.filter_alphanum(self.delimiter, keep=True) + .str.replace(flag, "_", regex=False) + ) + + # sklearn by default removes single char tokens + X = X.str.filter_tokens(2, delimiter=self.delimiter) + + # Drop any stop words specified + if self.stop_words is not None: + X = X.str.replace_tokens( + self._get_stop_words(), + " ", + delimiter=self.delimiter, + ) + + return X + + def _to_tokens(self, X): + """The main tokenization step. + + Parameters + ---------- + X : cudf.Series + A series of strings + + Returns + ------- + tokens : cudf.DataFrame + A dataframe with schema ``{"doc_id": int, "token": str}``, where + `doc_id` is the integer id of the document, and `token` is a token + present in that doc. + """ + X = self._preprocess(X) + parts = [] + + if self.analyzer == "word": + token_counts = X.str.token_count(delimiter=self.delimiter) + for ngram_size in range( + self.ngram_range[0], self.ngram_range[1] + 1 + ): + ngrams = X.str.ngrams_tokenize( + n=ngram_size, + delimiter=self.delimiter, + separator=" ", + ) + ngram_count = (token_counts - (ngram_size - 1)).clip(0) + parts.append( + cudf.DataFrame( + { + "doc_id": X.index.repeat(ngram_count), + "token": ngrams, + } + ) + ) + else: + if self.analyzer == "char_wb": + words = X.str.tokenize() + padding = cudf.Series(" ", index=words.index) + X = padding.str.cat([words, padding]) + for ngram_size in range( + self.ngram_range[0], self.ngram_range[1] + 1 + ): + if ngram_size == 1: + ngrams = X.str.character_tokenize() + else: + ngrams = X.str.character_ngrams(ngram_size) + ngrams.index.name = "doc_id" + ngrams.name = "token" + parts.append(ngrams.reset_index()) + + if len(parts) == 1: + return parts[0] + return cudf.concat(parts) + + def _to_sparse(self, X, values, n_features): + """Create a sparse matrix from vectorizer output. + + Parameters + ---------- + X : cudf.Series + The original X input series + values : cudf.DataFrame + A dataframe with schema ``{"doc_id": int, "feature_id": int, + "value": float}``. The input is assumed to have already been + sorted by ``(doc_id, feature_id)``. + n_features : int + The number of features (columns) in the output. + + Returns + ------- + out : cupyx.scipy.sparse.csr_matrix + """ + data = values["value"].values.astype(self.dtype, copy=False) + indices = values.feature_id.values + doc_id_counts = values.doc_id.value_counts().reindex( + X.index, fill_value=0 + ) + indptr = cp.zeros(len(doc_id_counts) + 1, dtype="int32") + cp.cumsum(doc_id_counts.values, out=indptr[1:]) + + return cp_sp.csr_matrix( + (data, indices, indptr), + shape=(len(X), n_features), + ) + + +class HashingVectorizer(_BaseVectorizer): + """Convert a collection of text documents to a matrix of token occurrences. + + It turns a collection of text documents into a sparse matrix holding + token occurrence counts (or binary occurrence information), possibly + normalized as token frequencies if norm='l1' or projected on the euclidean + unit sphere if norm='l2'. + + This text vectorizer implementation uses the hashing trick to find the + token string name to feature integer index mapping. + + This strategy has several advantages: + + - it is very low memory scalable to large datasets as there is no need to + store a vocabulary dictionary in memory. + + - it is fast to pickle and un-pickle as it holds no state besides the + constructor parameters. + + - it can be used in a streaming (partial fit) or parallel pipeline as there + is no state computed during fit. + + There are also a couple of cons (vs using a CountVectorizer with an + in-memory vocabulary): + + - there is no way to compute the inverse transform (from feature indices to + string feature names) which can be a problem when trying to introspect + which features are most important to a model. + + - there can be collisions: distinct tokens can be mapped to the same + feature index. However in practice this is rarely an issue if n_features + is large enough (e.g. 2 ** 18 for text classification problems). + + - no IDF weighting as this would render the transformer stateful. + + The hash function employed is the signed 32-bit version of Murmurhash3. + + Parameters + ---------- + lowercase : bool, default=True + Convert all characters to lowercase before tokenizing. + + preprocessor : callable, default=None + Override the preprocessing (string transformation) stage while + preserving the tokenizing and n-grams generation steps. + + stop_words : {'english'}, list, default=None + If 'english', a built-in stop word list for English is used. If a list, + that list is assumed to contain stop words, all of which will be + removed from the resulting tokens. If None, no stop words will be used. + Only applies if ``analyzer == 'word'``. + + ngram_range : tuple (min_n, max_n), default=(1, 1) + The lower and upper boundary of the range of n-values for different + n-grams to be extracted. All values of n such that min_n <= n <= max_n + will be used. For example an ``ngram_range`` of ``(1, 1)`` means only + unigrams, ``(1, 2)`` means unigrams and bigrams, and ``(2, 2)`` means + only bigrams. + + analyzer : {'word', 'char', 'char_wb'}, default='word' + Whether the feature should be made of word or character n-grams. + Option 'char_wb' creates character n-grams only from text inside + word boundaries; n-grams at the edges of words are padded with space. + + n_features : int, default=(2 ** 20) + The number of features (columns) in the output matrices. Small numbers + of features are likely to cause hash collisions, but large numbers + will cause larger coefficient dimensions in linear learners. + + binary : bool, default=False + If True, all non zero counts are set to 1. This is useful for discrete + probabilistic models that model binary events rather than integer + counts. + + norm : {'l1', 'l2', None}, default='l2' + Norm used to normalize term vectors. None for no normalization. + + alternate_sign : bool, default=True + When True, an alternating sign is added to the features as to + approximately conserve the inner product in the hashed space even for + small n_features. This approach is similar to sparse random projection. + + dtype : type, default=np.float32 + Type of the matrix returned by fit_transform() or transform(). + + delimiter : str, default=" " + String used as a replacement for stop words if `stop_words` is not + None. Typically the delimiting character between words is a good + choice. + + Examples + -------- + >>> from cuml.feature_extraction.text import HashingVectorizer + >>> corpus = [ + ... 'This is the first document.', + ... 'This document is the second document.', + ... 'And this is the third one.', + ... 'Is this the first document?', + ... ] + >>> vectorizer = HashingVectorizer(n_features=2**4) + >>> X = vectorizer.fit_transform(corpus) + >>> X.shape + (4, 16) + """ + + def __init__( + self, + *, + lowercase=True, + preprocessor=None, + stop_words=None, + ngram_range=(1, 1), + analyzer="word", + delimiter=" ", + alternate_sign=True, + n_features=2**20, + dtype=cp.float32, + binary=False, + norm="l2", + verbose=False, + output_type=None, + ): + super().__init__( + lowercase=lowercase, + preprocessor=preprocessor, + stop_words=stop_words, + ngram_range=ngram_range, + analyzer=analyzer, + delimiter=delimiter, + binary=binary, + dtype=dtype, + verbose=verbose, + output_type=output_type, + ) + self.alternate_sign = alternate_sign + self.n_features = n_features + self.norm = norm + + def _get_param_names(self): + return [ + "alternate_sign", + "n_features", + "norm", + *super()._get_param_names(), + ] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.requires_fit = False + return tags + + def _check_params(self): + super()._check_params() + _check_oneof(self, "norm", ["l1", "l2", None]) + if self.n_features < 1: + raise ValueError( + f"Expected `n_features >= 1`, got {self.n_features!r}" + ) + + @mlfunc(set_input_type=True) + def fit(self, X, y=None): + """Only validate's the estimator's parameters. + + This estimator is stateless, ``fit`` is a no-op. + + Parameters + ---------- + X : Iterable[str] + Training samples. Each sample must be a text document + which will be tokenized and hashed. + y : None + Ignored. Exists for API compatibility only. + + Returns + ------- + self : object + The instance itself. + """ + self._check_params() + return self + + def partial_fit(self, X, y=None): + """Only validate's the estimator's parameters. + + This estimator is stateless, ``fit`` is a no-op. + + Parameters + ---------- + X : Iterable[str] + Training samples. Each sample must be a text document + which will be tokenized and hashed. + y : None + Ignored. Exists for API compatibility only. + + Returns + ------- + self : object + The instance itself. + """ + if not hasattr(self, "_input_type"): + self._set_output_type(X) + self._check_params() + return self + + @mlfunc + def transform(self, X): + """Transform a sequence of documents to a document-term matrix. + + Parameters + ---------- + X : Iterable[str] + Training samples. Each sample must be a text document + which will be tokenized and hashed. + + Returns + ------- + X : sparse matrix of shape (n_samples, n_features) + Document-term matrix. + """ + X = check_cudf(X, ensure_ndim=1, input_name="X").reset_index(drop=True) + tokens = self._to_tokens(X) + hashes = tokens.token.hash_values().to_cupy().view("int32") + + if self.binary: + tokens = ( + tokens.assign(feature_id=cp.abs(hashes) % self.n_features) + .drop(columns="token") + .drop_duplicates() + .assign(value=cp.dtype(self.dtype).type(1)) + .sort_values(["doc_id", "feature_id"]) + ) + elif self.alternate_sign: + tokens = ( + tokens.assign( + value=cp.sign(hashes, dtype=self.dtype), + feature_id=cp.abs(hashes) % self.n_features, + ) + .drop(columns="token") + .groupby(["doc_id", "feature_id"], sort=True) + .value.sum() + .reset_index(name="value") + ) + else: + tokens = ( + tokens.assign(feature_id=cp.abs(hashes) % self.n_features) + .drop(columns="token") + .groupby(["doc_id", "feature_id"], sort=True) + .size() + .reset_index(name="value") + ) + + out = self._to_sparse(X, tokens, n_features=self.n_features) + + if self.norm: + if self.norm == "l1": + csr_row_normalize_l1(out, inplace=True) + elif self.norm == "l2": + csr_row_normalize_l2(out, inplace=True) + + return out + + @mlfunc(preserve_index=True) + def fit_transform(self, X, y=None): + """Transform a sequence of documents to a document-term matrix. + + Parameters + ---------- + X : Iterable[str] + Training samples. Each sample must be a text document + which will be tokenized and hashed. + y : None + Ignored. Exists for API compatibility only. + + Returns + ------- + X : sparse matrix of shape (n_samples, n_features) + Document-term matrix. + """ + return self.fit(X, y).transform(X) + + +class CountVectorizer(DeprecatedGetFeatureNamesMixin, _BaseVectorizer): + """Convert a collection of text documents to a matrix of token counts. + + If you do not provide an a-priori dictionary then the number of features + will be equal to the vocabulary size found by analyzing the data. + + Parameters + ---------- + lowercase : bool, default=True + Convert all characters to lowercase before tokenizing. + + preprocessor : callable, default=None + Override the preprocessing (string transformation) stage while + preserving the tokenizing and n-grams generation steps. + + stop_words : {'english'}, list, default=None + If 'english', a built-in stop word list for English is used. If a list, + that list is assumed to contain stop words, all of which will be + removed from the resulting tokens. If None, no stop words will be used. + Only applies if ``analyzer == 'word'``. + + ngram_range : tuple (min_n, max_n), default=(1, 1) + The lower and upper boundary of the range of n-values for different + word n-grams or char n-grams to be extracted. All values of n such + such that min_n <= n <= max_n will be used. For example an + ``ngram_range`` of ``(1, 1)`` means only unigrams, ``(1, 2)`` means + unigrams and bigrams, and ``(2, 2)`` means only bigrams. + + analyzer : {'word', 'char', 'char_wb'}, default='word' + Whether the feature should be made of word or character n-grams. + Option 'char_wb' creates character n-grams only from text inside + word boundaries; n-grams at the edges of words are padded with space. + + max_df : float in range [0.0, 1.0] or int, default=1.0 + When building the vocabulary ignore terms that have a document + frequency strictly higher than the given threshold (corpus-specific + stop words). + If float, the parameter represents a proportion of documents, integer + absolute counts. + This parameter is ignored if vocabulary is not None. + + min_df : float in range [0.0, 1.0] or int, default=1 + When building the vocabulary ignore terms that have a document + frequency strictly lower than the given threshold. This value is also + called cut-off in the literature. + If float, the parameter represents a proportion of documents, integer + absolute counts. + This parameter is ignored if vocabulary is not None. + + max_features : int, default=None + If not None, build a vocabulary that only consider the top + `max_features` ordered by term frequency across the corpus. + Otherwise, all features are used. + + This parameter is ignored if vocabulary is not None. + + vocabulary : array-like or mapping, default=None + Either an array-like of terms, or a mapping where keys are terms and + values are indices in the feature matrix. If not given, a vocabulary is + determined from the input documents. + + binary : bool, default=False + If True, all non zero counts are set to 1. This is useful for discrete + probabilistic models that model binary events rather than integer + counts. + + dtype : dtype, default=np.float32 + Type of the matrix returned by fit_transform() or transform(). + + delimiter : str, default=" " + String used as a replacement for stop words if `stop_words` is not + None. Typically the delimiting character between words is a good + choice. + + Attributes + ---------- + vocabulary_ : cudf.Series + The vocabulary used to map terms to feature indices. + + fixed_vocabulary_ : bool + True if a fixed vocabulary of term to indices mapping + is provided by the user. + + Examples + -------- + >>> from cuml.feature_extraction.text import CountVectorizer + >>> corpus = [ + ... 'This is the first document.', + ... 'This document is the second document.', + ... 'And this is the third one.', + ... 'Is this the first document?', + ... ] + >>> vectorizer = CountVectorizer() + >>> X = vectorizer.fit_transform(corpus) + >>> vectorizer.get_feature_names_out() + array(['and', 'document', 'first', 'is', 'one', 'second', 'the', 'third', + 'this'], ...) + >>> X.shape + (4, 9) + """ + + def __init__( + self, + *, + lowercase=True, + preprocessor=None, + stop_words=None, + ngram_range=(1, 1), + analyzer="word", + delimiter=" ", + max_df=1.0, + min_df=1, + max_features=None, + vocabulary=None, + binary=False, + dtype=cp.float32, + verbose=False, + output_type=None, + ): + super().__init__( + lowercase=lowercase, + preprocessor=preprocessor, + stop_words=stop_words, + ngram_range=ngram_range, + analyzer=analyzer, + delimiter=delimiter, + binary=binary, + dtype=dtype, + verbose=verbose, + output_type=output_type, + ) + self.max_df = max_df + self.min_df = min_df + self.max_features = max_features + self.vocabulary = vocabulary + + def _get_param_names(self): + return [ + "max_df", + "min_df", + "max_features", + "vocabulary", + *super()._get_param_names(), + ] + + def _check_params(self): + super()._check_params() + for name in ["max_df", "min_df"]: + val = getattr(self, name) + if not ( + val >= 1 if isinstance(val, Integral) else (0 <= val <= 1) + ): + raise ValueError( + f"Expected a float `(0.0 <= {name} <= 1.0)` or an int " + f"`({name} >= 1)`, got {val}." + ) + if self.max_features is not None and self.max_features < 1: + raise ValueError( + f"Expected `max_features >= 1`, got {self.max_features}" + ) + + def _fit(self, X, tokens): + self.fixed_vocabulary_ = self.vocabulary is not None + + if self.vocabulary is not None: + self.vocabulary_ = _check_vocabulary(self.vocabulary) + else: + n_doc = X.shape[0] + max_features = self.max_features + max_doc_count = ( + self.max_df + if isinstance(self.max_df, Integral) + else self.max_df * n_doc + ) + min_doc_count = ( + self.min_df + if isinstance(self.min_df, Integral) + else self.min_df * n_doc + ) + if max_doc_count < min_doc_count: + raise ValueError( + "max_df corresponds to < documents than min_df" + ) + + if ( + max_doc_count < n_doc + or min_doc_count > 1 + or max_features is not None + ): + counts = tokens.token.value_counts() + if max_doc_count < n_doc: + counts = counts[counts <= max_doc_count] + if min_doc_count > 1: + counts = counts[counts >= min_doc_count] + if max_features is not None: + counts = counts.iloc[:max_features] + self.vocabulary_ = cudf.Series(counts.index.sort_values()) + if not len(self.vocabulary_): + raise ValueError( + "After pruning, no terms remain. Try a lower min_df or " + "a higher max_df." + ) + else: + self.vocabulary_ = ( + tokens.token.drop_duplicates() + .sort_values() + .reset_index(drop=True) + ) + if not len(self.vocabulary_): + raise ValueError( + "empty vocabulary; perhaps the documents only contain stop words" + ) + return self + + def _transform(self, X, tokens): + tokens = ( + tokens.assign( + token=tokens.token.astype( + cudf.CategoricalDtype(self.vocabulary_) + ).cat.codes + ) + .rename(columns={"token": "feature_id"}) + .dropna() + ) + + if self.binary: + tokens = ( + tokens.drop_duplicates() + .assign(value=cp.dtype(self.dtype).type(1)) + .sort_values(["doc_id", "feature_id"]) + ) + else: + tokens = ( + tokens.groupby(["doc_id", "feature_id"], sort=True) + .size() + .reset_index(name="value") + ) + + return self._to_sparse(X, tokens, n_features=len(self.vocabulary_)) + + @mlfunc(set_input_type=True) + def fit(self, X, y=None): + """Fit the vectorizer. + + Parameters + ---------- + X : Iterable[str] + Training samples. Each sample must be a text document + which will be tokenized and hashed. + y : None + Ignored. Exists for API compatibility only. + + Returns + ------- + self : object + The instance itself. + """ + self._check_params() + X = check_cudf(X, ensure_ndim=1, input_name="X") + tokens = self._to_tokens(X) + self._fit(X, tokens) + return self + + @mlfunc(set_input_type=True) + def fit_transform(self, X, y=None): + """Fit the vectorizer and return a document-term matrix. + + Parameters + ---------- + X : Iterable[str] + Training samples. Each sample must be a text document + which will be tokenized and hashed. + y : None + Ignored. Exists for API compatibility only. + + Returns + ------- + X : sparse matrix of shape (n_samples, n_features) + Document-term matrix. + """ + self._check_params() + X = check_cudf(X, ensure_ndim=1, input_name="X") + tokens = self._to_tokens(X) + self._fit(X, tokens) + return self._transform(X, tokens) + + @mlfunc + def transform(self, X): + """Transform documents to document-term matrix. + + Parameters + ---------- + X : Iterable[str] + Training samples. Each sample must be a text document + which will be tokenized and hashed. + + Returns + ------- + X : sparse matrix of shape (n_samples, n_features) + Document-term matrix. + """ + check_is_fitted(self) + X = check_cudf(X, ensure_ndim=1, input_name="X") + tokens = self._to_tokens(X) + return self._transform(X, tokens) + + @mlfunc(preserve_index=True) + def inverse_transform(self, X): + """Return terms per document with nonzero entries in X. + + Parameters + ---------- + X : array-like or sparse matrix, shape=(n_samples, n_features) + Document-term matrix. + + Returns + ------- + X_original : list of arrays of shape (n_samples,) + List of arrays of terms. + """ + check_is_fitted(self) + X = check_array(X, accept_sparse="csr", mem_type="host") + n_samples = X.shape[0] + vocab = self.vocabulary_.to_numpy() + return [vocab[X[i, :].nonzero()[-1]].ravel() for i in range(n_samples)] + + def get_feature_names_out(self, input_features=None): + """Get output feature names for transformation. + + Parameters + ---------- + input_features : array-like of str or None, default=None + Not used, present here for API consistency by convention. + + Returns + ------- + feature_names_out : numpy.ndarray of str objects. + Transformed feature names. + """ + check_is_fitted(self) + return self.vocabulary_.to_numpy(dtype=object) + + +class TfidfTransformer(OneToOneFeatureMixin, Base): + """Transform a count matrix to a normalized tf or tf-idf representation. + + Tf means term-frequency while tf-idf means term-frequency times inverse + document-frequency. This is a common term weighting scheme in information + retrieval, that has also found good use in document classification. + + The goal of using tf-idf instead of the raw frequencies of occurrence of a + token in a given document is to scale down the impact of tokens that occur + very frequently in a given corpus and that are hence empirically less + informative than features that occur in a small fraction of the training + corpus. + + The formula that is used to compute the tf-idf for a term t of a document d + in a document set is tf-idf(t, d) = tf(t, d) * idf(t), and the idf is + computed as idf(t) = log [ n / df(t) ] + 1 (if ``smooth_idf=False``), where + n is the total number of documents in the document set and df(t) is the + document frequency of t; the document frequency is the number of documents + in the document set that contain the term t. The effect of adding "1" to + the idf in the equation above is that terms with zero idf, i.e., terms + that occur in all documents in a training set, will not be entirely + ignored. + (Note that the idf formula above differs from the standard textbook + notation that defines the idf as + idf(t) = log [ n / (df(t) + 1) ]). + + If ``smooth_idf=True`` (the default), the constant "1" is added to the + numerator and denominator of the idf as if an extra document was seen + containing every term in the collection exactly once, which prevents + zero divisions: idf(t) = log [ (1 + n) / (1 + df(t)) ] + 1. + + Furthermore, the formulas used to compute tf and idf depend + on parameter settings that correspond to the SMART notation used in IR + as follows: + + Tf is "n" (natural) by default, "l" (logarithmic) when + ``sublinear_tf=True``. + Idf is "t" when use_idf is given, "n" (none) otherwise. + Normalization is "c" (cosine) when ``norm='l2'``, "n" (none) + when ``norm=None``. + + Parameters + ---------- + norm : {'l1', 'l2', None}, default='l2' + Norm used to normalize term vectors. None for no normalization. + + use_idf : bool, default=True + Enable inverse-document-frequency reweighting. + + smooth_idf : bool, default=True + Smooth idf weights by adding one to document frequencies, as if an + extra document was seen containing every term in the collection + exactly once. Prevents zero divisions. + + sublinear_tf : bool, default=False + Apply sublinear tf scaling, i.e. replace tf with 1 + log(tf). + + Attributes + ---------- + idf_ : array of shape (n_features) + The inverse document frequency (IDF) vector; only defined + if ``use_idf`` is True. + + Examples + -------- + >>> from cuml.feature_extraction.text import TfidfTransformer + >>> from cuml.feature_extraction.text import CountVectorizer + >>> from sklearn.pipeline import Pipeline + >>> corpus = ['this is the first document', + ... 'this document is the second document', + ... 'and this is the third one', + ... 'is this the first document'] + >>> pipe = Pipeline([('count', CountVectorizer()), + ... ('tfid', TfidfTransformer())]) + >>> X = pipe.fit_transform(corpus) + >>> X.shape + (4, 9) + """ + + idf_ = ReflectedAttr() + + def __init__( + self, + *, + norm="l2", + use_idf=True, + smooth_idf=True, + sublinear_tf=False, + verbose=False, + output_type=None, + ): + super().__init__(verbose=verbose, output_type=output_type) + self.norm = norm + self.use_idf = use_idf + self.smooth_idf = smooth_idf + self.sublinear_tf = sublinear_tf + + def _get_param_names(self): + return [ + "norm", + "use_idf", + "smooth_idf", + "sublinear_tf", + *super()._get_param_names(), + ] + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.input_tags.sparse = True + tags.transformer_tags.preserves_dtype = ["float64", "float32"] + return tags + + def _check_params(self): + _check_oneof(self, "norm", ["l1", "l2", None]) + + def _check_X(self, X, reset=False, copy=False): + """Validate and normalize X to a CSR sparse matrix""" + X = check_inputs( + self, + X, + accept_sparse="csr", + dtype=("float32", "float64"), + reset=reset, + copy=copy, + ) + if not cp_sp.issparse(X): + X = cp_sp.csr_matrix(X) + return X + + def _fit(self, X): + assert cp_sp.issparse(X) + assert X.format == "csr" + if self.use_idf: + n_samples, _ = X.shape + + # Compute document frequency + df = cp.bincount(X.indices, minlength=X.shape[1]).astype( + X.dtype, copy=False + ) + + # perform idf smoothing if required + if self.smooth_idf: + df += 1.0 + n_samples += 1 + + # log + 1 instead of log makes sure terms with zero idf don't get + # suppressed entirely. + idf = cp.full_like(df, fill_value=n_samples, dtype=X.dtype) + idf /= df + cp.log(idf, out=idf) + idf += 1.0 + self.idf_ = idf + + return self + + def _transform(self, X): + assert cp_sp.issparse(X) + assert X.format == "csr" + if self.sublinear_tf: + cp.log(X.data, out=X.data) + X.data += 1.0 + + if hasattr(self, "idf_"): + # the columns of X (CSR matrix) can be accessed with `X.indices `and + # multiplied with the corresponding `idf` value + X.data *= self.idf_[X.indices] + + if self.norm is not None: + if self.norm == "l1": + csr_row_normalize_l1(X, inplace=True) + elif self.norm == "l2": + csr_row_normalize_l2(X, inplace=True) + + return X + + @mlfunc(set_input_type=True) + def fit(self, X, y=None): + """Fit the transformer. + + Parameters + ---------- + X : sparse matrix of shape (n_samples, n_features) + A matrix of term/token counts. + + y : None + Ignored. Exists for API compatibility only. + + Returns + ------- + self : object + The instance itself. + """ + self._check_params() + X = self._check_X(X, reset=True) + return self._fit(X) + + @mlfunc(set_input_type=True) + def fit_transform(self, X, y=None): + """Fit the transformer, then transform X. + + Parameters + ---------- + X : sparse matrix of shape (n_samples, n_features) + A matrix of term/token counts. + + y : None + Ignored. Exists for API compatibility only. + + Returns + ------- + X : sparse matrix of shape (n_samples, n_features) + Tf-idf weighted document-term matrix. + """ + self._check_params() + X = self._check_X(X, reset=True) + return self._fit(X)._transform(X) + + @mlfunc + def transform(self, X, copy=True): + """Transform a count matrix to tf or tf-idf representation. + + Parameters + ---------- + X : sparse matrix of shape (n_samples, n_features) + A matrix of term/token counts. + + copy : bool, default=True + If `copy=False,` then `transform` may mutate `X` in-place when + possible. + + Returns + ------- + X : sparse matrix of shape (n_samples, n_features) + Tf-idf weighted document-term matrix. + """ + check_is_fitted(self) + X = self._check_X(X, copy=copy) + return self._transform(X) + + +class TfidfVectorizer(CountVectorizer): + """Convert a collection of raw documents to a matrix of TF-IDF features. + + Equivalent to `CountVectorizer` followed by `TfidfTransformer`. + + Parameters + ---------- + lowercase : bool, default=True + Convert all characters to lowercase before tokenizing. + + preprocessor : callable, default=None + Override the preprocessing (string transformation) stage while + preserving the tokenizing and n-grams generation steps. + + stop_words : {'english'}, list, default=None + If 'english', a built-in stop word list for English is used. If a list, + that list is assumed to contain stop words, all of which will be + removed from the resulting tokens. If None, no stop words will be used. + Only applies if ``analyzer == 'word'``. + + ngram_range : tuple (min_n, max_n), default=(1, 1) + The lower and upper boundary of the range of n-values for different + word n-grams or char n-grams to be extracted. All values of n such + such that min_n <= n <= max_n will be used. For example an + ``ngram_range`` of ``(1, 1)`` means only unigrams, ``(1, 2)`` means + unigrams and bigrams, and ``(2, 2)`` means only bigrams. + + analyzer : {'word', 'char', 'char_wb'}, default='word' + Whether the feature should be made of word or character n-grams. + Option 'char_wb' creates character n-grams only from text inside + word boundaries; n-grams at the edges of words are padded with space. + + max_df : float in range [0.0, 1.0] or int, default=1.0 + When building the vocabulary ignore terms that have a document + frequency strictly higher than the given threshold (corpus-specific + stop words). + If float, the parameter represents a proportion of documents, integer + absolute counts. + This parameter is ignored if vocabulary is not None. + + min_df : float in range [0.0, 1.0] or int, default=1 + When building the vocabulary ignore terms that have a document + frequency strictly lower than the given threshold. This value is also + called cut-off in the literature. + If float, the parameter represents a proportion of documents, integer + absolute counts. + This parameter is ignored if vocabulary is not None. + + max_features : int, default=None + If not None, build a vocabulary that only consider the top + `max_features` ordered by term frequency across the corpus. + Otherwise, all features are used. + + This parameter is ignored if vocabulary is not None. + + vocabulary : array-like or mapping, default=None + Either an array-like of terms, or a mapping where keys are terms and + values are indices in the feature matrix. If not given, a vocabulary is + determined from the input documents. + + binary : bool, default=False + If True, all non zero counts are set to 1. This is useful for discrete + probabilistic models that model binary events rather than integer + counts. + + dtype : dtype, default=np.float32 + Type of the matrix returned by fit_transform() or transform(). + + delimiter : str, default=" " + String used as a replacement for stop words if `stop_words` is not + None. Typically the delimiting character between words is a good + choice. + + norm : {'l1', 'l2', None}, default='l2' + Norm used to normalize term vectors. None for no normalization. + + use_idf : bool, default=True + Enable inverse-document-frequency reweighting. + + smooth_idf : bool, default=True + Smooth idf weights by adding one to document frequencies, as if an + extra document was seen containing every term in the collection + exactly once. Prevents zero divisions. + + sublinear_tf : bool, default=False + Apply sublinear tf scaling, i.e. replace tf with 1 + log(tf). + + Attributes + ---------- + vocabulary_ : cudf.Series + The vocabulary used to map terms to feature indices. + + fixed_vocabulary_ : bool + True if a fixed vocabulary of term to indices mapping + is provided by the user. + + idf_ : array of shape (n_features) + The inverse document frequency (IDF) vector; only defined + if ``use_idf`` is True. + + Examples + -------- + >>> from cuml.feature_extraction.text import TfidfVectorizer + >>> corpus = [ + ... 'This is the first document.', + ... 'This document is the second document.', + ... 'And this is the third one.', + ... 'Is this the first document?', + ... ] + >>> vectorizer = TfidfVectorizer() + >>> X = vectorizer.fit_transform(corpus) + >>> vectorizer.get_feature_names_out() + array(['and', 'document', 'first', 'is', 'one', 'second', 'the', 'third', + 'this'], ...) + >>> X.shape + (4, 9) + """ + + def __init__( + self, + *, + lowercase=True, + preprocessor=None, + stop_words=None, + ngram_range=(1, 1), + analyzer="word", + delimiter=" ", + max_df=1.0, + min_df=1, + max_features=None, + vocabulary=None, + binary=False, + dtype=cp.float32, + norm="l2", + use_idf=True, + smooth_idf=True, + sublinear_tf=False, + verbose=False, + output_type=None, + ): + super().__init__( + lowercase=lowercase, + preprocessor=preprocessor, + stop_words=stop_words, + ngram_range=ngram_range, + analyzer=analyzer, + delimiter=delimiter, + max_df=max_df, + min_df=min_df, + max_features=max_features, + vocabulary=vocabulary, + binary=binary, + dtype=dtype, + verbose=verbose, + output_type=output_type, + ) + self.norm = norm + self.use_idf = use_idf + self.smooth_idf = smooth_idf + self.sublinear_tf = sublinear_tf + + def _get_param_names(self): + return [ + "norm", + "use_idf", + "smooth_idf", + "sublinear_tf", + *super()._get_param_names(), + ] + + @property + def idf_(self): + return self._tfidf.idf_ + + def _check_params(self): + super()._check_params() + _check_oneof(self, "norm", ["l1", "l2", None]) + + def _fit_transform(self, X, transform=True): + self._check_params() + self._tfidf = TfidfTransformer( + norm=self.norm, + use_idf=self.use_idf, + smooth_idf=self.smooth_idf, + sublinear_tf=self.sublinear_tf, + ) + X = super().fit_transform(X) + self._tfidf.fit(X) + if transform: + return self._tfidf.transform(X) + return self + + @mlfunc(set_input_type=True) + def fit_transform(self, X, y=None): + """Fit the vectorizer and return a document-term matrix. + + Parameters + ---------- + X : Iterable[str] + Training samples. Each sample must be a text document + which will be tokenized and hashed. + y : None + Ignored. Exists for API compatibility only. + + Returns + ------- + X : sparse matrix of shape (n_samples, n_features) + Tf-idf weighted document-term matrix. + """ + return self._fit_transform(X) + + @mlfunc(set_input_type=True) + def fit(self, X, y=None): + """Fit the vectorizer. + + Parameters + ---------- + X : Iterable[str] + Training samples. Each sample must be a text document + which will be tokenized and hashed. + y : None + Ignored. Exists for API compatibility only. + + Returns + ------- + self : object + The instance itself. + """ + return self._fit_transform(X, transform=False) + + @mlfunc + def transform(self, X): + """Transform documents to document-term matrix. + + Parameters + ---------- + X : Iterable[str] + Training samples. Each sample must be a text document + which will be tokenized and hashed. + + Returns + ------- + X : sparse matrix of shape (n_samples, n_features) + Tf-idf weighted document-term matrix. + """ + check_is_fitted(self) + + X = super().transform(X) + return self._tfidf.transform(X, copy=False) diff --git a/python/cuml/tests/test_text_feature_extraction.py b/python/cuml/tests/test_text_feature_extraction.py index 35f911005b..6ddad5a8ba 100644 --- a/python/cuml/tests/test_text_feature_extraction.py +++ b/python/cuml/tests/test_text_feature_extraction.py @@ -7,7 +7,6 @@ import numpy as np import pandas as pd import pytest -from cudf import Series from numpy.testing import assert_array_equal from sklearn.feature_extraction.text import CountVectorizer as SkCountVect from sklearn.feature_extraction.text import HashingVectorizer as SkHashVect @@ -28,9 +27,9 @@ def test_count_vectorizer(): "Is this the first document?", ] - res = CountVectorizer().fit_transform(Series(corpus)) + res = CountVectorizer().fit_transform(corpus) ref = SkCountVect().fit_transform(corpus) - cp.testing.assert_array_equal(res.todense(), ref.toarray()) + cp.testing.assert_array_equal(res.toarray(), ref.toarray()) JUNK_FOOD_DOCS = ( @@ -53,7 +52,6 @@ def test_count_vectorizer(): EMPTY_DOCS = ("",) DOCS = JUNK_FOOD_DOCS + EMPTY_DOCS + NOTJUNK_FOOD_DOCS + EMPTY_DOCS -DOCS_GPU = Series(DOCS) NGRAM_RANGES = [(1, 1), (1, 2), (2, 3)] NGRAM_IDS = [f"ngram_range={str(r)}" for r in NGRAM_RANGES] @@ -61,7 +59,7 @@ def test_count_vectorizer(): @pytest.mark.parametrize("ngram_range", NGRAM_RANGES, ids=NGRAM_IDS) def test_word_analyzer(ngram_range): - v = CountVectorizer(ngram_range=ngram_range).fit(DOCS_GPU) + v = CountVectorizer(ngram_range=ngram_range).fit(DOCS) ref = SkCountVect(ngram_range=ngram_range).fit(DOCS) assert_array_equal( ref.get_feature_names_out(), @@ -71,28 +69,27 @@ def test_word_analyzer(ngram_range): def test_countvectorizer_custom_vocabulary(): vocab = {"pizza": 0, "beer": 1} - vocab_gpu = Series(vocab.keys()) ref = SkCountVect(vocabulary=vocab).fit_transform(DOCS) - X = CountVectorizer(vocabulary=vocab_gpu).fit_transform(DOCS_GPU) - cp.testing.assert_array_equal(X.todense(), ref.toarray()) + X = CountVectorizer(vocabulary=vocab).fit_transform(DOCS) + cp.testing.assert_array_equal(X.toarray(), ref.toarray()) def test_countvectorizer_stop_words(): ref = SkCountVect(stop_words="english").fit_transform(DOCS) - X = CountVectorizer(stop_words="english").fit_transform(DOCS_GPU) - cp.testing.assert_array_equal(X.todense(), ref.toarray()) + X = CountVectorizer(stop_words="english").fit_transform(DOCS) + cp.testing.assert_array_equal(X.toarray(), ref.toarray()) def test_countvectorizer_empty_vocabulary(): v = CountVectorizer(max_df=1.0, stop_words="english") # fitting only on stopwords will result in an empty vocabulary with pytest.raises(ValueError): - v.fit(Series(["to be or not to be", "and me too", "and so do you"])) + v.fit(["to be or not to be", "and me too", "and so do you"]) def test_countvectorizer_stop_words_ngrams(): - stop_words_doc = Series(["and me too andy andy too"]) + stop_words_doc = ["and me too andy andy too"] expected_vocabulary = ["andy andy"] v = CountVectorizer(ngram_range=(2, 2), stop_words="english") @@ -103,33 +100,21 @@ def test_countvectorizer_stop_words_ngrams(): def test_countvectorizer_max_features(): expected_vocabulary = {"burger", "beer", "salad", "pizza"} - expected_stop_words = { - "celeri", - "tomato", - "copyright", - "coke", - "sparkling", - "water", - "the", - } # test bounded number of extracted features vec = CountVectorizer(max_df=0.6, max_features=4) - vec.fit(DOCS_GPU) + vec.fit(DOCS) assert set(vec.get_feature_names_out()) == expected_vocabulary - assert set(vec.stop_words_.to_arrow().to_pylist()) == expected_stop_words def test_countvectorizer_max_features_counts(): - JUNK_FOOD_DOCS_GPU = Series(JUNK_FOOD_DOCS) - cv_1 = CountVectorizer(max_features=1) cv_3 = CountVectorizer(max_features=3) cv_None = CountVectorizer(max_features=None) - counts_1 = cv_1.fit_transform(JUNK_FOOD_DOCS_GPU).sum(axis=0) - counts_3 = cv_3.fit_transform(JUNK_FOOD_DOCS_GPU).sum(axis=0) - counts_None = cv_None.fit_transform(JUNK_FOOD_DOCS_GPU).sum(axis=0) + counts_1 = cv_1.fit_transform(JUNK_FOOD_DOCS).sum(axis=0) + counts_3 = cv_3.fit_transform(JUNK_FOOD_DOCS).sum(axis=0) + counts_None = cv_None.fit_transform(JUNK_FOOD_DOCS).sum(axis=0) features_1 = cv_1.get_feature_names_out() features_3 = cv_3.get_feature_names_out() @@ -144,70 +129,60 @@ def test_countvectorizer_max_features_counts(): def as_index(x): return x.astype(cp.int32).item() - assert "the" == features_1[as_index(cp.argmax(counts_1))] - assert "the" == features_3[as_index(cp.argmax(counts_3))] - assert "the" == features_None[as_index(cp.argmax(counts_None))] + assert "the" == features_1[as_index(counts_1.argmax())] + assert "the" == features_3[as_index(counts_3.argmax())] + assert "the" == features_None[as_index(counts_None.argmax())] def test_countvectorizer_max_df(): - test_data = Series(["abc", "dea", "eat"]) + test_data = ["abc", "dea", "eat"] vect = CountVectorizer(analyzer="char", max_df=1.0) vect.fit(test_data) assert "a" in vect.vocabulary_.to_arrow().to_pylist() assert len(vect.vocabulary_.to_arrow().to_pylist()) == 6 - assert len(vect.stop_words_) == 0 vect.max_df = 0.5 # 0.5 * 3 documents -> max_doc_count == 1.5 vect.fit(test_data) assert "a" not in vect.vocabulary_.to_arrow().to_pylist() # {ae} ignored assert len(vect.vocabulary_.to_arrow().to_pylist()) == 4 # {bcdt} remain - assert "a" in vect.stop_words_.to_arrow().to_pylist() - assert len(vect.stop_words_) == 2 vect.max_df = 1 vect.fit(test_data) assert "a" not in vect.vocabulary_.to_arrow().to_pylist() # {ae} ignored assert len(vect.vocabulary_.to_arrow().to_pylist()) == 4 # {bcdt} remain - assert "a" in vect.stop_words_.to_arrow().to_pylist() - assert len(vect.stop_words_) == 2 def test_vectorizer_min_df(): - test_data = Series(["abc", "dea", "eat"]) + test_data = ["abc", "dea", "eat"] vect = CountVectorizer(analyzer="char", min_df=1) vect.fit(test_data) assert "a" in vect.vocabulary_.to_arrow().to_pylist() assert len(vect.vocabulary_.to_arrow().to_pylist()) == 6 - assert len(vect.stop_words_) == 0 vect.min_df = 2 vect.fit(test_data) assert "c" not in vect.vocabulary_.to_arrow().to_pylist() # {bcdt} ignored assert len(vect.vocabulary_.to_arrow().to_pylist()) == 2 # {ae} remain - assert "c" in vect.stop_words_.to_arrow().to_pylist() - assert len(vect.stop_words_) == 4 vect.min_df = 0.8 # 0.8 * 3 documents -> min_doc_count == 2.4 vect.fit(test_data) # {bcdet} ignored assert "c" not in vect.vocabulary_.to_arrow().to_pylist() assert len(vect.vocabulary_.to_arrow().to_pylist()) == 1 # {a} remains - assert "c" in vect.stop_words_.to_arrow().to_pylist() - assert len(vect.stop_words_) == 5 def test_count_binary_occurrences(): # by default multiple occurrences are counted as longs - test_data = Series(["aaabc", "abbde"]) + test_data = ["aaabc", "abbde"] vect = CountVectorizer(analyzer="char", max_df=1.0) - X = cp.asnumpy(vect.fit_transform(test_data).todense()) + X = cp.asnumpy(vect.fit_transform(test_data).toarray()) assert_array_equal(["a", "b", "c", "d", "e"], vect.get_feature_names_out()) assert_array_equal([[3, 1, 1, 0, 0], [1, 2, 0, 1, 1]], X) # using boolean features, we can fetch the binary occurrence info # instead. vect = CountVectorizer(analyzer="char", max_df=1.0, binary=True) - X = cp.asnumpy(vect.fit_transform(test_data).todense()) + X = cp.asnumpy(vect.fit_transform(test_data).toarray()) assert_array_equal([[1, 1, 1, 0, 0], [1, 1, 0, 1, 1]], X) # check the ability to change the dtype @@ -220,7 +195,7 @@ def test_count_binary_occurrences(): def test_vectorizer_inverse_transform(): vectorizer = CountVectorizer() - transformed_data = vectorizer.fit_transform(DOCS_GPU) + transformed_data = vectorizer.fit_transform(DOCS) inversed_data = vectorizer.inverse_transform(transformed_data) sk_vectorizer = SkCountVect() @@ -228,10 +203,8 @@ def test_vectorizer_inverse_transform(): sk_inversed_data = sk_vectorizer.inverse_transform(sk_transformed_data) for doc, sk_doc in zip(inversed_data, sk_inversed_data): - doc = np.sort(doc.to_arrow().to_pylist()) + doc = np.sort(doc) sk_doc = np.sort(sk_doc) - if len(doc) + len(sk_doc) == 0: - continue assert_array_equal(doc, sk_doc) @@ -243,8 +216,7 @@ def test_vectorizer_inverse_transform(): @pytest.mark.parametrize("ngram_range", NGRAM_RANGES, ids=NGRAM_IDS) def test_space_ngrams(ngram_range): data = ["abc def. 123 456 789"] - data_gpu = Series(data) - vec = CountVectorizer(ngram_range=ngram_range).fit(data_gpu) + vec = CountVectorizer(ngram_range=ngram_range).fit(data) ref = SkCountVect(ngram_range=ngram_range).fit(data) assert_array_equal( ref.get_feature_names_out(), @@ -254,47 +226,43 @@ def test_space_ngrams(ngram_range): def test_empty_doc_after_limit_features(): data = ["abc abc def", "def abc", "ghi"] - data_gpu = Series(data) - count = CountVectorizer(min_df=2).fit_transform(data_gpu) + count = CountVectorizer(min_df=2).fit_transform(data) ref = SkCountVect(min_df=2).fit_transform(data) - cp.testing.assert_array_equal(count.todense(), ref.toarray()) + cp.testing.assert_array_equal(count.toarray(), ref.toarray()) def test_countvectorizer_separate_fit_transform(): - res = CountVectorizer().fit(DOCS_GPU).transform(DOCS_GPU) + res = CountVectorizer().fit(DOCS).transform(DOCS) ref = SkCountVect().fit(DOCS).transform(DOCS) - cp.testing.assert_array_equal(res.todense(), ref.toarray()) + cp.testing.assert_array_equal(res.toarray(), ref.toarray()) def test_non_ascii(): non_ascii = ("This is ascii,", "but not this Αγγλικά.") - non_ascii_gpu = Series(non_ascii) cv = CountVectorizer() - res = cv.fit_transform(non_ascii_gpu) + res = cv.fit_transform(non_ascii) ref = SkCountVect().fit_transform(non_ascii) assert "αγγλικά" in set(cv.get_feature_names_out()) - cp.testing.assert_array_equal(res.todense(), ref.toarray()) + cp.testing.assert_array_equal(res.toarray(), ref.toarray()) -def test_sngle_len(): - single_token_ser = ["S I N G L E T 0 K E N Example", "1 2 3 4 5 eg"] - single_token_gpu = Series(single_token_ser) +def test_single_token_length(): + data = ["S I N G L E T 0 K E N Example", "1 2 3 4 5 eg"] cv = CountVectorizer() - res = cv.fit_transform(single_token_gpu) - ref = SkCountVect().fit_transform(single_token_ser) + res = cv.fit_transform(data) + ref = SkCountVect().fit_transform(data) - cp.testing.assert_array_equal(res.todense(), ref.toarray()) + cp.testing.assert_array_equal(res.toarray(), ref.toarray()) def test_only_delimiters(): data = ["abc def. 123", " ", "456 789"] - data_gpu = Series(data) - res = CountVectorizer().fit_transform(data_gpu) + res = CountVectorizer().fit_transform(data) ref = SkCountVect().fit_transform(data) - cp.testing.assert_array_equal(res.todense(), ref.toarray()) + cp.testing.assert_array_equal(res.toarray(), ref.toarray()) @pytest.mark.skip( @@ -307,9 +275,7 @@ def test_only_delimiters(): def test_character_ngrams(analyzer, ngram_range): data = ["ab c", "edf gh"] - res = CountVectorizer(analyzer=analyzer, ngram_range=ngram_range) - res.fit(Series(data)) - + res = CountVectorizer(analyzer=analyzer, ngram_range=ngram_range).fit(data) ref = SkCountVect(analyzer=analyzer, ngram_range=ngram_range).fit(data) assert_array_equal( @@ -321,9 +287,9 @@ def test_character_ngrams(analyzer, ngram_range): @pytest.mark.parametrize( "query", [ - Series(["science aa", "", "a aa aaa"]), - Series(["science aa", ""]), - Series(["science"]), + ["science aa", "", "a aa aaa"], + ["science aa", ""], + ["science"], ], ) def test_transform_unsigned_categories(query): @@ -334,7 +300,7 @@ def test_transform_unsigned_categories(query): token += "a" thousand_tokens[128] = "science" - vec = CountVectorizer().fit(Series(thousand_tokens)) + vec = CountVectorizer().fit(thousand_tokens) res = vec.transform(query) assert res.shape[0] == len(query) @@ -346,30 +312,6 @@ def test_transform_unsigned_categories(query): # ---------------------------------------------------------------- -def test_tfidf_vectorizer_setters(): - tv = TfidfVectorizer( - norm="l2", use_idf=False, smooth_idf=False, sublinear_tf=False - ) - tv.norm = "l1" - assert tv._tfidf.norm == "l1" - tv.use_idf = True - assert tv._tfidf.use_idf - tv.smooth_idf = True - assert tv._tfidf.smooth_idf - tv.sublinear_tf = True - assert tv._tfidf.sublinear_tf - - -def test_tfidf_vectorizer_idf_setter(): - orig = TfidfVectorizer(use_idf=True) - orig.fit(DOCS_GPU) - copy = TfidfVectorizer(vocabulary=orig.vocabulary_, use_idf=True) - copy.idf_ = orig.idf_[0] - cp.testing.assert_array_almost_equal( - copy.transform(DOCS_GPU).todense(), orig.transform(DOCS_GPU).todense() - ) - - @pytest.mark.parametrize("norm", ["l1", "l2", None]) @pytest.mark.parametrize("use_idf", [True, False]) @pytest.mark.parametrize("smooth_idf", [True, False]) @@ -380,7 +322,7 @@ def test_tfidf_vectorizer(norm, use_idf, smooth_idf, sublinear_tf): use_idf=use_idf, smooth_idf=smooth_idf, sublinear_tf=sublinear_tf, - ).fit_transform(DOCS_GPU) + ).fit_transform(DOCS) ref = SkTfidfVect( norm=norm, @@ -389,7 +331,7 @@ def test_tfidf_vectorizer(norm, use_idf, smooth_idf, sublinear_tf): sublinear_tf=sublinear_tf, ).fit_transform(DOCS) - cp.testing.assert_array_almost_equal(tfidf_mat.todense(), ref.toarray()) + cp.testing.assert_array_almost_equal(tfidf_mat.toarray(), ref.toarray()) def test_tfidf_vectorizer_get_feature_names_out(): @@ -400,7 +342,7 @@ def test_tfidf_vectorizer_get_feature_names_out(): "Is this the first document?", ] vectorizer = TfidfVectorizer() - vectorizer.fit_transform(Series(corpus)) + vectorizer.fit_transform(corpus) output = [ "and", "document", @@ -417,14 +359,12 @@ def test_tfidf_vectorizer_get_feature_names_out(): @pytest.mark.parametrize("cls", [TfidfVectorizer, CountVectorizer]) def test_vectorizer_get_feature_names_deprecated(cls): - X = Series( - [ - "This is the first document.", - "This document is the second document.", - "And this is the third one.", - "Is this the first document?", - ] - ) + X = [ + "This is the first document.", + "This document is the second document.", + "And this is the third one.", + "Is this the first document?", + ] model = cls().fit(X) with pytest.warns(FutureWarning, match="get_feature_names"): res = model.get_feature_names() @@ -437,7 +377,7 @@ def test_tfidf_vectorizer_char_wb_ngrams(): # across documents once index alignment relied on the original # per-document index instead of a reset range index. vectorizer = TfidfVectorizer(analyzer="char_wb", ngram_range=(2, 6)) - tfidf_mat = vectorizer.fit_transform(DOCS_GPU) + tfidf_mat = vectorizer.fit_transform(DOCS) ref_vectorizer = SkTfidfVect(analyzer="char_wb", ngram_range=(2, 6)) ref = ref_vectorizer.fit_transform(DOCS) @@ -482,9 +422,9 @@ def test_hashingvectorizer(): "Is this the first document?", ] - res = HashingVectorizer().fit_transform(Series(corpus)) + res = HashingVectorizer().fit_transform(corpus) ref = SkHashVect().fit_transform(corpus) - assert_almost_equal_hash_matrices(res.todense().get(), ref.toarray()) + assert_almost_equal_hash_matrices(res.toarray(), ref.toarray()) @pytest.mark.xfail @@ -503,21 +443,17 @@ def test_vectorizer_empty_token_case(): # we have extra null token here # we slightly diverge from sklearn here as not treating it as a token - res = CountVectorizer(preprocessor=lambda s: s).fit_transform( - Series(corpus) - ) + res = CountVectorizer(preprocessor=lambda s: s).fit_transform(corpus) ref = SkCountVect( preprocessor=lambda s: s, tokenizer=lambda s: s.split(" ") ).fit_transform(corpus) - cp.testing.assert_array_equal(res.todense(), ref.toarray()) + cp.testing.assert_array_equal(res.toarray(), ref.toarray()) - res = HashingVectorizer(preprocessor=lambda s: s).fit_transform( - Series(corpus) - ) + res = HashingVectorizer(preprocessor=lambda s: s).fit_transform(corpus) ref = SkHashVect( preprocessor=lambda s: s, tokenizer=lambda s: s.split(" ") ).fit_transform(corpus) - assert_almost_equal_hash_matrices(res.todense().get(), ref.toarray()) + assert_almost_equal_hash_matrices(res.toarray(), ref.toarray()) @pytest.mark.parametrize("lowercase", [False, True]) @@ -528,24 +464,21 @@ def test_hashingvectorizer_lowercase(lowercase): "And this document is the third one.", "and Is this the first document?", ] - res = HashingVectorizer(lowercase=lowercase).fit_transform(Series(corpus)) + res = HashingVectorizer(lowercase=lowercase).fit_transform(corpus) ref = SkHashVect(lowercase=lowercase).fit_transform(corpus) - assert_almost_equal_hash_matrices(res.todense().get(), ref.toarray()) + assert_almost_equal_hash_matrices(res.toarray(), ref.toarray()) def test_hashingvectorizer_stop_word(): ref = SkHashVect(stop_words="english").fit_transform(DOCS) - res = HashingVectorizer(stop_words="english").fit_transform(DOCS_GPU) - assert_almost_equal_hash_matrices(res.todense().get(), ref.toarray()) + res = HashingVectorizer(stop_words="english").fit_transform(DOCS) + assert_almost_equal_hash_matrices(res.toarray(), ref.toarray()) def test_hashingvectorizer_n_features(): n_features = 10 res = ( - HashingVectorizer(n_features=n_features) - .fit_transform(DOCS_GPU) - .todense() - .get() + HashingVectorizer(n_features=n_features).fit_transform(DOCS).toarray() ) ref = SkHashVect(n_features=n_features).fit_transform(DOCS).toarray() assert res.shape == ref.shape @@ -555,33 +488,32 @@ def test_hashingvectorizer_n_features(): def test_hashingvectorizer_norm(norm): if norm not in ["l1", "l2", None]: with pytest.raises(ValueError): - res = HashingVectorizer(norm=norm).fit_transform(DOCS_GPU) + res = HashingVectorizer(norm=norm).fit_transform(DOCS) else: - res = HashingVectorizer(norm=norm).fit_transform(DOCS_GPU) + res = HashingVectorizer(norm=norm).fit_transform(DOCS) ref = SkHashVect(norm=norm).fit_transform(DOCS) - assert_almost_equal_hash_matrices(res.todense().get(), ref.toarray()) + assert_almost_equal_hash_matrices(res.toarray(), ref.toarray()) -@pytest.mark.xfail(reason="https://github.com/NVIDIA/cuml/issues/4721") def test_hashingvectorizer_alternate_sign(): # if alternate_sign = True # we should have some negative and positive values - res = HashingVectorizer(alternate_sign=True).fit_transform(DOCS_GPU) - res_f_array = res.todense().get().flatten() + res = HashingVectorizer(alternate_sign=True).fit_transform(DOCS) + res_f_array = res.toarray().flatten() assert np.sum(res_f_array > 0, axis=0) > 0 assert np.sum(res_f_array < 0, axis=0) > 0 # if alternate_sign = False # we should have no negative values and some positive values - res = HashingVectorizer(alternate_sign=False).fit_transform(DOCS_GPU) - res_f_array = res.todense().get().flatten() + res = HashingVectorizer(alternate_sign=False).fit_transform(DOCS) + res_f_array = res.toarray().flatten() assert np.sum(res_f_array > 0, axis=0) > 0 assert np.sum(res_f_array < 0, axis=0) == 0 @pytest.mark.parametrize("dtype", [np.float32, np.float64, cp.float64]) def test_hashingvectorizer_dtype(dtype): - res = HashingVectorizer(dtype=dtype).fit_transform(DOCS_GPU) + res = HashingVectorizer(dtype=dtype).fit_transform(DOCS) assert res.dtype == dtype @@ -589,7 +521,7 @@ def test_hashingvectorizer_delimiter(): corpus = ["a0b0c", "a 0 b0e", "c0d0f"] res = HashingVectorizer( delimiter="0", norm=None, preprocessor=lambda s: s - ).fit_transform(Series(corpus)) + ).fit_transform(corpus) # equivalent logic for sklearn ref = SkHashVect( tokenizer=lambda s: s.split("0"), @@ -597,7 +529,7 @@ def test_hashingvectorizer_delimiter(): token_pattern=None, preprocessor=lambda s: s, ).fit_transform(corpus) - assert_almost_equal_hash_matrices(res.todense().get(), ref.toarray()) + assert_almost_equal_hash_matrices(res.toarray(), ref.toarray()) @pytest.mark.parametrize("vectorizer", ["tfidf", "hash_vec", "count_vec"]) @@ -616,4 +548,4 @@ def test_vectorizer_with_pandas_series(vectorizer): raw_documents = pd.Series(corpus) res = cuml_vec().fit_transform(raw_documents) ref = sklearn_vec().fit_transform(raw_documents) - assert_almost_equal_hash_matrices(res.todense().get(), ref.toarray()) + assert_almost_equal_hash_matrices(res.toarray(), ref.toarray()) From 632d9ad3d8b5a615c19745fc4b41447f09fbba86 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 8 Sep 2026 22:43:04 -0500 Subject: [PATCH 02/10] Add `cuml.feature_extraction` to the sklearn compat tests --- python/cuml/pyproject.toml | 1 + .../cuml/tests/test_sklearn_compatibility.py | 26 ++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/python/cuml/pyproject.toml b/python/cuml/pyproject.toml index 0e7874345d..b988e9e73c 100644 --- a/python/cuml/pyproject.toml +++ b/python/cuml/pyproject.toml @@ -44,6 +44,7 @@ filterwarnings = [ "ignore:(.*)alias(.*):DeprecationWarning:hdbscan[.*]", # From sklearn "ignore:A column-vector y was passed:sklearn.exceptions.DataConversionWarning", + "ignore::sklearn.exceptions.SkipTestWarning", # sklearn 1.6 renamed force_all_finite to ensure_all_finite "ignore:'force_all_finite' was renamed to 'ensure_all_finite'.*:FutureWarning", # From dask-glm diff --git a/python/cuml/tests/test_sklearn_compatibility.py b/python/cuml/tests/test_sklearn_compatibility.py index 22134c5078..46973432fb 100644 --- a/python/cuml/tests/test_sklearn_compatibility.py +++ b/python/cuml/tests/test_sklearn_compatibility.py @@ -25,7 +25,12 @@ RandomForestClassifier, RandomForestRegressor, ) -from cuml.feature_extraction.text import TfidfTransformer +from cuml.feature_extraction.text import ( + CountVectorizer, + HashingVectorizer, + TfidfTransformer, + TfidfVectorizer, +) from cuml.kernel_ridge import KernelRidge from cuml.linear_model import ( ElasticNet, @@ -126,6 +131,10 @@ StandardScaler(), OneHotEncoder(), OrdinalEncoder(), + CountVectorizer(), + HashingVectorizer(), + TfidfVectorizer(), + TfidfTransformer(), ] @@ -164,8 +173,6 @@ def _all_cuml_estimators(): OneVsOneClassifier: "Meta-estimator, requires an inner estimator", # Manifold SpectralEmbedding: "Not yet tested for sklearn compat", - # Feature extraction - TfidfTransformer: "Not yet tested for sklearn compat", # Preprocessing (cuml-native) LabelEncoder: "Not yet tested for sklearn compat", TargetEncoder: "Not yet tested for sklearn compat", @@ -349,6 +356,9 @@ def test_sklearn_compatible_estimator_coverage(): SimpleImputer(), MissingIndicator(), TargetEncoder(multi_feature_mode="independent"), + CountVectorizer(), + TfidfVectorizer(), + TfidfTransformer(), ] GET_FEATURE_NAMES_OUT_XFAILS = {} @@ -407,7 +417,15 @@ def test_sklearn_get_feature_names_out_all_estimators_covered(): @pytest.mark.parametrize( "transformer", - GET_FEATURE_NAMES_OUT_ESTIMATORS, + [ + est + for est in GET_FEATURE_NAMES_OUT_ESTIMATORS + # These estimators only return sparse data, and so can + # never have index/column names attached to their output + if not isinstance( + est, (CountVectorizer, TfidfVectorizer, TfidfTransformer) + ) + ], ids=lambda est: type(est).__name__, ) def test_transform_inverse_transform_set_index_and_column_names(transformer): From 46ec373f7949281376f55f782eff40867eab83df Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Wed, 9 Sep 2026 00:40:49 -0500 Subject: [PATCH 03/10] Some easy bot-noticed fixups --- python/cuml/cuml/feature_extraction/text.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/cuml/cuml/feature_extraction/text.py b/python/cuml/cuml/feature_extraction/text.py index 91545b701c..b69f6ca1cf 100644 --- a/python/cuml/cuml/feature_extraction/text.py +++ b/python/cuml/cuml/feature_extraction/text.py @@ -832,7 +832,7 @@ def fit(self, X, y=None): The instance itself. """ self._check_params() - X = check_cudf(X, ensure_ndim=1, input_name="X") + X = check_cudf(X, ensure_ndim=1, input_name="X").reset_index(drop=True) tokens = self._to_tokens(X) self._fit(X, tokens) return self @@ -855,7 +855,7 @@ def fit_transform(self, X, y=None): Document-term matrix. """ self._check_params() - X = check_cudf(X, ensure_ndim=1, input_name="X") + X = check_cudf(X, ensure_ndim=1, input_name="X").reset_index(drop=True) tokens = self._to_tokens(X) self._fit(X, tokens) return self._transform(X, tokens) @@ -876,7 +876,7 @@ def transform(self, X): Document-term matrix. """ check_is_fitted(self) - X = check_cudf(X, ensure_ndim=1, input_name="X") + X = check_cudf(X, ensure_ndim=1, input_name="X").reset_index(drop=True) tokens = self._to_tokens(X) return self._transform(X, tokens) @@ -1032,7 +1032,7 @@ def __sklearn_tags__(self): def _check_params(self): _check_oneof(self, "norm", ["l1", "l2", None]) - def _check_X(self, X, reset=False, copy=False): + def _check_X(self, X, reset=False, copy=True): """Validate and normalize X to a CSR sparse matrix""" X = check_inputs( self, @@ -1079,7 +1079,7 @@ def _transform(self, X): cp.log(X.data, out=X.data) X.data += 1.0 - if hasattr(self, "idf_"): + if self.use_idf: # the columns of X (CSR matrix) can be accessed with `X.indices `and # multiplied with the corresponding `idf` value X.data *= self.idf_[X.indices] From 02d9dd80e529138fd80d126ddb65bd9c39f248b1 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Wed, 9 Sep 2026 10:03:18 -0500 Subject: [PATCH 04/10] More fixups from feedback --- python/cuml/cuml/feature_extraction/text.py | 25 +++++++----- .../tests/test_text_feature_extraction.py | 38 +++++-------------- python/cuml/tests/test_tfidf.py | 7 +--- 3 files changed, 26 insertions(+), 44 deletions(-) diff --git a/python/cuml/cuml/feature_extraction/text.py b/python/cuml/cuml/feature_extraction/text.py index b69f6ca1cf..af069f6016 100644 --- a/python/cuml/cuml/feature_extraction/text.py +++ b/python/cuml/cuml/feature_extraction/text.py @@ -763,14 +763,17 @@ def _fit(self, X, tokens): or min_doc_count > 1 or max_features is not None ): - counts = tokens.token.value_counts() + doc_freq = tokens.drop_duplicates().token.value_counts() if max_doc_count < n_doc: - counts = counts[counts <= max_doc_count] + doc_freq = doc_freq[doc_freq <= max_doc_count] if min_doc_count > 1: - counts = counts[counts >= min_doc_count] + doc_freq = doc_freq[doc_freq >= min_doc_count] + keep = doc_freq.index if max_features is not None: - counts = counts.iloc[:max_features] - self.vocabulary_ = cudf.Series(counts.index.sort_values()) + term_freq = tokens.token.value_counts() + term_freq = term_freq[term_freq.index.isin(keep)] + keep = term_freq.iloc[:max_features].index + self.vocabulary_ = cudf.Series(keep.sort_values()) if not len(self.vocabulary_): raise ValueError( "After pruning, no terms remain. Try a lower min_df or " @@ -1114,7 +1117,7 @@ def fit(self, X, y=None): return self._fit(X) @mlfunc(set_input_type=True) - def fit_transform(self, X, y=None): + def fit_transform(self, X, y=None, copy=True): """Fit the transformer, then transform X. Parameters @@ -1125,13 +1128,17 @@ def fit_transform(self, X, y=None): y : None Ignored. Exists for API compatibility only. + copy : bool, default=True + If `copy=False,` then `fit_transform` may choose to mutate `X` + in-place if that would be more efficient. + Returns ------- X : sparse matrix of shape (n_samples, n_features) Tf-idf weighted document-term matrix. """ self._check_params() - X = self._check_X(X, reset=True) + X = self._check_X(X, reset=True, copy=copy) return self._fit(X)._transform(X) @mlfunc @@ -1144,8 +1151,8 @@ def transform(self, X, copy=True): A matrix of term/token counts. copy : bool, default=True - If `copy=False,` then `transform` may mutate `X` in-place when - possible. + If `copy=False,` then `fit_transform` may choose to mutate `X` + in-place if that would be more efficient. Returns ------- diff --git a/python/cuml/tests/test_text_feature_extraction.py b/python/cuml/tests/test_text_feature_extraction.py index 6ddad5a8ba..3d9486166c 100644 --- a/python/cuml/tests/test_text_feature_extraction.py +++ b/python/cuml/tests/test_text_feature_extraction.py @@ -392,26 +392,6 @@ def test_tfidf_vectorizer_char_wb_ngrams(): # ---------------------------------------------------------------- # HashingVectorizer tests # ---------------------------------------------------------------- -def assert_almost_equal_hash_matrices(mat_1, mat_2, ignore_sign=True): - """ - Currently if all the sorted values in the row is equal we - assume equality - TODO: Find better way to test ig hash matrices are equal - """ - assert mat_1.shape == mat_2.shape - for row_id in range(mat_1.shape[0]): - row_m1 = mat_1[row_id] - row_m2 = mat_2[row_id] - nz_row_m1 = np.sort(row_m1[row_m1 != 0]) - nz_row_m2 = np.sort(row_m2[row_m2 != 0]) - # print(nz_row_m1) - # print(nz_row_m2) - if ignore_sign: - nz_row_m1 = np.abs(nz_row_m1) - nz_row_m2 = np.abs(nz_row_m2) - nz_row_m1.sort() - nz_row_m2.sort() - np.testing.assert_almost_equal(nz_row_m1, nz_row_m2) def test_hashingvectorizer(): @@ -424,7 +404,7 @@ def test_hashingvectorizer(): res = HashingVectorizer().fit_transform(corpus) ref = SkHashVect().fit_transform(corpus) - assert_almost_equal_hash_matrices(res.toarray(), ref.toarray()) + np.testing.assert_allclose(res.toarray(), ref.toarray()) @pytest.mark.xfail @@ -453,7 +433,7 @@ def test_vectorizer_empty_token_case(): ref = SkHashVect( preprocessor=lambda s: s, tokenizer=lambda s: s.split(" ") ).fit_transform(corpus) - assert_almost_equal_hash_matrices(res.toarray(), ref.toarray()) + np.testing.assert_allclose(res.toarray(), ref.toarray()) @pytest.mark.parametrize("lowercase", [False, True]) @@ -466,13 +446,13 @@ def test_hashingvectorizer_lowercase(lowercase): ] res = HashingVectorizer(lowercase=lowercase).fit_transform(corpus) ref = SkHashVect(lowercase=lowercase).fit_transform(corpus) - assert_almost_equal_hash_matrices(res.toarray(), ref.toarray()) + np.testing.assert_allclose(res.toarray(), ref.toarray()) def test_hashingvectorizer_stop_word(): ref = SkHashVect(stop_words="english").fit_transform(DOCS) res = HashingVectorizer(stop_words="english").fit_transform(DOCS) - assert_almost_equal_hash_matrices(res.toarray(), ref.toarray()) + np.testing.assert_allclose(res.toarray(), ref.toarray()) def test_hashingvectorizer_n_features(): @@ -492,7 +472,7 @@ def test_hashingvectorizer_norm(norm): else: res = HashingVectorizer(norm=norm).fit_transform(DOCS) ref = SkHashVect(norm=norm).fit_transform(DOCS) - assert_almost_equal_hash_matrices(res.toarray(), ref.toarray()) + np.testing.assert_allclose(res.toarray(), ref.toarray()) def test_hashingvectorizer_alternate_sign(): @@ -529,7 +509,7 @@ def test_hashingvectorizer_delimiter(): token_pattern=None, preprocessor=lambda s: s, ).fit_transform(corpus) - assert_almost_equal_hash_matrices(res.toarray(), ref.toarray()) + np.testing.assert_allclose(res.toarray(), ref.toarray()) @pytest.mark.parametrize("vectorizer", ["tfidf", "hash_vec", "count_vec"]) @@ -546,6 +526,6 @@ def test_vectorizer_with_pandas_series(vectorizer): "count_vec": (CountVectorizer, SkCountVect), }[vectorizer] raw_documents = pd.Series(corpus) - res = cuml_vec().fit_transform(raw_documents) - ref = sklearn_vec().fit_transform(raw_documents) - assert_almost_equal_hash_matrices(res.toarray(), ref.toarray()) + res = cuml_vec(dtype=np.float32).fit_transform(raw_documents) + ref = sklearn_vec(dtype=np.float32).fit_transform(raw_documents) + np.testing.assert_allclose(res.toarray(), ref.toarray()) diff --git a/python/cuml/tests/test_tfidf.py b/python/cuml/tests/test_tfidf.py index 5867c23466..b0da7db086 100644 --- a/python/cuml/tests/test_tfidf.py +++ b/python/cuml/tests/test_tfidf.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -63,11 +63,6 @@ def test_tfidf_transformer(data, norm, use_idf, smooth_idf, sublinear_tf): @pytest.mark.parametrize("smooth_idf", [True, False]) @pytest.mark.parametrize("sublinear_tf", [True, False]) def test_tfidf_transformer_copy(norm, use_idf, smooth_idf, sublinear_tf): - if use_idf: - pytest.xfail( - "cupyx.scipy.sparse.csr does not support inplace multiply." - ) - data_gpu = cupyx.scipy.sparse.csr_matrix( cp.array([[0, 1, 1, 1], [0, 2, 0, 1]], dtype=cp.float64, order="F") ) From 8b4c678cabaf8d67f010e6000bb247d05d4d7177 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Wed, 9 Sep 2026 11:44:22 -0500 Subject: [PATCH 05/10] Add `tokenizer`, make `preprocessor`/`delimiter` more robust Also improves docstring to better describe what each does. --- python/cuml/cuml/feature_extraction/text.py | 160 ++++++++++++------ .../tests/test_text_feature_extraction.py | 59 +++++-- 2 files changed, 151 insertions(+), 68 deletions(-) diff --git a/python/cuml/cuml/feature_extraction/text.py b/python/cuml/cuml/feature_extraction/text.py index af069f6016..6603a01d95 100644 --- a/python/cuml/cuml/feature_extraction/text.py +++ b/python/cuml/cuml/feature_extraction/text.py @@ -89,10 +89,11 @@ def __init__( *, lowercase=True, preprocessor=None, + tokenizer=None, + delimiter=None, stop_words=None, ngram_range=(1, 1), analyzer="word", - delimiter=" ", binary=False, dtype=cp.float32, verbose=False, @@ -101,10 +102,11 @@ def __init__( super().__init__(verbose=verbose, output_type=output_type) self.lowercase = lowercase self.preprocessor = preprocessor + self.tokenizer = tokenizer + self.delimiter = delimiter self.stop_words = stop_words self.ngram_range = ngram_range self.analyzer = analyzer - self.delimiter = delimiter self.binary = binary self.dtype = dtype @@ -112,10 +114,11 @@ def _get_param_names(self): return [ "lowercase", "preprocessor", + "tokenizer", + "delimiter", "stop_words", "ngram_range", "analyzer", - "delimiter", "binary", "dtype", *super()._get_param_names(), @@ -161,24 +164,41 @@ def _get_stop_words(self): def _preprocess(self, X): """Preprocess `X`. - The default preprocessor lowercases all inputs. + Preprocessing has three stages: + + ## Whole document transforms: + + - Lowercases all inputs if ``lowercase`` is true. + + This stage may be overridden by specifying a ``preprocessor``. + + ## Token level transforms (if ``analyzer="word"``): - Additionally, if `analyzer="word"`, it: - Removes all non-alphanumeric characters (excluding "_") - Normalizes whitespace to " " - Removes any single character tokens - - Removes any tokens specified by `stop_words`. - The whole step may be overridden by specifying `preprocessor` - explicitly. + This stage may be overridden by specifying a ``tokenizer`` or a + ``delimiter``. + + ## Removal of stop words (if ``analyzer="word"``) + + - Any tokens matching ``stop_words`` are removed """ + # 1. Whole document transforms if self.preprocessor is not None: - return self.preprocessor(X) - - if self.lowercase: + X = self.preprocessor(X) + elif self.lowercase: X = X.str.lower() - if self.analyzer == "word": + if self.analyzer != "word": + return X + + delimiter = self.delimiter or " " + # 2. Token level transforms + if self.tokenizer is not None: + X = self.tokenizer(X).str.join(delimiter) + elif self.delimiter is None: # XXX: a filler string to take the place of _ temporarily # since `filter_alphanum` strips `_` but sklearn keeps `_`. # We can use `X` in the common case of lowercase normalization @@ -187,20 +207,20 @@ def _preprocess(self, X): flag = "X" if self.lowercase else "cuᵐl" X = ( X.str.replace("_", flag, regex=False) - .str.filter_alphanum(self.delimiter, keep=True) + .str.filter_alphanum(delimiter, keep=True) .str.replace(flag, "_", regex=False) ) # sklearn by default removes single char tokens - X = X.str.filter_tokens(2, delimiter=self.delimiter) - - # Drop any stop words specified - if self.stop_words is not None: - X = X.str.replace_tokens( - self._get_stop_words(), - " ", - delimiter=self.delimiter, - ) + X = X.str.filter_tokens(2, delimiter=delimiter) + + # 3. Remove stop words + if self.stop_words is not None: + X = X.str.replace_tokens( + self._get_stop_words(), + delimiter, + delimiter=delimiter, + ) return X @@ -223,13 +243,14 @@ def _to_tokens(self, X): parts = [] if self.analyzer == "word": - token_counts = X.str.token_count(delimiter=self.delimiter) + delimiter = self.delimiter or " " + token_counts = X.str.token_count(delimiter=delimiter) for ngram_size in range( self.ngram_range[0], self.ngram_range[1] + 1 ): ngrams = X.str.ngrams_tokenize( n=ngram_size, - delimiter=self.delimiter, + delimiter=delimiter, separator=" ", ) ngram_count = (token_counts - (ngram_size - 1)).clip(0) @@ -338,6 +359,20 @@ class HashingVectorizer(_BaseVectorizer): preprocessor : callable, default=None Override the preprocessing (string transformation) stage while preserving the tokenizing and n-grams generation steps. + This function receives a ``cudf.Series`` of strings and should + return a ``cudf.Series`` of strings. + + tokenizer : callable, default=None + Override the string tokenization step while preserving the + preprocessing and n-grams generation steps. This function + receives a ``cudf.Series`` of strings and should return + a ``cudf.Series`` of lists of strings. + Only applies if ``analyzer == 'word'``. + + delimiter : str, default=None + String used to delimit tokens in the document. If ``None``, then any + non-alphanumeric (or " ") character is treated as a delimiter. + Only applies if ``analyzer == "word'``. stop_words : {'english'}, list, default=None If 'english', a built-in stop word list for English is used. If a list, @@ -378,11 +413,6 @@ class HashingVectorizer(_BaseVectorizer): dtype : type, default=np.float32 Type of the matrix returned by fit_transform() or transform(). - delimiter : str, default=" " - String used as a replacement for stop words if `stop_words` is not - None. Typically the delimiting character between words is a good - choice. - Examples -------- >>> from cuml.feature_extraction.text import HashingVectorizer @@ -403,10 +433,11 @@ def __init__( *, lowercase=True, preprocessor=None, + tokenizer=None, + delimiter=None, stop_words=None, ngram_range=(1, 1), analyzer="word", - delimiter=" ", alternate_sign=True, n_features=2**20, dtype=cp.float32, @@ -418,10 +449,11 @@ def __init__( super().__init__( lowercase=lowercase, preprocessor=preprocessor, + tokenizer=tokenizer, + delimiter=delimiter, stop_words=stop_words, ngram_range=ngram_range, analyzer=analyzer, - delimiter=delimiter, binary=binary, dtype=dtype, verbose=verbose, @@ -588,6 +620,20 @@ class CountVectorizer(DeprecatedGetFeatureNamesMixin, _BaseVectorizer): preprocessor : callable, default=None Override the preprocessing (string transformation) stage while preserving the tokenizing and n-grams generation steps. + This function receives a ``cudf.Series`` of strings and should + return a ``cudf.Series`` of strings. + + tokenizer : callable, default=None + Override the string tokenization step while preserving the + preprocessing and n-grams generation steps. This function + receives a ``cudf.Series`` of strings and should return + a ``cudf.Series`` of lists of strings. + Only applies if ``analyzer == 'word'``. + + delimiter : str, default=None + String used to delimit tokens in the document. If ``None``, then any + non-alphanumeric (or " ") character is treated as a delimiter. + Only applies if ``analyzer == "word'``. stop_words : {'english'}, list, default=None If 'english', a built-in stop word list for English is used. If a list, @@ -597,10 +643,10 @@ class CountVectorizer(DeprecatedGetFeatureNamesMixin, _BaseVectorizer): ngram_range : tuple (min_n, max_n), default=(1, 1) The lower and upper boundary of the range of n-values for different - word n-grams or char n-grams to be extracted. All values of n such - such that min_n <= n <= max_n will be used. For example an - ``ngram_range`` of ``(1, 1)`` means only unigrams, ``(1, 2)`` means - unigrams and bigrams, and ``(2, 2)`` means only bigrams. + n-grams to be extracted. All values of n such that min_n <= n <= max_n + will be used. For example an ``ngram_range`` of ``(1, 1)`` means only + unigrams, ``(1, 2)`` means unigrams and bigrams, and ``(2, 2)`` means + only bigrams. analyzer : {'word', 'char', 'char_wb'}, default='word' Whether the feature should be made of word or character n-grams. @@ -643,11 +689,6 @@ class CountVectorizer(DeprecatedGetFeatureNamesMixin, _BaseVectorizer): dtype : dtype, default=np.float32 Type of the matrix returned by fit_transform() or transform(). - delimiter : str, default=" " - String used as a replacement for stop words if `stop_words` is not - None. Typically the delimiting character between words is a good - choice. - Attributes ---------- vocabulary_ : cudf.Series @@ -680,10 +721,11 @@ def __init__( *, lowercase=True, preprocessor=None, + tokenizer=None, + delimiter=None, stop_words=None, ngram_range=(1, 1), analyzer="word", - delimiter=" ", max_df=1.0, min_df=1, max_features=None, @@ -696,10 +738,11 @@ def __init__( super().__init__( lowercase=lowercase, preprocessor=preprocessor, + tokenizer=tokenizer, + delimiter=delimiter, stop_words=stop_words, ngram_range=ngram_range, analyzer=analyzer, - delimiter=delimiter, binary=binary, dtype=dtype, verbose=verbose, @@ -1177,6 +1220,20 @@ class TfidfVectorizer(CountVectorizer): preprocessor : callable, default=None Override the preprocessing (string transformation) stage while preserving the tokenizing and n-grams generation steps. + This function receives a ``cudf.Series`` of strings and should + return a ``cudf.Series`` of strings. + + tokenizer : callable, default=None + Override the string tokenization step while preserving the + preprocessing and n-grams generation steps. This function + receives a ``cudf.Series`` of strings and should return + a ``cudf.Series`` of lists of strings. + Only applies if ``analyzer == 'word'``. + + delimiter : str, default=None + String used to delimit tokens in the document. If ``None``, then any + non-alphanumeric (or " ") character is treated as a delimiter. + Only applies if ``analyzer == "word'``. stop_words : {'english'}, list, default=None If 'english', a built-in stop word list for English is used. If a list, @@ -1186,10 +1243,10 @@ class TfidfVectorizer(CountVectorizer): ngram_range : tuple (min_n, max_n), default=(1, 1) The lower and upper boundary of the range of n-values for different - word n-grams or char n-grams to be extracted. All values of n such - such that min_n <= n <= max_n will be used. For example an - ``ngram_range`` of ``(1, 1)`` means only unigrams, ``(1, 2)`` means - unigrams and bigrams, and ``(2, 2)`` means only bigrams. + n-grams to be extracted. All values of n such that min_n <= n <= max_n + will be used. For example an ``ngram_range`` of ``(1, 1)`` means only + unigrams, ``(1, 2)`` means unigrams and bigrams, and ``(2, 2)`` means + only bigrams. analyzer : {'word', 'char', 'char_wb'}, default='word' Whether the feature should be made of word or character n-grams. @@ -1232,11 +1289,6 @@ class TfidfVectorizer(CountVectorizer): dtype : dtype, default=np.float32 Type of the matrix returned by fit_transform() or transform(). - delimiter : str, default=" " - String used as a replacement for stop words if `stop_words` is not - None. Typically the delimiting character between words is a good - choice. - norm : {'l1', 'l2', None}, default='l2' Norm used to normalize term vectors. None for no normalization. @@ -1287,10 +1339,11 @@ def __init__( *, lowercase=True, preprocessor=None, + tokenizer=None, + delimiter=None, stop_words=None, ngram_range=(1, 1), analyzer="word", - delimiter=" ", max_df=1.0, min_df=1, max_features=None, @@ -1307,10 +1360,11 @@ def __init__( super().__init__( lowercase=lowercase, preprocessor=preprocessor, + tokenizer=tokenizer, + delimiter=delimiter, stop_words=stop_words, ngram_range=ngram_range, analyzer=analyzer, - delimiter=delimiter, max_df=max_df, min_df=min_df, max_features=max_features, diff --git a/python/cuml/tests/test_text_feature_extraction.py b/python/cuml/tests/test_text_feature_extraction.py index 3d9486166c..1493cba19f 100644 --- a/python/cuml/tests/test_text_feature_extraction.py +++ b/python/cuml/tests/test_text_feature_extraction.py @@ -67,6 +67,50 @@ def test_word_analyzer(ngram_range): ) +def test_preprocessor(): + corpus = ["aa bb cc", "aa bb ee", "cc dd ff"] + vec = CountVectorizer( + preprocessor=lambda s: s.str.upper(), + stop_words=["EE"], + ).fit(corpus) + res = vec.get_feature_names_out() + np.testing.assert_array_equal( + res, + ["AA", "BB", "CC", "DD", "FF"], + ) + + +def test_delimiter(): + corpus = ["aa0bb0cc", "aa 0 bb0ee", "c0d0f"] + vec = CountVectorizer( + delimiter="0", + stop_words=["ee"], + ).fit(corpus) + res = vec.get_feature_names_out() + np.testing.assert_array_equal( + res, + [" bb", "aa", "aa ", "bb", "c", "cc", "d", "f"], + ) + + +def test_tokenizer(): + corpus = [ + "fillerfiller ", + "fillerfiller ", + " but not no terms here", + ] + vec = CountVectorizer( + tokenizer=lambda s: s.str.findall(r"<([\w\s]*)>"), + delimiter="|", + stop_words=["term5"], + ).fit(corpus) + res = vec.get_feature_names_out() + np.testing.assert_array_equal( + res, + ["term1", "term2", "term3 ", "term4"], + ) + + def test_countvectorizer_custom_vocabulary(): vocab = {"pizza": 0, "beer": 1} @@ -497,21 +541,6 @@ def test_hashingvectorizer_dtype(dtype): assert res.dtype == dtype -def test_hashingvectorizer_delimiter(): - corpus = ["a0b0c", "a 0 b0e", "c0d0f"] - res = HashingVectorizer( - delimiter="0", norm=None, preprocessor=lambda s: s - ).fit_transform(corpus) - # equivalent logic for sklearn - ref = SkHashVect( - tokenizer=lambda s: s.split("0"), - norm=None, - token_pattern=None, - preprocessor=lambda s: s, - ).fit_transform(corpus) - np.testing.assert_allclose(res.toarray(), ref.toarray()) - - @pytest.mark.parametrize("vectorizer", ["tfidf", "hash_vec", "count_vec"]) def test_vectorizer_with_pandas_series(vectorizer): corpus = [ From 375b69ad4f9d53908696bae869c57cdf2855f19f Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Wed, 9 Sep 2026 12:20:18 -0500 Subject: [PATCH 06/10] Appease the nitpicky always-has-to-say-something bot --- python/cuml/cuml/feature_extraction/text.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cuml/cuml/feature_extraction/text.py b/python/cuml/cuml/feature_extraction/text.py index 6603a01d95..6a48e0fb96 100644 --- a/python/cuml/cuml/feature_extraction/text.py +++ b/python/cuml/cuml/feature_extraction/text.py @@ -21,8 +21,8 @@ ) __all__ = ( - "HashingVectorizer", "CountVectorizer", + "HashingVectorizer", "TfidfTransformer", "TfidfVectorizer", ) @@ -158,7 +158,7 @@ def _get_stop_words(self): return cudf.Series(ENGLISH_STOP_WORDS) elif isinstance(self.stop_words, str): - raise ValueError("not a built-in stop list: %s" % self.stop_words) + raise ValueError(f"not a built-in stop list: {self.stop_words}") return cudf.Series(self.stop_words) def _preprocess(self, X): From 07175bebd07ce15d415a914bdf5879d64945d889 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Wed, 9 Sep 2026 12:57:13 -0500 Subject: [PATCH 07/10] Fixup docstring verbose/output_type --- python/cuml/cuml/feature_extraction/text.py | 40 +++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/python/cuml/cuml/feature_extraction/text.py b/python/cuml/cuml/feature_extraction/text.py index 6a48e0fb96..ea319f317a 100644 --- a/python/cuml/cuml/feature_extraction/text.py +++ b/python/cuml/cuml/feature_extraction/text.py @@ -413,6 +413,16 @@ class HashingVectorizer(_BaseVectorizer): dtype : type, default=np.float32 Type of the matrix returned by fit_transform() or transform(). + verbose : int or boolean, default=False + Sets logging level. It must be one of `cuml.common.logger.level_*`. + See :ref:`verbosity-levels` for more info. + + output_type : {None, 'input', 'cupy', 'numpy', 'cudf', 'pandas'}, default=None + Return results and set estimator attributes to the indicated output + type. If None, the output type set at the module level + (`cuml.global_settings.output_type`) will be used. See + :ref:`output-data-type-configuration` for more info. + Examples -------- >>> from cuml.feature_extraction.text import HashingVectorizer @@ -689,6 +699,16 @@ class CountVectorizer(DeprecatedGetFeatureNamesMixin, _BaseVectorizer): dtype : dtype, default=np.float32 Type of the matrix returned by fit_transform() or transform(). + verbose : int or boolean, default=False + Sets logging level. It must be one of `cuml.common.logger.level_*`. + See :ref:`verbosity-levels` for more info. + + output_type : {None, 'input', 'cupy', 'numpy', 'cudf', 'pandas'}, default=None + Return results and set estimator attributes to the indicated output + type. If None, the output type set at the module level + (`cuml.global_settings.output_type`) will be used. See + :ref:`output-data-type-configuration` for more info. + Attributes ---------- vocabulary_ : cudf.Series @@ -1020,6 +1040,16 @@ class TfidfTransformer(OneToOneFeatureMixin, Base): sublinear_tf : bool, default=False Apply sublinear tf scaling, i.e. replace tf with 1 + log(tf). + verbose : int or boolean, default=False + Sets logging level. It must be one of `cuml.common.logger.level_*`. + See :ref:`verbosity-levels` for more info. + + output_type : {None, 'input', 'cupy', 'numpy', 'cudf', 'pandas'}, default=None + Return results and set estimator attributes to the indicated output + type. If None, the output type set at the module level + (`cuml.global_settings.output_type`) will be used. See + :ref:`output-data-type-configuration` for more info. + Attributes ---------- idf_ : array of shape (n_features) @@ -1303,6 +1333,16 @@ class TfidfVectorizer(CountVectorizer): sublinear_tf : bool, default=False Apply sublinear tf scaling, i.e. replace tf with 1 + log(tf). + verbose : int or boolean, default=False + Sets logging level. It must be one of `cuml.common.logger.level_*`. + See :ref:`verbosity-levels` for more info. + + output_type : {None, 'input', 'cupy', 'numpy', 'cudf', 'pandas'}, default=None + Return results and set estimator attributes to the indicated output + type. If None, the output type set at the module level + (`cuml.global_settings.output_type`) will be used. See + :ref:`output-data-type-configuration` for more info. + Attributes ---------- vocabulary_ : cudf.Series From 3ce242f4f0f4c52582db4d2aa518545117989a47 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Wed, 9 Sep 2026 13:53:32 -0500 Subject: [PATCH 08/10] Fixup dask implementation --- .../text/tfidf_transformer.py | 104 +++++++++--------- python/cuml/cuml/feature_extraction/text.py | 37 ++++--- 2 files changed, 71 insertions(+), 70 deletions(-) diff --git a/python/cuml/cuml/dask/feature_extraction/text/tfidf_transformer.py b/python/cuml/cuml/dask/feature_extraction/text/tfidf_transformer.py index 11b782d0b4..b429344e37 100644 --- a/python/cuml/cuml/dask/feature_extraction/text/tfidf_transformer.py +++ b/python/cuml/cuml/dask/feature_extraction/text/tfidf_transformer.py @@ -1,17 +1,44 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # +import cupy as cp +import cupyx.scipy.sparse as cp_sp import dask import dask.array -from toolz import first +import cuml.feature_extraction.text from cuml.dask.common.base import BaseEstimator, DelayedTransformMixin from cuml.dask.common.func import reduce from cuml.dask.common.input_utils import DistributedDataHandler from cuml.dask.common.utils import wait_and_raise_from_futures -from cuml.feature_extraction.text import TfidfTransformer as s_TfidfTransformer + + +def _get_df_and_n_samples(X): + """Compute doc frequencies and n_samples for X""" + X = cp_sp.csr_matrix(X) + df = cp.bincount(X.indices, minlength=X.shape[1]).astype( + X.dtype, copy=False + ) + return df, X.shape[0] + + +def _merge_df_and_n_samples(parts): + """Merge doc frequencies and n_samples for X""" + dfs, ns = zip(*parts) + df = cp.vstack(dfs).sum(axis=0) + n_samples = sum(ns) + return df, n_samples + + +def _build_fit_tfidf_transformer(df_and_n_samples, kwargs): + """Build a fit TfidfTransformer from df, n_samples, and kwargs""" + df, n_samples = df_and_n_samples + model = cuml.feature_extraction.text.TfidfTransformer(**kwargs) + model._set_idf(df, n_samples) + model.n_features_in_ = len(df) + return model class TfidfTransformer(BaseEstimator, DelayedTransformMixin): @@ -77,30 +104,9 @@ def __init__(self, *, client=None, verbose=False, **kwargs): # Make any potential model args available and catch any potential # ValueErrors before distributed training begins. - self._set_internal_model(s_TfidfTransformer(**kwargs)) - - @staticmethod - def _set_doc_stats(X, kwargs): - model = s_TfidfTransformer(**kwargs) - # Below is only required if we have to set stats - if model.use_idf: - model._set_doc_stats(X) - - return model - - @staticmethod - def _merge_stats_to_model(models): - modela = first(models) - if modela.use_idf: - for model in models[1:]: - modela.__n_samples += model.__n_samples - modela.__df += model.__df - return modela - - @staticmethod - def _set_idf_diag(model): - model._set_idf_diag() - return model + model = cuml.feature_extraction.text.TfidfTransformer(**kwargs) + model._check_params() + self._set_internal_model(model) def fit(self, X, y=None): """ @@ -126,39 +132,29 @@ def fit(self, X, y=None): "Multi-dimensional chunking is not supported" ) - # We don't' do anything if we don't need idf + # No need to compute if we don't need idf if not self.internal_model.use_idf: + self.internal_model.n_features_in_ = X.shape[1] return self - futures = DistributedDataHandler.create(X, self.client) - - models = [ - self.client.submit( - self._set_doc_stats, part, self.kwargs, pure=False - ) - for w, part in futures.gpu_futures - ] - - models = reduce(models, self._merge_stats_to_model, client=self.client) - - wait_and_raise_from_futures([models]) - - models = self.client.submit(self._set_idf_diag, models, pure=False) - - wait_and_raise_from_futures([models]) - - self._set_internal_model(models) + handler = DistributedDataHandler.create(X, self.client) + chunks = [chunk for _, chunk in handler.gpu_futures] + df_and_n_samples = reduce( + self.client.map(_get_df_and_n_samples, chunks, pure=False), + _merge_df_and_n_samples, + client=self.client, + ) + model = self.client.submit( + _build_fit_tfidf_transformer, + df_and_n_samples, + self.kwargs, + pure=False, + ) + wait_and_raise_from_futures([model]) + self._set_internal_model(model) return self - @staticmethod - def _get_part(parts, idx): - return parts[idx] - - @staticmethod - def _get_size(arrs): - return arrs.shape[0] - def fit_transform(self, X, y=None): """ Fit distributed TFIDFTransformer and then transform diff --git a/python/cuml/cuml/feature_extraction/text.py b/python/cuml/cuml/feature_extraction/text.py index ea319f317a..7f671dbc75 100644 --- a/python/cuml/cuml/feature_extraction/text.py +++ b/python/cuml/cuml/feature_extraction/text.py @@ -1122,29 +1122,34 @@ def _check_X(self, X, reset=False, copy=True): X = cp_sp.csr_matrix(X) return X + @mlfunc(convert_output=False) + def _set_idf(self, df, n_samples): + """Set `idf_` from the computed document frequencies & n_samples. + + Split out to support the dask implementation. + """ + assert self.use_idf + # perform idf smoothing if required + if self.smooth_idf: + df += 1.0 + n_samples += 1 + + # log + 1 instead of log makes sure terms with zero idf don't get + # suppressed entirely. + idf = cp.full_like(df, fill_value=n_samples) + idf /= df + cp.log(idf, out=idf) + idf += 1.0 + self.idf_ = idf + def _fit(self, X): assert cp_sp.issparse(X) assert X.format == "csr" if self.use_idf: - n_samples, _ = X.shape - - # Compute document frequency df = cp.bincount(X.indices, minlength=X.shape[1]).astype( X.dtype, copy=False ) - - # perform idf smoothing if required - if self.smooth_idf: - df += 1.0 - n_samples += 1 - - # log + 1 instead of log makes sure terms with zero idf don't get - # suppressed entirely. - idf = cp.full_like(df, fill_value=n_samples, dtype=X.dtype) - idf /= df - cp.log(idf, out=idf) - idf += 1.0 - self.idf_ = idf + self._set_idf(df, X.shape[0]) return self From 8b278005f95eb6d34d7eec8074185569b6de0357 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Thu, 10 Sep 2026 09:14:26 -0500 Subject: [PATCH 09/10] Respond to feedback --- python/cuml/cuml/feature_extraction/text.py | 55 +++++++++++-------- .../tests/test_text_feature_extraction.py | 52 ++++++++++++++++++ 2 files changed, 85 insertions(+), 22 deletions(-) diff --git a/python/cuml/cuml/feature_extraction/text.py b/python/cuml/cuml/feature_extraction/text.py index 7f671dbc75..b32b21f82b 100644 --- a/python/cuml/cuml/feature_extraction/text.py +++ b/python/cuml/cuml/feature_extraction/text.py @@ -204,7 +204,9 @@ def _preprocess(self, X): # We can use `X` in the common case of lowercase normalization # since no uppercase letters will remain. Otherwise pick # an unlikely key of unicode characters" - flag = "X" if self.lowercase else "cuᵐl" + flag = ( + "X" if self.preprocessor is None and self.lowercase else "cuᵐl" + ) X = ( X.str.replace("_", flag, regex=False) .str.filter_alphanum(delimiter, keep=True) @@ -799,10 +801,8 @@ def _check_params(self): ) def _fit(self, X, tokens): - self.fixed_vocabulary_ = self.vocabulary is not None - if self.vocabulary is not None: - self.vocabulary_ = _check_vocabulary(self.vocabulary) + vocabulary = _check_vocabulary(self.vocabulary) else: n_doc = X.shape[0] max_features = self.max_features @@ -821,37 +821,48 @@ def _fit(self, X, tokens): "max_df corresponds to < documents than min_df" ) - if ( - max_doc_count < n_doc - or min_doc_count > 1 - or max_features is not None - ): + pruned = False + keep = None + if max_doc_count < n_doc or min_doc_count > 1: doc_freq = tokens.drop_duplicates().token.value_counts() if max_doc_count < n_doc: doc_freq = doc_freq[doc_freq <= max_doc_count] if min_doc_count > 1: doc_freq = doc_freq[doc_freq >= min_doc_count] keep = doc_freq.index - if max_features is not None: - term_freq = tokens.token.value_counts() - term_freq = term_freq[term_freq.index.isin(keep)] - keep = term_freq.iloc[:max_features].index - self.vocabulary_ = cudf.Series(keep.sort_values()) - if not len(self.vocabulary_): - raise ValueError( - "After pruning, no terms remain. Try a lower min_df or " - "a higher max_df." - ) + if max_features is not None: + term_freq = ( + tokens.groupby("token") + .size() + .rename("count") + .reset_index() + .sort_values(["count", "token"], ascending=[False, True]) + ) + if keep is not None: + term_freq = term_freq[term_freq.token.isin(keep)] + keep = term_freq.iloc[:max_features].token + if keep is not None: + vocabulary = cudf.Series(keep.sort_values()) else: - self.vocabulary_ = ( + vocabulary = ( tokens.token.drop_duplicates() .sort_values() .reset_index(drop=True) ) - if not len(self.vocabulary_): + + if not len(vocabulary): + if pruned: raise ValueError( - "empty vocabulary; perhaps the documents only contain stop words" + "After pruning, no terms remain. Try a lower min_df or " + "a higher max_df." ) + raise ValueError( + "empty vocabulary; perhaps the documents only contain stop words" + ) + + self.fixed_vocabulary_ = self.vocabulary is not None + self.vocabulary_ = vocabulary + return self def _transform(self, X, tokens): diff --git a/python/cuml/tests/test_text_feature_extraction.py b/python/cuml/tests/test_text_feature_extraction.py index 1493cba19f..a7b3143d89 100644 --- a/python/cuml/tests/test_text_feature_extraction.py +++ b/python/cuml/tests/test_text_feature_extraction.py @@ -111,6 +111,26 @@ def test_tokenizer(): ) +@pytest.mark.parametrize("lowercase", [True, False]) +@pytest.mark.parametrize( + "preprocessor", [None, pytest.param(lambda s: s, id="identity")] +) +def test_default_tokenizer(lowercase, preprocessor): + r"""Default tokenizer matches the regex `\b\w\w+\b`""" + X = ["AX_B A B ZZ", "(fizz)\t\n\rbuzz-foo _bar_"] + cu_vec = CountVectorizer( + lowercase=lowercase, + preprocessor=preprocessor, + ).fit(X) + sk_vec = SkCountVect( + lowercase=lowercase, + preprocessor=preprocessor, + ).fit(X) + np.testing.assert_array_equal( + cu_vec.get_feature_names_out(), sk_vec.get_feature_names_out() + ) + + def test_countvectorizer_custom_vocabulary(): vocab = {"pizza": 0, "beer": 1} @@ -178,6 +198,16 @@ def as_index(x): assert "the" == features_None[as_index(counts_None.argmax())] +def test_max_features_tied_counts(): + docs = ["zz aa the", "yy bb the"] + cu_vec = CountVectorizer(max_features=3).fit(docs) + sk_vec = SkCountVect(max_features=3).fit(docs) + np.testing.assert_array_equal( + cu_vec.get_feature_names_out(), + sk_vec.get_feature_names_out(), + ) + + def test_countvectorizer_max_df(): test_data = ["abc", "dea", "eat"] vect = CountVectorizer(analyzer="char", max_df=1.0) @@ -215,6 +245,28 @@ def test_vectorizer_min_df(): assert len(vect.vocabulary_.to_arrow().to_pylist()) == 1 # {a} remains +@pytest.mark.parametrize( + "min_df, max_df, max_features", + [ + (2, 0.8, None), + (1, 0.5, 6), + (2, 0.8, 6), + ], +) +def test_vectorizer_mix_min_df_max_df_max_features( + min_df, max_df, max_features +): + cu_vec = CountVectorizer( + min_df=min_df, max_df=max_df, max_features=max_features + ).fit(DOCS) + sk_vec = SkCountVect( + min_df=min_df, max_df=max_df, max_features=max_features + ).fit(DOCS) + np.testing.assert_array_equal( + cu_vec.get_feature_names_out(), sk_vec.get_feature_names_out() + ) + + def test_count_binary_occurrences(): # by default multiple occurrences are counted as longs test_data = ["aaabc", "abbde"] From 4aab059c2d8f936acf59a8f716e1ee31efbcbecb Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Thu, 10 Sep 2026 09:34:23 -0500 Subject: [PATCH 10/10] Fixup --- python/cuml/cuml/feature_extraction/text.py | 2 ++ python/cuml/tests/test_text_feature_extraction.py | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/python/cuml/cuml/feature_extraction/text.py b/python/cuml/cuml/feature_extraction/text.py index b32b21f82b..46036f5858 100644 --- a/python/cuml/cuml/feature_extraction/text.py +++ b/python/cuml/cuml/feature_extraction/text.py @@ -830,6 +830,7 @@ def _fit(self, X, tokens): if min_doc_count > 1: doc_freq = doc_freq[doc_freq >= min_doc_count] keep = doc_freq.index + pruned = True if max_features is not None: term_freq = ( tokens.groupby("token") @@ -841,6 +842,7 @@ def _fit(self, X, tokens): if keep is not None: term_freq = term_freq[term_freq.token.isin(keep)] keep = term_freq.iloc[:max_features].token + pruned = True if keep is not None: vocabulary = cudf.Series(keep.sort_values()) else: diff --git a/python/cuml/tests/test_text_feature_extraction.py b/python/cuml/tests/test_text_feature_extraction.py index a7b3143d89..1aaf36167c 100644 --- a/python/cuml/tests/test_text_feature_extraction.py +++ b/python/cuml/tests/test_text_feature_extraction.py @@ -148,9 +148,14 @@ def test_countvectorizer_stop_words(): def test_countvectorizer_empty_vocabulary(): v = CountVectorizer(max_df=1.0, stop_words="english") # fitting only on stopwords will result in an empty vocabulary - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty vocabulary"): v.fit(["to be or not to be", "and me too", "and so do you"]) + # pruning may also result in an empty vocabulary + v = CountVectorizer(min_df=2) + with pytest.raises(ValueError, match="After pruning"): + v.fit(["unique", "words"]) + def test_countvectorizer_stop_words_ngrams(): stop_words_doc = ["and me too andy andy too"]