mirror of
https://github.com/GokuMohandas/Made-With-ML.git
synced 2026-07-19 00:13:02 -05:00
761 lines
22 KiB
HTML
761 lines
22 KiB
HTML
<!DOCTYPE html>
|
||
<html class="writer-html5" lang="en" >
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><link rel="canonical" href="https://madewithml.com/madewithml/data/" />
|
||
<link rel="shortcut icon" href="../../img/favicon.ico" />
|
||
<title>data - Made With ML</title>
|
||
<link rel="stylesheet" href="../../css/theme.css" />
|
||
<link rel="stylesheet" href="../../css/theme_extra.css" />
|
||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/github.min.css" />
|
||
<link href="../../assets/_mkdocstrings.css" rel="stylesheet" />
|
||
|
||
<script>
|
||
// Current page data
|
||
var mkdocs_page_name = "data";
|
||
var mkdocs_page_input_path = "madewithml/data.md";
|
||
var mkdocs_page_url = "/madewithml/data/";
|
||
</script>
|
||
|
||
<script src="../../js/jquery-3.6.0.min.js" defer></script>
|
||
<!--[if lt IE 9]>
|
||
<script src="../../js/html5shiv.min.js"></script>
|
||
<![endif]-->
|
||
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
|
||
<script>hljs.initHighlightingOnLoad();</script>
|
||
</head>
|
||
|
||
<body class="wy-body-for-nav" role="document">
|
||
|
||
<div class="wy-grid-for-nav">
|
||
<nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
|
||
<div class="wy-side-scroll">
|
||
<div class="wy-side-nav-search">
|
||
<a href="../.." class="icon icon-home"> Made With ML
|
||
</a>
|
||
</div>
|
||
|
||
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../..">Home</a>
|
||
</li>
|
||
</ul>
|
||
<p class="caption"><span class="caption-text">madewithml</span></p>
|
||
<ul class="current">
|
||
<li class="toctree-l1 current"><a class="reference internal current" href="./">data</a>
|
||
<ul class="current">
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../models/">models</a>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../train/">train</a>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../tune/">tune</a>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../evaluate/">evaluate</a>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../predict/">predict</a>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../serve/">serve</a>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../utils/">utils</a>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</nav>
|
||
|
||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
|
||
<nav class="wy-nav-top" role="navigation" aria-label="Mobile navigation menu">
|
||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||
<a href="../..">Made With ML</a>
|
||
|
||
</nav>
|
||
<div class="wy-nav-content">
|
||
<div class="rst-content"><div role="navigation" aria-label="breadcrumbs navigation">
|
||
<ul class="wy-breadcrumbs">
|
||
<li><a href="../.." class="icon icon-home" alt="Docs"></a> »</li>
|
||
<li>madewithml »</li>
|
||
<li>data</li>
|
||
<li class="wy-breadcrumbs-aside">
|
||
<a href="https://github.com/GokuMohandas/Made-With-ML/edit/master/docs/madewithml/data.md" class="icon icon-github"> Edit on GitHub</a>
|
||
</li>
|
||
</ul>
|
||
<hr/>
|
||
</div>
|
||
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
||
<div class="section" itemprop="articleBody">
|
||
|
||
<div class="doc doc-object doc-module">
|
||
|
||
|
||
|
||
<a id="madewithml.data"></a>
|
||
<div class="doc doc-contents first">
|
||
|
||
|
||
|
||
<div class="doc doc-children">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="doc doc-object doc-class">
|
||
|
||
|
||
|
||
|
||
<h2 id="madewithml.data.CustomPreprocessor" class="doc doc-heading">
|
||
<code>CustomPreprocessor</code>
|
||
|
||
|
||
</h2>
|
||
|
||
|
||
<div class="doc doc-contents ">
|
||
|
||
|
||
<p>Custom preprocessor class.</p>
|
||
|
||
<details class="quote">
|
||
<summary>Source code in <code>madewithml/data.py</code></summary>
|
||
<pre class="highlight"><code class="language-python">class CustomPreprocessor:
|
||
"""Custom preprocessor class."""
|
||
|
||
def __init__(self, class_to_index={}):
|
||
self.class_to_index = class_to_index or {} # mutable defaults
|
||
self.index_to_class = {v: k for k, v in self.class_to_index.items()}
|
||
|
||
def fit(self, ds):
|
||
tags = ds.unique(column="tag")
|
||
self.class_to_index = {tag: i for i, tag in enumerate(tags)}
|
||
self.index_to_class = {v: k for k, v in self.class_to_index.items()}
|
||
return self
|
||
|
||
def transform(self, ds):
|
||
return ds.map_batches(preprocess, fn_kwargs={"class_to_index": self.class_to_index}, batch_format="pandas")</code></pre>
|
||
</details>
|
||
|
||
|
||
|
||
<div class="doc doc-children">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="doc doc-object doc-function">
|
||
|
||
|
||
|
||
|
||
<h2 id="madewithml.data.clean_text" class="doc doc-heading">
|
||
<code class="highlight language-python">clean_text(text, stopwords=STOPWORDS)</code>
|
||
|
||
</h2>
|
||
|
||
|
||
<div class="doc doc-contents ">
|
||
|
||
<p>Clean raw text string.</p>
|
||
|
||
|
||
|
||
<table class="field-list">
|
||
<colgroup>
|
||
<col class="field-name" />
|
||
<col class="field-body" />
|
||
</colgroup>
|
||
<tbody valign="top">
|
||
<tr class="field">
|
||
<th class="field-name">Parameters:</th>
|
||
<td class="field-body">
|
||
<ul class="first simple">
|
||
<li>
|
||
<b><code>text</code></b>
|
||
(<code>str</code>)
|
||
–
|
||
<div class="doc-md-description">
|
||
<p>Raw text to clean.</p>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<b><code>stopwords</code></b>
|
||
(<code><span title="typing.List">List</span></code>, default:
|
||
<code><span title="madewithml.config.STOPWORDS">STOPWORDS</span></code>
|
||
)
|
||
–
|
||
<div class="doc-md-description">
|
||
<p>list of words to filter out. Defaults to STOPWORDS.</p>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
|
||
|
||
<table class="field-list">
|
||
<colgroup>
|
||
<col class="field-name" />
|
||
<col class="field-body" />
|
||
</colgroup>
|
||
<tbody valign="top">
|
||
<tr class="field">
|
||
<th class="field-name">Returns:</th>
|
||
<td class="field-body">
|
||
<ul class="first simple">
|
||
<li>
|
||
<b><code>str</code></b>( <code>str</code>
|
||
) –
|
||
<div class="doc-md-description">
|
||
<p>cleaned text.</p>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<details class="quote">
|
||
<summary>Source code in <code>madewithml/data.py</code></summary>
|
||
<pre class="highlight"><code class="language-python">def clean_text(text: str, stopwords: List = STOPWORDS) -> str:
|
||
"""Clean raw text string.
|
||
|
||
Args:
|
||
text (str): Raw text to clean.
|
||
stopwords (List, optional): list of words to filter out. Defaults to STOPWORDS.
|
||
|
||
Returns:
|
||
str: cleaned text.
|
||
"""
|
||
# Lower
|
||
text = text.lower()
|
||
|
||
# Remove stopwords
|
||
pattern = re.compile(r"\b(" + r"|".join(stopwords) + r")\b\s*")
|
||
text = pattern.sub(" ", text)
|
||
|
||
# Spacing and filters
|
||
text = re.sub(r"([!\"'#$%&()*\+,-./:;<=>?@\\\[\]^_`{|}~])", r" \1 ", text) # add spacing
|
||
text = re.sub("[^A-Za-z0-9]+", " ", text) # remove non alphanumeric chars
|
||
text = re.sub(" +", " ", text) # remove multiple spaces
|
||
text = text.strip() # strip white space at the ends
|
||
text = re.sub(r"http\S+", "", text) # remove links
|
||
|
||
return text</code></pre>
|
||
</details>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="doc doc-object doc-function">
|
||
|
||
|
||
|
||
|
||
<h2 id="madewithml.data.load_data" class="doc doc-heading">
|
||
<code class="highlight language-python">load_data(dataset_loc, num_samples=None)</code>
|
||
|
||
</h2>
|
||
|
||
|
||
<div class="doc doc-contents ">
|
||
|
||
<p>Load data from source into a Ray Dataset.</p>
|
||
|
||
|
||
|
||
<table class="field-list">
|
||
<colgroup>
|
||
<col class="field-name" />
|
||
<col class="field-body" />
|
||
</colgroup>
|
||
<tbody valign="top">
|
||
<tr class="field">
|
||
<th class="field-name">Parameters:</th>
|
||
<td class="field-body">
|
||
<ul class="first simple">
|
||
<li>
|
||
<b><code>dataset_loc</code></b>
|
||
(<code>str</code>)
|
||
–
|
||
<div class="doc-md-description">
|
||
<p>Location of the dataset.</p>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<b><code>num_samples</code></b>
|
||
(<code>int</code>, default:
|
||
<code>None</code>
|
||
)
|
||
–
|
||
<div class="doc-md-description">
|
||
<p>The number of samples to load. Defaults to None.</p>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
|
||
|
||
<table class="field-list">
|
||
<colgroup>
|
||
<col class="field-name" />
|
||
<col class="field-body" />
|
||
</colgroup>
|
||
<tbody valign="top">
|
||
<tr class="field">
|
||
<th class="field-name">Returns:</th>
|
||
<td class="field-body">
|
||
<ul class="first simple">
|
||
<li>
|
||
<b><code>Dataset</code></b>( <code><span title="ray.data.Dataset">Dataset</span></code>
|
||
) –
|
||
<div class="doc-md-description">
|
||
<p>Our dataset represented by a Ray Dataset.</p>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<details class="quote">
|
||
<summary>Source code in <code>madewithml/data.py</code></summary>
|
||
<pre class="highlight"><code class="language-python">def load_data(dataset_loc: str, num_samples: int = None) -> Dataset:
|
||
"""Load data from source into a Ray Dataset.
|
||
|
||
Args:
|
||
dataset_loc (str): Location of the dataset.
|
||
num_samples (int, optional): The number of samples to load. Defaults to None.
|
||
|
||
Returns:
|
||
Dataset: Our dataset represented by a Ray Dataset.
|
||
"""
|
||
ds = ray.data.read_csv(dataset_loc)
|
||
ds = ds.random_shuffle(seed=1234)
|
||
ds = ray.data.from_items(ds.take(num_samples)) if num_samples else ds
|
||
return ds</code></pre>
|
||
</details>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="doc doc-object doc-function">
|
||
|
||
|
||
|
||
|
||
<h2 id="madewithml.data.preprocess" class="doc doc-heading">
|
||
<code class="highlight language-python">preprocess(df, class_to_index)</code>
|
||
|
||
</h2>
|
||
|
||
|
||
<div class="doc doc-contents ">
|
||
|
||
<p>Preprocess the data in our dataframe.</p>
|
||
|
||
|
||
|
||
<table class="field-list">
|
||
<colgroup>
|
||
<col class="field-name" />
|
||
<col class="field-body" />
|
||
</colgroup>
|
||
<tbody valign="top">
|
||
<tr class="field">
|
||
<th class="field-name">Parameters:</th>
|
||
<td class="field-body">
|
||
<ul class="first simple">
|
||
<li>
|
||
<b><code>df</code></b>
|
||
(<code><span title="pandas.DataFrame">DataFrame</span></code>)
|
||
–
|
||
<div class="doc-md-description">
|
||
<p>Raw dataframe to preprocess.</p>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<b><code>class_to_index</code></b>
|
||
(<code><span title="typing.Dict">Dict</span></code>)
|
||
–
|
||
<div class="doc-md-description">
|
||
<p>Mapping of class names to indices.</p>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
|
||
|
||
<table class="field-list">
|
||
<colgroup>
|
||
<col class="field-name" />
|
||
<col class="field-body" />
|
||
</colgroup>
|
||
<tbody valign="top">
|
||
<tr class="field">
|
||
<th class="field-name">Returns:</th>
|
||
<td class="field-body">
|
||
<ul class="first simple">
|
||
<li>
|
||
<b><code>Dict</code></b>( <code><span title="typing.Dict">Dict</span></code>
|
||
) –
|
||
<div class="doc-md-description">
|
||
<p>preprocessed data (ids, masks, targets).</p>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<details class="quote">
|
||
<summary>Source code in <code>madewithml/data.py</code></summary>
|
||
<pre class="highlight"><code class="language-python">def preprocess(df: pd.DataFrame, class_to_index: Dict) -> Dict:
|
||
"""Preprocess the data in our dataframe.
|
||
|
||
Args:
|
||
df (pd.DataFrame): Raw dataframe to preprocess.
|
||
class_to_index (Dict): Mapping of class names to indices.
|
||
|
||
Returns:
|
||
Dict: preprocessed data (ids, masks, targets).
|
||
"""
|
||
df["text"] = df.title + " " + df.description # feature engineering
|
||
df["text"] = df.text.apply(clean_text) # clean text
|
||
df = df.drop(columns=["id", "created_on", "title", "description"], errors="ignore") # clean dataframe
|
||
df = df[["text", "tag"]] # rearrange columns
|
||
df["tag"] = df["tag"].map(class_to_index) # label encoding
|
||
outputs = tokenize(df)
|
||
return outputs</code></pre>
|
||
</details>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="doc doc-object doc-function">
|
||
|
||
|
||
|
||
|
||
<h2 id="madewithml.data.stratify_split" class="doc doc-heading">
|
||
<code class="highlight language-python">stratify_split(ds, stratify, test_size, shuffle=True, seed=1234)</code>
|
||
|
||
</h2>
|
||
|
||
|
||
<div class="doc doc-contents ">
|
||
|
||
<p>Split a dataset into train and test splits with equal
|
||
amounts of data points from each class in the column we
|
||
want to stratify on.</p>
|
||
|
||
|
||
|
||
<table class="field-list">
|
||
<colgroup>
|
||
<col class="field-name" />
|
||
<col class="field-body" />
|
||
</colgroup>
|
||
<tbody valign="top">
|
||
<tr class="field">
|
||
<th class="field-name">Parameters:</th>
|
||
<td class="field-body">
|
||
<ul class="first simple">
|
||
<li>
|
||
<b><code>ds</code></b>
|
||
(<code><span title="ray.data.Dataset">Dataset</span></code>)
|
||
–
|
||
<div class="doc-md-description">
|
||
<p>Input dataset to split.</p>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<b><code>stratify</code></b>
|
||
(<code>str</code>)
|
||
–
|
||
<div class="doc-md-description">
|
||
<p>Name of column to split on.</p>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<b><code>test_size</code></b>
|
||
(<code>float</code>)
|
||
–
|
||
<div class="doc-md-description">
|
||
<p>Proportion of dataset to split for test set.</p>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<b><code>shuffle</code></b>
|
||
(<code>bool</code>, default:
|
||
<code>True</code>
|
||
)
|
||
–
|
||
<div class="doc-md-description">
|
||
<p>whether to shuffle the dataset. Defaults to True.</p>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<b><code>seed</code></b>
|
||
(<code>int</code>, default:
|
||
<code>1234</code>
|
||
)
|
||
–
|
||
<div class="doc-md-description">
|
||
<p>seed for shuffling. Defaults to 1234.</p>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
|
||
|
||
<table class="field-list">
|
||
<colgroup>
|
||
<col class="field-name" />
|
||
<col class="field-body" />
|
||
</colgroup>
|
||
<tbody valign="top">
|
||
<tr class="field">
|
||
<th class="field-name">Returns:</th>
|
||
<td class="field-body">
|
||
<ul class="first simple">
|
||
<li>
|
||
<code><span title="typing.Tuple">Tuple</span>[<span title="ray.data.Dataset">Dataset</span>, <span title="ray.data.Dataset">Dataset</span>]</code>
|
||
–
|
||
<div class="doc-md-description">
|
||
<p>Tuple[Dataset, Dataset]: the stratified train and test datasets.</p>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<details class="quote">
|
||
<summary>Source code in <code>madewithml/data.py</code></summary>
|
||
<pre class="highlight"><code class="language-python">def stratify_split(
|
||
ds: Dataset,
|
||
stratify: str,
|
||
test_size: float,
|
||
shuffle: bool = True,
|
||
seed: int = 1234,
|
||
) -> Tuple[Dataset, Dataset]:
|
||
"""Split a dataset into train and test splits with equal
|
||
amounts of data points from each class in the column we
|
||
want to stratify on.
|
||
|
||
Args:
|
||
ds (Dataset): Input dataset to split.
|
||
stratify (str): Name of column to split on.
|
||
test_size (float): Proportion of dataset to split for test set.
|
||
shuffle (bool, optional): whether to shuffle the dataset. Defaults to True.
|
||
seed (int, optional): seed for shuffling. Defaults to 1234.
|
||
|
||
Returns:
|
||
Tuple[Dataset, Dataset]: the stratified train and test datasets.
|
||
"""
|
||
|
||
def _add_split(df: pd.DataFrame) -> pd.DataFrame: # pragma: no cover, used in parent function
|
||
"""Naively split a dataframe into train and test splits.
|
||
Add a column specifying whether it's the train or test split."""
|
||
train, test = train_test_split(df, test_size=test_size, shuffle=shuffle, random_state=seed)
|
||
train["_split"] = "train"
|
||
test["_split"] = "test"
|
||
return pd.concat([train, test])
|
||
|
||
def _filter_split(df: pd.DataFrame, split: str) -> pd.DataFrame: # pragma: no cover, used in parent function
|
||
"""Filter by data points that match the split column's value
|
||
and return the dataframe with the _split column dropped."""
|
||
return df[df["_split"] == split].drop("_split", axis=1)
|
||
|
||
# Train, test split with stratify
|
||
grouped = ds.groupby(stratify).map_groups(_add_split, batch_format="pandas") # group by each unique value in the column we want to stratify on
|
||
train_ds = grouped.map_batches(_filter_split, fn_kwargs={"split": "train"}, batch_format="pandas") # combine
|
||
test_ds = grouped.map_batches(_filter_split, fn_kwargs={"split": "test"}, batch_format="pandas") # combine
|
||
|
||
# Shuffle each split (required)
|
||
train_ds = train_ds.random_shuffle(seed=seed)
|
||
test_ds = test_ds.random_shuffle(seed=seed)
|
||
|
||
return train_ds, test_ds</code></pre>
|
||
</details>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="doc doc-object doc-function">
|
||
|
||
|
||
|
||
|
||
<h2 id="madewithml.data.tokenize" class="doc doc-heading">
|
||
<code class="highlight language-python">tokenize(batch)</code>
|
||
|
||
</h2>
|
||
|
||
|
||
<div class="doc doc-contents ">
|
||
|
||
<p>Tokenize the text input in our batch using a tokenizer.</p>
|
||
|
||
|
||
|
||
<table class="field-list">
|
||
<colgroup>
|
||
<col class="field-name" />
|
||
<col class="field-body" />
|
||
</colgroup>
|
||
<tbody valign="top">
|
||
<tr class="field">
|
||
<th class="field-name">Parameters:</th>
|
||
<td class="field-body">
|
||
<ul class="first simple">
|
||
<li>
|
||
<b><code>batch</code></b>
|
||
(<code><span title="typing.Dict">Dict</span></code>)
|
||
–
|
||
<div class="doc-md-description">
|
||
<p>batch of data with the text inputs to tokenize.</p>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
|
||
|
||
<table class="field-list">
|
||
<colgroup>
|
||
<col class="field-name" />
|
||
<col class="field-body" />
|
||
</colgroup>
|
||
<tbody valign="top">
|
||
<tr class="field">
|
||
<th class="field-name">Returns:</th>
|
||
<td class="field-body">
|
||
<ul class="first simple">
|
||
<li>
|
||
<b><code>Dict</code></b>( <code><span title="typing.Dict">Dict</span></code>
|
||
) –
|
||
<div class="doc-md-description">
|
||
<p>batch of data with the results of tokenization (<code>input_ids</code> and <code>attention_mask</code>) on the text inputs.</p>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<details class="quote">
|
||
<summary>Source code in <code>madewithml/data.py</code></summary>
|
||
<pre class="highlight"><code class="language-python">def tokenize(batch: Dict) -> Dict:
|
||
"""Tokenize the text input in our batch using a tokenizer.
|
||
|
||
Args:
|
||
batch (Dict): batch of data with the text inputs to tokenize.
|
||
|
||
Returns:
|
||
Dict: batch of data with the results of tokenization (`input_ids` and `attention_mask`) on the text inputs.
|
||
"""
|
||
tokenizer = BertTokenizer.from_pretrained("allenai/scibert_scivocab_uncased", return_dict=False)
|
||
encoded_inputs = tokenizer(batch["text"].tolist(), return_tensors="np", padding="longest")
|
||
return dict(ids=encoded_inputs["input_ids"], masks=encoded_inputs["attention_mask"], targets=np.array(batch["tag"]))</code></pre>
|
||
</details>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div><footer>
|
||
<div class="rst-footer-buttons" role="navigation" aria-label="Footer Navigation">
|
||
<a href="../.." class="btn btn-neutral float-left" title="Home"><span class="icon icon-circle-arrow-left"></span> Previous</a>
|
||
<a href="../models/" class="btn btn-neutral float-right" title="models">Next <span class="icon icon-circle-arrow-right"></span></a>
|
||
</div>
|
||
|
||
<hr/>
|
||
|
||
<div role="contentinfo">
|
||
<!-- Copyright etc -->
|
||
</div>
|
||
|
||
Built with <a href="https://www.mkdocs.org/">MkDocs</a> using a <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||
</footer>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</section>
|
||
|
||
</div>
|
||
|
||
<div class="rst-versions" role="note" aria-label="Versions">
|
||
<span class="rst-current-version" data-toggle="rst-current-version">
|
||
|
||
<span>
|
||
<a href="https://github.com/GokuMohandas/Made-With-ML/" class="fa fa-github" style="color: #fcfcfc"> GitHub</a>
|
||
</span>
|
||
|
||
|
||
<span><a href="../.." style="color: #fcfcfc">« Previous</a></span>
|
||
|
||
|
||
<span><a href="../models/" style="color: #fcfcfc">Next »</a></span>
|
||
|
||
</span>
|
||
</div>
|
||
<script>var base_url = '../..';</script>
|
||
<script src="../../js/theme_extra.js" defer></script>
|
||
<script src="../../js/theme.js" defer></script>
|
||
<script defer>
|
||
window.onload = function () {
|
||
SphinxRtdTheme.Navigation.enable(true);
|
||
};
|
||
</script>
|
||
|
||
</body>
|
||
</html>
|