Page Menu
Home
c4science
Search
Configure Global Search
Log In
Files
F66425418
tokenization_led.py
No One
Temporary
Actions
Download File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Subscribers
None
File Metadata
Details
File Info
Storage
Attached
Created
Mon, Jun 10, 10:44
Size
1 KB
Mime Type
text/x-python
Expires
Wed, Jun 12, 10:44 (2 d)
Engine
blob
Format
Raw Data
Handle
18220342
Attached To
R11484 ADDI
tokenization_led.py
View Options
# coding=utf-8
# Copyright 2021 Iz Beltagy, Matthew E. Peters, Arman Cohan and The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tokenization classes for LED."""
from
...utils
import
logging
from
..bart.tokenization_bart
import
BartTokenizer
logger
=
logging
.
get_logger
(
__name__
)
PRETRAINED_VOCAB_FILES_MAP
=
{
"vocab_file"
:
{
"allenai/led-base-16384"
:
"https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json"
,
},
"merges_file"
:
{
"allenai/led-base-16384"
:
"https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt"
,
},
"tokenizer_file"
:
{
"allenai/led-base-16384"
:
"https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json"
,
},
}
PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
=
{
"allenai/led-base-16384"
:
16384
,
}
class
LEDTokenizer
(
BartTokenizer
):
"""
Construct a LED tokenizer.
:class:`~transformers.LEDTokenizer` is identical to :class:`~transformers.BartTokenizer` and runs end-to-end
tokenization: punctuation splitting and wordpiece.
Refer to superclass :class:`~transformers.BartTokenizer` for usage examples and documentation concerning
parameters.
"""
pretrained_vocab_files_map
=
PRETRAINED_VOCAB_FILES_MAP
max_model_input_sizes
=
PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
Event Timeline
Log In to Comment