Page Menu
Home
c4science
Search
Configure Global Search
Log In
Files
F78597968
run_tf_multiple_choice.py
No One
Temporary
Actions
Download File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Subscribers
None
File Metadata
Details
File Info
Storage
Attached
Created
Wed, Aug 21, 21:23
Size
7 KB
Mime Type
text/x-python
Expires
Fri, Aug 23, 21:23 (2 d)
Engine
blob
Format
Raw Data
Handle
20070679
Attached To
R11484 ADDI
run_tf_multiple_choice.py
View Options
#!/usr/bin/env python
# coding=utf-8
# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Finetuning the library models for multiple choice (Bert, Roberta, XLNet)."""
import
logging
import
os
from
dataclasses
import
dataclass
,
field
from
typing
import
Dict
,
Optional
import
numpy
as
np
from
transformers
import
(
AutoConfig
,
AutoTokenizer
,
EvalPrediction
,
HfArgumentParser
,
TFAutoModelForMultipleChoice
,
TFTrainer
,
TFTrainingArguments
,
set_seed
,
)
from
transformers.utils
import
logging
as
hf_logging
from
utils_multiple_choice
import
Split
,
TFMultipleChoiceDataset
,
processors
hf_logging
.
set_verbosity_info
()
hf_logging
.
enable_default_handler
()
hf_logging
.
enable_explicit_format
()
logger
=
logging
.
getLogger
(
__name__
)
def
simple_accuracy
(
preds
,
labels
):
return
(
preds
==
labels
)
.
mean
()
@dataclass
class
ModelArguments
:
"""
Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
"""
model_name_or_path
:
str
=
field
(
metadata
=
{
"help"
:
"Path to pretrained model or model identifier from huggingface.co/models"
}
)
config_name
:
Optional
[
str
]
=
field
(
default
=
None
,
metadata
=
{
"help"
:
"Pretrained config name or path if not the same as model_name"
}
)
tokenizer_name
:
Optional
[
str
]
=
field
(
default
=
None
,
metadata
=
{
"help"
:
"Pretrained tokenizer name or path if not the same as model_name"
}
)
cache_dir
:
Optional
[
str
]
=
field
(
default
=
None
,
metadata
=
{
"help"
:
"Where do you want to store the pretrained models downloaded from huggingface.co"
},
)
@dataclass
class
DataTrainingArguments
:
"""
Arguments pertaining to what data we are going to input our model for training and eval.
"""
task_name
:
str
=
field
(
metadata
=
{
"help"
:
"The name of the task to train on: "
+
", "
.
join
(
processors
.
keys
())})
data_dir
:
str
=
field
(
metadata
=
{
"help"
:
"Should contain the data files for the task."
})
max_seq_length
:
int
=
field
(
default
=
128
,
metadata
=
{
"help"
:
"The maximum total input sequence length after tokenization. Sequences longer "
"than this will be truncated, sequences shorter will be padded."
},
)
overwrite_cache
:
bool
=
field
(
default
=
False
,
metadata
=
{
"help"
:
"Overwrite the cached training and evaluation sets"
}
)
def
main
():
# See all possible arguments in src/transformers/training_args.py
# or by passing the --help flag to this script.
# We now keep distinct sets of args, for a cleaner separation of concerns.
parser
=
HfArgumentParser
((
ModelArguments
,
DataTrainingArguments
,
TFTrainingArguments
))
model_args
,
data_args
,
training_args
=
parser
.
parse_args_into_dataclasses
()
if
(
os
.
path
.
exists
(
training_args
.
output_dir
)
and
os
.
listdir
(
training_args
.
output_dir
)
and
training_args
.
do_train
and
not
training_args
.
overwrite_output_dir
):
raise
ValueError
(
f
"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
)
# Setup logging
logging
.
basicConfig
(
format
=
"
%(asctime)s
-
%(levelname)s
-
%(name)s
-
%(message)s
"
,
datefmt
=
"%m/
%d
/%Y %H:%M:%S"
,
level
=
logging
.
INFO
,
)
logger
.
warning
(
"device:
%s
, n_replicas:
%s
, 16-bits training:
%s
"
,
training_args
.
device
,
training_args
.
n_replicas
,
training_args
.
fp16
,
)
logger
.
info
(
"Training/evaluation parameters
%s
"
,
training_args
)
# Set seed
set_seed
(
training_args
.
seed
)
try
:
processor
=
processors
[
data_args
.
task_name
]()
label_list
=
processor
.
get_labels
()
num_labels
=
len
(
label_list
)
except
KeyError
:
raise
ValueError
(
"Task not found:
%s
"
%
(
data_args
.
task_name
))
# Load pretrained model and tokenizer
#
# Distributed training:
# The .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
config
=
AutoConfig
.
from_pretrained
(
model_args
.
config_name
if
model_args
.
config_name
else
model_args
.
model_name_or_path
,
num_labels
=
num_labels
,
finetuning_task
=
data_args
.
task_name
,
cache_dir
=
model_args
.
cache_dir
,
)
tokenizer
=
AutoTokenizer
.
from_pretrained
(
model_args
.
tokenizer_name
if
model_args
.
tokenizer_name
else
model_args
.
model_name_or_path
,
cache_dir
=
model_args
.
cache_dir
,
)
with
training_args
.
strategy
.
scope
():
model
=
TFAutoModelForMultipleChoice
.
from_pretrained
(
model_args
.
model_name_or_path
,
from_pt
=
bool
(
".bin"
in
model_args
.
model_name_or_path
),
config
=
config
,
cache_dir
=
model_args
.
cache_dir
,
)
# Get datasets
train_dataset
=
(
TFMultipleChoiceDataset
(
data_dir
=
data_args
.
data_dir
,
tokenizer
=
tokenizer
,
task
=
data_args
.
task_name
,
max_seq_length
=
data_args
.
max_seq_length
,
overwrite_cache
=
data_args
.
overwrite_cache
,
mode
=
Split
.
train
,
)
if
training_args
.
do_train
else
None
)
eval_dataset
=
(
TFMultipleChoiceDataset
(
data_dir
=
data_args
.
data_dir
,
tokenizer
=
tokenizer
,
task
=
data_args
.
task_name
,
max_seq_length
=
data_args
.
max_seq_length
,
overwrite_cache
=
data_args
.
overwrite_cache
,
mode
=
Split
.
dev
,
)
if
training_args
.
do_eval
else
None
)
def
compute_metrics
(
p
:
EvalPrediction
)
->
Dict
:
preds
=
np
.
argmax
(
p
.
predictions
,
axis
=
1
)
return
{
"acc"
:
simple_accuracy
(
preds
,
p
.
label_ids
)}
# Initialize our Trainer
trainer
=
TFTrainer
(
model
=
model
,
args
=
training_args
,
train_dataset
=
train_dataset
.
get_dataset
()
if
train_dataset
else
None
,
eval_dataset
=
eval_dataset
.
get_dataset
()
if
eval_dataset
else
None
,
compute_metrics
=
compute_metrics
,
)
# Training
if
training_args
.
do_train
:
trainer
.
train
()
trainer
.
save_model
()
tokenizer
.
save_pretrained
(
training_args
.
output_dir
)
# Evaluation
results
=
{}
if
training_args
.
do_eval
:
logger
.
info
(
"*** Evaluate ***"
)
result
=
trainer
.
evaluate
()
output_eval_file
=
os
.
path
.
join
(
training_args
.
output_dir
,
"eval_results.txt"
)
with
open
(
output_eval_file
,
"w"
)
as
writer
:
logger
.
info
(
"***** Eval results *****"
)
for
key
,
value
in
result
.
items
():
logger
.
info
(
"
%s
=
%s
"
,
key
,
value
)
writer
.
write
(
"
%s
=
%s
\n
"
%
(
key
,
value
))
results
.
update
(
result
)
return
results
if
__name__
==
"__main__"
:
main
()
Event Timeline
Log In to Comment