From b244215eecff15ba3c508cc57434837dc309f102 Mon Sep 17 00:00:00 2001 From: Alan Braz Date: Wed, 30 Aug 2023 13:26:07 -0300 Subject: [PATCH 1/4] load dataset from local json file --- examples/run_peft_tuning.py | 52 ++++++++++++++++++++++++++++++++----- examples/utils.py | 21 +++++++++++++++ 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/examples/run_peft_tuning.py b/examples/run_peft_tuning.py index 44d7d28dc..5d50c2a5b 100644 --- a/examples/run_peft_tuning.py +++ b/examples/run_peft_tuning.py @@ -27,6 +27,7 @@ DatasetInfo, configure_random_seed_and_logging, print_colored, + load_json_file_dataset ) import datasets @@ -166,7 +167,7 @@ def register_common_arguments(subparsers: Tuple[argparse.ArgumentParser]) -> Non subparser.add_argument( "--dataset", help="Dataset to use to train prompt vectors. Options: {}".format( - list(SUPPORTED_DATASETS.keys()) + list(SUPPORTED_DATASETS.keys())+["json_file"] ), default="twitter_complaints", ) @@ -235,6 +236,32 @@ def register_common_arguments(subparsers: Tuple[argparse.ArgumentParser]) -> Non default=1, type=int, ) + subparser.add_argument( + "--json_file_path", + help="File path of the local JSON file to be loaded as dataset. It should be a JSON array or JSONL format.", + default=None, + type=pathlib.Path, + ) + subparser.add_argument( + "--json_file_input_field", + help="The input field to be used at the JSON file dataset", + default="input", + ) + subparser.add_argument( + "--json_file_output_field", + help="The output field to be used at the JSON file dataset", + default="output", + ) + subparser.add_argument( + "--json_file_init_text", + help="The init text to be used by the JSON file dataset", + default="", + ) + subparser.add_argument( + "--json_file_verbalizer", + help="The custom verbalizer to be used by the JSON file dataset", + default="{{input}}", + ) def register_multitask_prompt_tuning_args(subparser: argparse.ArgumentParser): @@ -281,12 +308,14 @@ def validate_common_args(args: argparse.Namespace): Parsed args corresponding to one tuning task. """ # Validate that the dataset is one of our allowed values - if args.dataset not in SUPPORTED_DATASETS: + if args.dataset != "json_file" and args.dataset not in SUPPORTED_DATASETS: raise KeyError( "[{}] is not a supported dataset; see --help for options.".format( args.dataset ) ) + if args.dataset == "json_file" and args.json_file_path is None: + raise argparse.ArgumentError(None, "--json_file_path is required when dataset value is json_file.") # Purge our output directory if one already exists. if os.path.isdir(args.output_dir): print("Existing model directory found; purging it now.") @@ -378,13 +407,22 @@ def show_experiment_configuration(args, dataset_info, model_type) -> None: configure_random_seed_and_logging() args = parse_args() model_type = get_resource_type(args.model_name) - # Unpack the dataset dictionary into a loaded dataset & verbalizer - dataset_info = SUPPORTED_DATASETS[args.dataset] - show_experiment_configuration(args, dataset_info, model_type) # Convert the loaded dataset to a stream print_colored("[Loading the dataset...]") - # TODO - conditionally enable validation stream - train_stream = dataset_info.dataset_loader()[0] + # Unpack the dataset dictionary into a loaded dataset & verbalizer + if args.dataset != "json_file": + dataset_info = SUPPORTED_DATASETS[args.dataset] + # TODO - conditionally enable validation stream + train_stream = dataset_info.dataset_loader()[0] + else: + print(args.json_file_path, args.json_file_input_field, args.json_file_output_field) + dataset_info = DatasetInfo( + verbalizer=args.json_file_verbalizer, + dataset_loader=load_json_file_dataset(str(args.json_file_path), str(args.json_file_input_field), str(args.json_file_output_field)), + init_text=args.json_file_init_text, + ) + train_stream = dataset_info.dataset_loader[0] + show_experiment_configuration(args, dataset_info, model_type) if args.num_shots is not None: train_stream = subsample_stream(train_stream, args.num_shots) # Init the resource & Build the tuning config from our dataset/arg info diff --git a/examples/utils.py b/examples/utils.py index cfb944fd9..9de2937e2 100644 --- a/examples/utils.py +++ b/examples/utils.py @@ -279,6 +279,27 @@ def to_generation_fmt(x): test_stream = build_stream("test") return (train_stream, validation_stream, test_stream) +def load_json_file_dataset(file_path, input_field, output_field, test_size=0.1, validation_size=0.1) -> Tuple[caikit.core.data_model.DataStream]: + """Load the dataset from local JSON file.""" + + def to_generation_fmt(x): + return GenerationTrainRecord(input=x[input_field], output=str(x[output_field])) + + dataset = datasets.load_dataset('json', data_files=file_path) + train_test_dataset = dataset["train"].train_test_split(test_size=test_size) + # # # Split the 10% test + valid into half test, half valid + test_valid = train_test_dataset['train'].train_test_split(test_size=validation_size) + train_test_dataset["train"] = test_valid["train"] + train_test_dataset["validation"] = test_valid["test"] + + build_stream = lambda split: caikit.core.data_model.DataStream.from_iterable( + [to_generation_fmt(datum) for datum in train_test_dataset[split]] + ) + train_stream = build_stream("train") + validation_stream = build_stream("validation") + test_stream = build_stream("test") + return (train_stream, validation_stream, test_stream) + def get_wrapped_evaluate_metric(metric_name: str, convert_to_numeric: bool) -> Callable: """Wrapper for running metrics out of evaluate which operate on numeric arrays From 241705b28fa23a3edc129870869064797e680083 Mon Sep 17 00:00:00 2001 From: Alan Braz Date: Wed, 30 Aug 2023 14:56:32 -0300 Subject: [PATCH 2/4] fmt --- examples/run_peft_tuning.py | 20 ++++++++++++++------ examples/utils.py | 9 ++++++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/examples/run_peft_tuning.py b/examples/run_peft_tuning.py index 5d50c2a5b..fcdbb0279 100644 --- a/examples/run_peft_tuning.py +++ b/examples/run_peft_tuning.py @@ -26,8 +26,8 @@ SUPPORTED_DATASETS, DatasetInfo, configure_random_seed_and_logging, + load_json_file_dataset, print_colored, - load_json_file_dataset ) import datasets @@ -167,7 +167,7 @@ def register_common_arguments(subparsers: Tuple[argparse.ArgumentParser]) -> Non subparser.add_argument( "--dataset", help="Dataset to use to train prompt vectors. Options: {}".format( - list(SUPPORTED_DATASETS.keys())+["json_file"] + list(SUPPORTED_DATASETS.keys()) + ["json_file"] ), default="twitter_complaints", ) @@ -315,7 +315,9 @@ def validate_common_args(args: argparse.Namespace): ) ) if args.dataset == "json_file" and args.json_file_path is None: - raise argparse.ArgumentError(None, "--json_file_path is required when dataset value is json_file.") + raise argparse.ArgumentError( + None, "--json_file_path is required when dataset value is json_file." + ) # Purge our output directory if one already exists. if os.path.isdir(args.output_dir): print("Existing model directory found; purging it now.") @@ -412,13 +414,19 @@ def show_experiment_configuration(args, dataset_info, model_type) -> None: # Unpack the dataset dictionary into a loaded dataset & verbalizer if args.dataset != "json_file": dataset_info = SUPPORTED_DATASETS[args.dataset] - # TODO - conditionally enable validation stream + # TODO - conditionally enable validation stream train_stream = dataset_info.dataset_loader()[0] else: - print(args.json_file_path, args.json_file_input_field, args.json_file_output_field) + print( + args.json_file_path, args.json_file_input_field, args.json_file_output_field + ) dataset_info = DatasetInfo( verbalizer=args.json_file_verbalizer, - dataset_loader=load_json_file_dataset(str(args.json_file_path), str(args.json_file_input_field), str(args.json_file_output_field)), + dataset_loader=load_json_file_dataset( + str(args.json_file_path), + str(args.json_file_input_field), + str(args.json_file_output_field), + ), init_text=args.json_file_init_text, ) train_stream = dataset_info.dataset_loader[0] diff --git a/examples/utils.py b/examples/utils.py index 9de2937e2..a1159776c 100644 --- a/examples/utils.py +++ b/examples/utils.py @@ -279,16 +279,19 @@ def to_generation_fmt(x): test_stream = build_stream("test") return (train_stream, validation_stream, test_stream) -def load_json_file_dataset(file_path, input_field, output_field, test_size=0.1, validation_size=0.1) -> Tuple[caikit.core.data_model.DataStream]: + +def load_json_file_dataset( + file_path, input_field, output_field, test_size=0.1, validation_size=0.1 +) -> Tuple[caikit.core.data_model.DataStream]: """Load the dataset from local JSON file.""" def to_generation_fmt(x): return GenerationTrainRecord(input=x[input_field], output=str(x[output_field])) - dataset = datasets.load_dataset('json', data_files=file_path) + dataset = datasets.load_dataset("json", data_files=file_path) train_test_dataset = dataset["train"].train_test_split(test_size=test_size) # # # Split the 10% test + valid into half test, half valid - test_valid = train_test_dataset['train'].train_test_split(test_size=validation_size) + test_valid = train_test_dataset["train"].train_test_split(test_size=validation_size) train_test_dataset["train"] = test_valid["train"] train_test_dataset["validation"] = test_valid["test"] From 5c0c1b5f340ac48f6ecf562bec2390f38fb6e70c Mon Sep 17 00:00:00 2001 From: Alan Braz Date: Fri, 1 Sep 2023 13:30:26 -0300 Subject: [PATCH 3/4] test and validation size --- examples/run_peft_tuning.py | 15 ++++++++++++++- examples/utils.py | 15 +++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/examples/run_peft_tuning.py b/examples/run_peft_tuning.py index fcdbb0279..53221516c 100644 --- a/examples/run_peft_tuning.py +++ b/examples/run_peft_tuning.py @@ -262,7 +262,18 @@ def register_common_arguments(subparsers: Tuple[argparse.ArgumentParser]) -> Non help="The custom verbalizer to be used by the JSON file dataset", default="{{input}}", ) - + subparser.add_argument( + "--json_file_test_size", + help="The percentage of the dataset that will be extracted as test set", + default=0.1, + type=float + ) + subparser.add_argument( + "--json_file_validation_size", + help="The percentage of the dataset that will be extracted as validation set", + default=0.1, + type=float + ) def register_multitask_prompt_tuning_args(subparser: argparse.ArgumentParser): """Register additional configuration options for MP(rompt)T subtask. @@ -426,6 +437,8 @@ def show_experiment_configuration(args, dataset_info, model_type) -> None: str(args.json_file_path), str(args.json_file_input_field), str(args.json_file_output_field), + test_size=args.json_file_test_size, + validation_size=args.json_file_validation_size ), init_text=args.json_file_init_text, ) diff --git a/examples/utils.py b/examples/utils.py index a1159776c..b688064de 100644 --- a/examples/utils.py +++ b/examples/utils.py @@ -289,11 +289,18 @@ def to_generation_fmt(x): return GenerationTrainRecord(input=x[input_field], output=str(x[output_field])) dataset = datasets.load_dataset("json", data_files=file_path) - train_test_dataset = dataset["train"].train_test_split(test_size=test_size) + if test_size > 0: + train_test_dataset = dataset["train"].train_test_split(test_size=test_size) + else: + train_test_dataset = dataset + train_test_dataset["test"] = [] # # # Split the 10% test + valid into half test, half valid - test_valid = train_test_dataset["train"].train_test_split(test_size=validation_size) - train_test_dataset["train"] = test_valid["train"] - train_test_dataset["validation"] = test_valid["test"] + if validation_size > 0: + test_valid = train_test_dataset["train"].train_test_split(test_size=validation_size) + train_test_dataset["train"] = test_valid["train"] + train_test_dataset["validation"] = test_valid["test"] + else: + train_test_dataset["validation"] = [] build_stream = lambda split: caikit.core.data_model.DataStream.from_iterable( [to_generation_fmt(datum) for datum in train_test_dataset[split]] From ef8462f8de9816e76cf1e882d59eca12da5ae4b6 Mon Sep 17 00:00:00 2001 From: Alan Braz Date: Tue, 12 Sep 2023 17:53:18 -0300 Subject: [PATCH 4/4] evaluate from json file Signed-off-by: Alan Braz --- examples/evaluate_model.py | 72 ++++++++++++++++++++++++++++++++++---- examples/utils.py | 4 +-- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/examples/evaluate_model.py b/examples/evaluate_model.py index 7e7f9410c..d2bff821b 100644 --- a/examples/evaluate_model.py +++ b/examples/evaluate_model.py @@ -18,6 +18,8 @@ load_model, print_colored, string_to_float, + DatasetInfo, + load_json_file_dataset, ) # Local @@ -49,7 +51,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--dataset", help="Dataset to use to train prompt vectors. Options: {}".format( - list(SUPPORTED_DATASETS.keys()) + list(SUPPORTED_DATASETS.keys()) + ["json_file"] ), default="twitter_complaints", ) @@ -64,6 +66,46 @@ def parse_args() -> argparse.Namespace: help="JSON file to dump raw source / target texts to.", default="model_preds.json", ) + + parser.add_argument( + "--json_file_path", + help="File path of the local JSON file to be loaded as dataset. It should be a JSON array or JSONL format.", + default=None, + type=pathlib.Path, + ) + parser.add_argument( + "--json_file_input_field", + help="The input field to be used at the JSON file dataset", + default="input", + ) + parser.add_argument( + "--json_file_output_field", + help="The output field to be used at the JSON file dataset", + default="output", + ) + parser.add_argument( + "--json_file_init_text", + help="The init text to be used by the JSON file dataset", + default="", + ) + parser.add_argument( + "--json_file_verbalizer", + help="The custom verbalizer to be used by the JSON file dataset", + default="{{input}}", + ) + parser.add_argument( + "--json_file_test_size", + help="The percentage of the dataset that will be extracted as test set", + default=0.1, + type=float + ) + parser.add_argument( + "--json_file_validation_size", + help="The percentage of the dataset that will be extracted as validation set", + default=0.1, + type=float + ) + args = parser.parse_args() return args @@ -89,10 +131,11 @@ def get_model_preds_and_references(model, validation_stream): for datum in tqdm(validation_stream): # Local .run() currently prepends the input text to the generated string; # Ensure that we're just splitting the first predicted token & beyond. - raw_model_text = model.run(datum.input).text - parse_pred_text = raw_model_text.split(datum.input)[-1].strip() - model_preds.append(parse_pred_text) - targets.append(datum.output) + if len(datum.input) > 0: + raw_model_text = model.run(datum.input).text + parse_pred_text = raw_model_text.split(datum.input)[-1].strip() + model_preds.append(parse_pred_text) + targets.append(datum.output) return ( model_preds, targets, @@ -148,8 +191,23 @@ def export_model_preds(preds_file, predictions, validation_stream, verbalizer): model = load_model(args.tgis, str(args.model_path)) # Load the validation stream with marked target sequences print_colored("Grabbing validation data...") - dataset_info = SUPPORTED_DATASETS[args.dataset] - validation_stream = dataset_info.dataset_loader()[1] + if args.dataset != "json_file": + dataset_info = SUPPORTED_DATASETS[args.dataset] + validation_stream = dataset_info.dataset_loader()[1] + else: + dataset_info = DatasetInfo( + verbalizer=args.json_file_verbalizer, + dataset_loader=load_json_file_dataset( + str(args.json_file_path), + str(args.json_file_input_field), + str(args.json_file_output_field), + test_size=args.json_file_test_size, + validation_size=args.json_file_validation_size + ), + init_text=args.json_file_init_text, + ) + validation_stream = dataset_info.dataset_loader[1] + if validation_stream is None: raise ValueError( "Selected dataset does not have a validation dataset available!" diff --git a/examples/utils.py b/examples/utils.py index b688064de..fc1506a2f 100644 --- a/examples/utils.py +++ b/examples/utils.py @@ -290,13 +290,13 @@ def to_generation_fmt(x): dataset = datasets.load_dataset("json", data_files=file_path) if test_size > 0: - train_test_dataset = dataset["train"].train_test_split(test_size=test_size) + train_test_dataset = dataset["train"].train_test_split(test_size=test_size, shuffle=False) else: train_test_dataset = dataset train_test_dataset["test"] = [] # # # Split the 10% test + valid into half test, half valid if validation_size > 0: - test_valid = train_test_dataset["train"].train_test_split(test_size=validation_size) + test_valid = train_test_dataset["train"].train_test_split(test_size=validation_size, shuffle=False) train_test_dataset["train"] = test_valid["train"] train_test_dataset["validation"] = test_valid["test"] else: