-
Notifications
You must be signed in to change notification settings - Fork 47
[AMD] support sd with VitisAIExecutionProvider. #263
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
liujij
wants to merge
32
commits into
microsoft:main
Choose a base branch
from
liujij:feat-vitisai-ep
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from 1 commit
Commits
Show all changes
32 commits
Select commit
Hold shift + click to select a range
c0175e8
[feat][vitis_ai_ep] support sd15 with VitisAIExecutionProvider.
liujij e2b2af7
add doc.
liujij 39fe41a
up.
liujij f712ebf
up.
liujij a103a2a
Merge branch 'main' into feat-vitisai-ep
liujij 86113a0
up doc.
liujij 1d78de5
up dir structure.
liujij 4e91b29
Merge branch 'feat-vitisai-ep' of https://github.com/liujij/olive-rec…
liujij 0177fb8
Update README.md
liujij 9ad22b7
Update README.md
liujij a144939
Update requirements_vitisai_sd.txt
liujij f128c3e
update json.
liujij d1db4d9
Update config_safety_checker.json
liujij 7fec719
Update config_unet.json
liujij 8e02f95
Update config_vae_decoder.json
liujij 79659ef
Update config_vae_encoder.json
liujij 5ec74a0
Update vai.py
liujij 40a7bd9
up sd_utils/vai.py.
liujij 7f3adb7
update requirements_vitisai_sd.txt.
liujij b00716a
Update README.md
liujij 50974df
workround requirements_vitisai_sd.txt
liujij 8743fed
Merge branch 'main' into feat-vitisai-ep
liujij 8002267
Merge branch 'feat-vitisai-ep' of https://github.com/liujij/olive-rec…
liujij e7c3b68
update README.md and requirements_vitisai_sd.txt.
liujij b6aea57
Merge branch 'main' into feat-vitisai-ep
liujij 6d2573a
Remove unrelated files.
liujij 42f49dc
fix pre-commit issue.
liujij 00fd9dc
Merge branch 'feat-vitisai-ep' of https://github.com/liujij/olive-rec…
liujij 4db2514
remove useless codes.
liujij c1252a0
auto set DD_PLUGINS_ROOT.
liujij f07aeba
Merge branch 'main' into feat-vitisai-ep
xieofxie 50e2cd4
Update requirements_vitisai_sd.txt
liujij File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
98 changes: 98 additions & 0 deletions
98
sd-legacy-stable-diffusion-v1-5/VitisAI/config_safety_checker.json
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| { | ||
| "input_model": { | ||
| "type": "PyTorchModel", | ||
| "model_path": "stable-diffusion-v1-5/stable-diffusion-v1-5", | ||
| "model_loader": "safety_checker_load", | ||
| "model_script": "user_script.py", | ||
| "io_config": { | ||
| "input_names": [ "clip_input", "images" ], | ||
| "output_names": [ "out_images", "has_nsfw_concepts" ], | ||
| "dynamic_axes": { | ||
| "clip_input": { "0": "batch", "1": "channels", "2": "height", "3": "width" }, | ||
| "images": { "0": "batch", "1": "height", "2": "width", "3": "channels" } | ||
| } | ||
| }, | ||
| "dummy_inputs_func": "safety_checker_conversion_inputs" | ||
| }, | ||
| "systems": { | ||
| "local_system": { | ||
| "type": "LocalSystem", | ||
| "accelerators": [ { "device": "gpu", "execution_providers": [ "CUDAExecutionProvider" ] } ] | ||
| } | ||
| }, | ||
| "data_configs": [ | ||
| { | ||
| "name": "latency_data_config", | ||
| "user_script": "user_script.py", | ||
| "load_dataset_config": { "type": "local_dataset" }, | ||
| "dataloader_config": { "type": "safety_checker_data_loader", "batch_size": 1 } | ||
| } | ||
| ], | ||
| "evaluators": { | ||
| "common_evaluator": { | ||
| "metrics": [ | ||
| { | ||
| "name": "latency", | ||
| "type": "latency", | ||
| "data_config": "latency_data_config", | ||
| "sub_types": [ { "name": "avg" } ] | ||
| } | ||
| ] | ||
| } | ||
| }, | ||
| "passes": { | ||
| "convert": { "type": "OnnxConversion", "target_opset": 14 }, | ||
| "ov_convert": { | ||
| "type": "OpenVINOConversion", | ||
| "user_script": "user_script.py", | ||
| "example_input_func": "safety_checker_conversion_inputs", | ||
| "output_model": "safety_checker" | ||
| }, | ||
| "optimize": { | ||
| "type": "OrtTransformersOptimization", | ||
| "model_type": "unet", | ||
| "opt_level": 0, | ||
| "float16": true, | ||
| "use_gpu": true, | ||
| "keep_io_types": false, | ||
| "optimization_options": { | ||
| "enable_gelu": true, | ||
| "enable_layer_norm": true, | ||
| "enable_attention": true, | ||
| "use_multi_head_attention": true, | ||
| "enable_skip_layer_norm": false, | ||
| "enable_embed_layer_norm": true, | ||
| "enable_bias_skip_layer_norm": false, | ||
| "enable_bias_gelu": true, | ||
| "enable_gelu_approximation": false, | ||
| "enable_qordered_matmul": false, | ||
| "enable_shape_inference": true, | ||
| "enable_gemm_fast_gelu": false, | ||
| "enable_nhwc_conv": false, | ||
| "enable_group_norm": true, | ||
| "enable_bias_splitgelu": false, | ||
| "enable_packed_qkv": true, | ||
| "enable_packed_kv": true, | ||
| "enable_bias_add": false, | ||
| "group_norm_channels_last": false | ||
| }, | ||
| "force_fp32_ops": [ "RandomNormalLike" ], | ||
| "force_fp16_inputs": { "GroupNorm": [ 0, 1, 2 ] } | ||
| }, | ||
| "optimize_cuda": { | ||
| "type": "OrtTransformersOptimization", | ||
| "model_type": "unet", | ||
| "opt_level": 0, | ||
| "float16": true, | ||
| "use_gpu": true, | ||
| "keep_io_types": false | ||
| } | ||
| }, | ||
| "log_severity_level": 0, | ||
| "evaluator": "common_evaluator", | ||
| "evaluate_input_model": false, | ||
| "host": "local_system", | ||
| "target": "local_system", | ||
| "cache_dir": "cache", | ||
| "output_dir": "footprints/safety_checker" | ||
| } | ||
147 changes: 147 additions & 0 deletions
147
sd-legacy-stable-diffusion-v1-5/VitisAI/config_text_encoder.json
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| { | ||
| "input_model": { | ||
| "type": "PyTorchModel", | ||
| "model_path": "stable-diffusion-v1-5/stable-diffusion-v1-5", | ||
| "model_loader": "text_encoder_load", | ||
| "model_script": "user_script.py", | ||
| "io_config": { | ||
| "input_names": [ "input_ids" ], | ||
| "output_names": [ "last_hidden_state", "pooler_output" ], | ||
| "dynamic_axes": { "input_ids": { "0": "batch", "1": "sequence" } } | ||
| }, | ||
| "dummy_inputs_func": "text_encoder_conversion_inputs" | ||
| }, | ||
| "systems": { | ||
| "local_system": { | ||
| "type": "LocalSystem", | ||
| "accelerators": [ { "device": "gpu", "execution_providers": [ "CUDAExecutionProvider" ] } ] | ||
| } | ||
| }, | ||
| "data_configs": [ | ||
| { | ||
| "name": "latency_data_config", | ||
| "user_script": "user_script.py", | ||
| "load_dataset_config": { "type": "local_dataset" }, | ||
| "dataloader_config": { "type": "text_encoder_data_loader", "batch_size": 1 } | ||
| }, | ||
| { | ||
| "name": "quantize_data_config", | ||
| "user_script": "user_script.py", | ||
| "load_dataset_config": { "type": "local_dataset" }, | ||
| "dataloader_config": { "type": "text_encoder_quantize_data_loader", "data_num": 100 } | ||
| } | ||
| ], | ||
| "evaluators": { | ||
| "common_evaluator": { | ||
| "metrics": [ | ||
| { | ||
| "name": "latency", | ||
| "type": "latency", | ||
| "data_config": "latency_data_config", | ||
| "sub_types": [ { "name": "avg" } ] | ||
| } | ||
| ] | ||
| } | ||
| }, | ||
| "passes": { | ||
| "convert": { "type": "OnnxConversion", "target_opset": 17 }, | ||
| "ov_convert": { | ||
| "type": "OpenVINOConversion", | ||
| "user_script": "user_script.py", | ||
| "example_input_func": "text_encoder_conversion_inputs", | ||
| "output_model": "text_encoder" | ||
| }, | ||
| "optimize": { | ||
| "type": "OrtTransformersOptimization", | ||
| "model_type": "clip", | ||
| "opt_level": 0, | ||
| "float16": true, | ||
| "use_gpu": true, | ||
| "keep_io_types": false, | ||
| "optimization_options": { | ||
| "enable_gelu": true, | ||
| "enable_layer_norm": true, | ||
| "enable_attention": true, | ||
| "use_multi_head_attention": true, | ||
| "enable_skip_layer_norm": false, | ||
| "enable_embed_layer_norm": true, | ||
| "enable_bias_skip_layer_norm": false, | ||
| "enable_bias_gelu": true, | ||
| "enable_gelu_approximation": false, | ||
| "enable_qordered_matmul": false, | ||
| "enable_shape_inference": true, | ||
| "enable_gemm_fast_gelu": false, | ||
| "enable_nhwc_conv": false, | ||
| "enable_group_norm": true, | ||
| "enable_bias_splitgelu": false, | ||
| "enable_packed_qkv": true, | ||
| "enable_packed_kv": true, | ||
| "enable_bias_add": false, | ||
| "group_norm_channels_last": false | ||
| }, | ||
| "force_fp32_ops": [ "RandomNormalLike" ], | ||
| "force_fp16_inputs": { "GroupNorm": [ 0, 1, 2 ] } | ||
| }, | ||
| "optimize_cuda": { | ||
| "type": "OrtTransformersOptimization", | ||
| "model_type": "clip", | ||
| "opt_level": 0, | ||
| "float16": true, | ||
| "use_gpu": true, | ||
| "keep_io_types": false | ||
| }, | ||
| "dynamic_shape_to_fixed": { | ||
| "type": "DynamicToFixedShape", | ||
| "dim_param": [ "batch", "sequence" ], | ||
| "dim_value": [ 1, 77 ] | ||
| }, | ||
| "surgery": { "type": "GraphSurgeries", "surgeries": [ { "surgeon": "ReplaceAttentionMaskValue", "replacement": -200.0 } ] }, | ||
| "optimize_qdq": { | ||
| "type": "OrtTransformersOptimization", | ||
| "model_type": "clip", | ||
| "opt_level": 0, | ||
| "optimization_options": { | ||
| "enable_gelu": true, | ||
| "enable_layer_norm": true, | ||
| "enable_attention": false, | ||
| "use_multi_head_attention": false, | ||
| "enable_skip_layer_norm": false, | ||
| "enable_embed_layer_norm": false, | ||
| "enable_bias_skip_layer_norm": false, | ||
| "enable_bias_gelu": false, | ||
| "enable_gelu_approximation": false, | ||
| "enable_qordered_matmul": false, | ||
| "enable_shape_inference": false, | ||
| "enable_gemm_fast_gelu": false, | ||
| "enable_nhwc_conv": false, | ||
| "enable_group_norm": false, | ||
| "enable_bias_splitgelu": false, | ||
| "enable_packed_qkv": false, | ||
| "enable_packed_kv": false, | ||
| "enable_bias_add": false, | ||
| "group_norm_channels_last": false | ||
| } | ||
| }, | ||
| "quantization": { | ||
| "type": "OnnxStaticQuantization", | ||
| "data_config": "quantize_data_config", | ||
| "activation_type": "uint16", | ||
| "precision": "uint8", | ||
| "calibrate_method": "MinMax", | ||
| "quant_preprocess": true | ||
| }, | ||
| "cb": { | ||
| "type": "EPContextBinaryGenerator", | ||
| "provider_options": { | ||
| "htp_graph_finalization_optimization_mode": "3" | ||
| } | ||
| } | ||
| }, | ||
| "log_severity_level": 0, | ||
| "evaluator": "common_evaluator", | ||
| "evaluate_input_model": false, | ||
| "host": "local_system", | ||
| "target": "local_system", | ||
| "cache_dir": "cache", | ||
| "output_dir": "footprints/text_encoder" | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| { | ||
| "input_model": { | ||
| "type": "PyTorchModel", | ||
| "model_path": "sd-legacy/stable-diffusion-v1-5", | ||
| "model_loader": "unet_load", | ||
| "script_dir": ".", | ||
| "model_script": "user_script.py", | ||
| "io_config": { | ||
| "input_names": [ "sample", "timestep", "encoder_hidden_states", "return_dict" ], | ||
| "output_names": [ "out_sample" ], | ||
| "dynamic_axes": { | ||
| "sample": { | ||
| "0": "batch", | ||
| "1": "channels", | ||
| "2": "height", | ||
| "3": "width" | ||
| }, | ||
| "encoder_hidden_states": { "0": "batch", "1": "sequence" } | ||
| } | ||
| }, | ||
| "dummy_inputs_func": "unet_conversion_inputs" | ||
| }, | ||
| "passes": { | ||
| "convert": { | ||
| "type": "OnnxConversion", | ||
| "target_opset": 17, | ||
| "save_as_external_data": true, | ||
| "all_tensors_to_one_file": true, | ||
| "external_data_name": "weights.pb" | ||
| }, | ||
| "model_generation": { | ||
| "type": "VitisGenerateModelSD", | ||
| "model_type": "sd_unet", | ||
| "fixed_shapes_dim_param": [ "graphs", "batch", "channels", "height", "width", "sequence" ], | ||
| "fixed_shapes_dim_value": [ 1, 2, 4, 64, 64, 77 ] | ||
| } | ||
| }, | ||
| "log_severity_level": 0, | ||
| "cache_dir": "vai_cache", | ||
| "output_dir": "footprints/unet" | ||
| } |
34 changes: 34 additions & 0 deletions
34
sd-legacy-stable-diffusion-v1-5/VitisAI/config_vae_decoder.json
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| { | ||
| "input_model": { | ||
| "type": "PyTorchModel", | ||
| "model_path": "sd-legacy/stable-diffusion-v1-5", | ||
| "model_loader": "vae_decoder_load", | ||
| "script_dir": ".", | ||
| "model_script": "user_script.py", | ||
| "io_config": { | ||
| "input_names": [ "latent_sample", "return_dict" ], | ||
| "output_names": [ "sample" ], | ||
| "dynamic_axes": { | ||
| "latent_sample": { | ||
| "0": "batch", | ||
| "1": "channels", | ||
| "2": "height", | ||
| "3": "width" | ||
| } | ||
| } | ||
| }, | ||
| "dummy_inputs_func": "vae_decoder_conversion_inputs" | ||
| }, | ||
| "passes": { | ||
| "convert": { "type": "OnnxConversion", "target_opset": 17 }, | ||
| "model_generation": { | ||
| "type": "VitisGenerateModelSD", | ||
| "model_type": "sd_vae_decoder", | ||
| "fixed_shapes_dim_param": [ "graphs", "batch", "channels", "height", "width" ], | ||
| "fixed_shapes_dim_value": [ 1, 1, 4, 64, 64 ] | ||
| } | ||
| }, | ||
| "log_severity_level": 0, | ||
| "cache_dir": "vai_cache", | ||
| "output_dir": "footprints/vae_decoder" | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.