diff --git a/js/packages/openinference-core/src/trace/trace-config/constants.ts b/js/packages/openinference-core/src/trace/trace-config/constants.ts index a9b878e7c4..ec0e94f815 100644 --- a/js/packages/openinference-core/src/trace/trace-config/constants.ts +++ b/js/packages/openinference-core/src/trace/trace-config/constants.ts @@ -23,6 +23,8 @@ export const OPENINFERENCE_HIDE_EMBEDDING_VECTORS = /** Limits characters of a base64 encoding of an image */ export const OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH = "OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH"; +/** Hides LLM prompts */ +export const OPENINFERENCE_HIDE_PROMPTS = "OPENINFERENCE_HIDE_PROMPTS"; export const DEFAULT_HIDE_INPUTS = false; export const DEFAULT_HIDE_OUTPUTS = false; @@ -37,6 +39,8 @@ export const DEFAULT_HIDE_OUTPUT_TEXT = false; export const DEFAULT_HIDE_EMBEDDING_VECTORS = false; export const DEFAULT_BASE64_IMAGE_MAX_LENGTH = 32000; +export const DEFAULT_HIDE_PROMPTS = false; + /** When a value is hidden, it will be replaced by this redacted value */ export const REDACTED_VALUE = "__REDACTED__"; @@ -92,6 +96,11 @@ export const traceConfigMetadata: Readonly< envKey: OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH, type: "number", }, + hidePrompts: { + default: DEFAULT_HIDE_PROMPTS, + envKey: OPENINFERENCE_HIDE_PROMPTS, + type: "boolean", + }, }; export const DefaultTraceConfig: TraceConfig = { @@ -104,4 +113,5 @@ export const DefaultTraceConfig: TraceConfig = { hideOutputText: DEFAULT_HIDE_OUTPUT_TEXT, hideEmbeddingVectors: DEFAULT_HIDE_EMBEDDING_VECTORS, base64ImageMaxLength: DEFAULT_BASE64_IMAGE_MAX_LENGTH, + hidePrompts: DEFAULT_HIDE_PROMPTS, }; diff --git a/js/packages/openinference-core/src/trace/trace-config/index.ts b/js/packages/openinference-core/src/trace/trace-config/index.ts index f325bc598a..68aa693444 100644 --- a/js/packages/openinference-core/src/trace/trace-config/index.ts +++ b/js/packages/openinference-core/src/trace/trace-config/index.ts @@ -4,6 +4,7 @@ export { OPENINFERENCE_HIDE_EMBEDDING_VECTORS, DEFAULT_BASE64_IMAGE_MAX_LENGTH, DEFAULT_HIDE_EMBEDDING_VECTORS, + OPENINFERENCE_HIDE_PROMPTS, DEFAULT_HIDE_INPUTS, DEFAULT_HIDE_INPUT_IMAGES, DEFAULT_HIDE_INPUT_MESSAGES, diff --git a/js/packages/openinference-core/src/trace/trace-config/maskingRules.ts b/js/packages/openinference-core/src/trace/trace-config/maskingRules.ts index c4cbed0f9d..d4fc0c54cc 100644 --- a/js/packages/openinference-core/src/trace/trace-config/maskingRules.ts +++ b/js/packages/openinference-core/src/trace/trace-config/maskingRules.ts @@ -140,6 +140,21 @@ const maskEmbeddingVectorsRule: MaskingRule = { action: () => undefined, }; +/** + * Masks (redacts) llm.prompts if hidePrompts is true. + * @example + * ```typescript + * maskPromptsRule.condition({ + * config: {hidePrompts: true}, + * key: "llm.prompts" + * }) // returns true so the rule applies and the value will be redacted + */ +const maskPromptsRule: MaskingRule = { + condition: ({ config, key }) => + config.hidePrompts && key === SemanticConventions.LLM_PROMPTS, + action: () => REDACTED_VALUE, +}; + /** * A list of {@link MaskingRule}s that are applied to span attributes to either redact or remove sensitive information. * The order of these rules is important as it can ensure appropriate masking of information @@ -185,6 +200,7 @@ const maskingRules: MaskingRule[] = [ maskInputImagesRule, maskLongBase64ImageRule, maskEmbeddingVectorsRule, + maskPromptsRule, ]; /** diff --git a/js/packages/openinference-core/src/trace/trace-config/types.ts b/js/packages/openinference-core/src/trace/trace-config/types.ts index cb18e8439b..d5c9c3c488 100644 --- a/js/packages/openinference-core/src/trace/trace-config/types.ts +++ b/js/packages/openinference-core/src/trace/trace-config/types.ts @@ -14,6 +14,7 @@ export type TraceConfigOptions = { hideOutputText?: boolean; hideEmbeddingVectors?: boolean; base64ImageMaxLength?: number; + hidePrompts?: boolean; }; /** diff --git a/js/packages/openinference-core/test/trace-config/maskingRules.test.ts b/js/packages/openinference-core/test/trace-config/maskingRules.test.ts index 1f95fc4583..bd238cc1cd 100644 --- a/js/packages/openinference-core/test/trace-config/maskingRules.test.ts +++ b/js/packages/openinference-core/test/trace-config/maskingRules.test.ts @@ -112,6 +112,15 @@ const generateMaskTestCases = (): MaskTestCases[] => { "data:image/base64,verylongbase64string", ]); break; + case "hidePrompts": + testCases.push([ + `should return ${REDACTED_VALUE} for "llm.prompts" when hidePrompts is set to true`, + { ...DefaultTraceConfig, hidePrompts: true }, + REDACTED_VALUE, + "llm.prompts", + "some prompt", + ]); + break; default: assertUnreachable(configKey); } diff --git a/python/openinference-instrumentation/src/openinference/instrumentation/config.py b/python/openinference-instrumentation/src/openinference/instrumentation/config.py index 28d8623b86..3bc1cf134a 100644 --- a/python/openinference-instrumentation/src/openinference/instrumentation/config.py +++ b/python/openinference-instrumentation/src/openinference/instrumentation/config.py @@ -83,10 +83,13 @@ def __aexit__( # Hides embedding vectors OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH = "OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH" # Limits characters of a base64 encoding of an image +OPENINFERENCE_HIDE_PROMPTS = "OPENINFERENCE_HIDE_PROMPTS" +# Hides LLM prompts REDACTED_VALUE = "__REDACTED__" # When a value is hidden, it will be replaced by this redacted value DEFAULT_HIDE_LLM_INVOCATION_PARAMETERS = False +DEFAULT_HIDE_PROMPTS = False DEFAULT_HIDE_INPUTS = False DEFAULT_HIDE_OUTPUTS = False @@ -185,6 +188,14 @@ class TraceConfig: }, ) """Hides embedding vectors""" + hide_prompts: Optional[bool] = field( + default=None, + metadata={ + "env_var": OPENINFERENCE_HIDE_PROMPTS, + "default_value": DEFAULT_HIDE_PROMPTS, + }, + ) + """Hides LLM prompts""" base64_image_max_length: Optional[int] = field( default=None, metadata={ @@ -213,6 +224,8 @@ def mask( ) -> Optional[AttributeValue]: if self.hide_llm_invocation_parameters and key == SpanAttributes.LLM_INVOCATION_PARAMETERS: return None + elif self.hide_prompts and key == SpanAttributes.LLM_PROMPTS: + value = REDACTED_VALUE elif self.hide_inputs and key == SpanAttributes.INPUT_VALUE: value = REDACTED_VALUE elif self.hide_inputs and key == SpanAttributes.INPUT_MIME_TYPE: diff --git a/python/openinference-instrumentation/tests/test_config.py b/python/openinference-instrumentation/tests/test_config.py index 28c0cd4148..62b33e542a 100644 --- a/python/openinference-instrumentation/tests/test_config.py +++ b/python/openinference-instrumentation/tests/test_config.py @@ -23,6 +23,7 @@ DEFAULT_HIDE_OUTPUT_MESSAGES, DEFAULT_HIDE_OUTPUT_TEXT, DEFAULT_HIDE_OUTPUTS, + DEFAULT_HIDE_PROMPTS, OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH, OPENINFERENCE_HIDE_INPUT_IMAGES, OPENINFERENCE_HIDE_INPUT_MESSAGES, @@ -31,6 +32,7 @@ OPENINFERENCE_HIDE_OUTPUT_MESSAGES, OPENINFERENCE_HIDE_OUTPUT_TEXT, OPENINFERENCE_HIDE_OUTPUTS, + OPENINFERENCE_HIDE_PROMPTS, REDACTED_VALUE, ) from openinference.semconv.trace import SpanAttributes @@ -46,6 +48,7 @@ def test_default_settings() -> None: assert config.hide_input_images == DEFAULT_HIDE_INPUT_IMAGES assert config.hide_input_text == DEFAULT_HIDE_INPUT_TEXT assert config.hide_output_text == DEFAULT_HIDE_OUTPUT_TEXT + assert config.hide_prompts == DEFAULT_HIDE_PROMPTS assert config.base64_image_max_length == DEFAULT_BASE64_IMAGE_MAX_LENGTH @@ -117,6 +120,7 @@ def test_attribute_priority(k: str, in_memory_span_exporter: InMemorySpanExporte @pytest.mark.parametrize("hide_input_images", [False, True]) @pytest.mark.parametrize("hide_input_text", [False, True]) @pytest.mark.parametrize("hide_output_text", [False, True]) +@pytest.mark.parametrize("hide_prompts", [False, True]) @pytest.mark.parametrize("base64_image_max_length", [10_000]) def test_settings_from_env_vars_and_code( hide_inputs: bool, @@ -126,6 +130,7 @@ def test_settings_from_env_vars_and_code( hide_input_images: bool, hide_input_text: bool, hide_output_text: bool, + hide_prompts: bool, base64_image_max_length: int, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -135,6 +140,7 @@ def test_settings_from_env_vars_and_code( monkeypatch.setenv(OPENINFERENCE_HIDE_INPUT_MESSAGES, str(hide_input_messages)) monkeypatch.setenv(OPENINFERENCE_HIDE_OUTPUT_MESSAGES, str(hide_output_messages)) monkeypatch.setenv(OPENINFERENCE_HIDE_INPUT_IMAGES, str(hide_input_images)) + monkeypatch.setenv(OPENINFERENCE_HIDE_PROMPTS, str(hide_prompts)) monkeypatch.setenv(OPENINFERENCE_HIDE_INPUT_TEXT, str(hide_input_text)) monkeypatch.setenv(OPENINFERENCE_HIDE_OUTPUT_TEXT, str(hide_output_text)) monkeypatch.setenv(OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH, str(base64_image_max_length)) @@ -147,6 +153,7 @@ def test_settings_from_env_vars_and_code( assert config.hide_input_images is parse_bool_from_env(OPENINFERENCE_HIDE_INPUT_IMAGES) assert config.hide_input_text is parse_bool_from_env(OPENINFERENCE_HIDE_INPUT_TEXT) assert config.hide_output_text is parse_bool_from_env(OPENINFERENCE_HIDE_OUTPUT_TEXT) + assert config.hide_prompts is parse_bool_from_env(OPENINFERENCE_HIDE_PROMPTS) assert config.base64_image_max_length == int( os.getenv(OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH, default=-1) ) @@ -161,6 +168,7 @@ def test_settings_from_env_vars_and_code( new_hide_input_images = not hide_input_images new_hide_input_text = not hide_input_text new_hide_output_text = not hide_output_text + new_hide_prompts = not hide_prompts config = TraceConfig( hide_inputs=new_hide_inputs, hide_outputs=new_hide_outputs, @@ -169,6 +177,7 @@ def test_settings_from_env_vars_and_code( hide_input_images=new_hide_input_images, hide_input_text=new_hide_input_text, hide_output_text=new_hide_output_text, + hide_prompts=new_hide_prompts, base64_image_max_length=new_base64_image_max_length, ) assert config.hide_inputs is new_hide_inputs @@ -178,6 +187,7 @@ def test_settings_from_env_vars_and_code( assert config.hide_input_images is new_hide_input_images assert config.hide_input_text is new_hide_input_text assert config.hide_output_text is new_hide_output_text + assert config.hide_prompts is new_hide_prompts assert config.base64_image_max_length == new_base64_image_max_length diff --git a/spec/configuration.md b/spec/configuration.md index 159041c9fd..d9b01eb7fe 100644 --- a/spec/configuration.md +++ b/spec/configuration.md @@ -12,8 +12,9 @@ The possible settings are: | OPENINFERENCE_HIDE_OUTPUTS | Hides output value & all output messages | bool | False | | OPENINFERENCE_HIDE_INPUT_MESSAGES | Hides all input messages & embedding input text | bool | False | | OPENINFERENCE_HIDE_OUTPUT_MESSAGES | Hides all output messages | bool | False | -| PENINFERENCE_HIDE_INPUT_IMAGES | Hides images from input messages | bool | False | +| OPENINFERENCE_HIDE_INPUT_IMAGES | Hides images from input messages | bool | False | | OPENINFERENCE_HIDE_INPUT_TEXT | Hides text from input messages & input embeddings | bool | False | +| OPENINFERENCE_HIDE_PROMPTS | Hides LLM prompts | bool | False | | OPENINFERENCE_HIDE_OUTPUT_TEXT | Hides text from output messages | bool | False | | OPENINFERENCE_HIDE_EMBEDDING_VECTORS | Hides returned embedding vectors | bool | False | | OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH | Limits characters of a base64 encoding of an image | int | 32,000 | @@ -44,6 +45,7 @@ If you are working in Python, and want to set up a configuration different than hide_output_text=..., hide_embedding_vectors=..., base64_image_max_length=..., + hide_prompts=..., ) from openinference.instrumentation.openai import OpenAIInstrumentor