feat(v21): Learned Policy decision outcome closed loop successor

THIRD_PARTY_NOTICES.md

@@ -295,3 +295,12 @@ Exact Voice Brain source, model and build-tool pins are recorded in `config/upst
 - License copy: `third_party/licenses/agent-lightning-MIT.txt`
 - Upstream dependency lock: exact `uv.lock` Git blob `5a98a2ac121b050b0a82f6ac8dc207577ce3af4e`
 - Yance integration: source-module CORE + APO only, downstream of Learning and Model Brain authority, returning `CANDIDATE_ONLY`.
+## Vowpal Wabbit
+
+- Project: Vowpal Wabbit
+- Upstream: `VowpalWabbit/vowpal_wabbit`
+- Version: `9.11.2`
+- Frozen commit: `122bae254a5b8bc2b774d13b33d53e6dbc2cfba7`
+- License: `BSD-3-Clause`
+- License copy: `third_party/licenses/vowpal-wabbit-BSD-3-Clause.txt`
+- Yance integration: sealed Learning runtime contextual-bandit ADF policy head only. P1 is deterministic (`actionProbability=1.0`, `exploration=false`); Model Brain/LiteLLM remains the final reply-generation authority.

backend/services/contextAwareReplyBrain.js

@@ -22,6 +22,8 @@ const aiDirectorStrategyAuthority = require('./aiDirectorStrategyAuthority').sin
 const aiWorkbenchDirectorRuleAuthority = require('./aiWorkbenchDirectorRuleAuthority');
 const goalDrivenMemoryRecall = require('./goalDrivenMemoryRecallService');
 const { singleton: platformCoreRepository } = require('../repositories/platformCoreRepository');
+const { createLearningPolicyRuntimeAdapter } = require('./learningPolicyRuntimeAdapter');
+const { createLearningPolicyDecisionContract } = require('./learningPolicyDecisionContract');
 
 function clean(value) {
   return String(value == null ? '' : value).trim();
@@ -98,6 +100,9 @@ function memoryCandidatesForRecall(packet = {}) {
 
 function branchNameForVariant(value = '') {
   const variant = clean(value).toLowerCase();
+  const exactAction = ['natural_hook', 'playful_attraction', 'direct_advance', 'screen_and_advance', 'leave_aftertaste']
+    .find(action => action === variant);
+  if (exactAction) return exactAction;
   if (/情趣|暧昧|俏皮|妩媚|女人味|feminine|flirt/u.test(variant)) return 'playful_attraction';
   if (/边界|筛选|screen/u.test(variant)) return 'screen_and_advance';
   if (/直接|强势|direct/u.test(variant)) return 'direct_advance';
@@ -106,6 +111,17 @@ function branchNameForVariant(value = '') {
   return 'natural_hook';
 }
 
+function learnedPolicyInteractionBand(context = {}) {
+  const interaction = context.interaction && typeof context.interaction === 'object' ? context.interaction : {};
+  const explicit = clean(interaction.engagementBand || interaction.responseBand || interaction.band).toLowerCase();
+  if (['low', 'balanced', 'high'].includes(explicit)) return explicit;
+  const score = Number(interaction.engagementScore ?? interaction.responseRate ?? interaction.reciprocity);
+  if (!Number.isFinite(score)) return 'balanced';
+  if (score < 0.35) return 'low';
+  if (score > 0.7) return 'high';
+  return 'balanced';
+}
+
 function candidateBranchPlanForCount(value = 3) {
   const count = Math.max(1, Math.min(5, Number(value || 3)));
   return ['natural_hook', 'playful_attraction', 'direct_advance', 'screen_and_advance', 'leave_aftertaste'].slice(0, count);
@@ -674,10 +690,19 @@ async function contextStillCurrent(previous, current) {
   return clean(previous.contactId) === clean(current.contactId);
 }
 
-function createContextAwareReplyBrain({ storeManager, aiGateway, personaBrain, resolveContactId }) {
+function createContextAwareReplyBrain({
+  storeManager,
+  aiGateway,
+  personaBrain,
+  resolveContactId,
+  learningPolicyRuntimeAdapter,
+  learningPolicyDecisionContract
+}) {
   if (!storeManager?.select || !storeManager?.dispatch) throw new TypeError('storeManager is required');
   if (!aiGateway?.execute) throw new TypeError('aiGateway is required');
   const persona = personaBrain || personaBrainModule.createPersonaBrain();
+  const learnedPolicyRuntime = learningPolicyRuntimeAdapter || createLearningPolicyRuntimeAdapter();
+  const learnedPolicyDecisions = learningPolicyDecisionContract || createLearningPolicyDecisionContract();
 
   function currentConversationRevision(conversationId) {
     const selected = storeManager.select(state => Number(state.conversations?.byId?.[conversationId]?.version || 0));
@@ -1081,7 +1106,75 @@ function createContextAwareReplyBrain({ storeManager, aiGateway, personaBrain, r
         targetLanguage: languageAuthority.code,
         learningWeights: {},
       });
-      const branchApplication = applyCandidateBranch(effectiveDirector, candidatePlan.plan, variant);
+      const policyFeatureBundle = Object.freeze({
+        interactionBand: learnedPolicyInteractionBand(socialContext),
+        performanceMode,
+        questionPolicy: Number(automaticDirectorPlan.maxQuestions) === 0 ? 'none' : 'optional',
+        relationshipStage: clean(socialContext.relationshipPotential?.relationshipStage || 'unknown') || 'unknown',
+        targetLanguage: clean(languageAuthority.code || 'unknown') || 'unknown'
+      });
+      const allowedPolicyActions = (candidatePlan.plan.branches || [])
+        .map(row => clean(row.strategy))
+        .filter(Boolean);
+      const requestedBaselineAction = branchNameForVariant(variant);
+      const baselinePolicyAction = allowedPolicyActions.includes(requestedBaselineAction)
+        ? requestedBaselineAction
+        : allowedPolicyActions[0];
+      const learnedPolicySelection = await learnedPolicyRuntime.selectLearnedPolicyAction({
+        featureBundle: policyFeatureBundle,
+        allowedActions: allowedPolicyActions,
+        baselineAction: baselinePolicyAction
+      });
+      const branchApplication = applyCandidateBranch(
+        effectiveDirector,
+        candidatePlan.plan,
+        learnedPolicySelection.candidateStrategyBranch
+      );
+      let decisionRecord = null;
+      let learningPolicyEvidenceEligible = true;
+      let learningPolicyDegradation = learnedPolicySelection.degradation || null;
+      try {
+        decisionRecord = learnedPolicyDecisions.createDecisionRecord({
+          contactId,
+          conversationId,
+          personaProfileId: personaCtx.profileId || 'owner',
+          featureBundle: policyFeatureBundle,
+          allowedActionSet: allowedPolicyActions,
+          candidateStrategyBranch: learnedPolicySelection.candidateStrategyBranch,
+          policyVersion: learnedPolicySelection.policyVersion,
+          behaviorPolicyVersion: learnedPolicySelection.policyVersion,
+          policyArtifactId: learnedPolicySelection.policyArtifactId,
+          generation: {
+            candidatePlanId: candidatePlan.plan.planId,
+            directorStrategyId: directorStrategy.strategy.strategyId,
+            contextVersion: socialContext.contextVersion,
+            conversationRevision
+          },
+          contributingPolicyVersions: {
+            relationship: `relationship-v${Number(socialContext.entityVersions?.relationship || 0)}`,
+            memory: `memory-v${Number(socialContext.entityVersions?.memory || 0)}`,
+            strategy: `director-strategy-v${Number(directorStrategy.strategy.strategyVersion || 1)}`,
+            candidateRanker: 'candidate-strategy-branch-v1',
+            routing: 'model-brain-routing-current-v1',
+            promptProgram: 'context-aware-reply-current-v1'
+          }
+        });
+      } catch (error) {
+        learningPolicyEvidenceEligible = false;
+        learningPolicyDegradation = Object.freeze({
+          reasonCode: clean(error.reasonCode || error.code) || 'LEARNING_POLICY_DECISION_EVIDENCE_BLOCKED'
+        });
+      }
+      const learningPolicyReceipt = Object.freeze({
+        authority: 'LearningPolicyRuntimeAdapter',
+        policyVersion: clean(learnedPolicySelection.policyVersion),
+        policyArtifactId: clean(learnedPolicySelection.policyArtifactId),
+        candidateStrategyBranch: clean(learnedPolicySelection.candidateStrategyBranch),
+        actionProbability: 1,
+        exploration: false,
+        evidenceEligible: learningPolicyEvidenceEligible,
+        degradation: learningPolicyDegradation
+      });
       effectiveDirector = {
         ...branchApplication.director,
         strategyId: directorStrategy.strategy.strategyId,
@@ -1265,6 +1358,8 @@ function createContextAwareReplyBrain({ storeManager, aiGateway, personaBrain, r
         emergencyMode,
         learningEligible,
         highCapabilityPath,
+        decisionRecord,
+        learningPolicy: learningPolicyReceipt,
         directorRuleStackReceipt: directorRuleStack.receipt,
         director: {
           plan: automaticDirectorPlan,

backend/services/learningDeepTrainingContract.js

@@ -27,15 +27,15 @@ function createLearningDeepTrainingContract(options = {}) {
   }
 
   function isLearningEligible(signal = {}) {
-    return signal.learning_eligible === true || signal.learning_eligible === 1;
+    return signal.learning_eligible === true || signal.learning_eligible === 1 || signal.learningEligible === true;
   }
 
   function isDoNotLearn(signal = {}) {
     return signal.do_not_learn === true || signal.doNotLearn === true || signal.signal?.doNotLearn === true || signal.signal?.metadata?.doNotLearn === true;
   }
 
   function hasRawPrivatePersistence(signal = {}) {
-    return signal.signal?.metadata?.rawPrivateChatPersisted === true;
+    return signal.signal?.metadata?.rawPrivateChatPersisted === true || signal.signal?.rawPrivateChatPersisted === true;
   }
 
   function hasValidScoreSubject(score = {}) {
@@ -195,6 +195,110 @@ function createLearningDeepTrainingContract(options = {}) {
     });
   }
 
+  function exactOutcomeIds(rows = []) {
+    return [...new Set(rows.flatMap(row => Array.isArray(row.signal?.outcomes) ? row.signal.outcomes : [])
+      .map(outcome => clean(outcome?.outcomeId)).filter(Boolean))].sort();
+  }
+
+  function hasReplayableDecision(decision = {}) {
+    const probability = Number(decision.actionProbability);
+    return Boolean(
+      clean(decision.decisionId) && clean(decision.candidateStrategyBranch) &&
+      decision.featureBundle && typeof decision.featureBundle === 'object' && !Array.isArray(decision.featureBundle) &&
+      clean(decision.actionId) && clean(decision.actionSetRef) && clean(decision.actionEncodingVersion) &&
+      clean(decision.behaviorPolicyVersion || decision.policyVersion) &&
+      Number.isFinite(probability) && probability > 0 && probability <= 1 && decision.exploration !== true
+    );
+  }
+
+  async function listPolicyOutcomes(decisionIds, scopeType, scopeId) {
+    if (typeof repository.listPolicyOutcomeSignals === 'function') {
+      const listed = await repository.listPolicyOutcomeSignals({ decisionIds });
+      return Array.isArray(listed) ? listed : [];
+    }
+    const listed = await repository.listLearningSignals({ scopeType, scopeId, learningLevel: 'L1', learningEligible: false });
+    return (Array.isArray(listed) ? listed : []).filter(row =>
+      clean(row.signal_type || row.signalType) === 'policy_outcome_observed' && decisionIds.includes(clean(row.signal?.decisionId))
+    );
+  }
+
+  async function projectPolicy(input = {}) {
+    requireProjectionDependencies();
+    const scopeType = clean(input.scopeType);
+    const scopeId = clean(input.scopeId);
+    const listed = await repository.listLearningSignals({ scopeType, scopeId, learningLevel: 'L1', learningEligible: true });
+    const sources = (Array.isArray(listed) ? listed : []).filter(signal =>
+      isLearningEligible(signal) && !isDoNotLearn(signal) && !hasRawPrivatePersistence(signal)
+      && clean(signal.signal_type || signal.signalType) === 'candidate_sent'
+      && clean(signal.signal?.decisionRecord?.decisionId)
+    );
+    assertCanonicalScope(sources, scopeType, scopeId, 'LEARNING_POLICY_SOURCE_SCOPE_MISMATCH');
+    const decisionIds = sources.map(row => clean(row.signal.decisionRecord.decisionId));
+    const rawOutcomes = await listPolicyOutcomes(decisionIds, scopeType, scopeId);
+    const trajectory = [];
+
+    for (const source of sources) {
+      const sourceSignalId = clean(source.signal_id || source.signalId);
+      const decision = source.signal.decisionRecord;
+      const decisionId = clean(decision.decisionId);
+      const joined = rawOutcomes.filter(row => {
+        const raw = row.signal || {};
+        const rawFalseEligible = row.learning_eligible === false || row.learning_eligible === 0 || row.learningEligible === false;
+        if (!rawFalseEligible || clean(row.signal_type || row.signalType) !== 'policy_outcome_observed') return false;
+        if (clean(raw.decisionId) !== decisionId) return false;
+        if (clean(raw.sourceSignalId) && clean(raw.sourceSignalId) !== sourceSignalId) return false;
+        if (clean(raw.personId) && clean(decision.personId) && clean(raw.personId) !== clean(decision.personId)) return false;
+        if (clean(raw.conversationId) && clean(decision.conversationId) && clean(raw.conversationId) !== clean(decision.conversationId)) return false;
+        return true;
+      });
+      const outcomes = Object.freeze(joined.flatMap(row => Array.isArray(row.signal?.outcomes) ? row.signal.outcomes.map(value => Object.freeze({ ...value })) : []));
+      const outcomeIds = exactOutcomeIds(joined);
+      const score = approvedScoreFor(sourceSignalId, input.approvedScoresBySignalId);
+      const scoreSourceId = clean(score.sourceSignalId || score.eligibleSourceSignalId);
+      const scoreOutcomeIds = [...new Set((Array.isArray(score.outcomeIds) ? score.outcomeIds : []).map(clean).filter(Boolean))].sort();
+      const exactOutcomeSet = outcomeIds.length === scoreOutcomeIds.length && outcomeIds.every((id, index) => id === scoreOutcomeIds[index]);
+      if (
+        scoreSourceId !== sourceSignalId || clean(score.decisionId) !== decisionId || !exactOutcomeSet ||
+        !clean(score.outcomeEvidenceSetRef) || !clean(score.rewardPolicyVersion)
+      ) {
+        throw contractError('LEARNING_POLICY_SCORE_EVIDENCE_BINDING_REQUIRED', `Score for ${sourceSignalId} must bind exact source/decision/outcome evidence.`);
+      }
+      const minimized = await dataPolicy.minimize({
+        text: clean(input.contentBySignalId?.[sourceSignalId]),
+        signalId: sourceSignalId,
+        scopeType,
+        scopeId,
+        learningEligible: true,
+        featureBundle: decision.featureBundle
+      });
+      if (!minimized || minimized.allowed !== true) continue;
+      trajectory.push(Object.freeze({
+        sourceSignalId,
+        signalId: sourceSignalId,
+        decisionId,
+        decision: Object.freeze({ ...decision }),
+        featureBundle: Object.freeze({ ...(decision.featureBundle || {}) }),
+        outcomes,
+        approvedScore: score,
+        score,
+        minimizedContent: String(minimized.text ?? minimized.minimizedText ?? ''),
+        vwTrainingEligible: hasReplayableDecision(decision)
+      }));
+    }
+
+    const projection = Object.freeze({
+      authority: 'Learning',
+      readOnly: true,
+      scopeType,
+      scopeId,
+      learningLevel: 'L1',
+      policyProjection: true,
+      trajectory: Object.freeze(trajectory)
+    });
+    issuedProjections.add(projection);
+    return projection;
+  }
+
   async function bindExperimentEvidence(input = {}) {
     if (!evidenceAdapter || typeof evidenceAdapter.bindTrainingEvidence !== 'function') {
       throw contractError('LEARNING_DEEP_TRAINING_LANGFUSE_EVIDENCE_REQUIRED', 'Langfuse Dataset/Score evidence adapter is required.');
@@ -231,10 +335,11 @@ function createLearningDeepTrainingContract(options = {}) {
   return Object.freeze({
     projectRelationship,
     projectGlobal,
+    projectPolicy,
     bindExperimentEvidence,
     rollbackPromotion,
     authority: 'Learning read-only Deep Training projection'
   });
 }
 
-module.exports = { createLearningDeepTrainingContract };
+module.exports = { createLearningDeepTrainingContract };