multi identifier preference record uploads

examples/pm-test/cli-upload-preferences-example.csv

@@ -0,0 +1,5 @@
+email_id,person_id,member_id,timestamp,Sales,source_system
+test-acme10@gmail.com,p1,m1,2025-08-07T05:09:40.317Z,true,
+test-acme11@gmail.com,p2,m2,2025-08-07T17:49:21.192Z,true,
+test-acme12@gmail.com,p3,m3,2025-08-07T18:49:21.192Z,false,
+test-acme13@gmail.com,p4,m4,2025-08-07T17:49:21.192Z,false,

examples/pm-test/cli-upload-preferences-example2.csv

@@ -0,0 +1,5 @@
+email_id,person_id,member_id,timestamp,Sales,source_system
+,p5,m5,2025-08-07T05:09:40.317Z,true,
+test-acme14@gmail.com,p6,m6,2025-08-07T17:49:21.192Z,true,
+test-acme15@gmail.com,p7,m7,2025-08-07T18:49:21.192Z,false,
+test-acme16@gmail.com,p8,m8,2025-08-07T17:49:21.192Z,false,

package.json

@@ -130,7 +130,8 @@
     "query-string": "=7.0.0",
     "semver": "^7.6.0",
     "undici": "^5.22.1",
-    "yargs-parser": "^21.1.1"
+    "yargs-parser": "^21.1.1",
+    "bluebird": "^3.7.2"
   },
   "devDependencies": {
     "@types/JSONStream": "npm:@types/jsonstream@^0.8.33",
@@ -165,7 +166,8 @@
     "tsx": "^4.20.3",
     "typescript": "^5.0.4",
     "vite-tsconfig-paths": "^5.1.4",
-    "vitest": "^3.2.4"
+    "vitest": "^3.2.4",
+    "@types/bluebird": "^3.5.38"
   },
   "packageManager": "pnpm@10.12.4+sha512.5ea8b0deed94ed68691c9bad4c955492705c5eeb8a87ef86bc62c74a26b037b08ff9570f108b2e4dbd1dd1a9186fea925e527f141c648e85af45631074680184"
 }

src/commands/consent/upload-preferences/command.ts

@@ -97,7 +97,26 @@ export const uploadPreferencesCommand = buildCommand({
         kind: 'parsed',
         parse: numberParser,
         brief: 'The concurrency to use when uploading in parallel',
-        default: '10',
+        default: '1',
+      },
+      allowedIdentifierNames: {
+        kind: 'parsed',
+        parse: (value: string) => value.split(',').map((s) => s.trim()),
+        brief:
+          'Identifiers configured for the run. Comma-separated list of identifier names.',
+      },
+      identifierColumns: {
+        kind: 'parsed',
+        parse: (value: string) => value.split(',').map((s) => s.trim()),
+        brief:
+          'Columns in the CSV that should be used as identifiers. Comma-separated list of column names.',
+      },
+      columnsToIgnore: {
+        kind: 'parsed',
+        parse: (value: string) => value.split(',').map((s) => s.trim()),
+        brief:
+          'Columns in the CSV that should be ignored. Comma-separated list of column names.',
+        optional: true,
       },
     },
   },

src/commands/consent/upload-preferences/impl.ts

@@ -26,6 +26,9 @@ export interface UploadPreferencesCommandFlags {
   attributes: string;
   receiptFilepath: string;
   concurrency: number;
+  allowedIdentifierNames: string[];
+  identifierColumns: string[];
+  columnsToIgnore?: string[];
 }
 
 export async function uploadPreferences(
@@ -46,6 +49,9 @@ export async function uploadPreferences(
     isSilent,
     attributes,
     concurrency,
+    allowedIdentifierNames,
+    identifierColumns,
+    columnsToIgnore = [],
   }: UploadPreferencesCommandFlags,
 ): Promise<void> {
   if (!!directory && !!file) {
@@ -121,8 +127,15 @@ export async function uploadPreferences(
 
   await map(
     files,
-    async (filePath) => {
+    async (filePath, index) => {
       const fileName = basename(filePath).replace('.csv', '');
+      const oldReceiptFilepath =
+        index > 0
+          ? join(
+              receiptFileDir,
+              `${basename(files[0]).replace('.csv', '')}-receipts.json`,
+            )
+          : undefined;
       await uploadPreferenceManagementPreferencesInteractive({
         receiptFilepath: join(receiptFileDir, `${fileName}-receipts.json`),
         auth,
@@ -137,6 +150,10 @@ export async function uploadPreferences(
         dryRun,
         attributes: splitCsvToList(attributes),
         forceTriggerWorkflows,
+        allowedIdentifierNames,
+        identifierColumns,
+        oldReceiptFilepath,
+        columnsToIgnore,
       });
     },
     { concurrency },

src/lib/graphql/fetchIdentifiers.ts

@@ -42,6 +42,8 @@ export interface Identifier {
   };
   /** Display order */
   displayOrder: number;
+  /** does this identifier uniquely identify a consent record */
+  isUniqueOnPreferenceStore: boolean;
 }
 
 const PAGE_SIZE = 20;

src/lib/graphql/gqls/identifier.ts

@@ -32,6 +32,7 @@ export const IDENTIFIERS = gql`
           defaultMessage
         }
         displayOrder
+        isUniqueOnPreferenceStore
       }
     }
   }

src/lib/preference-management/codecs.ts

@@ -72,12 +72,20 @@ export const FileMetadataState = t.intersection([
      * their preferences are already in the store
      */
     skippedUpdates: t.record(t.string, t.record(t.string, t.string)),
+    /** The column name that maps to the identifier */
+    columnToIdentifier: t.record(
+      t.string,
+      t.type({
+        /** The identifier name */
+        name: t.string,
+        /** Is unique on preference store */
+        isUniqueOnPreferenceStore: t.boolean,
+      }),
+    ),
   }),
   t.partial({
-    /** Determine which column name in file maps to consent record identifier to upload on  */
-    identifierColumn: t.string,
     /** Determine which column name in file maps to the timestamp  */
-    timestampColum: t.string,
+    timestampColumn: t.string,
   }),
 ]);
 

src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts

@@ -25,22 +25,26 @@ export async function parsePreferenceAndPurposeValuesFromCsv(
     purposeSlugs,
     preferenceTopics,
     forceTriggerWorkflows,
+    columnsToIgnore,
   }: {
     /** The purpose slugs that are allowed to be updated */
     purposeSlugs: string[];
     /** The preference topics */
     preferenceTopics: PreferenceTopic[];
     /** Force workflow triggers */
     forceTriggerWorkflows: boolean;
+    /** Columns to ignore in the CSV file */
+    columnsToIgnore: string[];
   },
 ): Promise<FileMetadataState> {
   // Determine columns to map
   const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat());
 
   // Determine the columns that could potentially be used for identifier
   const otherColumns = difference(columnNames, [
-    ...(currentState.identifierColumn ? [currentState.identifierColumn] : []),
-    ...(currentState.timestampColum ? [currentState.timestampColum] : []),
+    ...Object.keys(currentState.columnToIdentifier),
+    ...(currentState.timestampColumn ? [currentState.timestampColumn] : []),
+    ...columnsToIgnore,
   ]);
   if (otherColumns.length === 0) {
     if (forceTriggerWorkflows) {