FINERACT-2602: Remove orphaned COB locks after job completion

fineract-client-feign/src/main/java/org/apache/fineract/client/feign/FineractFeignClient.java

@@ -155,6 +155,7 @@
 import org.apache.fineract.client.feign.services.TwoFactorApi;
 import org.apache.fineract.client.feign.services.UsersApi;
 import org.apache.fineract.client.feign.services.WorkingCapitalBreachApi;
+import org.apache.fineract.client.feign.services.WorkingCapitalLoanAccountLockApi;
 import org.apache.fineract.client.feign.services.WorkingCapitalLoanBreachScheduleApi;
 import org.apache.fineract.client.feign.services.WorkingCapitalLoanCobCatchUpApi;
 import org.apache.fineract.client.feign.services.WorkingCapitalLoanDelinquencyActionsApi;
@@ -756,6 +757,10 @@ public WorkingCapitalLoanProductsApi workingCapitalLoanProducts() {
         return create(WorkingCapitalLoanProductsApi.class);
     }
 
+    public WorkingCapitalLoanAccountLockApi workingCapitalLoanAccountLock() {
+        return create(WorkingCapitalLoanAccountLockApi.class);
+    }
+
     public WorkingCapitalLoanCobCatchUpApi workingCapitalLoanCobCatchUpApi() {
         return create(WorkingCapitalLoanCobCatchUpApi.class);
     }

fineract-provider/src/main/java/org/apache/fineract/cob/api/LockRequest.java → fineract-cob/src/main/java/org/apache/fineract/cob/api/LockRequest.java

@@ -18,10 +18,13 @@
  */
 package org.apache.fineract.cob.api;
 
+import java.time.LocalDate;
 import lombok.Data;
 
 @Data
 public class LockRequest {
 
     private String error;
+    private LocalDate cobBusinessDate;
+    private Boolean nullCobBusinessDate;
 }

fineract-cob/src/main/java/org/apache/fineract/cob/domain/AccountLockRepository.java

@@ -41,6 +41,8 @@ public interface AccountLockRepository<T extends AccountLock> {
 
     void removeByLockOwnerInAndErrorIsNotNullAndLockPlacedOnCobBusinessDateIsNotNull(List<LockOwner> lockOwners);
 
+    int deleteOrphanedLocksForProcessedAccounts(List<LockOwner> lockOwners);
+
     Page<T> findAll(Pageable loanAccountLockPage);
 
     T saveAndFlush(T entity);

fineract-cob/src/main/java/org/apache/fineract/cob/service/AbstractAccountLockService.java

@@ -33,6 +33,9 @@
 @RequiredArgsConstructor
 public abstract class AbstractAccountLockService<T extends AccountLock> implements AccountLockService<T> {
 
+    protected static final List<LockOwner> COB_LOCK_OWNERS = List.of(LockOwner.LOAN_COB_CHUNK_PROCESSING,
+            LockOwner.LOAN_INLINE_COB_PROCESSING);
+
     private final AccountLockRepository<T> loanAccountLockRepository;
     private final CustomLoanAccountLockRepository<T> customLoanAccountLockRepository;
 
@@ -59,8 +62,13 @@ public boolean isLockOverrulable(Long loanId) {
     @Transactional(propagation = Propagation.REQUIRES_NEW)
     public void updateCobAndRemoveLocks() {
         customLoanAccountLockRepository.updateLoanFromAccountLocks();
-        loanAccountLockRepository.removeByLockOwnerInAndErrorIsNotNullAndLockPlacedOnCobBusinessDateIsNotNull(
-                List.of(LockOwner.LOAN_COB_CHUNK_PROCESSING, LockOwner.LOAN_INLINE_COB_PROCESSING));
+        loanAccountLockRepository.removeByLockOwnerInAndErrorIsNotNullAndLockPlacedOnCobBusinessDateIsNotNull(COB_LOCK_OWNERS);
+    }
+
+    @Override
+    @Transactional(propagation = Propagation.REQUIRES_NEW)
+    public int removeOrphanedLocksForProcessedAccounts() {
+        return loanAccountLockRepository.deleteOrphanedLocksForProcessedAccounts(COB_LOCK_OWNERS);
     }
 
 }

fineract-cob/src/main/java/org/apache/fineract/cob/service/AccountLockService.java

@@ -30,4 +30,6 @@ public interface AccountLockService<T extends AccountLock> {
     boolean isLockOverrulable(Long loanId);
 
     void updateCobAndRemoveLocks();
+
+    int removeOrphanedLocksForProcessedAccounts();
 }

fineract-cob/src/main/java/org/apache/fineract/cob/tasklet/UnlockProcessedAccountsTasklet.java

@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.cob.tasklet;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.fineract.cob.domain.AccountLock;
+import org.apache.fineract.cob.service.AccountLockService;
+import org.springframework.batch.core.StepContribution;
+import org.springframework.batch.core.scope.context.ChunkContext;
+import org.springframework.batch.core.step.tasklet.Tasklet;
+import org.springframework.batch.repeat.RepeatStatus;
+import org.springframework.lang.NonNull;
+
+/**
+ * Tasklet that unlocks accounts which were successfully processed during COB but whose locks were not removed.
+ *
+ * An account is considered successfully processed when its {@code last_closed_business_date} matches the
+ * {@code lock_placed_on_cob_business_date} on the lock record, proving all COB business steps completed. If the lock
+ * still exists with no error, it is orphaned and should be removed.
+ */
+@Slf4j
+@RequiredArgsConstructor
+public abstract class UnlockProcessedAccountsTasklet<T extends AccountLock> implements Tasklet {
+
+    private final AccountLockService<T> accountLockService;
+
+    @Override
+    public RepeatStatus execute(@NonNull final StepContribution contribution, @NonNull final ChunkContext chunkContext) throws Exception {
+        final int removedCount = accountLockService.removeOrphanedLocksForProcessedAccounts();
+        if (removedCount > 0) {
+            log.info("Unlocked {} account(s) that completed COB processing but remained locked", removedCount);
+        } else {
+            log.debug("No orphaned account locks found after COB processing");
+        }
+        return RepeatStatus.FINISHED;
+    }
+}

fineract-doc/src/docs/en/chapters/architecture/loan-locking.adoc

@@ -21,4 +21,16 @@ And when a loan account is locked, loan related write API calls will be either r
 
 Since the fixing might involve making changes to the loan account via API (for example doing a repayment to fix the loan account's inconsistent state), we need to allow those API calls. Hence, the lock table includes a `bypass_enabled` column which disables the lock checks on the loan write APIs.
 
+== Orphaned COB lock cleanup
+
+A separate failure mode exists where a loan account is fully processed by COB but its lock row stays in `m_loan_account_locks` (no `error` was recorded, yet the lock was never removed). Such locks are orphaned: the loan is in a consistent state, but real-time write APIs are still rejected.
+
+To remove these, the Loan COB job runs a final step `unlockProcessedLoansStep` (after `stayedLockedStep`) which executes `UnlockProcessedLoansTasklet`. The tasklet delegates to `AccountLockService#removeOrphanedLocksForProcessedAccounts`, which deletes lock rows that meet all of the following:
+
+* `lock_owner` is `LOAN_COB_CHUNK_PROCESSING` or `LOAN_INLINE_COB_PROCESSING`
+* `error IS NULL`
+* `lock_placed_on_cob_business_date IS NOT NULL`
+* the referenced loan's `last_closed_business_date` equals the lock's `lock_placed_on_cob_business_date` (i.e. all business steps for that COB date completed)
+
+The same mechanism is wired into the Working Capital Loan COB job via `UnlockProcessedWorkingCapitalLoansTasklet`. Locks that carry an `error` are intentionally left in place – they still represent a loan that needs manual fixing and `bypass_enabled` handling continues to apply.
 

fineract-e2e-tests-core/src/test/java/org/apache/fineract/test/helper/ErrorMessageHelper.java

@@ -681,6 +681,11 @@ public static String listOfLockedLoansContainsLoan(Long loanId, LoanAccountLockR
         return String.format("List of locked loan accounts contains the loan with loanId %s. List of locked loans: %n%s", loanId, bodyStr);
     }
 
+    public static String expectedLoanToRemainLocked(Long loanId, LoanAccountLockResponseDTO response) {
+        String bodyStr = response.toString();
+        return String.format("Expected loan %s to remain locked after COB but it is not present in the lock list: %n%s", loanId, bodyStr);
+    }
+
     public static String wrongValueInLineDelinquencyActions(int line, List<String> actual, List<String> expected) {
         String lineStr = String.valueOf(line);
         String expectedStr = expected.toString();

fineract-e2e-tests-core/src/test/java/org/apache/fineract/test/stepdef/common/WorkingCapitalLoanCobStepDef.java

@@ -28,19 +28,20 @@
 import io.cucumber.java.en.Given;
 import io.cucumber.java.en.Then;
 import io.cucumber.java.en.When;
-import java.io.IOException;
 import java.time.Duration;
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.fineract.client.feign.FineractFeignClient;
 import org.apache.fineract.client.feign.util.CallFailedRuntimeException;
 import org.apache.fineract.client.models.BusinessDateResponse;
 import org.apache.fineract.client.models.InlineJobRequest;
 import org.apache.fineract.client.models.IsCatchUpRunningDTO;
+import org.apache.fineract.client.models.LockRequest;
 import org.apache.fineract.client.models.OldestCOBProcessedLoanDTO;
 import org.apache.fineract.client.models.PostClientsResponse;
 import org.apache.fineract.client.models.PostWorkingCapitalLoanProductsResponse;
@@ -51,17 +52,15 @@
 import org.apache.fineract.test.stepdef.AbstractStepDef;
 import org.apache.fineract.test.support.TestContextKey;
 import org.junit.jupiter.api.Assertions;
-import org.springframework.beans.factory.annotation.Autowired;
 
 @Slf4j
+@RequiredArgsConstructor
 public class WorkingCapitalLoanCobStepDef extends AbstractStepDef {
 
     private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("dd MMMM yyyy");
 
-    @Autowired
-    private WorkingCapitalLoanTestHelper wcLoanHelper;
-    @Autowired
-    private FineractFeignClient fineractClient;
+    private final WorkingCapitalLoanTestHelper wcLoanHelper;
+    private final FineractFeignClient fineractClient;
 
     @Before(value = "@WCCOBFeature")
     public void beforeWcCobScenario() {
@@ -87,24 +86,24 @@ public void afterWcCobScenario() {
     }
 
     @When("Admin runs inline COB job for Working Capital Loan")
-    public void runWorkingCapitalInlineCOB() throws IOException {
+    public void runWorkingCapitalInlineCOB() {
         InlineJobRequest inlineJobRequest = new InlineJobRequest().addLoanIdsItem(getTrackedLoanIds().getLast());
         ok(() -> fineractClient.inlineJob().executeInlineJob("WC_LOAN_COB", inlineJobRequest));
     }
 
     @When("Admin runs inline COB job for Working Capital Loan by loanId")
-    public void runWorkingCapitalInlineCOBByLoanId() throws IOException {
+    public void runWorkingCapitalInlineCOBByLoanId() {
         PostWorkingCapitalLoansResponse loanResponse = testContext().get(TestContextKey.LOAN_CREATE_RESPONSE);
         Assertions.assertNotNull(loanResponse);
-        long loanId = loanResponse.getLoanId();
+        Long loanId = loanResponse.getLoanId();
 
         InlineJobRequest inlineJobRequest = new InlineJobRequest().addLoanIdsItem(loanId);
 
         ok(() -> fineractClient.inlineJob().executeInlineJob("WC_LOAN_COB", inlineJobRequest));
     }
 
     @When("Admin runs inline COB job for all Working Capital Loans")
-    public void runWorkingCapitalInlineCOBForAll() throws IOException {
+    public void runWorkingCapitalInlineCOBForAll() {
         InlineJobRequest inlineJobRequest = new InlineJobRequest();
         for (Long loanId : getTrackedLoanIds()) {
             inlineJobRequest.addLoanIdsItem(loanId);
@@ -200,6 +199,110 @@ public void verifyAllLoansHaveNoAccountLocks() {
         }
     }
 
+    @Then("Admin verifies all inserted WC loans have at least one account lock")
+    public void verifyAllLoansHaveAtLeastOneAccountLock() {
+        final List<Long> loanIds = getTrackedLoanIds();
+        assertThat(loanIds).as("No WC loan IDs tracked in test context").isNotEmpty();
+        for (final Long loanId : loanIds) {
+            final int lockCount = wcLoanHelper.countLocksByLoanId(loanId);
+            log.debug("WC loan id={} lock count={}", loanId, lockCount);
+            assertThat(lockCount)//
+                    .as("WC loan id=%d — expected at least one account lock but got %d", loanId, lockCount)//
+                    .isPositive();
+        }
+    }
+
+    @When("Admin places a chunk-processing lock without an error message on the last inserted WC loan")
+    public void placeChunkLockWithoutErrorOnLastWcLoan() {
+        final Long loanId = getTrackedLoanIds().getLast();
+        executeVoid(() -> fineractClient.workingCapitalLoanAccountLock().placeLockOnWorkingCapitalLoanAccount(loanId,
+                "LOAN_COB_CHUNK_PROCESSING", new LockRequest()));
+        log.debug("Placed chunk-processing lock without error on WC loan id={}", loanId);
+    }
+
+    @When("Admin places a chunk-processing lock with error {string} on the last inserted WC loan")
+    public void placeChunkLockWithErrorOnLastWcLoan(final String error) {
+        final Long loanId = getTrackedLoanIds().getLast();
+        executeVoid(() -> fineractClient.workingCapitalLoanAccountLock().placeLockOnWorkingCapitalLoanAccount(loanId,
+                "LOAN_COB_CHUNK_PROCESSING", new LockRequest().error(error)));
+        log.debug("Placed chunk-processing lock with error '{}' on WC loan id={}", error, loanId);
+    }
+
+    @When("Admin places an inline-COB lock without an error message on the last inserted WC loan")
+    public void placeInlineLockWithoutErrorOnLastWcLoan() {
+        final Long loanId = getTrackedLoanIds().getLast();
+        executeVoid(() -> fineractClient.workingCapitalLoanAccountLock().placeLockOnWorkingCapitalLoanAccount(loanId,
+                "LOAN_INLINE_COB_PROCESSING", new LockRequest()));
+        log.debug("Placed inline-COB lock without error on WC loan id={}", loanId);
+    }
+
+    @When("Admin places an inline-COB lock with error {string} on the last inserted WC loan")
+    public void placeInlineLockWithErrorOnLastWcLoan(final String error) {
+        final Long loanId = getTrackedLoanIds().getLast();
+        executeVoid(() -> fineractClient.workingCapitalLoanAccountLock().placeLockOnWorkingCapitalLoanAccount(loanId,
+                "LOAN_INLINE_COB_PROCESSING", new LockRequest().error(error)));
+        log.debug("Placed inline-COB lock with error '{}' on WC loan id={}", error, loanId);
+    }
+
+    @When("Admin places a chunk-processing lock without an error message and null cob business date on the last inserted WC loan")
+    public void placeChunkLockWithNullCobDateOnLastWcLoan() {
+        final Long loanId = getTrackedLoanIds().getLast();
+        executeVoid(() -> fineractClient.workingCapitalLoanAccountLock().placeLockOnWorkingCapitalLoanAccount(loanId,
+                "LOAN_COB_CHUNK_PROCESSING", new LockRequest().nullCobBusinessDate(true)));
+        log.debug("Placed chunk-processing lock with null cob date on WC loan id={}", loanId);
+    }
+
+    @When("Admin places a chunk-processing lock without an error message and cob business date {string} on the last inserted WC loan")
+    public void placeChunkLockWithExplicitCobDateOnLastWcLoan(final String cobBusinessDate) {
+        final Long loanId = getTrackedLoanIds().getLast();
+        final LocalDate parsed = LocalDate.parse(cobBusinessDate, DATE_FORMAT);
+        executeVoid(() -> fineractClient.workingCapitalLoanAccountLock().placeLockOnWorkingCapitalLoanAccount(loanId,
+                "LOAN_COB_CHUNK_PROCESSING", new LockRequest().cobBusinessDate(parsed)));
+        log.debug("Placed chunk-processing lock with explicit cob date {} on WC loan id={}", parsed, loanId);
+    }
+
+    @When("Admin places a chunk-processing lock without an error message on WC loan {int}")
+    public void placeChunkLockWithoutErrorOnWcLoanAtIndex(final int index) {
+        final Long loanId = loanAtIndex(index);
+        executeVoid(() -> fineractClient.workingCapitalLoanAccountLock().placeLockOnWorkingCapitalLoanAccount(loanId,
+                "LOAN_COB_CHUNK_PROCESSING", new LockRequest()));
+        log.debug("Placed chunk-processing lock without error on WC loan index={} id={}", index, loanId);
+    }
+
+    @When("Admin places a chunk-processing lock with error {string} on WC loan {int}")
+    public void placeChunkLockWithErrorOnWcLoanAtIndex(final String error, final int index) {
+        final Long loanId = loanAtIndex(index);
+        executeVoid(() -> fineractClient.workingCapitalLoanAccountLock().placeLockOnWorkingCapitalLoanAccount(loanId,
+                "LOAN_COB_CHUNK_PROCESSING", new LockRequest().error(error)));
+        log.debug("Placed chunk-processing lock with error '{}' on WC loan index={} id={}", error, index, loanId);
+    }
+
+    @Then("Admin verifies inserted WC loan {int} has no account locks")
+    public void verifyLoanAtIndexHasNoLocks(final int index) {
+        final Long loanId = loanAtIndex(index);
+        final int lockCount = wcLoanHelper.countLocksByLoanId(loanId);
+        log.debug("WC loan index={} id={} lock count={}", index, loanId, lockCount);
+        assertThat(lockCount)//
+                .as("WC loan index=%d id=%d — expected 0 account locks but got %d", index, loanId, lockCount)//
+                .isZero();
+    }
+
+    @Then("Admin verifies inserted WC loan {int} has at least one account lock")
+    public void verifyLoanAtIndexHasAtLeastOneLock(final int index) {
+        final Long loanId = loanAtIndex(index);
+        final int lockCount = wcLoanHelper.countLocksByLoanId(loanId);
+        log.debug("WC loan index={} id={} lock count={}", index, loanId, lockCount);
+        assertThat(lockCount)//
+                .as("WC loan index=%d id=%d — expected at least one account lock but got %d", index, loanId, lockCount)//
+                .isPositive();
+    }
+
+    private Long loanAtIndex(final int index) {
+        final List<Long> loanIds = getTrackedLoanIds();
+        assertThat(index).as("Loan index %d out of range (1..%d)", index, loanIds.size()).isBetween(1, loanIds.size());
+        return loanIds.get(index - 1);
+    }
+
     @Then("Admin verifies inserted WC loan {int} has lastClosedBusinessDate {string}")
     public void verifyLoanAtIndexHasLastClosedBusinessDate(int index, String expectedDate) {
         LocalDate expected = LocalDate.parse(expectedDate, DATE_FORMAT);