diff --git a/marge/job.py b/marge/job.py
index 0646518f..5eeffd32 100644
--- a/marge/job.py
+++ b/marge/job.py
@@ -343,7 +343,7 @@ def update_from_target_branch_and_push(
             # and likely explains any previous failure, so it will better to just
             # raise a GitError
             if source_branch != 'master':
-                repo.checkout_branch('master')
+                repo.checkout_branch('HEAD^')
                 repo.remove_branch(source_branch)
 
             if not branch_update_done: