From 6b97a3694189e533620ab0cc993090ff388c8354 Mon Sep 17 00:00:00 2001 From: Dave Rolsky Date: Thu, 23 Jul 2026 09:42:19 -0400 Subject: [PATCH] TOOLS-4263 Convert mongorestore replica-set oplog-replay tests to Go This adds a new `TestOplogReplayFromLocalOplogRS` test in `mongorestore/oplog_replset_test.go`. This ports two JS tests into one Go test. It dumps `local.oplog.rs` with a `$gt` timestamp query and replays the dump via `--oplogReplay --oplogFile ...`, verifying only the ops after the checkpoint are applied. JS -> Go mapping: - test/qa-tests/jstests/restore/oplog_replay_local_rs.js -> the --oplogReplay --oplogFile replay of a dumped local/oplog.rs.bson - test/legacy42/jstests/tool/dumprestore7.js -> the mongodump of local.oplog.rs with a {ts:{$gt:...}} timestamp query --- mongorestore/oplog_replset_test.go | 120 ++++++++++++++++++ test/legacy42/jstests/tool/dumprestore7.js | 97 -------------- .../jstests/restore/oplog_replay_local_rs.js | 62 --------- 3 files changed, 120 insertions(+), 159 deletions(-) create mode 100644 mongorestore/oplog_replset_test.go delete mode 100644 test/legacy42/jstests/tool/dumprestore7.js delete mode 100644 test/qa-tests/jstests/restore/oplog_replay_local_rs.js diff --git a/mongorestore/oplog_replset_test.go b/mongorestore/oplog_replset_test.go new file mode 100644 index 000000000..46fa11a4c --- /dev/null +++ b/mongorestore/oplog_replset_test.go @@ -0,0 +1,120 @@ +// Copyright (C) MongoDB, Inc. 2014-present. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +package mongorestore + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/mongodb/mongo-tools/common/options" + "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/common/testutil" + "github.com/mongodb/mongo-tools/mongodump" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo" + mopts "go.mongodb.org/mongo-driver/v2/mongo/options" +) + +// TestOplogReplayFromLocalOplogRS converts oplog_replay_local_rs.js and +// dumprestore7.js into a single replica-set test: it dumps local.oplog.rs with +// a $gt timestamp query (capturing only the ops written after a checkpoint), +// then replays that dumped oplog with --oplogReplay --oplogFile and verifies +// exactly those ops are applied. +func TestOplogReplayFromLocalOplogRS(t *testing.T) { + testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) + // local.oplog.rs only exists on a replica set. + testtype.SkipUnlessTestType(t, testtype.ReplSetTestType) + + const ( + dbName = "testdr_oplogrs" + collName = "coll" + ) + + client, err := testutil.GetBareSession() + require.NoError(t, err) + + ctx := context.Background() + testDB := client.Database(dbName) + coll := testDB.Collection(collName) + ns := dbName + "." + collName + require.NoError(t, testDB.Drop(ctx), "dropping test db") + t.Cleanup(func() { _ = testDB.Drop(context.Background()) }) + + // First batch: these ops precede the checkpoint and must NOT be replayed. + for i := 0; i < 5; i++ { + _, err = coll.InsertOne(ctx, bson.D{{Key: "_id", Value: i}, {Key: "batch", Value: 1}}) + require.NoError(t, err, "inserting first batch") + } + + checkpoint := latestOplogTimestamp(t, client) + + // Second batch: these ops follow the checkpoint and must be replayed. + const secondBatch = 7 + for i := 0; i < secondBatch; i++ { + _, err = coll.InsertOne(ctx, bson.D{{Key: "_id", Value: 100 + i}, {Key: "batch", Value: 2}}) + require.NoError(t, err, "inserting second batch") + } + + dumpDir := t.TempDir() + query := fmt.Sprintf( + `{"ts":{"$gt":{"$timestamp":{"t":%d,"i":%d}}},"ns":%q,"op":"i"}`, + checkpoint.T, checkpoint.I, ns, + ) + require.NoError(t, runDump(t, baseToolOpts(t), dumpDir, func(md *mongodump.MongoDump) { + md.ToolOptions.Namespace = &options.Namespace{DB: "local", Collection: "oplog.rs"} + md.InputOptions.Query = query + }), "dumping local.oplog.rs with a timestamp query") + + require.NoError(t, coll.Drop(ctx), "dropping collection before replay") + require.NoError( + t, + testDB.CreateCollection(ctx, collName), + "recreating collection before replay", + ) + + oplogFile := filepath.Join(dumpDir, "local", "oplog.rs.bson") + result := restoreFromArgs( + t, + OplogReplayOption, + OplogFileOption, oplogFile, + DirectoryOption, t.TempDir(), + ) + require.NoError(t, result.Err, "replaying the dumped oplog succeeds") + + count, err := coll.CountDocuments(ctx, bson.M{}) + require.NoError(t, err, "counting replayed documents") + assert.EqualValues( + t, secondBatch, count, + "only the ops after the checkpoint are replayed", + ) + + batch1Count, err := coll.CountDocuments(ctx, bson.M{"batch": 1}) + require.NoError(t, err, "counting first-batch documents") + assert.EqualValues(t, 0, batch1Count, "ops before the checkpoint are not replayed") +} + +// latestOplogTimestamp returns the ts of the most recent entry in +// local.oplog.rs. +func latestOplogTimestamp(t *testing.T, client *mongo.Client) bson.Timestamp { + t.Helper() + var entry struct { + TS bson.Timestamp `bson:"ts"` + } + err := client.Database("local").Collection("oplog.rs"). + FindOne( + context.Background(), + bson.D{}, + mopts.FindOne().SetSort(bson.D{{Key: "$natural", Value: -1}}), + ). + Decode(&entry) + require.NoError(t, err, "reading latest oplog entry") + return entry.TS +} diff --git a/test/legacy42/jstests/tool/dumprestore7.js b/test/legacy42/jstests/tool/dumprestore7.js deleted file mode 100644 index 782529f03..000000000 --- a/test/legacy42/jstests/tool/dumprestore7.js +++ /dev/null @@ -1,97 +0,0 @@ -(function() { -"use strict"; - -// Skip this test if running with --nojournal and WiredTiger. -if (jsTest.options().noJournal && - (!jsTest.options().storageEngine || jsTest.options().storageEngine === "wiredTiger")) { - print("Skipping test because running WiredTiger without journaling isn't a valid" + - " replica set configuration"); - return; -} - -var name = "dumprestore7"; - -var step = (function() { - var n = 0; - return function(msg) { - msg = msg || ""; - print('\n' + name + ".js step " + (++n) + ' ' + msg); - }; -})(); - -step("starting the replset test"); - -var replTest = new ReplSetTest({name: name, nodes: 1}); -var nodes = replTest.startSet(); -replTest.initiate(); - -step("inserting first chunk of data"); -var foo = replTest.getPrimary().getDB("foo"); -for (var i = 0; i < 20; i++) { - foo.bar.insert({x: i, y: "abc"}); -} - -step("waiting for replication"); -replTest.awaitReplication(); -assert.eq(foo.bar.count(), 20, "should have inserted 20 documents"); - -// The time of the last oplog entry. -var time = replTest.getPrimary() - .getDB("local") - .getCollection("oplog.rs") - .find() - .limit(1) - .sort({$natural: -1}) - .next() - .ts; -step("got time of last oplog entry: " + time); - -step("inserting second chunk of data"); -for (var i = 30; i < 50; i++) { - foo.bar.insert({x: i, y: "abc"}); -} -assert.eq(foo.bar.count(), 40, "should have inserted 40 total documents"); - -step("try mongodump with $timestamp"); - -var data = MongoRunner.dataDir + "/dumprestore7-dump1/"; -var query = {ts: {$gt: time}}; -var queryJSON = '{"ts":{"$gt":{"$timestamp":{"t":' + time.t + ',"i":' + time.i + '}}}}'; -print("mongodump query: " + queryJSON); -if (_isWindows()) { - queryJSON = '"' + queryJSON.replace(/"/g, '\\"') + '"'; -} -var testQueryCount = - replTest.getPrimary().getDB("local").getCollection("oplog.rs").find(query).itcount(); -assert.eq(testQueryCount, 20, "the query should match 20 documents"); - -var exitCode = MongoRunner.runMongoTool("mongodump", { - host: "127.0.0.1:" + replTest.ports[0], - db: "local", - collection: "oplog.rs", - query: queryJSON, - out: data, -}); -assert.eq(0, exitCode, "monogdump failed to dump the oplog"); - -step("try mongorestore from $timestamp"); - -var restoreMongod = MongoRunner.runMongod({}); -exitCode = MongoRunner.runMongoTool("mongorestore", { - host: "127.0.0.1:" + restoreMongod.port, - dir: data, - writeConcern: 1, -}); -assert.eq(0, exitCode, "mongorestore failed to restore the oplog"); - -var count = restoreMongod.getDB("local").getCollection("oplog.rs").count(); -if (count != 20) { - print("mongorestore restored too many documents"); - restoreMongod.getDB("local").getCollection("oplog.rs").find().pretty().shellPrint(); - assert.eq(count, 20, "mongorestore should only have inserted the latter 20 entries"); -} - -MongoRunner.stopMongod(restoreMongod); -step("stopping replset test"); -replTest.stopSet(); -})(); diff --git a/test/qa-tests/jstests/restore/oplog_replay_local_rs.js b/test/qa-tests/jstests/restore/oplog_replay_local_rs.js deleted file mode 100644 index a0aeb5de6..000000000 --- a/test/qa-tests/jstests/restore/oplog_replay_local_rs.js +++ /dev/null @@ -1,62 +0,0 @@ -/** - * oplog_replay_local_rs.js - * - * This file tests mongorestore with --oplogReplay where the oplog file is in the 'oplog.rs' - * collection of the 'local' database. This occurs when using a replica-set for replication. - */ -(function() { - 'use strict'; - if (typeof getToolTest === 'undefined') { - load('jstests/configs/plain_28.config.js'); - } - - var commonToolArgs = getCommonToolArguments(); - var dumpTarget = 'oplog_replay_local_rs'; - - var toolTest = getToolTest('oplog_replay_local_rs'); - - // Set the test db to 'local' and collection to 'oplog.rs' to fake a replica set oplog - var testDB = toolTest.db.getSiblingDB('local'); - var testColl = testDB['oplog.rs']; - var testRestoreDB = toolTest.db.getSiblingDB('test'); - var testRestoreColl = testRestoreDB.op; - resetDbpath(dumpTarget); - - var oplogSize = 100; - testDB.createCollection('oplog.rs', {capped: true, size: 100000}); - - // Create a fake oplog consisting of 100 inserts. - for (var i = 0; i < oplogSize; i++) { - var r = testColl.insert({ - ts: new Timestamp(0, i), - op: "i", - o: {_id: i, x: 'a' + i}, - ns: "test.op", - }); - assert.eq(1, r.nInserted, "insert failed"); - } - - // Dump the fake oplog. - var ret = toolTest.runTool.apply(toolTest, ['dump', - '--db', 'local', - '-c', 'oplog.rs', - '--out', dumpTarget] - .concat(commonToolArgs)); - assert.eq(0, ret, "dump operation failed"); - - // Create the test.op collection. - testRestoreColl.drop(); - testRestoreDB.createCollection("op"); - assert.eq(0, testRestoreColl.count()); - - // Replay the oplog from the provided oplog - ret = toolTest.runTool.apply(toolTest, ['restore', - '--oplogReplay', - dumpTarget] - .concat(commonToolArgs)); - assert.eq(0, ret, "restore operation failed"); - - assert.eq(oplogSize, testRestoreColl.count(), - "all oplog entries should be inserted"); - toolTest.stop(); -}());