diff --git a/common/testutil/testutil.go b/common/testutil/testutil.go index fde068335..6a08ca1d6 100644 --- a/common/testutil/testutil.go +++ b/common/testutil/testutil.go @@ -195,6 +195,20 @@ func SkipIfFCVLessThan(t *testing.T, versionStr string, reason string) { } } +// SkipUnlessStandalone skips the test unless it is connected to a standalone +// server. Some operations (e.g. writing local.oplog.rs as a collection) are +// only permitted on a standalone: a replica set rejects direct oplog writes and +// mongos rejects writes to the local database. +func SkipUnlessStandalone(t *testing.T) { + sessionProvider, _, err := GetBareSessionProvider() + require.NoError(t, err) + nodeType, err := sessionProvider.GetNodeType() + require.NoError(t, err) + if nodeType != db.Standalone { + t.Skipf("Skipping test because it requires a standalone server, not %q", nodeType) + } +} + func dottedStringToSlice(s string) ([]int, error) { parts := make([]int, 0, 2) for _, v := range strings.Split(s, ".") { diff --git a/mongorestore/oplog_edges_test.go b/mongorestore/oplog_edges_test.go new file mode 100644 index 000000000..610ad2992 --- /dev/null +++ b/mongorestore/oplog_edges_test.go @@ -0,0 +1,259 @@ +// Copyright (C) MongoDB, Inc. 2014-present. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +package mongorestore + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/mongodb/mongo-tools/common/db" + "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/common/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/v2/bson" +) + +// TestOplogReplayConflict converts oplog_replay_conflict.js: when a dump +// directory already contains an oplog.bson AND --oplogFile points at another +// oplog, mongorestore must fail and apply no data. +func TestOplogReplayConflict(t *testing.T) { + testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) + + client, err := testutil.GetBareSession() + require.NoError(t, err) + coll := client.Database("test").Collection("data") + require.NoError(t, coll.Drop(context.Background()), "dropping target collection") + t.Cleanup(func() { _ = coll.Drop(context.Background()) }) + + result := restoreFromArgs( + t, + OplogReplayOption, + OplogFileOption, "testdata/extra_oplog.bson", + DirectoryOption, "testdata/dump_oplog_conflict", + ) + require.ErrorContains( + t, result.Err, "cannot provide both an oplog.bson file and an oplog file", + "providing two top-priority oplogs errors", + ) + + count, err := coll.CountDocuments(context.Background(), bson.M{}) + require.NoError(t, err, "counting documents") + assert.EqualValues(t, 0, count, "no entries are restored when the oplogs conflict") +} + +// TestOplogReplayPriorityOplog converts oplog_replay_priority_oplog.js: when a +// dump directory contains a local/oplog.rs.bson AND --oplogFile points at a +// higher-priority oplog, only the priority oplog's entries are applied. +func TestOplogReplayPriorityOplog(t *testing.T) { + testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) + + // This test restores the dump_local_oplog fixture, which contains a + // local/oplog.rs.bson. Writing local.oplog.rs is only allowed on a + // standalone. Like the JS test this replaces, it is only meaningful there. + testutil.SkipUnlessStandalone(t) + + client, err := testutil.GetBareSession() + require.NoError(t, err) + testDB := client.Database("test") + dataColl := testDB.Collection("data") + opColl := testDB.Collection("op") + require.NoError(t, dataColl.Drop(context.Background()), "dropping data collection") + require.NoError(t, opColl.Drop(context.Background()), "dropping op collection") + t.Cleanup(func() { + _ = dataColl.Drop(context.Background()) + _ = opColl.Drop(context.Background()) + }) + // On a standalone, applyOps will not auto-create the target namespaces, so + // they must exist before the oplog is replayed. + require.NoError( + t, + testDB.CreateCollection(context.Background(), "data"), + "creating data collection", + ) + require.NoError( + t, + testDB.CreateCollection(context.Background(), "op"), + "creating op collection", + ) + + result := restoreFromArgs( + t, + OplogReplayOption, + OplogFileOption, "testdata/extra_oplog.bson", + DirectoryOption, "testdata/dump_local_oplog", + ) + require.NoError(t, result.Err, "restoring with a priority oplog succeeds") + + dataCount, err := dataColl.CountDocuments(context.Background(), bson.M{}) + require.NoError(t, err, "counting data documents") + assert.EqualValues( + t, 5, dataCount, + "all entries from the high-priority --oplogFile are restored", + ) + + opCount, err := opColl.CountDocuments(context.Background(), bson.M{}) + require.NoError(t, err, "counting op documents") + assert.EqualValues( + t, 0, opCount, + "no entries from the low-priority local oplog are restored", + ) +} + +// TestOplogReplayNoop converts oplog_replay_noop.js: noop ("n") entries +// interleaved with inserts are skipped, while the inserts are applied. +func TestOplogReplayNoop(t *testing.T) { + testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) + + client, err := testutil.GetBareSession() + require.NoError(t, err) + coll := client.Database("test").Collection("data") + require.NoError(t, coll.Drop(context.Background()), "dropping target collection") + t.Cleanup(func() { _ = coll.Drop(context.Background()) }) + + result := restoreFromArgs( + t, + OplogReplayOption, + DirectoryOption, "testdata/dump_with_noop_in_oplog", + ) + require.NoError(t, result.Err, "restoring an oplog with noops succeeds") + + count, err := coll.CountDocuments(context.Background(), bson.M{}) + require.NoError(t, err, "counting documents") + assert.EqualValues(t, 1, count, "the insert after the noops is applied") + + aCount, err := coll.CountDocuments(context.Background(), bson.M{"a": 1}) + require.NoError(t, err, "counting {a:1} documents") + assert.EqualValues(t, 1, aCount, "the inserted document has the expected contents") +} + +// TestOplogReplayPreservesComplexIDOrder converts +// preserve_oplog_structure_order.js: replaying an update op whose _id is a +// multi-field subdocument must preserve the field order, otherwise the server +// rejects the op (the o._id and o2._id must match). +func TestOplogReplayPreservesComplexIDOrder(t *testing.T) { + testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) + + sessionProvider, _, err := testutil.GetBareSessionProvider() + require.NoError(t, err) + serverVersion, err := sessionProvider.ServerVersionArray() + require.NoError(t, err) + + client, err := testutil.GetBareSession() + require.NoError(t, err) + testDB := client.Database("test") + coll := testDB.Collection("foobar") + require.NoError(t, coll.Drop(context.Background()), "dropping target collection") + t.Cleanup(func() { _ = coll.Drop(context.Background()) }) + require.NoError( + t, testDB.CreateCollection(context.Background(), "foobar"), + "creating target collection", + ) + + complexID := bson.D{ + {Key: "a", Value: 1.0}, + {Key: "b", Value: 2.0}, + {Key: "c", Value: 3.0}, + {Key: "d", Value: 5.0}, + {Key: "e", Value: 6.0}, + {Key: "f", Value: 7.0}, + {Key: "g", Value: 8.0}, + } + + // As of SERVER-88158 (8.1+), applyOps no longer upserts by default, so the + // document the update op targets must already exist. + if serverVersion.GTE(db.Version{8, 1, 0}) { + _, err = coll.InsertOne(context.Background(), bson.D{{Key: "_id", Value: complexID}}) + require.NoError(t, err, "pre-inserting the document for the update op") + } + + result := restoreFromArgs( + t, + OplogReplayOption, + DirectoryOption, "testdata/dump_with_complex_id_oplog", + ) + require.NoError( + t, result.Err, + "replaying an update op with a subdocument _id preserves field order", + ) + + count, err := coll.CountDocuments(context.Background(), bson.D{{Key: "_id", Value: complexID}}) + require.NoError(t, err, "counting documents by the exact subdocument _id") + assert.EqualValues(t, 1, count, "the document is found by its ordered subdocument _id") +} + +// TestOplogReplaySizeSafety converts oplog_replay_size_safety.js with a reduced +// matrix (the JS swept up to a million ops): a large batch of small ops plus a +// batch of ~1MB ops all replay successfully. This guards the batching that +// keeps oplog replay under the 16MB message limit (TOOLS-939). +func TestOplogReplaySizeSafety(t *testing.T) { + testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) + + client, err := testutil.GetBareSession() + require.NoError(t, err) + testDB := client.Database("test") + coll := testDB.Collection("op") + require.NoError(t, coll.Drop(context.Background()), "dropping target collection") + t.Cleanup(func() { _ = coll.Drop(context.Background()) }) + require.NoError( + t, testDB.CreateCollection(context.Background(), "op"), + "creating target collection", + ) + + const ( + smallOps = 50000 + largeOps = 8 + oneMB = 1024 * 1024 + ) + + dir := t.TempDir() + oplogPath := filepath.Join(dir, "oplog.bson") + writeOplogInserts(t, oplogPath, "test.op", smallOps, largeOps, oneMB) + + result := restoreFromArgs(t, OplogReplayOption, DirectoryOption, dir) + require.NoError(t, result.Err, "replaying a large oplog succeeds") + + count, err := coll.CountDocuments(context.Background(), bson.M{}) + require.NoError(t, err, "counting restored documents") + assert.EqualValues(t, smallOps+largeOps, count, "all oplog entries are inserted") +} + +// writeOplogInserts writes an oplog.bson file to path containing smallOps tiny +// insert ops followed by largeOps insert ops each carrying a value of +// largeSize bytes, all targeting namespace ns. +func writeOplogInserts(t *testing.T, path, ns string, smallOps, largeOps, largeSize int) { + t.Helper() + f, err := os.Create(path) + require.NoError(t, err, "creating oplog fixture file") + defer func() { require.NoError(t, f.Close(), "closing oplog fixture file") }() + + writeOp := func(id int, value any) { + op := db.Oplog{ + Version: 2, + Operation: "i", + Namespace: ns, + Object: bson.D{{Key: "_id", Value: id}, {Key: "x", Value: value}}, + } + marshaled, err := bson.Marshal(op) + require.NoError(t, err, "marshaling oplog entry") + _, err = f.Write(marshaled) + require.NoError(t, err, "writing oplog entry") + } + + for i := 0; i < smallOps; i++ { + writeOp(i, "x") + } + big := make([]byte, largeSize) + for i := range big { + big[i] = 'x' + } + for i := 0; i < largeOps; i++ { + writeOp(smallOps+i, string(big)) + } +} diff --git a/test/qa-tests/jstests/restore/testdata/dump_local_oplog/local/oplog.rs.bson b/mongorestore/testdata/dump_local_oplog/local/oplog.rs.bson similarity index 100% rename from test/qa-tests/jstests/restore/testdata/dump_local_oplog/local/oplog.rs.bson rename to mongorestore/testdata/dump_local_oplog/local/oplog.rs.bson diff --git a/test/qa-tests/jstests/restore/testdata/dump_local_oplog/local/oplog.rs.metadata.json b/mongorestore/testdata/dump_local_oplog/local/oplog.rs.metadata.json similarity index 100% rename from test/qa-tests/jstests/restore/testdata/dump_local_oplog/local/oplog.rs.metadata.json rename to mongorestore/testdata/dump_local_oplog/local/oplog.rs.metadata.json diff --git a/test/qa-tests/jstests/restore/testdata/dump_oplog_conflict/oplog.bson b/mongorestore/testdata/dump_oplog_conflict/oplog.bson similarity index 100% rename from test/qa-tests/jstests/restore/testdata/dump_oplog_conflict/oplog.bson rename to mongorestore/testdata/dump_oplog_conflict/oplog.bson diff --git a/test/qa-tests/jstests/restore/testdata/dump_with_complex_id_oplog/oplog.bson b/mongorestore/testdata/dump_with_complex_id_oplog/oplog.bson similarity index 100% rename from test/qa-tests/jstests/restore/testdata/dump_with_complex_id_oplog/oplog.bson rename to mongorestore/testdata/dump_with_complex_id_oplog/oplog.bson diff --git a/test/qa-tests/jstests/restore/testdata/dump_with_noop_in_oplog/oplog.bson b/mongorestore/testdata/dump_with_noop_in_oplog/oplog.bson similarity index 100% rename from test/qa-tests/jstests/restore/testdata/dump_with_noop_in_oplog/oplog.bson rename to mongorestore/testdata/dump_with_noop_in_oplog/oplog.bson diff --git a/test/qa-tests/jstests/restore/testdata/extra_oplog.bson b/mongorestore/testdata/extra_oplog.bson similarity index 100% rename from test/qa-tests/jstests/restore/testdata/extra_oplog.bson rename to mongorestore/testdata/extra_oplog.bson diff --git a/test/qa-tests/jstests/restore/oplog_replay_conflict.js b/test/qa-tests/jstests/restore/oplog_replay_conflict.js deleted file mode 100644 index b7399df66..000000000 --- a/test/qa-tests/jstests/restore/oplog_replay_conflict.js +++ /dev/null @@ -1,33 +0,0 @@ -/** - * oplog_replay_conflict.js - * - * This file tests mongorestore with --oplogReplay where the user provides two top priority - * oplogs and mongorestore should exit with an error. - */ -(function() { - 'use strict'; - if (typeof getToolTest === 'undefined') { - load('jstests/configs/plain_28.config.js'); - } - - var commonToolArgs = getCommonToolArguments(); - var restoreTarget = 'jstests/restore/testdata/dump_oplog_conflict'; - - var toolTest = getToolTest('oplog_replay_conflict'); - - // The test db and collections we'll be using. - var testDB = toolTest.db.getSiblingDB('test'); - testDB.createCollection('data'); - var testColl = testDB.data; - - // Replay the oplog from the provided oplog - var ret = toolTest.runTool.apply(toolTest, ['restore', - '--oplogReplay', - '--oplogFile', 'jstests/restore/testdata/extra_oplog.bson', - restoreTarget].concat(commonToolArgs)); - - assert.eq(0, testColl.count(), - "no original entries should be restored"); - assert.eq(1, ret, "restore operation succeeded when it shouldn't have"); - toolTest.stop(); -}()); diff --git a/test/qa-tests/jstests/restore/oplog_replay_noop.js b/test/qa-tests/jstests/restore/oplog_replay_noop.js deleted file mode 100644 index 9d4934284..000000000 --- a/test/qa-tests/jstests/restore/oplog_replay_noop.js +++ /dev/null @@ -1,37 +0,0 @@ -(function() { - - if (typeof getToolTest === 'undefined') { - load('jstests/configs/plain_28.config.js'); - } - - if (dump_targets !== "standard") { - print('skipping test incompatible with archiving or compression'); - return assert(true); - } - - // Tests using mongorestore with --oplogReplay and noops in the oplog.bson, - // making sure the noops are ignored. - - jsTest.log('Testing restoration with --oplogReplay and noops'); - - var toolTest = getToolTest('oplog_replay_noop'); - var commonToolArgs = getCommonToolArguments(); - - // the db and collection we'll be using - var testDB = toolTest.db.getSiblingDB('test'); - var testColl = testDB.data; - - // restore the data, with --oplogReplay - var ret = toolTest.runTool.apply(toolTest, ['restore', '--oplogReplay'] - .concat(getRestoreTarget('jstests/restore/testdata/dump_with_noop_in_oplog')) - .concat(commonToolArgs)); - assert.eq(0, ret); - - // make sure the document appearing in the oplog, which shows up - // after the noops, was added successfully - assert.eq(1, testColl.count()); - assert.eq(1, testColl.count({a: 1})); - - toolTest.stop(); - -}()); diff --git a/test/qa-tests/jstests/restore/oplog_replay_priority_oplog.js b/test/qa-tests/jstests/restore/oplog_replay_priority_oplog.js deleted file mode 100644 index eb2eeb113..000000000 --- a/test/qa-tests/jstests/restore/oplog_replay_priority_oplog.js +++ /dev/null @@ -1,40 +0,0 @@ -/** - * oplog_replay_priority_oplog.js - * - * This file tests mongorestore with --oplogReplay where the user provides two oplogs and - * mongorestore only restores the higher priority one. - */ -(function() { - 'use strict'; - if (typeof getToolTest === 'undefined') { - load('jstests/configs/plain_28.config.js'); - } - - var commonToolArgs = getCommonToolArguments(); - var restoreTarget = 'jstests/restore/testdata/dump_local_oplog'; - - var toolTest = getToolTest('oplog_replay_priority_oplog'); - - // The test db and collections we'll be using. - var testDB = toolTest.db.getSiblingDB('test'); - testDB.createCollection('data'); - var testColl = testDB.data; - testDB.createCollection('op'); - var restoreColl = testDB.op; - - // Replay the oplog from the provided oplog - var ret = toolTest.runTool.apply(toolTest, ['restore', - '--oplogReplay', - '--oplogFile', 'jstests/restore/testdata/extra_oplog.bson', - restoreTarget] - .concat(commonToolArgs)); - assert.eq(0, ret, "restore operation failed"); - - // Extra oplog has 5 entries as explained in oplog_replay_and_limit.js - assert.eq(5, testColl.count(), - "all original entries from high priority oplog should be restored"); - assert.eq(0, restoreColl.count(), - "no original entries from low priority oplog should be restored"); - toolTest.stop(); -}()); - diff --git a/test/qa-tests/jstests/restore/oplog_replay_size_safety.js b/test/qa-tests/jstests/restore/oplog_replay_size_safety.js deleted file mode 100644 index 775da7fbc..000000000 --- a/test/qa-tests/jstests/restore/oplog_replay_size_safety.js +++ /dev/null @@ -1,71 +0,0 @@ -(function() { - - if (typeof getToolTest === 'undefined') { - load('jstests/configs/plain_28.config.js'); - } - - var commonToolArgs = getCommonToolArguments(); - var dumpTarget = 'oplog_replay_sizes'; - - // Helper for using mongorestore with --oplogReplay and a large oplog.bson - function tryOplogReplay(oplogSize, documentSize) { - var toolTest = getToolTest('oplog_replay_sizes'); - // the test db and collections we'll be using - var testDB = toolTest.db.getSiblingDB('test_oplog'); - var testColl = testDB.oplog; - var testRestoreDB = toolTest.db.getSiblingDB('test'); - var testRestoreColl = testRestoreDB.op; - resetDbpath(dumpTarget); - - var debugString = 'with ' + oplogSize + ' ops of size ' + documentSize; - jsTest.log('Testing --oplogReplay ' + debugString); - - - // create a fake oplog consisting of a large number of inserts - var xStr = new Array(documentSize).join("x"); // ~documentSize bytes string - var data = []; - for (var i = 0; i < oplogSize; i++) { - data.push({ - ts: new Timestamp(0, i), - op: "i", - o: {_id: i, x: xStr}, - ns: "test.op" - }); - if (data.length === 1000) { - testColl.insertMany(data); - data = []; - } - } - testColl.insertMany(data); - - // dump the fake oplog - var ret = toolTest.runTool.apply(toolTest, ['dump', - '--db', 'test_oplog', - '-c', 'oplog', - '--out', dumpTarget] - .concat(commonToolArgs)); - assert.eq(0, ret, "dump operation failed " + debugString); - - // create the test.op collection - testRestoreColl.drop(); - testRestoreDB.createCollection("op"); - assert.eq(0, testRestoreColl.count()); - - // trick restore into replaying the "oplog" we forged above - ret = toolTest.runTool.apply(toolTest, ['restore', - '--oplogReplay', dumpTarget+'/test_oplog'] - .concat(commonToolArgs)); - assert.eq(0, ret, "restore operation failed " + debugString); - assert.eq(oplogSize, testRestoreColl.count(), - "all oplog entries should be inserted " + debugString); - toolTest.stop(); - } - - // run the test on various oplog and op sizes - tryOplogReplay(1024, 1024); // sanity check - tryOplogReplay(1024*1024, 1); // millions of micro ops - tryOplogReplay(8, 16*1024*1023); // 8 ~16MB ops - tryOplogReplay(32, 1024*1024); // 32 ~1MB ops - tryOplogReplay(32*1024, 1024); // many ~1KB ops - -}()); diff --git a/test/qa-tests/jstests/restore/preserve_oplog_structure_order.js b/test/qa-tests/jstests/restore/preserve_oplog_structure_order.js deleted file mode 100644 index b6503f193..000000000 --- a/test/qa-tests/jstests/restore/preserve_oplog_structure_order.js +++ /dev/null @@ -1,60 +0,0 @@ -(function() { - - load('jstests/common/check_version.js'); - load("jstests/configs/standard_dump_targets.config.js"); - - jsTest.log('Testing that the order of fields is preserved in the oplog'); - - var TOOLS_TEST_CONFIG = {}; - if (TestData.useTLS) { - TOOLS_TEST_CONFIG = { - tlsMode: "requireTLS", - tlsCertificateKeyFile: "jstests/libs/server.pem", - tlsCAFile: "jstests/libs/ca.pem", - tlsAllowInvalidHostnames: "", - }; - } - var toolTest = new ToolTest('ordered_oplog', TOOLS_TEST_CONFIG); - toolTest.startDB('foo'); - - var testDb = toolTest.db.getSiblingDB('test'); - testDb.createCollection("foobar"); - - // this test is directly trying to apply the oplog. The document with _id that it's applying does not exist in the - // collection. After SERVER-88158, applyOps does not upsert by default, and therefore we need to insert the document - // into the collection first. - if (isAtLeastVersion(testDb.version(), "8.1.0")) { - testDb.foobar.insert({ - "_id": { - "a": 1.0, - "b": 2.0, - "c": 3.0, - "d": 5.0, - "e": 6.0, - "f": 7.0, - "g": 8.0 - } - }); - } - - // run restore, with an "update" oplog with a _id field that is a subdocument with several fields - // { "h":{"$numberLong":"7987029173745013482"},"ns":"test.foobar", - // "o":{"_id":{"a":1,"b":2,"c":3,"d":5,"e":6,"f":7,"g":8},"foo":"bar"}, - // "o2":{"_id":{"a":1,"b":2,"c":3,"d":5,"e":6,"f":7,"g":8}},"op":"u","ts":{"$timestamp":{"t":1439225650,"i":1}},"v":NumberInt(2) - // } - // if the _id from the "o" and the _id from the "o2" don't match then mongod complains - // run it several times, because with just one execution there is a chance that restore randomly selects the correct order - // With several executions the chances of all false positives diminishes. - for (var i=0; i<10; i++) { - var ret = toolTest.runTool('restore', '--oplogReplay', - 'jstests/restore/testdata/dump_with_complex_id_oplog', - '--ssl', - '--sslPEMKeyFile=jstests/libs/client.pem', - '--sslCAFile=jstests/libs/ca.pem', - '--sslAllowInvalidHostnames'); - assert.eq(0, ret); - } - - toolTest.stop(); - -}());