From 1612785bdaf21fabf03340c33ac501eeab47ddcd Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Thu, 22 Sep 2022 22:44:34 +0200 Subject: [PATCH 01/36] run MongoDB tests on multiple Mongo versions --- .github/workflows/ci.yml | 6 +---- .github/workflows/mongo.yml | 53 +++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/mongo.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c05243607..83edc71fe3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,10 +41,6 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - mongodb: - image: mongo - ports: - - 27017:27017 steps: - uses: actions/checkout@v3 @@ -57,7 +53,7 @@ jobs: - name: '[POSIX] Run tests' env: VIBED_DRIVER: vibe-core - PARTS: builds,unittests,examples,tests,mongo + PARTS: builds,unittests,examples,tests run: | ./run-ci.sh diff --git a/.github/workflows/mongo.yml b/.github/workflows/mongo.yml new file mode 100644 index 0000000000..a4a14a8754 --- /dev/null +++ b/.github/workflows/mongo.yml @@ -0,0 +1,53 @@ +name: MongoDB Tests + +on: [push, pull_request] + +jobs: + main: + name: Run + strategy: + fail-fast: false + matrix: + os: [ ubuntu-20.04 ] + dc: [ dmd-latest ] + mongo: + - '3.6' + - '4.0' + - '4.2' + - '4.4' + - '5.0' + - '6.0' + + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v3 + + - name: Prepare compiler + uses: dlang-community/setup-dlang@v1 + with: + compiler: ${{ matrix.dc }} + + - name: variable-mapper + uses: kanga333/variable-mapper@v0.2.2 + with: + key: "${{ matrix.mongo }}" + map: | + { + "^3\\.6$": {"MONGO_URL":"https://repo.mongodb.org/apt/ubuntu/dists/bionic/mongodb-org/3.6/multiverse/binary-amd64/mongodb-org-server_3.6.23_amd64.deb"}, + "^4\\.0$": {"MONGO_URL":"https://repo.mongodb.org/apt/ubuntu/dists/bionic/mongodb-org/4.0/multiverse/binary-amd64/mongodb-org-server_4.0.28_amd64.deb"}, + "^4\\.2$": {"MONGO_URL":"https://repo.mongodb.org/apt/ubuntu/dists/bionic/mongodb-org/4.2/multiverse/binary-amd64/mongodb-org-server_4.2.22_amd64.deb"}, + "^4\\.4$": {"MONGO_URL":"https://repo.mongodb.org/apt/ubuntu/dists/focal/mongodb-org/4.4/multiverse/binary-amd64/mongodb-org-server_4.4.16_amd64.deb"}, + "^5\\.0$": {"MONGO_URL":"https://repo.mongodb.org/apt/ubuntu/dists/focal/mongodb-org/5.0/multiverse/binary-amd64/mongodb-org-server_5.0.12_amd64.deb"}, + "^6\\.0$": {"MONGO_URL":"https://repo.mongodb.org/apt/ubuntu/dists/focal/mongodb-org/6.0/multiverse/binary-amd64/mongodb-org-server_6.0.1_amd64.deb"} + } + + - name: 'Install MongoDB' + run: wget "$MONGO_URL" && sudo dpkg -i "$(basename "$MONGO_URL")" + - name: 'Run tests' + env: + VIBED_DRIVER: vibe-core + PARTS: mongo + run: | + ./run-ci.sh \ No newline at end of file From d77e3744b03e87991af74a556786d5130910dde6 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Thu, 22 Sep 2022 22:46:34 +0200 Subject: [PATCH 02/36] support MongoDB connectTimeout and socketTimeout --- mongodb/vibe/db/mongo/connection.d | 3 ++- mongodb/vibe/db/mongo/settings.d | 10 ++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index 27aaab3566..975e784636 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -163,8 +163,9 @@ final class MongoConnection { * options such as connect timeouts and so on. */ try { - m_conn = connectTCP(m_settings.hosts[0].name, m_settings.hosts[0].port); + m_conn = connectTCP(m_settings.hosts[0].name, m_settings.hosts[0].port, null, 0, m_settings.connectTimeoutMS.msecs); m_conn.tcpNoDelay = true; + m_conn.readTimeout = m_settings.socketTimeoutMS.msecs; if (m_settings.ssl) { auto ctx = createTLSContext(TLSContextKind.client); if (!m_settings.sslverifycertificate) { diff --git a/mongodb/vibe/db/mongo/settings.d b/mongodb/vibe/db/mongo/settings.d index bcd2f3caa9..e1a15a3d94 100644 --- a/mongodb/vibe/db/mongo/settings.d +++ b/mongodb/vibe/db/mongo/settings.d @@ -160,8 +160,8 @@ bool parseMongoDBUrl(out MongoClientSettings cfg, string url) case "safe": setBool(cfg.safe); break; case "fsync": setBool(cfg.fsync); break; case "journal": setBool(cfg.journal); break; - case "connecttimeoutms": setLong(cfg.connectTimeoutMS); warnNotImplemented(); break; - case "sockettimeoutms": setLong(cfg.socketTimeoutMS); warnNotImplemented(); break; + case "connecttimeoutms": setLong(cfg.connectTimeoutMS); break; + case "sockettimeoutms": setLong(cfg.socketTimeoutMS); break; case "tls": setBool(cfg.ssl); break; case "ssl": setBool(cfg.ssl); break; case "sslverifycertificate": setBool(cfg.sslverifycertificate); break; @@ -454,16 +454,14 @@ class MongoClientSettings /** * The time in milliseconds to attempt a connection before timing out. - * - * Bugs: Not yet implemented */ - long connectTimeoutMS; + long connectTimeoutMS = 10_000; /** * The time in milliseconds to attempt a send or receive on a socket before * the attempt times out. * - * Bugs: Not yet implemented + * Bugs: Not implemented for sending */ long socketTimeoutMS; From 738286c4dcfa6844afce604fb643a3ce6f9f1796 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Thu, 22 Sep 2022 23:34:35 +0200 Subject: [PATCH 03/36] support MongoDB authSource database setting --- mongodb/vibe/db/mongo/connection.d | 7 +++--- mongodb/vibe/db/mongo/settings.d | 40 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index 975e784636..cb930351f2 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -553,6 +553,7 @@ final class MongoConnection { private void certAuthenticate() { + string cn = m_settings.getAuthDatabase ~ ".$cmd"; Bson cmd = Bson.emptyObject; cmd["authenticate"] = Bson(1); cmd["mechanism"] = Bson("MONGODB-X509"); @@ -568,7 +569,7 @@ final class MongoConnection { cmd["user"] = Bson(m_settings.username); } - query!Bson("$external.$cmd", QueryFlags.None, 0, -1, cmd, Bson(null), + query!Bson(cn, QueryFlags.None, 0, -1, cmd, Bson(null), (cursor, flags, first_doc, num_docs) { if ((flags & ReplyFlags.QueryFailure) || num_docs != 1) throw new MongoDriverException("Calling authenticate failed."); @@ -582,7 +583,7 @@ final class MongoConnection { private void authenticate() { - string cn = (m_settings.database == string.init ? "admin" : m_settings.database) ~ ".$cmd"; + string cn = m_settings.getAuthDatabase ~ ".$cmd"; string nonce, key; @@ -621,7 +622,7 @@ final class MongoConnection { private void scramAuthenticate() { import vibe.db.mongo.sasl; - string cn = (m_settings.database == string.init ? "admin" : m_settings.database) ~ ".$cmd"; + string cn = m_settings.getAuthDatabase ~ ".$cmd"; ScramState state; string payload = state.createInitialRequest(m_settings.username); diff --git a/mongodb/vibe/db/mongo/settings.d b/mongodb/vibe/db/mongo/settings.d index e1a15a3d94..f656cd0899 100644 --- a/mongodb/vibe/db/mongo/settings.d +++ b/mongodb/vibe/db/mongo/settings.d @@ -153,6 +153,8 @@ bool parseMongoDBUrl(out MongoClientSettings cfg, string url) } switch( option.toLower() ){ + import std.string : split; + default: logWarn("Unknown MongoDB option %s", option); break; case "appname": cfg.appName = value; break; case "slaveok": bool v; if( setBool(v) && v ) cfg.defQueryFlags |= QueryFlags.SlaveOk; break; @@ -166,6 +168,8 @@ bool parseMongoDBUrl(out MongoClientSettings cfg, string url) case "ssl": setBool(cfg.ssl); break; case "sslverifycertificate": setBool(cfg.sslverifycertificate); break; case "authmechanism": cfg.authMechanism = parseAuthMechanism(value); break; + case "authmechanismproperties": cfg.authMechanismProperties = value.split(","); warnNotImplemented(); break; + case "authsource": cfg.authSource = value; break; case "wtimeoutms": setLong(cfg.wTimeoutMS); break; case "w": try { @@ -490,6 +494,20 @@ class MongoClientSettings */ string sslCAFile; + /** + * Specify the database name associated with the user's credentials. If + * `authSource` is unspecified, `authSource` defaults to the `defaultauthdb` + * specified in the connection string. If `defaultauthdb` is unspecified, + * then `authSource` defaults to `admin`. + * + * The `PLAIN` (LDAP), `GSSAPI` (Kerberos), and `MONGODB-AWS` (IAM) + * authentication mechanisms require that `authSource` be set to `$external`, + * as these mechanisms delegate credential storage to external services. + * + * Ignored if no username is provided. + */ + string authSource; + /** * Use the given authentication mechanism when connecting to the server. If * unsupported by the server, throw a MongoAuthException. @@ -499,6 +517,14 @@ class MongoClientSettings */ MongoAuthMechanism authMechanism; + /** + * Specify properties for the specified authMechanism as a comma-separated + * list of colon-separated key-value pairs. + * + * Currently none are used by the vibe.d Mongo driver. + */ + string[] authMechanismProperties; + /** * Application name for the connection information when connected. * @@ -553,6 +579,20 @@ class MongoClientSettings this.sslPEMKeyFile = sslPEMKeyFile; this.sslCAFile = sslCAFile; } + + /** + * Resolves the database to run authentication commands on. + * (authSource if set, otherwise the URI's database if set, otherwise "admin") + */ + string getAuthDatabase() @safe @nogc nothrow pure + { + if (authSource.length) + return authSource; + else if (database.length) + return database; + else + return "admin"; + } } /// Describes a host we might be able to connect to From b5286d3859e3669894e47f64a8ebf57f8fb7ab43 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Thu, 22 Sep 2022 23:35:58 +0200 Subject: [PATCH 04/36] start on opMsg implementation --- mongodb/vibe/db/mongo/connection.d | 158 ++++++++++++++++++++++++++--- 1 file changed, 142 insertions(+), 16 deletions(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index cb930351f2..5def3fef5c 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -133,6 +133,7 @@ final class MongoConnection { /// Flag to prevent recursive connections when server closes connection while connecting bool m_allowReconnect; bool m_isAuthenticating; + bool m_supportsOpMsg; } enum ushort defaultPort = MongoClientSettings.defaultPort; @@ -163,6 +164,8 @@ final class MongoConnection { * options such as connect timeouts and so on. */ try { + import core.time : msecs; + m_conn = connectTCP(m_settings.hosts[0].name, m_settings.hosts[0].port, null, 0, m_settings.connectTimeoutMS.msecs); m_conn.tcpNoDelay = true; m_conn.readTimeout = m_settings.socketTimeoutMS.msecs; @@ -196,7 +199,22 @@ final class MongoConnection { m_allowReconnect = true; Bson handshake = Bson.emptyObject; - handshake["isMaster"] = Bson(1); + static assert(!is(typeof(m_settings.loadBalanced)), "loadBalanced was added to the API, set legacy if it's true here!"); + // TODO: must use legacy handshake if m_settings.loadBalanced is true + // and also once we allow configuring a server API version in the driver + // (https://github.com/mongodb/specifications/blob/master/source/versioned-api/versioned-api.rst) + m_supportsOpMsg = false; + bool legacyHandshake = false; + if (legacyHandshake) + { + handshake["isMaster"] = Bson(1); + handshake["helloOk"] = Bson(1); + } + else + { + handshake["hello"] = Bson(1); + m_supportsOpMsg = true; + } import os = std.system; import compiler = std.compiler; @@ -216,7 +234,7 @@ final class MongoConnection { handshake["client"]["application"] = Bson(["name": Bson(m_settings.appName)]); } - query!Bson("$external.$cmd", QueryFlags.none, 0, -1, handshake, Bson(null), + runCommand!Bson("admin", handshake, (cursor, flags, first_doc, num_docs) { enforce!MongoDriverException(!(flags & ReplyFlags.QueryFailure) && num_docs == 1, "Authentication handshake failed."); @@ -226,6 +244,9 @@ final class MongoConnection { m_description = deserializeBson!ServerDescription(doc); }); + if (m_description.satisfiesVersion(WireVersion.v36)) + m_supportsOpMsg = true; + m_bytesRead = 0; auto authMechanism = m_settings.authMechanism; if (authMechanism == MongoAuthMechanism.none) @@ -311,6 +332,7 @@ final class MongoConnection { if (m_settings.safe) checkForError(collection_name); } + deprecated("Non-functional since MongoDB 5.1: use `find` to query collections instead - instead of `$cmd` use `runCommand` to send commands - use listIndices and listCollections instead of `.system.indexes` and `.system.namsepsaces`") void query(T)(string collection_name, QueryFlags flags, int nskip, int nret, Bson query, Bson returnFieldSelector, scope ReplyDelegate on_msg, scope DocDelegate!T on_doc) { scope(failure) disconnect(); @@ -323,6 +345,22 @@ final class MongoConnection { recvReply!T(id, on_msg, on_doc); } + void runCommand(T)(string database, Bson command, scope ReplyDelegate on_msg, scope DocDelegate!T on_doc) + { + scope (failure) disconnect(); + if (m_supportsOpMsg) + { + command["$db"] = Bson(database); + auto id = sendMsg(-1, 0, command); + recvMsg!T(id, on_msg, on_doc); + } + else + { + auto id = send(OpCode.Query, -1, 0, database ~ ".$cmd", 0, -1, command, Bson(null)); + recvReply!T(id, on_msg, on_doc); + } + } + void getMore(T)(string collection_name, int nret, long cursor_id, scope ReplyDelegate on_msg, scope DocDelegate!T on_doc) { scope(failure) disconnect(); @@ -430,6 +468,50 @@ final class MongoConnection { return result.byValue.map!toInfo; } + private int recvMsg(T)(int reqid, scope ReplyDelegate on_msg, scope DocDelegate!T on_doc) @trusted + { + import std.traits; + + auto bytes_read = m_bytesRead; + int msglen = recvInt(); + int resid = recvInt(); + int respto = recvInt(); + int opcode = recvInt(); + + enforce(respto == reqid, "Reply is not for the expected message on a sequential connection!"); + enforce(opcode == OpCode.Msg, "Got wrong reply type! (must be OP_MSG)"); + + import std.stdio : stderr; + uint flagBits = recvUInt(); + stderr.writefln!"flag bits: %b"(flagBits); + int sectionLength = cast(int)(msglen - 4 * int.sizeof - flagBits.sizeof); + if ((flagBits & (1 << 16)) != 0) + sectionLength -= uint.sizeof; // CRC present + while (m_bytesRead - bytes_read < sectionLength) { + ubyte[256] buf = void; + ubyte[] bufsl = buf; + ubyte payloadType = recvUByte(); + switch (payloadType) { + case 0: + stderr.writeln("payload 0: ", recvBson(bufsl)); + break; + case 1: + auto section_bytes_read = m_bytesRead; + int size = recvInt(); + auto identifier = recvCString(); + stderr.writeln("payload 1 = \"", identifier, "\" (size ", size, ")"); + while (m_bytesRead - section_bytes_read < size) { + stderr.writeln("payload 1 section: ", recvBson(bufsl)); + } + break; + default: + throw new MongoDriverException("Received unexpected payload section type"); + } + } + + assert(false); + } + private int recvReply(T)(int reqid, scope ReplyDelegate on_msg, scope DocDelegate!T on_doc) { import std.traits; @@ -504,10 +586,32 @@ final class MongoConnection { return id; } + private int sendMsg(int response_to, uint flagBits, Bson document) + { + if( !connected() ) { + if (m_allowReconnect) connect(); + else if (m_isAuthenticating) throw new MongoAuthException("Connection got closed while authenticating"); + else throw new MongoDriverException("Connection got closed while connecting"); + } + int id = nextMessageId(); + // sendValue!int to make sure we don't accidentally send other types after arithmetic operations/changing types + sendValue!int(21 + sendLength(document)); + sendValue!int(id); + sendValue!int(response_to); + sendValue!int(cast(int)OpCode.Msg); + sendValue!uint(flagBits); + sendValue!ubyte(0); + sendValue(document); + m_outRange.flush(); + // logDebugV("Sent mongo opcode %s (id %s) in response to %s with args %s", code, id, response_to, tuple(args)); + return id; + } + private void sendValue(T)(T value) { import std.traits; - static if (is(T == int)) sendBytes(toBsonData(value)); + static if (is(T == ubyte)) m_outRange.put(value); + else static if (is(T == int) || is(T == uint)) sendBytes(toBsonData(value)); else static if (is(T == long)) sendBytes(toBsonData(value)); else static if (is(T == Bson)) sendBytes(value.data); else static if (is(T == string)) { @@ -521,8 +625,11 @@ final class MongoConnection { private void sendBytes(in ubyte[] data){ m_outRange.put(data); } - private int recvInt() { ubyte[int.sizeof] ret; recv(ret); return fromBsonData!int(ret); } - private long recvLong() { ubyte[long.sizeof] ret; recv(ret); return fromBsonData!long(ret); } + private T recvInteger(T)() { ubyte[T.sizeof] ret; recv(ret); return fromBsonData!T(ret); } + private alias recvUByte = recvInteger!ubyte; + private alias recvInt = recvInteger!int; + private alias recvUInt = recvInteger!uint; + private alias recvLong = recvInteger!long; private Bson recvBson(ref ubyte[] buf) @system { int len = recvInt(); @@ -537,6 +644,18 @@ final class MongoConnection { return Bson(Bson.Type.Object, cast(immutable)dst); } private void recv(ubyte[] dst) { enforce(m_stream); m_stream.read(dst); m_bytesRead += dst.length; } + private const(char)[] recvCString() + { + auto buf = new ubyte[32]; + ptrdiff_t i = -1; + do + { + i++; + if (i == buf.length) buf.length *= 2; + recv(buf[i .. i + 1]); + } while (buf[i] != 0); + return cast(const(char)[])buf[0 .. i]; + } private int nextMessageId() { return m_msgid++; } @@ -679,14 +798,16 @@ final class MongoConnection { private enum OpCode : int { Reply = 1, // sent only by DB - Msg = 1000, Update = 2001, Insert = 2002, Reserved1 = 2003, Query = 2004, GetMore = 2005, Delete = 2006, - KillCursors = 2007 + KillCursors = 2007, + + Compressed = 2012, + Msg = 2013, } alias ReplyDelegate = void delegate(long cursor, ReplyFlags flags, int first_doc, int num_docs) @safe; @@ -759,15 +880,20 @@ struct ServerDescription enum WireVersion : int { - old, - v26, - v26_2, - v30, - v32, - v34, - v36, - v40, - v42 + old = 0, + v26 = 1, + v26_2 = 2, + v30 = 3, + v32 = 4, + v34 = 5, + v36 = 6, + v40 = 7, + v42 = 8, + v44 = 9, + v50 = 13, + v51 = 14, + v52 = 15, + v53 = 16 } private string getHostArchitecture() From 579294ad26ec4f49eeaa22e09b9c960278279681 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Thu, 22 Sep 2022 23:51:11 +0200 Subject: [PATCH 05/36] fixes --- mongodb/vibe/db/mongo/connection.d | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index 5def3fef5c..b40a00742e 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -164,11 +164,16 @@ final class MongoConnection { * options such as connect timeouts and so on. */ try { - import core.time : msecs; + import core.time : Duration, msecs; - m_conn = connectTCP(m_settings.hosts[0].name, m_settings.hosts[0].port, null, 0, m_settings.connectTimeoutMS.msecs); + auto connectTimeout = m_settings.connectTimeoutMS.msecs; + if (m_settings.connectTimeoutMS == 0) + connectTimeout = Duration.max; + + m_conn = connectTCP(m_settings.hosts[0].name, m_settings.hosts[0].port, null, 0, connectTimeout); m_conn.tcpNoDelay = true; - m_conn.readTimeout = m_settings.socketTimeoutMS.msecs; + if (m_settings.socketTimeoutMS) + m_conn.readTimeout = m_settings.socketTimeoutMS.msecs; if (m_settings.ssl) { auto ctx = createTLSContext(TLSContextKind.client); if (!m_settings.sslverifycertificate) { @@ -603,7 +608,10 @@ final class MongoConnection { sendValue!ubyte(0); sendValue(document); m_outRange.flush(); - // logDebugV("Sent mongo opcode %s (id %s) in response to %s with args %s", code, id, response_to, tuple(args)); + (() @trusted { + import std.stdio : stderr; + stderr.writefln("Sent mongo msg in response to %s with args %s", id, response_to, flagBits, document); + })(); return id; } From 0be86d9e10a8bdbd36c8df866ecf7f54a929d83c Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 23 Sep 2022 00:45:19 +0200 Subject: [PATCH 06/36] MongoDB runCommand unification --- mongodb/vibe/db/mongo/connection.d | 93 +++++++++++++++++++----------- 1 file changed, 58 insertions(+), 35 deletions(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index b40a00742e..aceda1cbd0 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -239,15 +239,9 @@ final class MongoConnection { handshake["client"]["application"] = Bson(["name": Bson(m_settings.appName)]); } - runCommand!Bson("admin", handshake, - (cursor, flags, first_doc, num_docs) { - enforce!MongoDriverException(!(flags & ReplyFlags.QueryFailure) && num_docs == 1, - "Authentication handshake failed."); - }, - (idx, ref doc) { - enforce!MongoAuthException(doc["ok"].get!double == 1.0, "Authentication failed."); - m_description = deserializeBson!ServerDescription(doc); - }); + auto reply = runCommand!Bson("admin", handshake); + enforce!MongoAuthException(reply["ok"].get!double == 1.0, "Authentication failed."); + m_description = deserializeBson!ServerDescription(reply); if (m_description.satisfiesVersion(WireVersion.v36)) m_supportsOpMsg = true; @@ -350,19 +344,47 @@ final class MongoConnection { recvReply!T(id, on_msg, on_doc); } - void runCommand(T)(string database, Bson command, scope ReplyDelegate on_msg, scope DocDelegate!T on_doc) + Bson runCommand(T)(string database, Bson command) { + import std.array; + scope (failure) disconnect(); if (m_supportsOpMsg) { command["$db"] = Bson(database); auto id = sendMsg(-1, 0, command); - recvMsg!T(id, on_msg, on_doc); + Bson ret; + Appender!(Bson[])[string] docs; + recvMsg(id, (flags, root) { + ret = root; + }, (ident, size) { + docs[ident] = appender!(Bson[]); + }, (ident, push) { + docs[ident].put(push); + }); + + foreach (ident, app; docs) + ret[ident] = Bson(app.data); + + static if (is(T == Bson)) return ret; + else { + T doc = deserializeBson!T(bson); + return doc; + } } else { auto id = send(OpCode.Query, -1, 0, database ~ ".$cmd", 0, -1, command, Bson(null)); - recvReply!T(id, on_msg, on_doc); + T ret; + recvReply!T(id, + (cursor, flags, first_doc, num_docs) { + enforce!MongoDriverException(!(flags & ReplyFlags.QueryFailure) && num_docs == 1, + "command failed or returned more than one document"); + }, + (idx, ref doc) { + ret = deserializeBson!T(doc); + }); + return ret; } } @@ -473,7 +495,7 @@ final class MongoConnection { return result.byValue.map!toInfo; } - private int recvMsg(T)(int reqid, scope ReplyDelegate on_msg, scope DocDelegate!T on_doc) @trusted + private int recvMsg(int reqid, scope MsgReplyDelegate on_sec0, scope MsgSection1StartDelegate on_sec1_start, scope MsgSection1Delegate on_sec1_doc) { import std.traits; @@ -486,27 +508,28 @@ final class MongoConnection { enforce(respto == reqid, "Reply is not for the expected message on a sequential connection!"); enforce(opcode == OpCode.Msg, "Got wrong reply type! (must be OP_MSG)"); - import std.stdio : stderr; uint flagBits = recvUInt(); - stderr.writefln!"flag bits: %b"(flagBits); int sectionLength = cast(int)(msglen - 4 * int.sizeof - flagBits.sizeof); if ((flagBits & (1 << 16)) != 0) sectionLength -= uint.sizeof; // CRC present + bool gotSec0; while (m_bytesRead - bytes_read < sectionLength) { - ubyte[256] buf = void; - ubyte[] bufsl = buf; ubyte payloadType = recvUByte(); switch (payloadType) { case 0: - stderr.writeln("payload 0: ", recvBson(bufsl)); + gotSec0 = true; + on_sec0(flagBits, recvBsonDup()); break; case 1: + if (!gotSec0) + throw new MongoDriverException("Got OP_MSG section 1 before section 0, which is not supported by vibe.d"); + auto section_bytes_read = m_bytesRead; int size = recvInt(); auto identifier = recvCString(); - stderr.writeln("payload 1 = \"", identifier, "\" (size ", size, ")"); + on_sec1_start(identifier, size); while (m_bytesRead - section_bytes_read < size) { - stderr.writeln("payload 1 section: ", recvBson(bufsl)); + on_sec1_doc(identifier, recvBsonDup()); } break; default: @@ -514,7 +537,7 @@ final class MongoConnection { } } - assert(false); + return resid; } private int recvReply(T)(int reqid, scope ReplyDelegate on_msg, scope DocDelegate!T on_doc) @@ -551,14 +574,7 @@ final class MongoConnection { static if (hasIndirections!T || is(T == Bson)) auto buf = new ubyte[msglen - cast(size_t)(m_bytesRead - bytes_read)]; foreach (i; 0 .. cast(size_t)numret) { - // TODO: directly deserialize from the wire - static if (!hasIndirections!T && !is(T == Bson)) { - ubyte[256] buf = void; - ubyte[] bufsl = buf; - auto bson = () @trusted { return recvBson(bufsl); } (); - } else { - auto bson = () @trusted { return recvBson(buf); } (); - } + auto bson = recvBsonDup(); // logDebugV("Received mongo response on %s:%s: %s", reqid, i, bson); @@ -608,10 +624,6 @@ final class MongoConnection { sendValue!ubyte(0); sendValue(document); m_outRange.flush(); - (() @trusted { - import std.stdio : stderr; - stderr.writefln("Sent mongo msg in response to %s with args %s", id, response_to, flagBits, document); - })(); return id; } @@ -651,6 +663,13 @@ final class MongoConnection { recv(dst[4 .. $]); return Bson(Bson.Type.Object, cast(immutable)dst); } + private Bson recvBsonDup() + @trusted { + ubyte[4] size; + recv(size[]); + ubyte[] dst = new ubyte[fromBsonData!uint(size)]; + return Bson(Bson.Type.Object, cast(immutable)dst); + } private void recv(ubyte[] dst) { enforce(m_stream); m_stream.read(dst); m_bytesRead += dst.length; } private const(char)[] recvCString() { @@ -818,8 +837,12 @@ private enum OpCode : int { Msg = 2013, } -alias ReplyDelegate = void delegate(long cursor, ReplyFlags flags, int first_doc, int num_docs) @safe; -template DocDelegate(T) { alias DocDelegate = void delegate(size_t idx, ref T doc) @safe; } +private alias ReplyDelegate = void delegate(long cursor, ReplyFlags flags, int first_doc, int num_docs) @safe; +private template DocDelegate(T) { alias DocDelegate = void delegate(size_t idx, ref T doc) @safe; } + +private alias MsgReplyDelegate = void delegate(uint flags, Bson document) @safe; +private alias MsgSection1StartDelegate = void delegate(scope const(char)[] identifier, int size) @safe; +private alias MsgSection1Delegate = void delegate(scope const(char)[] identifier, Bson document) @safe; struct MongoDBInfo { From e408d19a498e0f6287cb9161dd54bcabe96f3cf5 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 23 Sep 2022 00:50:21 +0200 Subject: [PATCH 07/36] support CRC --- mongodb/vibe/db/mongo/connection.d | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index aceda1cbd0..cf95c36691 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -533,10 +533,16 @@ final class MongoConnection { } break; default: - throw new MongoDriverException("Received unexpected payload section type"); + throw new MongoDriverException("Received unexpected payload section type " ~ payloadType.to!string); } } + if ((flagBits & (1 << 16)) != 0) + { + uint crc = recvUInt(); + // TODO: validate CRC + } + return resid; } From eee21de8315345753a866e674246c7471d192792 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 23 Sep 2022 00:53:33 +0200 Subject: [PATCH 08/36] add assert to avoid mongo packet corruption --- mongodb/vibe/db/mongo/connection.d | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index cf95c36691..b9081dc971 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -543,6 +543,10 @@ final class MongoConnection { // TODO: validate CRC } + assert(bytes_read + msglen == m_bytesRead, + format!"Packet size mismatch! Expected %s bytes, but read %s."( + msglen, m_bytesRead - bytes_read)); + return resid; } From 99923b5473fc6f5e41faa83431c61985c2818a29 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 23 Sep 2022 01:30:23 +0200 Subject: [PATCH 09/36] fix recvBsonDup --- mongodb/vibe/db/mongo/connection.d | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index b9081dc971..ffa152f150 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -505,8 +505,8 @@ final class MongoConnection { int respto = recvInt(); int opcode = recvInt(); - enforce(respto == reqid, "Reply is not for the expected message on a sequential connection!"); - enforce(opcode == OpCode.Msg, "Got wrong reply type! (must be OP_MSG)"); + enforce!MongoDriverException(respto == reqid, "Reply is not for the expected message on a sequential connection!"); + enforce!MongoDriverException(opcode == OpCode.Msg, "Got wrong reply type! (must be OP_MSG)"); uint flagBits = recvUInt(); int sectionLength = cast(int)(msglen - 4 * int.sizeof - flagBits.sizeof); @@ -560,8 +560,8 @@ final class MongoConnection { int respto = recvInt(); int opcode = recvInt(); - enforce(respto == reqid, "Reply is not for the expected message on a sequential connection!"); - enforce(opcode == OpCode.Reply, "Got a non-'Reply' reply!"); + enforce!MongoDriverException(respto == reqid, "Reply is not for the expected message on a sequential connection!"); + enforce!MongoDriverException(opcode == OpCode.Reply, "Got a non-'Reply' reply!"); auto flags = cast(ReplyFlags)recvInt(); long cursor = recvLong(); @@ -678,6 +678,7 @@ final class MongoConnection { ubyte[4] size; recv(size[]); ubyte[] dst = new ubyte[fromBsonData!uint(size)]; + recv(dst); return Bson(Bson.Type.Object, cast(immutable)dst); } private void recv(ubyte[] dst) { enforce(m_stream); m_stream.read(dst); m_bytesRead += dst.length; } From 31cdc55bf196fbd3504e5bf03e4291bf610b221a Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 23 Sep 2022 01:43:16 +0200 Subject: [PATCH 10/36] use connect options timeouts --- tests/mongodb/_connection/source/app.d | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/mongodb/_connection/source/app.d b/tests/mongodb/_connection/source/app.d index b4bcc2b1fa..49f3c490e6 100644 --- a/tests/mongodb/_connection/source/app.d +++ b/tests/mongodb/_connection/source/app.d @@ -20,10 +20,12 @@ int main(string[] args) return 1; } - runTask({ sleep(10.seconds); assert(false, "Timeout exceeded"); }); + runTask({ sleep(20.seconds); assert(false, "Timeout exceeded"); }); port = args[1].to!ushort; MongoClientSettings settings = new MongoClientSettings; + settings.connectTimeoutMS = 5_000; + settings.socketTimeoutMS = 2_000; int authStep = 0; foreach (arg; args[2 .. $]) From 0f3f5d9567be9fe7fd523d304ba37a314d54e731 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 23 Sep 2022 01:46:08 +0200 Subject: [PATCH 11/36] fix recvBsonDup 2 --- mongodb/vibe/db/mongo/connection.d | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index ffa152f150..8bd4d9cd33 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -678,7 +678,8 @@ final class MongoConnection { ubyte[4] size; recv(size[]); ubyte[] dst = new ubyte[fromBsonData!uint(size)]; - recv(dst); + dst[0 .. 4] = size; + recv(dst[4 .. $]); return Bson(Bson.Type.Object, cast(immutable)dst); } private void recv(ubyte[] dst) { enforce(m_stream); m_stream.read(dst); m_bytesRead += dst.length; } From 71763f0c14dee4213a3010f3b69fe7c0b77b5e62 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 23 Sep 2022 16:38:53 +0200 Subject: [PATCH 12/36] fix unittests --- mongodb/vibe/db/mongo/settings.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb/vibe/db/mongo/settings.d b/mongodb/vibe/db/mongo/settings.d index f656cd0899..33941e819f 100644 --- a/mongodb/vibe/db/mongo/settings.d +++ b/mongodb/vibe/db/mongo/settings.d @@ -215,7 +215,7 @@ unittest assert(cfg.wTimeoutMS == long.init); assert(cfg.fsync == false); assert(cfg.journal == false); - assert(cfg.connectTimeoutMS == long.init); + assert(cfg.connectTimeoutMS == 10_000); assert(cfg.socketTimeoutMS == long.init); assert(cfg.ssl == bool.init); assert(cfg.sslverifycertificate == true); From 5e87e5fb7b75067c7edb0218345439d046f891f1 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 23 Sep 2022 17:39:28 +0200 Subject: [PATCH 13/36] use new Mongo runCommand method --- mongodb/vibe/db/mongo/collection.d | 34 ++---- mongodb/vibe/db/mongo/connection.d | 183 +++++++++-------------------- mongodb/vibe/db/mongo/database.d | 33 ++++-- 3 files changed, 93 insertions(+), 157 deletions(-) diff --git a/mongodb/vibe/db/mongo/collection.d b/mongodb/vibe/db/mongo/collection.d index dae19dd66a..53baf2cc48 100644 --- a/mongodb/vibe/db/mongo/collection.d +++ b/mongodb/vibe/db/mongo/collection.d @@ -205,8 +205,7 @@ struct MongoCollection { cmd.query = query; cmd.update = update; cmd.fields = returnFieldSelector; - auto ret = database.runCommand(cmd); - if( !ret["ok"].get!double ) throw new Exception("findAndModify failed."); + auto ret = database.runCommand(cmd, true); return ret["value"]; } @@ -245,8 +244,7 @@ struct MongoCollection { cmd[key] = value; return 0; }); - auto ret = database.runCommand(cmd); - enforce(ret["ok"].get!double != 0, "findAndModifyExt failed: "~ret["errmsg"].opt!string); + auto ret = database.runCommand(cmd, true); return ret["value"]; } @@ -336,8 +334,7 @@ struct MongoCollection { continue; cmd[k] = v; } - auto ret = database.runCommand(cmd); - enforce(ret["ok"].get!double == 1, "Aggregate command failed: "~ret["errmsg"].opt!string); + auto ret = database.runCommand(cmd, true); R[] existing; static if (is(R == Bson)) existing = ret["cursor"]["firstBatch"].get!(Bson[]); @@ -404,10 +401,8 @@ struct MongoCollection { cmd.distinct = m_name; cmd.key = key; cmd.query = query; - auto res = m_db.runCommand(cmd); - - enforce(res["ok"].get!double != 0, "Distinct query failed: "~res["errmsg"].opt!string); + auto res = m_db.runCommand(cmd, true); static if (is(R == Bson)) return res["values"].byValue; else return res["values"].byValue.map!(b => deserializeBson!R(b)); } @@ -487,8 +482,7 @@ struct MongoCollection { CMD cmd; cmd.dropIndexes = m_name; cmd.index = name; - auto reply = database.runCommand(cmd); - enforce(reply["ok"].get!double == 1, "dropIndex command failed: "~reply["errmsg"].opt!string); + database.runCommand(cmd, true); } /// ditto @@ -536,8 +530,7 @@ struct MongoCollection { CMD cmd; cmd.dropIndexes = m_name; cmd.index = "*"; - auto reply = database.runCommand(cmd); - enforce(reply["ok"].get!double == 1, "dropIndexes command failed: "~reply["errmsg"].opt!string); + database.runCommand(cmd, true); } /// Unofficial API extension, more efficient multi-index removal on @@ -554,8 +547,7 @@ struct MongoCollection { CMD cmd; cmd.dropIndexes = m_name; cmd.index = names; - auto reply = database.runCommand(cmd); - enforce(reply["ok"].get!double == 1, "dropIndexes command failed: "~reply["errmsg"].opt!string); + database.runCommand(cmd, true); } else { foreach (name; names) dropIndex(name); @@ -653,9 +645,7 @@ struct MongoCollection { indexes ~= index; } cmd["indexes"] = Bson(indexes); - auto reply = database.runCommand(cmd); - enforce(reply["ok"].get!double == 1, "createIndex command failed: " - ~ reply["errmsg"].opt!string); + database.runCommand(cmd, true); } else { foreach (model; models) { // trusted to support old compilers which think opt_dup has @@ -688,8 +678,7 @@ struct MongoCollection { CMD cmd; cmd.listIndexes = m_name; - auto reply = database.runCommand(cmd); - enforce(reply["ok"].get!double == 1, "getIndexes command failed: "~reply["errmsg"].opt!string); + auto reply = database.runCommand(cmd, true); return MongoCursor!R(m_client, reply["cursor"]["ns"].get!string, reply["cursor"]["id"].get!long, reply["cursor"]["firstBatch"].get!(Bson[])); } else { return database["system.indexes"].find!R(); @@ -723,8 +712,7 @@ struct MongoCollection { CMD cmd; cmd.drop = m_name; - auto reply = database.runCommand(cmd); - enforce(reply["ok"].get!double == 1, "drop command failed: "~reply["errmsg"].opt!string); + database.runCommand(cmd, true); } } @@ -975,7 +963,7 @@ struct AggregateOptions { /// Specifies the initial batch size for the cursor. ref inout(Nullable!int) batchSize() - @property inout @safe pure nothrow @nogc @ignore { + return @property inout @safe pure nothrow @nogc @ignore { return cursor.batchSize; } diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index 8bd4d9cd33..4536a8ce97 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -344,16 +344,24 @@ final class MongoConnection { recvReply!T(id, on_msg, on_doc); } - Bson runCommand(T)(string database, Bson command) + Bson runCommand(T, CommandFailException = MongoDriverException)(string database, Bson command, bool testOk = true, + string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__) { import std.array; scope (failure) disconnect(); + + string formatErrorInfo(string msg) + { + return text(msg, " in ", errorInfo, " (", errorFile, ":", errorLine, ")"); + } + + Bson ret; + if (m_supportsOpMsg) { command["$db"] = Bson(database); auto id = sendMsg(-1, 0, command); - Bson ret; Appender!(Bson[])[string] docs; recvMsg(id, (flags, root) { ret = root; @@ -365,26 +373,29 @@ final class MongoConnection { foreach (ident, app; docs) ret[ident] = Bson(app.data); - - static if (is(T == Bson)) return ret; - else { - T doc = deserializeBson!T(bson); - return doc; - } } else { auto id = send(OpCode.Query, -1, 0, database ~ ".$cmd", 0, -1, command, Bson(null)); - T ret; recvReply!T(id, (cursor, flags, first_doc, num_docs) { - enforce!MongoDriverException(!(flags & ReplyFlags.QueryFailure) && num_docs == 1, - "command failed or returned more than one document"); + logTrace("runCommand(%s) flags: %s, cursor: %s, documents: %s", database, flags, cursor, num_docs); + enforce!MongoDriverException(!(flags & ReplyFlags.QueryFailure), formatErrorInfo("command query failed")); + enforce!MongoDriverException(num_docs == 1, formatErrorInfo("received more than one document in command response")); }, (idx, ref doc) { - ret = deserializeBson!T(doc); + ret = doc; }); - return ret; + } + + if (testOk && ret["ok"].get!double != 1.0) + throw new CommandFailException(formatErrorInfo("command failed: " + ~ ret["errmsg"].opt!string("(no message)"))); + + static if (is(T == Bson)) return ret; + else { + T doc = deserializeBson!T(bson); + return doc; } } @@ -428,32 +439,19 @@ final class MongoConnection { _MongoErrorDescription ret; - query!Bson(db ~ ".$cmd", QueryFlags.NoCursorTimeout | m_settings.defQueryFlags, - 0, -1, command_and_options, Bson(null), - (cursor, flags, first_doc, num_docs) { - logTrace("getLastEror(%s) flags: %s, cursor: %s, documents: %s", db, flags, cursor, num_docs); - enforce(!(flags & ReplyFlags.QueryFailure), - new MongoDriverException(format("MongoDB error: getLastError(%s) call failed.", db)) - ); - enforce( - num_docs == 1, - new MongoDriverException(format("getLastError(%s) returned %s documents instead of one.", db, num_docs)) - ); - }, - (idx, ref error) { - try { - ret = MongoErrorDescription( - error["err"].opt!string(""), - error["code"].opt!int(-1), - error["connectionId"].opt!int(-1), - error["n"].get!int(), - error["ok"].get!double() - ); - } catch (Exception e) { - throw new MongoDriverException(e.msg); - } - } - ); + auto error = runCommand!Bson(db, command_and_options); + + try { + ret = MongoErrorDescription( + error["errmsg"].opt!string(error["err"].opt!string("")), + error["code"].opt!int(-1), + error["connectionId"].opt!int(-1), + error["n"].get!int(), + error["ok"].get!double() + ); + } catch (Exception e) { + throw new MongoDriverException(e.msg); + } return ret; } @@ -465,15 +463,10 @@ final class MongoConnection { */ auto listDatabases() { - string cn = (m_settings.database == string.init ? "admin" : m_settings.database) ~ ".$cmd"; + string cn = m_settings.database == string.init ? "admin" : m_settings.database; auto cmd = Bson(["listDatabases":Bson(1)]); - void on_msg(long cursor, ReplyFlags flags, int first_doc, int num_docs) { - if ((flags & ReplyFlags.QueryFailure)) - throw new MongoDriverException("Calling listDatabases failed."); - } - static MongoDBInfo toInfo(const(Bson) db_doc) { return MongoDBInfo( db_doc["name"].get!string, @@ -482,15 +475,7 @@ final class MongoConnection { ); } - Bson result; - void on_doc(size_t idx, ref Bson doc) { - if (doc["ok"].get!double != 1.0) - throw new MongoAuthException("listDatabases failed."); - - result = doc["databases"]; - } - - query!Bson(cn, QueryFlags.None, 0, -1, cmd, Bson(null), &on_msg, &on_doc); + auto result = runCommand!Bson(cn, cmd)["databases"]; return result.byValue.map!toInfo; } @@ -711,7 +696,6 @@ final class MongoConnection { private void certAuthenticate() { - string cn = m_settings.getAuthDatabase ~ ".$cmd"; Bson cmd = Bson.emptyObject; cmd["authenticate"] = Bson(1); cmd["mechanism"] = Bson("MONGODB-X509"); @@ -727,37 +711,17 @@ final class MongoConnection { cmd["user"] = Bson(m_settings.username); } - query!Bson(cn, QueryFlags.None, 0, -1, cmd, Bson(null), - (cursor, flags, first_doc, num_docs) { - if ((flags & ReplyFlags.QueryFailure) || num_docs != 1) - throw new MongoDriverException("Calling authenticate failed."); - }, - (idx, ref doc) { - if (doc["ok"].get!double != 1.0) - throw new MongoAuthException("Authentication failed."); - } - ); + runCommand!(Bson, MongoAuthException)(m_settings.getAuthDatabase, cmd); } private void authenticate() { - string cn = m_settings.getAuthDatabase ~ ".$cmd"; - - string nonce, key; - - auto cmd = Bson(["getnonce":Bson(1)]); - query!Bson(cn, QueryFlags.None, 0, -1, cmd, Bson(null), - (cursor, flags, first_doc, num_docs) { - if ((flags & ReplyFlags.QueryFailure) || num_docs != 1) - throw new MongoDriverException("Calling getNonce failed."); - }, - (idx, ref doc) { - if (doc["ok"].get!double != 1.0) - throw new MongoDriverException("getNonce failed."); - nonce = doc["nonce"].get!string; - key = toLower(toHexString(md5Of(nonce ~ m_settings.username ~ m_settings.digest)).idup); - } - ); + string cn = m_settings.getAuthDatabase; + + auto cmd = Bson(["getnonce": Bson(1)]); + auto result = runCommand!(Bson, MongoAuthException)(cn, cmd); + string nonce = result["nonce"].get!string; + string key = toLower(toHexString(md5Of(nonce ~ m_settings.username ~ m_settings.digest)).idup); cmd = Bson.emptyObject; cmd["authenticate"] = Bson(1); @@ -765,22 +729,14 @@ final class MongoConnection { cmd["nonce"] = Bson(nonce); cmd["user"] = Bson(m_settings.username); cmd["key"] = Bson(key); - query!Bson(cn, QueryFlags.None, 0, -1, cmd, Bson(null), - (cursor, flags, first_doc, num_docs) { - if ((flags & ReplyFlags.QueryFailure) || num_docs != 1) - throw new MongoDriverException("Calling authenticate failed."); - }, - (idx, ref doc) { - if (doc["ok"].get!double != 1.0) - throw new MongoAuthException("Authentication failed."); - } - ); + runCommand!(Bson, MongoAuthException)(cn, cmd); } private void scramAuthenticate() { import vibe.db.mongo.sasl; - string cn = m_settings.getAuthDatabase ~ ".$cmd"; + + string cn = m_settings.getAuthDatabase; ScramState state; string payload = state.createInitialRequest(m_settings.username); @@ -789,49 +745,26 @@ final class MongoConnection { cmd["saslStart"] = Bson(1); cmd["mechanism"] = Bson("SCRAM-SHA-1"); cmd["payload"] = Bson(BsonBinData(BsonBinData.Type.generic, payload.representation)); - string response; - Bson conversationId; - query!Bson(cn, QueryFlags.None, 0, -1, cmd, Bson(null), - (cursor, flags, first_doc, num_docs) { - if ((flags & ReplyFlags.QueryFailure) || num_docs != 1) - throw new MongoDriverException("SASL start failed."); - }, - (idx, ref doc) { - if (doc["ok"].get!double != 1.0) - throw new MongoAuthException("Authentication failed."); - response = cast(string)doc["payload"].get!BsonBinData().rawData; - conversationId = doc["conversationId"]; - }); + + auto doc = runCommand!Bson(cn, cmd); + string response = cast(string)doc["payload"].get!BsonBinData().rawData; + Bson conversationId = doc["conversationId"]; + payload = state.update(m_settings.digest, response); cmd = Bson.emptyObject; cmd["saslContinue"] = Bson(1); cmd["conversationId"] = conversationId; cmd["payload"] = Bson(BsonBinData(BsonBinData.Type.generic, payload.representation)); - query!Bson(cn, QueryFlags.None, 0, -1, cmd, Bson(null), - (cursor, flags, first_doc, num_docs) { - if ((flags & ReplyFlags.QueryFailure) || num_docs != 1) - throw new MongoDriverException("SASL continue failed."); - }, - (idx, ref doc) { - if (doc["ok"].get!double != 1.0) - throw new MongoAuthException("Authentication failed."); - response = cast(string)doc["payload"].get!BsonBinData().rawData; - }); + + doc = runCommand!Bson(cn, cmd); + response = cast(string)doc["payload"].get!BsonBinData().rawData; payload = state.finalize(response); cmd = Bson.emptyObject; cmd["saslContinue"] = Bson(1); cmd["conversationId"] = conversationId; cmd["payload"] = Bson(BsonBinData(BsonBinData.Type.generic, payload.representation)); - query!Bson(cn, QueryFlags.None, 0, -1, cmd, Bson(null), - (cursor, flags, first_doc, num_docs) { - if ((flags & ReplyFlags.QueryFailure) || num_docs != 1) - throw new MongoDriverException("SASL finish failed."); - }, - (idx, ref doc) { - if (doc["ok"].get!double != 1.0) - throw new MongoAuthException("Authentication failed."); - }); + runCommand!Bson(cn, cmd); } } diff --git a/mongodb/vibe/db/mongo/database.d b/mongodb/vibe/db/mongo/database.d index 64b82e32bb..c3e06ff7d6 100644 --- a/mongodb/vibe/db/mongo/database.d +++ b/mongodb/vibe/db/mongo/database.d @@ -23,7 +23,6 @@ struct MongoDatabase private { string m_name; - string m_commandCollection; MongoClient m_client; } @@ -41,7 +40,6 @@ struct MongoDatabase "Compound collection path provided to MongoDatabase constructor instead of single database name" ); m_name = name; - m_commandCollection = m_name ~ ".$cmd"; } /// The name of this database @@ -94,7 +92,7 @@ struct MongoDatabase } CMD cmd; cmd.getLog = mask; - return runCommand(cmd); + return runCommand(cmd, true); } /** Performs a filesystem/disk sync of the database on the server. @@ -111,7 +109,7 @@ struct MongoDatabase } CMD cmd; cmd.async = async; - return runCommand(cmd); + return runCommand(cmd, true); } @@ -127,20 +125,37 @@ struct MongoDatabase Params: command_and_options = Bson object containing the command to be executed as well as the command parameters as fields + checkOk = usually commands respond with a `double ok` field in them, + which is not checked if this parameter is false. Explicitly + specify this parameter to avoid issues with error checking. + Currently defaults to `false` (meaning don't check "ok" field), + omitting the argument may change to `true` in the future. Returns: The raw response of the MongoDB server */ - Bson runCommand(T)(T command_and_options) + deprecated("use runCommand with explicit checkOk overload") + Bson runCommand(T)(T command_and_options, + string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__) { - return m_client.getCollection(m_commandCollection).findOne(command_and_options); + return runCommand(command_and_options, false, errorInfo, errorFile, errorLine); + } + /// ditto + Bson runCommand(T)(T command_and_options, bool checkOk, + string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__) + { + Bson cmd; + static if (is(T : Bson)) + cmd = command_and_options; + else + cmd = command_and_options.serializeToBson; + return m_client.lockConnection().runCommand!(Bson, MongoException)(m_name, cmd, checkOk, errorInfo, errorFile, errorLine); } /// ditto MongoCursor!R runListCommand(R = Bson, T)(T command_and_options) { - auto cur = runCommand(command_and_options); - if (cur["ok"].get!double != 1.0) - throw new MongoException("MongoDB list command failed: " ~ cur["errmsg"].get!string); + auto cur = runCommand(command_and_options, true); + // TODO: use cursor API auto cursorid = cur["cursor"]["id"].get!long; static if (is(R == Bson)) auto existing = cur["cursor"]["firstBatch"].get!(Bson[]); From ccc17f00ff1a7485a22e71caf9e4579e3889e495 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 23 Sep 2022 17:41:21 +0200 Subject: [PATCH 14/36] fix cursor for now --- mongodb/vibe/db/mongo/database.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb/vibe/db/mongo/database.d b/mongodb/vibe/db/mongo/database.d index c3e06ff7d6..5e9280ab3b 100644 --- a/mongodb/vibe/db/mongo/database.d +++ b/mongodb/vibe/db/mongo/database.d @@ -160,6 +160,6 @@ struct MongoDatabase static if (is(R == Bson)) auto existing = cur["cursor"]["firstBatch"].get!(Bson[]); else auto existing = cur["cursor"]["firstBatch"].deserializeBson!(R[]); - return MongoCursor!R(m_client, m_commandCollection, cursorid, existing); + return MongoCursor!R(m_client, m_name ~ ".$cmd", cursorid, existing); } } From 4e4b5c0ab9bed261859154441fd5dc86916dabda Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 23 Sep 2022 17:43:43 +0200 Subject: [PATCH 15/36] fix exception types on auth methods --- mongodb/vibe/db/mongo/connection.d | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index 4536a8ce97..49eb7311dd 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -746,7 +746,7 @@ final class MongoConnection { cmd["mechanism"] = Bson("SCRAM-SHA-1"); cmd["payload"] = Bson(BsonBinData(BsonBinData.Type.generic, payload.representation)); - auto doc = runCommand!Bson(cn, cmd); + auto doc = runCommand!(Bson, MongoAuthException)(cn, cmd); string response = cast(string)doc["payload"].get!BsonBinData().rawData; Bson conversationId = doc["conversationId"]; @@ -756,7 +756,7 @@ final class MongoConnection { cmd["conversationId"] = conversationId; cmd["payload"] = Bson(BsonBinData(BsonBinData.Type.generic, payload.representation)); - doc = runCommand!Bson(cn, cmd); + doc = runCommand!(Bson, MongoAuthException)(cn, cmd); response = cast(string)doc["payload"].get!BsonBinData().rawData; payload = state.finalize(response); @@ -764,7 +764,7 @@ final class MongoConnection { cmd["saslContinue"] = Bson(1); cmd["conversationId"] = conversationId; cmd["payload"] = Bson(BsonBinData(BsonBinData.Type.generic, payload.representation)); - runCommand!Bson(cn, cmd); + runCommand!(Bson, MongoAuthException)(cn, cmd); } } From 0d160422968c7407e0d0a74c2a5d18400073f7dd Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Sat, 24 Sep 2022 04:56:28 +0200 Subject: [PATCH 16/36] update CRUD & Cursor API --- mongodb/vibe/db/mongo/collection.d | 430 ++++++++----- mongodb/vibe/db/mongo/connection.d | 240 +++++++- mongodb/vibe/db/mongo/cursor.d | 259 +++++++- mongodb/vibe/db/mongo/database.d | 17 +- mongodb/vibe/db/mongo/flags.d | 47 +- mongodb/vibe/db/mongo/impl/crud.d | 875 +++++++++++++++++++++++++++ mongodb/vibe/db/mongo/sessionstore.d | 10 +- mongodb/vibe/db/mongo/settings.d | 14 +- 8 files changed, 1625 insertions(+), 267 deletions(-) create mode 100644 mongodb/vibe/db/mongo/impl/crud.d diff --git a/mongodb/vibe/db/mongo/collection.d b/mongodb/vibe/db/mongo/collection.d index 53baf2cc48..0c6ffaf061 100644 --- a/mongodb/vibe/db/mongo/collection.d +++ b/mongodb/vibe/db/mongo/collection.d @@ -12,6 +12,7 @@ public import vibe.db.mongo.connection; public import vibe.db.mongo.flags; public import vibe.db.mongo.impl.index; +public import vibe.db.mongo.impl.crud; import vibe.core.log; import vibe.db.mongo.client; @@ -78,6 +79,7 @@ struct MongoCollection { Throws: Exception if a DB communication error occurred. See_Also: $(LINK http://www.mongodb.org/display/DOCS/Updating) */ + deprecated("Use updateOne or updateMany taking UpdateOptions instead, this method breaks in MongoDB 5.1 and onwards.") void update(T, U)(T selector, U update, UpdateFlags flags = UpdateFlags.None) { assert(m_client !is null, "Updating uninitialized MongoCollection."); @@ -97,6 +99,7 @@ struct MongoCollection { Throws: Exception if a DB communication error occurred. See_Also: $(LINK http://www.mongodb.org/display/DOCS/Inserting) */ + deprecated("Use the overload taking options, this method breaks in MongoDB 5.1 and onwards.") void insert(T)(T document_or_documents, InsertFlags flags = InsertFlags.None) { assert(m_client !is null, "Inserting into uninitialized MongoCollection."); @@ -108,6 +111,73 @@ struct MongoCollection { conn.insert(m_fullPath, flags, docs); } + /** + Inserts the provided document(s). If a document is missing an identifier, + one is generated automatically by vibe.d. + + See_Also: $(LINK https://www.mongodb.com/docs/manual/reference/method/db.collection.insertOne/#mongodb-method-db.collection.insertOne) + + Standards: $(LINK https://www.mongodb.com/docs/manual/reference/command/insert/) + */ + InsertOneResult insertOne(T)(T document, InsertOneOptions options = InsertOneOptions.init) + { + assert(m_client !is null, "Querying uninitialized MongoCollection."); + + Bson cmd = Bson.emptyObject; // empty object because order is important + cmd["insert"] = Bson(m_name); + auto doc = serializeToBson(document); + enforce(doc.type == Bson.Type.object, "Can only insert objects into collections"); + InsertOneResult res; + if ("_id" !in doc.get!(Bson[string])) + { + doc["_id"] = Bson(res.insertedId = BsonObjectID.generate); + } + cmd["documents"] = Bson([doc]); + MongoConnection conn = m_client.lockConnection(); + enforceWireVersionConstraints(options, conn.description.maxWireVersion); + foreach (string k, v; serializeToBson(options).byKeyValue) + cmd[k] = v; + + database.runCommand(cmd, true); + return res; + } + + /// ditto + InsertOneResult insertMany(T)(T[] documents, InsertManyOptions options = InsertManyOptions.init) + { + assert(m_client !is null, "Querying uninitialized MongoCollection."); + + Bson cmd = Bson.emptyObject; // empty object because order is important + cmd["insert"] = Bson(m_name); + Bson[] arr = new Bson[documents.length]; + BsonObjectID[size_t] insertedIds; + foreach (i, document; documents) + { + auto doc = serializeToBson(document); + arr[i] = doc; + enforce(doc.type == Bson.Type.object, "Can only insert objects into collections"); + if ("_id" !in doc.get!(Bson[string])) + { + doc["_id"] = Bson(insertedIds[i] = BsonObjectID.generate); + } + } + cmd["documents"] = Bson(arr); + MongoConnection conn = m_client.lockConnection(); + enforceWireVersionConstraints(options, conn.description.maxWireVersion); + foreach (string k, v; serializeToBson(options).byKeyValue) + cmd[k] = v; + + database.runCommand(cmd, true); + return InsertManyResult(insertedIds); + } + + deprecated("Use the overload taking FindOptions instead, this method breaks in MongoDB 5.1 and onwards. Note: using a `$query` / `query` member to override the query arguments is no longer supported in the new overload.") + MongoCursor!R find(R = Bson, T, U)(T query, U returnFieldSelector, QueryFlags flags = QueryFlags.None, int num_skip = 0, int num_docs_per_chunk = 0) + { + assert(m_client !is null, "Querying uninitialized MongoCollection."); + return MongoCursor!R(m_client, m_fullPath, flags, num_skip, num_docs_per_chunk, query, returnFieldSelector); + } + /** Queries the collection for existing documents. @@ -115,17 +185,39 @@ struct MongoCollection { See_Also: $(LINK http://www.mongodb.org/display/DOCS/Querying) */ - MongoCursor!R find(R = Bson, T, U)(T query, U returnFieldSelector, QueryFlags flags = QueryFlags.None, int num_skip = 0, int num_docs_per_chunk = 0) + MongoCursor!R find(R = Bson, Q)(Q query, FindOptions options) { - assert(m_client !is null, "Querying uninitialized MongoCollection."); - return MongoCursor!R(m_client, m_fullPath, flags, num_skip, num_docs_per_chunk, query, returnFieldSelector); + return MongoCursor!R(m_client, m_fullPath, query, options); } /// ditto - MongoCursor!R find(R = Bson, T)(T query) { return find!R(query, null); } + MongoCursor!R find(R = Bson, Q)(Q query) { return find!R(query, FindOptions.init); } /// ditto - MongoCursor!R find(R = Bson)() { return find!R(Bson.emptyObject, null); } + MongoCursor!R find(R = Bson)() { return find!R(Bson.emptyObject, FindOptions.init); } + + deprecated("Use the overload taking FindOptions instead, this method breaks in MongoDB 5.1 and onwards. Note: using a `$query` / `query` member to override the query arguments is no longer supported in the new overload.") + auto findOne(R = Bson, T, U)(T query, U returnFieldSelector, QueryFlags flags = QueryFlags.None) + { + import std.traits; + import std.typecons; + + auto c = find!R(query, returnFieldSelector, flags, 0, 1); + static if (is(R == Bson)) { + foreach (doc; c) return doc; + return Bson(null); + } else static if (is(R == class) || isPointer!R || isDynamicArray!R || isAssociativeArray!R) { + foreach (doc; c) return doc; + return null; + } else { + foreach (doc; c) { + Nullable!R ret; + ret = doc; + return ret; + } + return Nullable!R.init; + } + } /** Queries the collection for existing documents. @@ -137,12 +229,13 @@ struct MongoCollection { Throws: Exception if a DB communication error or a query error occurred. See_Also: $(LINK http://www.mongodb.org/display/DOCS/Querying) */ - auto findOne(R = Bson, T, U)(T query, U returnFieldSelector, QueryFlags flags = QueryFlags.None) + auto findOne(R = Bson, T)(T query, FindOptions options = FindOptions.init) { import std.traits; import std.typecons; - auto c = find!R(query, returnFieldSelector, flags, 0, 1); + options.limit = 1; + auto c = find!R(query, options); static if (is(R == Bson)) { foreach (doc; c) return doc; return Bson(null); @@ -158,8 +251,6 @@ struct MongoCollection { return Nullable!R.init; } } - /// ditto - auto findOne(R = Bson, T)(T query) { return findOne!R(query, Bson(null)); } /** Removes documents from the collection. @@ -167,6 +258,7 @@ struct MongoCollection { Throws: Exception if a DB communication error occurred. See_Also: $(LINK http://www.mongodb.org/display/DOCS/Removing) */ + deprecated("Use deleteOne or deleteMany taking DeleteOptions instead, this method breaks in MongoDB 5.1 and onwards.") void remove(T)(T selector, DeleteFlags flags = DeleteFlags.None) { assert(m_client !is null, "Removing from uninitialized MongoCollection."); @@ -176,6 +268,7 @@ struct MongoCollection { } /// ditto + deprecated("Use deleteOne or deleteMany taking DeleteOptions instead, this method breaks in MongoDB 5.1 and onwards.") void remove()() { remove(Bson.emptyObject); } /** @@ -259,26 +352,15 @@ struct MongoCollection { } } - /** - Counts the results of the specified query expression. + deprecated("deprecated since MongoDB v4.0, use countDocuments or estimatedDocumentCount instead") + alias count = countImpl; - Throws Exception if a DB communication error occurred. - See_Also: $(LINK http://www.mongodb.org/display/DOCS/Advanced+Queries#AdvancedQueries-{{count%28%29}}) - */ - ulong count(T)(T query) + private ulong countImpl(T)(T query) { - static struct Empty {} - static struct CMD { - string count; - T query; - Empty fields; - } - - CMD cmd; - cmd.count = m_name; - cmd.query = query; - auto reply = database.runCommand(cmd); - enforce(reply["ok"].opt!double == 1 || reply["ok"].opt!int == 1, "Count command failed: "~reply["errmsg"].opt!string); + Bson cmd = Bson.emptyObject; + cmd["count"] = m_name; + cmd["query"] = serializeToBson(query); + auto reply = database.runCommand(cmd, true); switch (reply["n"].type) with (Bson.Type) { default: assert(false, "Unsupported data type in BSON reply for COUNT"); case double_: return cast(ulong)reply["n"].get!double; // v2.x @@ -287,6 +369,68 @@ struct MongoCollection { } } + /** + Returns the count of documents that match the query for a collection or + view. + + The method wraps the `$group` aggregation stage with a `$sum` expression + to perform the count. + + Throws Exception if a DB communication error occurred. + + See_Also: $(LINK https://www.mongodb.com/docs/manual/reference/method/db.collection.countDocuments/) + */ + ulong countDocuments(T)(T filter, CountOptions options = CountOptions.init) + { + // https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/crud/crud.rst#count-api-details + Bson[] pipeline = [Bson(["$match": serializeToBson(filter)])]; + if (!options.skip.isNull) + pipeline ~= Bson(["$skip": Bson(options.skip.get)]); + if (!options.limit.isNull) + pipeline ~= Bson(["$limit": Bson(options.limit.get)]); + pipeline ~= Bson(["$group": Bson([ + "_id": Bson(1), + "n": Bson(["$sum": Bson(1)]) + ])]); + AggregateOptions aggOptions; + foreach (i, field; options.tupleof) + { + enum name = CountOptions.tupleof[i].stringof; + if (name != "filter" && name != "skip" && name != "limit") + __traits(getMember, aggOptions, name) = field; + } + auto reply = aggregate(pipeline, aggOptions).front; + return reply["n"].get!long; + } + + /** + Returns the count of all documents in a collection or view. + + Throws Exception if a DB communication error occurred. + + See_Also: $(LINK https://www.mongodb.com/docs/manual/reference/method/db.collection.estimatedDocumentCount/) + */ + ulong estimatedDocumentCount(EstimatedDocumentCountOptions options = EstimatedDocumentCountOptions.init) + { + // https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/crud/crud.rst#count-api-details + MongoConnection conn = m_client.lockConnection(); + if (conn.description.satisfiesVersion(WireVersion.v49)) { + Bson[] pipeline = [ + Bson(["$collStats": Bson(["count": Bson.emptyObject])]), + Bson(["$group": Bson([ + "_id": Bson(1), + "n": Bson(["$sum": Bson("$count")]) + ])]) + ]; + AggregateOptions aggOptions; + aggOptions.maxTimeMS = options.maxTimeMS; + auto reply = aggregate(pipeline, aggOptions).front; + return reply["n"].get!long; + } else { + return countImpl(null); + } + } + /** Calculates aggregate values for the data in a collection. @@ -327,6 +471,8 @@ struct MongoCollection { Bson cmd = Bson.emptyObject; // empty object because order is important cmd["aggregate"] = Bson(m_name); cmd["pipeline"] = serializeToBson(pipeline); + MongoConnection conn = m_client.lockConnection(); + enforceWireVersionConstraints(options, conn.description.maxWireVersion); foreach (string k, v; serializeToBson(options).byKeyValue) { // spec recommends to omit cursor field when explain is true @@ -334,13 +480,11 @@ struct MongoCollection { continue; cmd[k] = v; } - auto ret = database.runCommand(cmd, true); - R[] existing; - static if (is(R == Bson)) - existing = ret["cursor"]["firstBatch"].get!(Bson[]); - else - existing = ret["cursor"]["firstBatch"].deserializeBson!(R[]); - return MongoCursor!R(m_client, ret["cursor"]["ns"].get!string, ret["cursor"]["id"].get!long, existing); + return MongoCursor!R(m_client, cmd, + !options.batchSize.isNull ? options.batchSize.get : 0, + !options.maxAwaitTimeMS.isNull ? options.maxAwaitTimeMS.get + : !options.maxTimeMS.isNull ? options.maxTimeMS.get + : long.max); } /// Example taken from the MongoDB documentation @@ -381,26 +525,28 @@ struct MongoCollection { records matching the given query. Params: - key = Name of the field for which to collect unique values + fieldName = Name of the field for which to collect unique values query = The query used to select records + options = Options to apply Returns: An input range with items of type `R` (`Bson` by default) is returned. */ - auto distinct(R = Bson, Q)(string key, Q query) + auto distinct(R = Bson, Q)(string fieldName, Q query, DistinctOptions options = DistinctOptions.init) { - import std.algorithm : map; + assert(m_client !is null, "Querying uninitialized MongoCollection."); - static struct CMD { - string distinct; - string key; - Q query; - } - CMD cmd; - cmd.distinct = m_name; - cmd.key = key; - cmd.query = query; + Bson cmd = Bson.emptyObject; // empty object because order is important + cmd["distinct"] = Bson(m_name); + cmd["key"] = Bson(fieldName); + cmd["query"] = serializeToBson(query); + MongoConnection conn = m_client.lockConnection(); + enforceWireVersionConstraints(options, conn.description.maxWireVersion); + foreach (string k, v; serializeToBson(options).byKeyValue) + cmd[k] = v; + + import std.algorithm : map; auto res = m_db.runCommand(cmd, true); static if (is(R == Bson)) return res["values"].byValue; @@ -418,11 +564,11 @@ struct MongoCollection { auto coll = db["collection"]; coll.drop(); - coll.insert(["a": "first", "b": "foo"]); - coll.insert(["a": "first", "b": "bar"]); - coll.insert(["a": "first", "b": "bar"]); - coll.insert(["a": "second", "b": "baz"]); - coll.insert(["a": "second", "b": "bam"]); + coll.insertOne(["a": "first", "b": "foo"]); + coll.insertOne(["a": "first", "b": "bar"]); + coll.insertOne(["a": "first", "b": "bar"]); + coll.insertOne(["a": "second", "b": "baz"]); + coll.insertOne(["a": "second", "b": "bam"]); auto result = coll.distinct!string("b", ["a": "first"]); @@ -625,7 +771,8 @@ struct MongoCollection { See_Also: $(LINK https://docs.mongodb.com/manual/reference/command/createIndexes/) */ string[] createIndexes(scope const(IndexModel)[] models, - CreateIndexesOptions options = CreateIndexesOptions.init) + CreateIndexesOptions options = CreateIndexesOptions.init, + string file = __FILE__, size_t line = __LINE__) @safe { string[] keys = new string[models.length]; @@ -638,7 +785,7 @@ struct MongoCollection { // trusted to support old compilers which think opt_dup has // longer lifetime than model.options IndexOptions opt_dup = (() @trusted => model.options)(); - enforceWireVersionConstraints(opt_dup, conn.description.maxWireVersion); + enforceWireVersionConstraints(opt_dup, conn.description.maxWireVersion, file, line); Bson index = serializeToBson(opt_dup); index["key"] = model.keys; index["name"] = model.name; @@ -651,13 +798,13 @@ struct MongoCollection { // trusted to support old compilers which think opt_dup has // longer lifetime than model.options IndexOptions opt_dup = (() @trusted => model.options)(); - enforceWireVersionConstraints(opt_dup, WireVersion.old); + enforceWireVersionConstraints(opt_dup, WireVersion.old, file, line); Bson doc = serializeToBson(opt_dup); doc["v"] = 1; doc["key"] = model.keys; doc["ns"] = m_fullPath; doc["name"] = model.name; - database["system.indexes"].insert(doc); + database["system.indexes"].insertOne(doc); } } @@ -671,17 +818,11 @@ struct MongoCollection { @safe { MongoConnection conn = m_client.lockConnection(); if (conn.description.satisfiesVersion(WireVersion.v30)) { - static struct CMD { - string listIndexes; - } - - CMD cmd; - cmd.listIndexes = m_name; - - auto reply = database.runCommand(cmd, true); - return MongoCursor!R(m_client, reply["cursor"]["ns"].get!string, reply["cursor"]["id"].get!long, reply["cursor"]["firstBatch"].get!(Bson[])); + Bson command = Bson.emptyObject; + command["listIndexes"] = Bson(m_name); + return MongoCursor!R(m_client, command); } else { - return database["system.indexes"].find!R(); + throw new MongoDriverException("listIndexes not supported on MongoDB <3.0"); } } @@ -728,11 +869,11 @@ unittest { MongoCollection users = client.getCollection("myapp.users"); // canonical version using a Bson object - users.insert(Bson(["name": Bson("admin"), "password": Bson("secret")])); + users.insertOne(Bson(["name": Bson("admin"), "password": Bson("secret")])); // short version using a string[string] AA that is automatically // serialized to Bson - users.insert(["name": "admin", "password": "secret"]); + users.insertOne(["name": "admin", "password": "secret"]); // BSON specific types are also serialized automatically auto uid = BsonObjectID.fromString("507f1f77bcf86cd799439011"); @@ -740,7 +881,7 @@ unittest { // JSON is another possibility Json jusr = parseJsonString(`{"name": "admin", "password": "secret"}`); - users.insert(jusr); + users.insertOne(jusr); } } @@ -776,7 +917,7 @@ unittest { usr.id = BsonObjectID.generate(); usr.loginName = "admin"; usr.password = "secret"; - users.insert(usr); + users.insertOne(usr); // find supports direct de-serialization of the returned documents foreach (usr2; users.find!User()) { @@ -812,6 +953,36 @@ struct ReadConcern { string level; } +struct WriteConcern { + /** + If true, wait for the the write operation to get committed to the + + See_Also: $(LINK http://docs.mongodb.org/manual/core/write-concern/#journaled) + */ + @embedNullable @name("j") + Nullable!bool journal; + + /** + When an integer, specifies the number of nodes that should acknowledge + the write and MUST be greater than or equal to 0. + + When a string, indicates tags. "majority" is defined, but users could + specify other custom error modes. + */ + @embedNullable + Nullable!Bson w; + + /** + If provided, and the write concern is not satisfied within the specified + timeout (in milliseconds), the server will return an error for the + operation. + + See_Also: $(LINK http://docs.mongodb.org/manual/core/write-concern/#timeouts) + */ + @embedNullable @name("wtimeout") + Nullable!long wtimeoutMS; +} + /** Collation allows users to specify language-specific rules for string comparison, such as rules for letter-case and accent marks. @@ -884,6 +1055,32 @@ struct MinWireVersion /// ditto MinWireVersion since(WireVersion v) @safe { return MinWireVersion(v); } +/// UDA to warn when a nullable field is set and the server wire version matches +/// the given version. (inclusive) +/// +/// Use with $(LREF enforceWireVersionConstraints) +struct DeprecatedSinceWireVersion +{ + /// + WireVersion v; +} + +/// ditto +DeprecatedSinceWireVersion deprecatedSince(WireVersion v) @safe { return DeprecatedSinceWireVersion(v); } + +/// UDA to throw a MongoException when a nullable field is set and the server +/// wire version doesn't match the version. (inclusive) +/// +/// Use with $(LREF enforceWireVersionConstraints) +struct ErrorBeforeWireVersion +{ + /// + WireVersion v; +} + +/// ditto +ErrorBeforeWireVersion errorBefore(WireVersion v) @safe { return ErrorBeforeWireVersion(v); } + /// UDA to unset a nullable field if the server wire version is newer than the /// given version. (inclusive) /// @@ -897,13 +1094,28 @@ struct MaxWireVersion MaxWireVersion until(WireVersion v) @safe { return MaxWireVersion(v); } /// Unsets nullable fields not matching the server version as defined per UDAs. -void enforceWireVersionConstraints(T)(ref T field, WireVersion serverVersion) +void enforceWireVersionConstraints(T)(ref T field, WireVersion serverVersion, + string file = __FILE__, size_t line = __LINE__) @safe { import std.traits : getUDAs; + string exception; + foreach (i, ref v; field.tupleof) { enum minV = getUDAs!(field.tupleof[i], MinWireVersion); enum maxV = getUDAs!(field.tupleof[i], MaxWireVersion); + enum deprecateV = getUDAs!(field.tupleof[i], DeprecatedSinceWireVersion); + enum errorV = getUDAs!(field.tupleof[i], ErrorBeforeWireVersion); + + static foreach (depr; deprecateV) + if (serverVersion >= depr.v && !v.isNull) + logInfo("User-set field '%s' is deprecated since MongoDB %s (from %s:%s)", + T.tupleof[i].stringof, depr.v, file, line); + + static foreach (err; errorV) + if (serverVersion < err.v && !v.isNull) + exception ~= format("User-set field '%s' is not supported before MongoDB %s\n", + T.tupleof[i].stringof, err.v); static foreach (min; minV) if (serverVersion < min.v) @@ -913,6 +1125,9 @@ void enforceWireVersionConstraints(T)(ref T field, WireVersion serverVersion) if (serverVersion > max.v) v.nullify(); } + + if (exception.length) + throw new MongoException(exception ~ "from " ~ file ~ ":" ~ line.to!string); } /// @@ -948,78 +1163,3 @@ unittest assert(!test.a.isNull); assert(test.b.isNull); } - -/** - Represents available options for an aggregate call - - See_Also: $(LINK https://docs.mongodb.com/manual/reference/method/db.collection.aggregate/) - - Standards: $(LINK https://github.com/mongodb/specifications/blob/0c6e56141c867907aacf386e0cbe56d6562a0614/source/crud/crud.rst#api) - */ -struct AggregateOptions { - // non-optional since 3.6 - // get/set by `batchSize`, undocumented in favor of that field - CursorInitArguments cursor; - - /// Specifies the initial batch size for the cursor. - ref inout(Nullable!int) batchSize() - return @property inout @safe pure nothrow @nogc @ignore { - return cursor.batchSize; - } - - // undocumented because this field isn't a spec field because it is - // out-of-scope for a driver - @embedNullable Nullable!bool explain; - - /** - Enables writing to temporary files. When set to true, aggregation - operations can write data to the _tmp subdirectory in the dbPath - directory. - */ - @embedNullable Nullable!bool allowDiskUse; - - /** - Specifies a time limit in milliseconds for processing operations on a - cursor. If you do not specify a value for maxTimeMS, operations will not - time out. - */ - @embedNullable Nullable!long maxTimeMS; - - /** - If true, allows the write to opt-out of document level validation. - This only applies when the $out or $merge stage is specified. - */ - @embedNullable Nullable!bool bypassDocumentValidation; - - /** - Specifies the read concern. Only compatible with a write stage. (e.g. - `$out`, `$merge`) - - Aggregate commands do not support the $(D ReadConcern.Level.linearizable) - level. - - Standards: $(LINK https://github.com/mongodb/specifications/blob/7745234f93039a83ae42589a6c0cdbefcffa32fa/source/read-write-concern/read-write-concern.rst) - */ - @embedNullable Nullable!ReadConcern readConcern; - - /// Specifies a collation. - @embedNullable Nullable!Collation collation; - - /** - The index to use for the aggregation. The index is on the initial - collection / view against which the aggregation is run. - - The hint does not apply to $lookup and $graphLookup stages. - - Specify the index either by the index name as a string or the index key - pattern. If specified, then the query system will only consider plans - using the hinted index. - */ - @embedNullable Nullable!Bson hint; - - /** - Users can specify an arbitrary string to help trace the operation - through the database profiler, currentOp, and logs. - */ - @embedNullable Nullable!string comment; -} diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index 49eb7311dd..3cb89efa27 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -12,8 +12,9 @@ public import vibe.data.bson; import vibe.core.core : vibeVersionString; import vibe.core.log; import vibe.core.net; -import vibe.db.mongo.settings; +import vibe.data.bson; import vibe.db.mongo.flags; +import vibe.db.mongo.settings; import vibe.inet.webform; import vibe.stream.tls; @@ -25,7 +26,9 @@ import std.exception; import std.range; import std.string; import std.typecons; +import std.traits : hasIndirections; +import core.time; private struct _MongoErrorDescription { @@ -316,14 +319,14 @@ final class MongoConnection { @property const(ServerDescription) description() const { return m_description; } - void update(string collection_name, UpdateFlags flags, Bson selector, Bson update) + deprecated("Non-functional since MongoDB 5.1") void update(string collection_name, UpdateFlags flags, Bson selector, Bson update) { scope(failure) disconnect(); send(OpCode.Update, -1, cast(int)0, collection_name, cast(int)flags, selector, update); if (m_settings.safe) checkForError(collection_name); } - void insert(string collection_name, InsertFlags flags, Bson[] documents) + deprecated("Non-functional since MongoDB 5.1") void insert(string collection_name, InsertFlags flags, Bson[] documents) { scope(failure) disconnect(); foreach (d; documents) if (d["_id"].isNull()) d["_id"] = Bson(BsonObjectID.generate()); @@ -331,7 +334,7 @@ final class MongoConnection { if (m_settings.safe) checkForError(collection_name); } - deprecated("Non-functional since MongoDB 5.1: use `find` to query collections instead - instead of `$cmd` use `runCommand` to send commands - use listIndices and listCollections instead of `.system.indexes` and `.system.namsepsaces`") + deprecated("Non-functional since MongoDB 5.1: use `find` to query collections instead - instead of `$cmd` use `runCommand` to send commands - use listIndexes and listCollections instead of `.system.indexes` and `.system.namsepsaces`") void query(T)(string collection_name, QueryFlags flags, int nskip, int nret, Bson query, Bson returnFieldSelector, scope ReplyDelegate on_msg, scope DocDelegate!T on_doc) { scope(failure) disconnect(); @@ -360,10 +363,12 @@ final class MongoConnection { if (m_supportsOpMsg) { - command["$db"] = Bson(database); + if (database !is null) + command["$db"] = Bson(database); + auto id = sendMsg(-1, 0, command); Appender!(Bson[])[string] docs; - recvMsg(id, (flags, root) { + recvMsg!true(id, (flags, root) { ret = root; }, (ident, size) { docs[ident] = appender!(Bson[]); @@ -376,6 +381,8 @@ final class MongoConnection { } else { + if (database is null) + database = "$external"; auto id = send(OpCode.Query, -1, 0, database ~ ".$cmd", 0, -1, command, Bson(null)); recvReply!T(id, (cursor, flags, first_doc, num_docs) { @@ -399,26 +406,182 @@ final class MongoConnection { } } - void getMore(T)(string collection_name, int nret, long cursor_id, scope ReplyDelegate on_msg, scope DocDelegate!T on_doc) + template getMore(T) { - scope(failure) disconnect(); - auto id = send(OpCode.GetMore, -1, cast(int)0, collection_name, nret, cursor_id); - recvReply!T(id, on_msg, on_doc); + deprecated("use the modern overload instead") + void getMore(string collection_name, int nret, long cursor_id, scope ReplyDelegate on_msg, scope DocDelegate!T on_doc) + { + scope(failure) disconnect(); + auto id = send(OpCode.GetMore, -1, cast(int)0, collection_name, nret, cursor_id); + recvReply!T(id, on_msg, on_doc); + } + + /** + * Modern (MongoDB 3.2+ compatible) getMore implementation using the getMore + * command and OP_MSG. (if supported) + * + * Falls back to compatibility for older MongoDB versions, but those are not + * officially supported anymore. + * + * Upgrade_notes: + * - error checking is now done inside this function + * - document index is no longer sent, instead the callback is called sequentially + * + * Throws: $(LREF MongoDriverException) in case the command fails. + */ + void getMore(long cursor_id, string collection_name, long nret, + scope GetMoreHeaderDelegate on_header, + scope GetMoreDocumentDelegate!T on_doc, + Duration timeout = Duration.max, + string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__) + { + Bson command = Bson.emptyObject; + command["getMore"] = Bson(cursor_id); + command["collection"] = Bson(collection_name); + command["batchSize"] = Bson(nret); + if (timeout != Duration.max && timeout.total!"msecs" < int.max) + command["maxTimeMS"] = Bson(cast(int)timeout.total!"msecs"); + + string formatErrorInfo(string msg) + { + return text(msg, " in ", errorInfo, " (", errorFile, ":", errorLine, ")"); + } + + scope (failure) disconnect(); + + if (m_supportsOpMsg) + { + enum needsDup = hasIndirections!T || is(T == Bson); + + auto id = sendMsg(-1, 0, command); + recvMsg!needsDup(id, (flags, root) { + if (root["ok"].get!double != 1.0) + throw new MongoDriverException(formatErrorInfo("getMore failed: " + ~ root["errmsg"].opt!string("(no message)"))); + + auto cursor = root["cursor"]; + auto batch = cursor["nextBatch"].get!(Bson[]); + on_header(cursor["id"].get!long, cursor["ns"].get!string, batch.length); + + foreach (ref push; batch) + { + T doc = deserializeBson!T(push); + on_doc(doc); + } + }, (ident, size) {}, (ident, push) { + throw new MongoDriverException(formatErrorInfo("unexpected section type 1 in getMore response")); + }); + } + else + { + int brokenId = 0; + int nextId = 0; + int num_docs; + // array to store out-of-order items, to push them into the callback properly + T[] compatibilitySort; + auto id = send(OpCode.GetMore, -1, cast(int)0, collection_name, nret, cursor_id); + recvReply!T(id, (long cursor, ReplyFlags flags, int first_doc, int num_docs) + { + enforce!MongoDriverException(!(flags & ReplyFlags.CursorNotFound), + formatErrorInfo("Invalid cursor handle.")); + enforce!MongoDriverException(!(flags & ReplyFlags.QueryFailure), + formatErrorInfo("Query failed. Does the database exist?")); + + on_header(cursor, collection_name, num_docs); + }, (size_t idx, ref T doc) { + if (cast(int)idx == nextId) { + on_doc(doc); + nextId++; + brokenId = nextId; + } else { + enforce!MongoDriverException(idx >= brokenId, + formatErrorInfo("Got legacy document with same id after having already processed it!")); + enforce!MongoDriverException(idx < num_docs, + formatErrorInfo("Received more documents than the database reported to us")); + + size_t arrayIndex = cast(int)idx - brokenId; + if (!compatibilitySort.length) + compatibilitySort.length = num_docs - brokenId; + compatibilitySort[arrayIndex] = doc; + } + }); + + foreach (doc; compatibilitySort) + on_doc(doc); + } + } + } + + /// Forwards the `find` command passed in to the database, handles the + /// callbacks like with getMore. This exists for easier integration with + /// MongoCursor!T. + package void startFind(T)(Bson command, + scope GetMoreHeaderDelegate on_header, + scope GetMoreDocumentDelegate!T on_doc, + string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__) + { + string formatErrorInfo(string msg) + { + return text(msg, " in ", errorInfo, " (", errorFile, ":", errorLine, ")"); + } + + scope (failure) disconnect(); + + enforce!MongoDriverException(m_supportsOpMsg, formatErrorInfo("Database does not support required OP_MSG for new style queries")); + + enum needsDup = hasIndirections!T || is(T == Bson); + + auto id = sendMsg(-1, 0, command); + recvMsg!needsDup(id, (flags, root) { + if (root["ok"].get!double != 1.0) + throw new MongoDriverException(formatErrorInfo("find failed: " + ~ root["errmsg"].opt!string("(no message)"))); + + auto cursor = root["cursor"]; + auto batch = cursor["firstBatch"].get!(Bson[]); + on_header(cursor["id"].get!long, cursor["ns"].get!string, batch.length); + + foreach (ref push; batch) + { + T doc = deserializeBson!T(push); + on_doc(doc); + } + }, (ident, size) {}, (ident, push) { + throw new MongoDriverException(formatErrorInfo("unexpected section type 1 in find response")); + }); } - void delete_(string collection_name, DeleteFlags flags, Bson selector) + deprecated("Non-functional since MongoDB 5.1") void delete_(string collection_name, DeleteFlags flags, Bson selector) { scope(failure) disconnect(); send(OpCode.Delete, -1, cast(int)0, collection_name, cast(int)flags, selector); if (m_settings.safe) checkForError(collection_name); } - void killCursors(long[] cursors) + deprecated("Non-functional since MongoDB 5.1, use the overload taking the collection as well") + void killCursors(scope long[] cursors) { scope(failure) disconnect(); send(OpCode.KillCursors, -1, cast(int)0, cast(int)cursors.length, cursors); } + void killCursors(string collection, scope long[] cursors) + { + scope(failure) disconnect(); + // TODO: could add special case to runCommand to not return anything + if (m_supportsOpMsg) + { + Bson command = Bson.emptyObject; + command["killCursors"] = Bson(collection); + command["cursors"] = cursors.serializeToBson; + runCommand!Bson(null, command); + } + else + { + send(OpCode.KillCursors, -1, cast(int)0, cast(int)cursors.length, cursors); + } + } + MongoErrorDescription getLastError(string db) { // Though higher level abstraction level by concept, this function @@ -480,7 +643,10 @@ final class MongoConnection { return result.byValue.map!toInfo; } - private int recvMsg(int reqid, scope MsgReplyDelegate on_sec0, scope MsgSection1StartDelegate on_sec1_start, scope MsgSection1Delegate on_sec1_doc) + private int recvMsg(bool dupBson = true)(int reqid, + scope MsgReplyDelegate!dupBson on_sec0, + scope MsgSection1StartDelegate on_sec1_start, + scope MsgSection1Delegate!dupBson on_sec1_doc) { import std.traits; @@ -494,16 +660,28 @@ final class MongoConnection { enforce!MongoDriverException(opcode == OpCode.Msg, "Got wrong reply type! (must be OP_MSG)"); uint flagBits = recvUInt(); + const bool hasCRC = (flagBits & (1 << 16)) != 0; + int sectionLength = cast(int)(msglen - 4 * int.sizeof - flagBits.sizeof); - if ((flagBits & (1 << 16)) != 0) + if (hasCRC) sectionLength -= uint.sizeof; // CRC present + bool gotSec0; while (m_bytesRead - bytes_read < sectionLength) { + // TODO: directly deserialize from the wire + static if (!dupBson) { + ubyte[256] buf = void; + ubyte[] bufsl = buf; + } + ubyte payloadType = recvUByte(); switch (payloadType) { case 0: gotSec0 = true; - on_sec0(flagBits, recvBsonDup()); + static if (dupBson) + on_sec0(flagBits, recvBsonDup()); + else + on_sec0(flagBits, recvBson(bufsl)); break; case 1: if (!gotSec0) @@ -514,7 +692,10 @@ final class MongoConnection { auto identifier = recvCString(); on_sec1_start(identifier, size); while (m_bytesRead - section_bytes_read < size) { - on_sec1_doc(identifier, recvBsonDup()); + static if (dupBson) + on_sec1_doc(identifier, recvBsonDup()); + else + on_sec1_doc(identifier, recvBson(bufsl)); } break; default: @@ -522,7 +703,7 @@ final class MongoConnection { } } - if ((flagBits & (1 << 16)) != 0) + if (hasCRC) { uint crc = recvUInt(); // TODO: validate CRC @@ -537,8 +718,6 @@ final class MongoConnection { private int recvReply(T)(int reqid, scope ReplyDelegate on_msg, scope DocDelegate!T on_doc) { - import std.traits; - auto bytes_read = m_bytesRead; int msglen = recvInt(); int resid = recvInt(); @@ -569,7 +748,14 @@ final class MongoConnection { static if (hasIndirections!T || is(T == Bson)) auto buf = new ubyte[msglen - cast(size_t)(m_bytesRead - bytes_read)]; foreach (i; 0 .. cast(size_t)numret) { - auto bson = recvBsonDup(); + // TODO: directly deserialize from the wire + static if (!hasIndirections!T && !is(T == Bson)) { + ubyte[256] buf = void; + ubyte[] bufsl = buf; + auto bson = () @trusted { return recvBson(bufsl); } (); + } else { + auto bson = () @trusted { return recvBson(buf); } (); + } // logDebugV("Received mongo response on %s:%s: %s", reqid, i, bson); @@ -583,7 +769,7 @@ final class MongoConnection { return resid; } - private int send(ARGS...)(OpCode code, int response_to, ARGS args) + private int send(ARGS...)(OpCode code, int response_to, scope ARGS args) { if( !connected() ) { if (m_allowReconnect) connect(); @@ -622,7 +808,7 @@ final class MongoConnection { return id; } - private void sendValue(T)(T value) + private void sendValue(T)(scope T value) { import std.traits; static if (is(T == ubyte)) m_outRange.put(value); @@ -785,9 +971,14 @@ private enum OpCode : int { private alias ReplyDelegate = void delegate(long cursor, ReplyFlags flags, int first_doc, int num_docs) @safe; private template DocDelegate(T) { alias DocDelegate = void delegate(size_t idx, ref T doc) @safe; } -private alias MsgReplyDelegate = void delegate(uint flags, Bson document) @safe; +private alias MsgReplyDelegate(bool dupBson : true) = void delegate(uint flags, Bson document) @safe; +private alias MsgReplyDelegate(bool dupBson : false) = void delegate(uint flags, scope Bson document) @safe; private alias MsgSection1StartDelegate = void delegate(scope const(char)[] identifier, int size) @safe; -private alias MsgSection1Delegate = void delegate(scope const(char)[] identifier, Bson document) @safe; +private alias MsgSection1Delegate(bool dupBson : true) = void delegate(scope const(char)[] identifier, Bson document) @safe; +private alias MsgSection1Delegate(bool dupBson : false) = void delegate(scope const(char)[] identifier, scope Bson document) @safe; + +alias GetMoreHeaderDelegate = void delegate(long id, string ns, size_t count) @safe; +alias GetMoreDocumentDelegate(T) = void delegate(ref T document) @safe; struct MongoDBInfo { @@ -866,6 +1057,7 @@ enum WireVersion : int v40 = 7, v42 = 8, v44 = 9, + v49 = 12, v50 = 13, v51 = 14, v52 = 15, diff --git a/mongodb/vibe/db/mongo/cursor.d b/mongodb/vibe/db/mongo/cursor.d index a5db41e7c9..7b6f2ca5d7 100644 --- a/mongodb/vibe/db/mongo/cursor.d +++ b/mongodb/vibe/db/mongo/cursor.d @@ -8,6 +8,7 @@ module vibe.db.mongo.cursor; public import vibe.data.bson; +public import vibe.db.mongo.impl.crud; import vibe.db.mongo.connection; import vibe.db.mongo.client; @@ -16,6 +17,7 @@ import std.array : array; import std.algorithm : map, max, min; import std.exception; +import core.time; /** Represents a cursor for a MongoDB query. @@ -25,29 +27,94 @@ import std.exception; This struct uses reference counting to destroy the underlying MongoDB cursor. */ struct MongoCursor(DocType = Bson) { - private MongoCursorData!DocType m_data; + private IMongoCursorData!DocType m_data; - package this(Q, S)(MongoClient client, string collection, QueryFlags flags, int nskip, int nret, Q query, S return_field_selector) + package deprecated this(Q, S)(MongoClient client, string collection, QueryFlags flags, int nskip, int nret, Q query, S return_field_selector) { // TODO: avoid memory allocation, if possible - m_data = new MongoFindCursor!(Q, DocType, S)(client, collection, flags, nskip, nret, query, return_field_selector); + m_data = new MongoQueryCursor!(Q, DocType, S)(client, collection, flags, nskip, nret, query, return_field_selector); } - package this(MongoClient client, string collection, long cursor, DocType[] existing_documents) + package deprecated this(MongoClient client, string collection, long cursor, DocType[] existing_documents) { // TODO: avoid memory allocation, if possible m_data = new MongoGenericCursor!DocType(client, collection, cursor, existing_documents); } + this(Q)(MongoClient client, string collection, Q query, FindOptions options) + { + Bson command = Bson.emptyObject; + command["find"] = Bson(collection); + static if (is(Q == Bson)) + command["filter"] = query; + else + command["filter"] = serializeToBson(query); + + MongoConnection conn = client.lockConnection(); + enforceWireVersionConstraints(options, conn.description.maxWireVersion); + + // https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/find_getmore_killcursors_commands.rst#mapping-op_query-behavior-to-the-find-command-limit-and-batchsize-fields + bool singleBatch; + if (!options.limit.isNull && options.limit.get < 0) + { + singleBatch = true; + options.limit = -options.limit.get; + options.batchSize = cast(int)options.limit.get; + } + if (!options.batchSize.isNull && options.batchSize.get < 0) + { + singleBatch = true; + options.batchSize = -options.batchSize.get; + } + if (singleBatch) + command["singleBatch"] = Bson(true); + + // https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/find_getmore_killcursors_commands.rst#semantics-of-maxtimems-for-a-driver + bool allowMaxTime = true; + if (options.cursorType == CursorType.tailable + || options.cursorType == CursorType.tailableAwait) + command["tailable"] = Bson(true); + else + { + options.maxAwaitTimeMS.nullify(); + allowMaxTime = false; + } + + if (options.cursorType == CursorType.tailableAwait) + command["awaitData"] = Bson(true); + else + { + options.maxAwaitTimeMS.nullify(); + allowMaxTime = false; + } + + // see table: https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/find_getmore_killcursors_commands.rst#find + auto optionsBson = serializeToBson(options); + foreach (string key, value; optionsBson.byKeyValue) + command[key] = value; + + this(client, command, + options.batchSize.isNull ? 0 : options.batchSize.get, + !options.maxAwaitTimeMS.isNull ? options.maxAwaitTimeMS.get + : allowMaxTime && !options.maxTimeMS.isNull ? options.maxTimeMS.get + : long.max); + } + + this(MongoClient client, Bson command, int batchSize = 0, long getMoreMaxTimeMS = long.max) + { + // TODO: avoid memory allocation, if possible + m_data = new MongoFindCursor!DocType(client, command, batchSize, getMoreMaxTimeMS); + } + this(this) { - if( m_data ) m_data.m_refCount++; + if( m_data ) m_data.refCount++; } ~this() { - if( m_data && --m_data.m_refCount == 0 ){ - m_data.destroy(); + if( m_data && --m_data.refCount == 0 ){ + m_data.killCursors(); } } @@ -129,7 +196,7 @@ struct MongoCursor(DocType = Bson) { See_Also: $(LINK http://docs.mongodb.org/manual/reference/method/cursor.limit) */ - MongoCursor limit(size_t count) + MongoCursor limit(long count) { m_data.limit(count); return this; @@ -149,7 +216,7 @@ struct MongoCursor(DocType = Bson) { See_Also: $(LINK http://docs.mongodb.org/manual/reference/method/cursor.skip) */ - MongoCursor skip(int count) + MongoCursor skip(long count) { m_data.skip(count); return this; @@ -167,7 +234,7 @@ struct MongoCursor(DocType = Bson) { try { coll.drop(); } catch (Exception) {} for (int i = 0; i < 10000; i++) - coll.insert(["i": i]); + coll.insertOne(["i": i]); static struct Order { int i; } auto data = coll.find().sort(Order(1)).skip(2000).limit(2000).array; @@ -197,41 +264,53 @@ struct MongoCursor(DocType = Bson) { { import std.typecons : Tuple, tuple; static struct Rng { - private MongoCursorData!DocType data; + private IMongoCursorData!DocType data; @property bool empty() { return data.empty; } - @property Tuple!(size_t, DocType) front() { return tuple(data.index, data.front); } + @property Tuple!(long, DocType) front() { return tuple(data.index, data.front); } void popFront() { data.popFront(); } } return Rng(m_data); } } +private interface IMongoCursorData(DocType) { + @property bool empty() @safe; + @property long index() @safe; + @property DocType front() @safe; + void sort(Bson order) @safe; + void limit(long count) @safe; + void skip(long count) @safe; + void popFront() @safe; + void startIterating() @safe; + void killCursors() @safe; + ref int refCount() @safe; +} + /** - Internal class exposed through MongoCursor. + Deprecated query internals exposed through MongoCursor. */ -private abstract class MongoCursorData(DocType) { +private deprecated abstract class LegacyMongoCursorData(DocType) : IMongoCursorData!DocType { private { int m_refCount = 1; MongoClient m_client; string m_collection; long m_cursor; - int m_nskip; + long m_nskip; int m_nret; Bson m_sort = Bson(null); int m_offset; size_t m_currentDoc = 0; DocType[] m_documents; bool m_iterationStarted = false; - size_t m_limit = 0; - bool m_needDestroy = false; + long m_limit = 0; } final @property bool empty() @safe { if (!m_iterationStarted) startIterating(); if (m_limit > 0 && index >= m_limit) { - destroy(); + killCursors(); return true; } if( m_currentDoc < m_documents.length ) @@ -244,7 +323,7 @@ private abstract class MongoCursorData(DocType) { return m_currentDoc >= m_documents.length; } - final @property size_t index() + final @property long index() @safe { return m_offset + m_currentDoc; } @@ -262,19 +341,19 @@ private abstract class MongoCursorData(DocType) { m_sort = order; } - final void limit(size_t count) + final void limit(long count) @safe { // A limit() value of 0 (e.g. “.limit(0)”) is equivalent to setting no limit. if (count > 0) { if (m_nret == 0 || m_nret > count) - m_nret = min(count, 1024); + m_nret = cast(int)min(count, 1024); if (m_limit == 0 || m_limit > count) m_limit = count; } } - final void skip(int count) + final void skip(long count) @safe { // A skip() value of 0 (e.g. “.skip(0)”) is equivalent to setting no skip. m_nskip = max(m_nskip, count); @@ -289,15 +368,15 @@ private abstract class MongoCursorData(DocType) { abstract void startIterating() @safe; - final private void destroy() + final void killCursors() @safe { if (m_cursor == 0) return; auto conn = m_client.lockConnection(); - conn.killCursors(() @trusted { return (&m_cursor)[0 .. 1]; } ()); + conn.killCursors(m_collection, () @trusted { return (&m_cursor)[0 .. 1]; } ()); m_cursor = 0; } - final private void handleReply(long cursor, ReplyFlags flags, int first_doc, int num_docs) + final void handleReply(long cursor, ReplyFlags flags, int first_doc, int num_docs) { enforce!MongoDriverException(!(flags & ReplyFlags.CursorNotFound), "Invalid cursor handle."); enforce!MongoDriverException(!(flags & ReplyFlags.QueryFailure), "Query failed. Does the database exist?"); @@ -308,16 +387,138 @@ private abstract class MongoCursorData(DocType) { m_currentDoc = 0; } - final private void handleDocument(size_t idx, ref DocType doc) + final void handleDocument(size_t idx, ref DocType doc) { m_documents[idx] = doc; } + + final ref int refCount() { return m_refCount; } +} + +/** + Find + getMore internals exposed through MongoCursor. Unifies the old + LegacyMongoCursorData approach, so it can be used both for find queries and + for custom commands. +*/ +private class MongoFindCursor(DocType) : IMongoCursorData!DocType { + private { + int m_refCount = 1; + MongoClient m_client; + Bson m_findQuery; + string m_collection; + long m_cursor; + int m_batchSize; + long m_maxTimeMS; + long m_totalReceived; + size_t m_readDoc; + size_t m_insertDoc; + DocType[] m_documents; + bool m_iterationStarted = false; + long m_queryLimit; + } + + this(MongoClient client, Bson command, int batchSize = 0, long getMoreMaxTimeMS = long.max) + { + m_client = client; + m_findQuery = command; + m_batchSize = batchSize; + m_maxTimeMS = getMoreMaxTimeMS; + } + + @property bool empty() + @safe { + if (!m_iterationStarted) startIterating(); + if (m_queryLimit > 0 && index >= m_queryLimit) { + killCursors(); + return true; + } + if( m_readDoc < m_documents.length ) + return false; + if( m_cursor == 0 ) + return true; + + auto conn = m_client.lockConnection(); + conn.getMore!DocType(m_cursor, m_collection, m_batchSize, &handleReply, &handleDocument, + m_maxTimeMS >= int.max ? Duration.max : m_maxTimeMS.msecs); + return m_readDoc >= m_documents.length; + } + + final @property long index() + @safe { + return m_totalReceived + m_readDoc; + } + + final @property DocType front() + @safe { + if (!m_iterationStarted) startIterating(); + assert(!empty(), "Cursor has no more data."); + return m_documents[m_readDoc]; + } + + final void sort(Bson order) + @safe { + assert(!m_iterationStarted, "Cursor cannot be modified after beginning iteration"); + m_findQuery["sort"] = order; + } + + final void limit(long count) + @safe { + assert(!m_iterationStarted, "Cursor cannot be modified after beginning iteration"); + m_findQuery["limit"] = Bson(count); + } + + final void skip(long count) + @safe { + assert(!m_iterationStarted, "Cursor cannot be modified after beginning iteration"); + m_findQuery["skip"] = Bson(count); + } + + final void popFront() + @safe { + if (!m_iterationStarted) startIterating(); + assert(!empty(), "Cursor has no more data."); + m_readDoc++; + } + + void startIterating() + @safe { + auto conn = m_client.lockConnection(); + m_totalReceived = 0; + m_queryLimit = m_findQuery["limit"].opt!long(0); + conn.startFind!DocType(m_findQuery, &handleReply, &handleDocument); + m_iterationStarted = true; + } + + final void killCursors() + @safe { + if (m_cursor == 0) return; + auto conn = m_client.lockConnection(); + conn.killCursors(m_collection, () @trusted { return (&m_cursor)[0 .. 1]; } ()); + m_cursor = 0; + } + + final void handleReply(long id, string ns, size_t count) + { + m_cursor = id; + m_collection = ns; + m_documents.length = count; + m_readDoc = 0; + m_insertDoc = 0; + } + + final void handleDocument(ref DocType doc) + { + m_documents[m_insertDoc++] = doc; + m_totalReceived++; + } + + final ref int refCount() { return m_refCount; } } /** Internal class implementing MongoCursorData for find queries */ -private class MongoFindCursor(Q, R, S) : MongoCursorData!R { +private deprecated class MongoQueryCursor(Q, R, S) : LegacyMongoCursorData!R { private { QueryFlags m_flags; Q m_query; @@ -363,7 +564,7 @@ private class MongoFindCursor(Q, R, S) : MongoCursorData!R { if (!m_sort.isNull()) full_query["orderby"] = m_sort; - conn.query!R(m_collection, m_flags, m_nskip, m_nret, full_query, selector, &handleReply, &handleDocument); + conn.query!R(m_collection, m_flags, cast(int)m_nskip, cast(int)m_nret, full_query, selector, &handleReply, &handleDocument); m_iterationStarted = true; } @@ -372,7 +573,7 @@ private class MongoFindCursor(Q, R, S) : MongoCursorData!R { /** Internal class implementing MongoCursorData for already initialized generic cursors */ -private class MongoGenericCursor(DocType) : MongoCursorData!DocType { +private deprecated class MongoGenericCursor(DocType) : LegacyMongoCursorData!DocType { this(MongoClient client, string collection, long cursor, DocType[] existing_documents) { m_client = client; diff --git a/mongodb/vibe/db/mongo/database.d b/mongodb/vibe/db/mongo/database.d index 5e9280ab3b..205bc9d514 100644 --- a/mongodb/vibe/db/mongo/database.d +++ b/mongodb/vibe/db/mongo/database.d @@ -151,15 +151,14 @@ struct MongoDatabase return m_client.lockConnection().runCommand!(Bson, MongoException)(m_name, cmd, checkOk, errorInfo, errorFile, errorLine); } /// ditto - MongoCursor!R runListCommand(R = Bson, T)(T command_and_options) + MongoCursor!R runListCommand(R = Bson, T)(T command_and_options, int batchSize = 0, long getMoreMaxTimeMS = long.max) { - auto cur = runCommand(command_and_options, true); - - // TODO: use cursor API - auto cursorid = cur["cursor"]["id"].get!long; - static if (is(R == Bson)) - auto existing = cur["cursor"]["firstBatch"].get!(Bson[]); - else auto existing = cur["cursor"]["firstBatch"].deserializeBson!(R[]); - return MongoCursor!R(m_client, m_name ~ ".$cmd", cursorid, existing); + Bson cmd; + static if (is(T : Bson)) + cmd = command_and_options; + else + cmd = command_and_options.serializeToBson; + + return MongoCursor!R(m_client, cmd, batchSize, getMoreMaxTimeMS); } } diff --git a/mongodb/vibe/db/mongo/flags.d b/mongodb/vibe/db/mongo/flags.d index 367b8d1c9f..29195ef342 100644 --- a/mongodb/vibe/db/mongo/flags.d +++ b/mongodb/vibe/db/mongo/flags.d @@ -8,52 +8,7 @@ module vibe.db.mongo.flags; deprecated public import vibe.db.mongo.impl.index : IndexFlags; - -enum UpdateFlags { - none = 0, /// Normal update of a single document. - upsert = 1<<0, /// Creates a document if none exists. - multiUpdate = 1<<1, /// Updates all matching documents. - - None = none, /// Deprecated compatibility alias - Upsert = upsert, /// Deprecated compatibility alias - MultiUpdate = multiUpdate /// Deprecated compatibility alias -} - -enum InsertFlags { - none = 0, /// Normal insert. - continueOnError = 1<<0, /// For multiple inserted documents, continues inserting further documents after a failure. - - None = none, /// Deprecated compatibility alias - ContinueOnError = continueOnError /// Deprecated compatibility alias -} - -enum QueryFlags { - none = 0, /// Normal query - tailableCursor = 1<<1, /// - slaveOk = 1<<2, /// - oplogReplay = 1<<3, /// - noCursorTimeout = 1<<4, /// - awaitData = 1<<5, /// - exhaust = 1<<6, /// - partial = 1<<7, /// - - None = none, /// Deprecated compatibility alias - TailableCursor = tailableCursor, /// Deprecated compatibility alias - SlaveOk = slaveOk, /// Deprecated compatibility alias - OplogReplay = oplogReplay, /// Deprecated compatibility alias - NoCursorTimeout = noCursorTimeout, /// Deprecated compatibility alias - AwaitData = awaitData, /// Deprecated compatibility alias - Exhaust = exhaust, /// Deprecated compatibility alias - Partial = partial /// Deprecated compatibility alias -} - -enum DeleteFlags { - none = 0, - singleRemove = 1<<0, - - None = none, /// Deprecated compatibility alias - SingleRemove = singleRemove /// Deprecated compatibility alias -} +deprecated public import vibe.db.mongo.impl.crud : UpdateFlags, InsertFlags, QueryFlags, DeleteFlags; enum ReplyFlags { none = 0, diff --git a/mongodb/vibe/db/mongo/impl/crud.d b/mongodb/vibe/db/mongo/impl/crud.d new file mode 100644 index 0000000000..6319eae697 --- /dev/null +++ b/mongodb/vibe/db/mongo/impl/crud.d @@ -0,0 +1,875 @@ +module vibe.db.mongo.impl.crud; + +import core.time; + +import vibe.db.mongo.collection; +import vibe.data.bson; + +import std.typecons; + +@safe: + +enum UpdateFlags { + none = 0, /// Normal update of a single document. + upsert = 1<<0, /// Creates a document if none exists. + multiUpdate = 1<<1, /// Updates all matching documents. + + None = none, /// Deprecated compatibility alias + Upsert = upsert, /// Deprecated compatibility alias + MultiUpdate = multiUpdate /// Deprecated compatibility alias +} + +enum InsertFlags { + none = 0, /// Normal insert. + continueOnError = 1<<0, /// For multiple inserted documents, continues inserting further documents after a failure. + + None = none, /// Deprecated compatibility alias + ContinueOnError = continueOnError /// Deprecated compatibility alias +} + +deprecated("Use FindOptions instead") +enum QueryFlags { + none = 0, /// Normal query + tailableCursor = 1<<1, /// + slaveOk = 1<<2, /// + oplogReplay = 1<<3, /// + noCursorTimeout = 1<<4, /// + awaitData = 1<<5, /// + exhaust = 1<<6, /// + partial = 1<<7, /// + + None = none, /// Deprecated compatibility alias + TailableCursor = tailableCursor, /// Deprecated compatibility alias + SlaveOk = slaveOk, /// Deprecated compatibility alias + OplogReplay = oplogReplay, /// Deprecated compatibility alias + NoCursorTimeout = noCursorTimeout, /// Deprecated compatibility alias + AwaitData = awaitData, /// Deprecated compatibility alias + Exhaust = exhaust, /// Deprecated compatibility alias + Partial = partial /// Deprecated compatibility alias +} + +enum DeleteFlags { + none = 0, + singleRemove = 1<<0, + + None = none, /// Deprecated compatibility alias + SingleRemove = singleRemove /// Deprecated compatibility alias +} + +/** + See_Also: $(LINK https://docs.mongodb.com/manual/reference/command/find/) + + Standards: $(LINK https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/crud/crud.rst#id16) +*/ +struct FindOptions +{ + /** + Enables writing to temporary files on the server. When set to true, the server + can write temporary data to disk while executing the find operation. + + This option is only supported by servers >= 4.4. + */ + @embedNullable @errorBefore(WireVersion.v44) + Nullable!bool allowDiskUse; + + /** + Get partial results from a mongos if some shards are down (instead of throwing an error). + */ + @embedNullable + Nullable!bool allowPartialResults; + + /** + The number of documents to return per batch. + */ + @embedNullable + Nullable!int batchSize; + + /** + Determines whether to close the cursor after the first batch. + + Set automatically if limit < 0 || batchSize < 0. + */ + @embedNullable + package Nullable!bool singleBatch; + + /** + Collation allows users to specify language-specific rules for string + comparison, such as rules for letter-case and accent marks. + */ + @embedNullable @errorBefore(WireVersion.v34) + Nullable!Collation collation; + + /** + Users can specify an arbitrary string to help trace the operation + through the database profiler, currentOp, and logs. + */ + @embedNullable + Nullable!string comment; + + /** + Indicates the type of cursor to use. This value includes both + the tailable and awaitData options. + */ + @ignore CursorType cursorType; + + /** + The index to use. Specify either the index name as a string or the index + key pattern. + + If specified, then the query system will only consider plans using the + hinted index. + */ + @embedNullable + Nullable!Bson hint; + + /** + The maximum number of documents to return. + + A negative limit only returns a single batch of results. + */ + @embedNullable + Nullable!long limit; + + /** + The exclusive upper bound for a specific index. + */ + @embedNullable + Nullable!Bson max; + + /** + The maximum amount of time for the server to wait on new documents to + satisfy a tailable cursor query. This only applies to a TAILABLE_AWAIT + cursor. When the cursor is not a TAILABLE_AWAIT cursor, this option is + ignored. + + Note: This option is specified as "maxTimeMS" in the getMore command and + not provided as part of the initial find command. + */ + @embedNullable @since(WireVersion.v32) + Nullable!long maxAwaitTimeMS; + + /// ditto + void maxAwaitTime(Duration d) + @safe { + maxAwaitTimeMS = cast(long)d.total!"msecs"; + } + + /** + Maximum number of documents or index keys to scan when executing the query. + */ + @embedNullable @deprecatedSince(WireVersion.v40) + Nullable!long maxScan; + + /** + The maximum amount of time to allow the query to run. + */ + @embedNullable + Nullable!long maxTimeMS; + + /// ditto + void maxTime(Duration d) + @safe { + maxTimeMS = cast(long)d.total!"msecs"; + } + + /** + The exclusive lower bound for a specific index. + */ + @embedNullable + Nullable!Bson min; + + /** + The server normally times out idle cursors after an inactivity period + (10 minutes) to prevent excess memory use. Set this option to prevent + that. + */ + @embedNullable + Nullable!bool noCursorTimeout; + + /** + Enables optimization when querying the oplog for a range of ts values. + + Note: this option is intended for internal replication use only. + */ + @embedNullable @deprecatedSince(WireVersion.v44) + Nullable!bool oplogReplay; + + /** + Limits the fields to return for all matching documents. + */ + @embedNullable + Nullable!Bson projection; + + /** + If true, returns only the index keys in the resulting documents. + */ + @embedNullable + Nullable!bool returnKey; + + /** + Determines whether to return the record identifier for each document. If + true, adds a field $recordId to the returned documents. + */ + @embedNullable + Nullable!bool showRecordId; + + /** + The number of documents to skip before returning. + */ + @embedNullable + Nullable!long skip; + + /** + Prevents the cursor from returning a document more than once because of + an intervening write operation. + */ + @embedNullable @deprecatedSince(WireVersion.v40) + Nullable!bool snapshot; + + /** + The order in which to return matching documents. + */ + @embedNullable + Nullable!Bson sort; + + /** + Specifies the read concern. Only compatible with a write stage. (e.g. + `$out`, `$merge`) + + Aggregate commands do not support the $(D ReadConcern.Level.linearizable) + level. + + Standards: $(LINK https://github.com/mongodb/specifications/blob/7745234f93039a83ae42589a6c0cdbefcffa32fa/source/read-write-concern/read-write-concern.rst) + */ + @embedNullable Nullable!ReadConcern readConcern; +} + +/// +enum CursorType +{ + /** + The default value. A vast majority of cursors will be of this type. + */ + nonTailable, + /** + Tailable means the cursor is not closed when the last data is retrieved. + Rather, the cursor marks the final object’s position. You can resume + using the cursor later, from where it was located, if more data were + received. Like any “latent cursor”, the cursor may become invalid at + some point (CursorNotFound) – for example if the final object it + references were deleted. + */ + tailable, + /** + Combines the tailable option with awaitData, as defined below. + + Use with TailableCursor. If we are at the end of the data, block for a + while rather than returning no data. After a timeout period, we do + return as normal. The default is true. + */ + tailableAwait, +} + +/** + See_Also: $(LINK https://www.mongodb.com/docs/manual/reference/command/distinct/) + + Standards: $(LINK https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/crud/crud.rst#id16) +*/ +struct DistinctOptions +{ + /** + Collation allows users to specify language-specific rules for string + comparison, such as rules for letter-case and accent marks. + */ + @embedNullable @errorBefore(WireVersion.v34) + Nullable!Collation collation; + + /** + The maximum amount of time to allow the query to run. + */ + @embedNullable + Nullable!long maxTimeMS; + + /// ditto + void maxTime(Duration d) + @safe { + maxTimeMS = cast(long)d.total!"msecs"; + } + + /** + Specifies the read concern. Only compatible with a write stage. (e.g. + `$out`, `$merge`) + + Aggregate commands do not support the $(D ReadConcern.Level.linearizable) + level. + + Standards: $(LINK https://github.com/mongodb/specifications/blob/7745234f93039a83ae42589a6c0cdbefcffa32fa/source/read-write-concern/read-write-concern.rst) + */ + @embedNullable Nullable!ReadConcern readConcern; + + /** + Users can specify an arbitrary string to help trace the operation + through the database profiler, currentOp, and logs. + */ + @embedNullable + Nullable!string comment; +} + +/** + See_Also: $(LINK https://www.mongodb.com/docs/manual/reference/command/count/) + and $(LINK https://www.mongodb.com/docs/manual/reference/method/db.collection.countDocuments/) + + Standards: $(LINK https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/crud/crud.rst#id16) +*/ +struct CountOptions +{ + /** + Collation allows users to specify language-specific rules for string + comparison, such as rules for letter-case and accent marks. + */ + @embedNullable @errorBefore(WireVersion.v34) + Nullable!Collation collation; + + /** + The index to use. Specify either the index name as a string or the index + key pattern. + + If specified, then the query system will only consider plans using the + hinted index. + */ + @embedNullable + Nullable!Bson hint; + + /** + The maximum number of documents to return. + + A negative limit only returns a single batch of results. + */ + @embedNullable + Nullable!long limit; + + /** + The maximum amount of time to allow the query to run. + */ + @embedNullable + Nullable!long maxTimeMS; + + /// ditto + void maxTime(Duration d) + @safe { + maxTimeMS = cast(long)d.total!"msecs"; + } + + /** + The number of documents to skip before returning. + */ + @embedNullable + Nullable!long skip; + + /** + Specifies the read concern. Only compatible with a write stage. (e.g. + `$out`, `$merge`) + + Aggregate commands do not support the $(D ReadConcern.Level.linearizable) + level. + + Standards: $(LINK https://github.com/mongodb/specifications/blob/7745234f93039a83ae42589a6c0cdbefcffa32fa/source/read-write-concern/read-write-concern.rst) + */ + @embedNullable Nullable!ReadConcern readConcern; +} + +/** + See_Also: $(LINK https://www.mongodb.com/docs/manual/reference/method/db.collection.estimatedDocumentCount/) + + Standards: $(LINK https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/crud/crud.rst#id16) +*/ +struct EstimatedDocumentCountOptions +{ + /** + The maximum amount of time to allow the query to run. + */ + @embedNullable + Nullable!long maxTimeMS; + + /// ditto + void maxTime(Duration d) + @safe { + maxTimeMS = cast(long)d.total!"msecs"; + } +} + +/** + Represents available options for an aggregate call + + See_Also: $(LINK https://www.mongodb.com/docs/manual/reference/command/aggregate/#dbcmd.aggregate) + + Standards: $(LINK https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/crud/crud.rst#id16) +*/ +struct AggregateOptions +{ + // undocumented because this field isn't a spec field because it is + // out-of-scope for a driver + @embedNullable Nullable!bool explain; + + /** + Enables writing to temporary files. When set to true, aggregation + operations can write data to the _tmp subdirectory in the dbPath + directory. + */ + @embedNullable + Nullable!bool allowDiskUse; + + // non-optional since 3.6 + // get/set by `batchSize`, undocumented in favor of that field + CursorInitArguments cursor; + + /// Specifies the initial batch size for the cursor. + ref inout(Nullable!int) batchSize() + return @property inout @safe pure nothrow @nogc @ignore { + return cursor.batchSize; + } + + /** + If true, allows the write to opt-out of document level validation. + This only applies when the $out or $merge stage is specified. + */ + @embedNullable @since(WireVersion.v32) + Nullable!bool bypassDocumentValidation; + + /** + Collation allows users to specify language-specific rules for string + comparison, such as rules for letter-case and accent marks. + */ + @embedNullable @errorBefore(WireVersion.v34) + Nullable!Collation collation; + + /** + Users can specify an arbitrary string to help trace the operation + through the database profiler, currentOp, and logs. + */ + @embedNullable + Nullable!string comment; + + /** + The maximum amount of time for the server to wait on new documents to + satisfy a tailable cursor query. This only applies to a TAILABLE_AWAIT + cursor. When the cursor is not a TAILABLE_AWAIT cursor, this option is + ignored. + + Note: This option is specified as "maxTimeMS" in the getMore command and + not provided as part of the initial find command. + */ + @embedNullable @since(WireVersion.v32) + Nullable!long maxAwaitTimeMS; + + /// ditto + void maxAwaitTime(Duration d) + @safe { + maxAwaitTimeMS = cast(long)d.total!"msecs"; + } + + /** + Specifies a time limit in milliseconds for processing operations on a + cursor. If you do not specify a value for maxTimeMS, operations will not + time out. + */ + @embedNullable + Nullable!long maxTimeMS; + + /// ditto + void maxTime(Duration d) + @safe { + maxTimeMS = cast(long)d.total!"msecs"; + } + + /** + The index to use for the aggregation. The index is on the initial + collection / view against which the aggregation is run. + + The hint does not apply to $lookup and $graphLookup stages. + + Specify the index either by the index name as a string or the index key + pattern. If specified, then the query system will only consider plans + using the hinted index. + */ + @embedNullable + Nullable!Bson hint; + + /** + Map of parameter names and values. Values must be constant or closed + expressions that do not reference document fields. Parameters can then + be accessed as variables in an aggregate expression context + (e.g. `"$$var"`). + + This option is only supported by servers >= 5.0. Older servers >= 2.6 (and possibly earlier) will report an error for using this option. + */ + @embedNullable + Nullable!Bson let; + + /** + Specifies the read concern. Only compatible with a write stage. (e.g. + `$out`, `$merge`) + + Aggregate commands do not support the $(D ReadConcern.Level.linearizable) + level. + + Standards: $(LINK https://github.com/mongodb/specifications/blob/7745234f93039a83ae42589a6c0cdbefcffa32fa/source/read-write-concern/read-write-concern.rst) + */ + @embedNullable Nullable!ReadConcern readConcern; +} + +/** + Standards: $(LINK https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/crud/crud.rst#insert-update-replace-delete-and-bulk-writes) +*/ +struct BulkWriteOptions { + + /** + If true, when a write fails, return without performing the remaining + writes. If false, when a write fails, continue with the remaining writes, + if any. + + Defaults to true. + */ + bool ordered = true; + + /** + If true, allows the write to opt-out of document level validation. + + For servers < 3.2, this option is ignored and not sent as document + validation is not available. + + For unacknowledged writes using OP_INSERT, OP_UPDATE, or OP_DELETE, the + driver MUST raise an error if the caller explicitly provides a value. + */ + @embedNullable + Nullable!bool bypassDocumentValidation; + + /** + A document that expresses the + $(LINK2 https://www.mongodb.com/docs/manual/reference/write-concern/,write concern) + of the insert command. Omit to use the default write concern. + */ + @embedNullable + Nullable!WriteConcern writeConcern; + + /** + Users can specify an arbitrary string to help trace the operation + through the database profiler, currentOp, and logs. + */ + @embedNullable + Nullable!string comment; +} + +/** + See_Also: $(LINK https://docs.mongodb.com/manual/reference/command/insert/) + + Standards: $(LINK https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/crud/crud.rst#insert-update-replace-delete-and-bulk-writes) +*/ +struct InsertOneOptions { + /** + If true, allows the write to opt-out of document level validation. + + For servers < 3.2, this option is ignored and not sent as document + validation is not available. + */ + @embedNullable + Nullable!bool bypassDocumentValidation; + + /** + A document that expresses the + $(LINK2 https://www.mongodb.com/docs/manual/reference/write-concern/,write concern) + of the insert command. Omit to use the default write concern. + */ + @embedNullable + Nullable!WriteConcern writeConcern; + + /** + Users can specify an arbitrary string to help trace the operation + through the database profiler, currentOp, and logs. + */ + @embedNullable + Nullable!string comment; +} + +/** + See_Also: $(LINK https://docs.mongodb.com/manual/reference/command/insert/) + + Standards: $(LINK https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/crud/crud.rst#insert-update-replace-delete-and-bulk-writes) +*/ +struct InsertManyOptions { + /** + If true, allows the write to opt-out of document level validation. + + For servers < 3.2, this option is ignored and not sent as document + validation is not available. + */ + @embedNullable + Nullable!bool bypassDocumentValidation; + + /** + If true, when an insert fails, return without performing the remaining + writes. If false, when a write fails, continue with the remaining writes, + if any. + + Defaults to true. + */ + bool ordered = true; + + /** + A document that expresses the + $(LINK2 https://www.mongodb.com/docs/manual/reference/write-concern/,write concern) + of the insert command. Omit to use the default write concern. + */ + @embedNullable + Nullable!WriteConcern writeConcern; + + /** + Users can specify an arbitrary string to help trace the operation + through the database profiler, currentOp, and logs. + */ + @embedNullable + Nullable!string comment; +} + +/** + See_Also: $(LINK https://docs.mongodb.com/manual/reference/command/update/) + + Standards: $(LINK https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/crud/crud.rst#insert-update-replace-delete-and-bulk-writes) +*/ +struct UpdateOptions { + /** + A set of filters specifying to which array elements an update should + apply. + + This option is sent only if the caller explicitly provides a value. The + default is to not send a value. + */ + @embedNullable @errorBefore(WireVersion.v36) + Nullable!(Bson[]) arrayFilters; + + /** + If true, allows the write to opt-out of document level validation. + + For servers < 3.2, this option is ignored and not sent as document + validation is not available. + */ + @embedNullable + Nullable!bool bypassDocumentValidation; + + /** + Collation allows users to specify language-specific rules for string + comparison, such as rules for letter-case and accent marks. + */ + @embedNullable @errorBefore(WireVersion.v34) + Nullable!Collation collation; + + /** + The index to use. Specify either the index name as a string or the index + key pattern. + + If specified, then the query system will only consider plans using the + hinted index. + */ + @embedNullable + Nullable!Bson hint; + + /** + When true, creates a new document if no document matches the query. + */ + @embedNullable + Nullable!bool upsert; + + /** + A document that expresses the + $(LINK2 https://www.mongodb.com/docs/manual/reference/write-concern/,write concern) + of the insert command. Omit to use the default write concern. + */ + @embedNullable + Nullable!WriteConcern writeConcern; + + /** + Users can specify an arbitrary string to help trace the operation + through the database profiler, currentOp, and logs. + */ + @embedNullable + Nullable!string comment; +} + +/** + See_Also: $(LINK https://docs.mongodb.com/manual/reference/command/update/) + + Standards: $(LINK https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/crud/crud.rst#insert-update-replace-delete-and-bulk-writes) +*/ +struct ReplaceOptions { + /** + If true, allows the write to opt-out of document level validation. + + For servers < 3.2, this option is ignored and not sent as document + validation is not available. + */ + @embedNullable + Nullable!bool bypassDocumentValidation; + + /** + Collation allows users to specify language-specific rules for string + comparison, such as rules for letter-case and accent marks. + */ + @embedNullable @errorBefore(WireVersion.v34) + Nullable!Collation collation; + + /** + The index to use. Specify either the index name as a string or the index + key pattern. + + If specified, then the query system will only consider plans using the + hinted index. + */ + @embedNullable + Nullable!Bson hint; + + /** + When true, creates a new document if no document matches the query. + */ + @embedNullable + Nullable!bool upsert; + + /** + A document that expresses the + $(LINK2 https://www.mongodb.com/docs/manual/reference/write-concern/,write concern) + of the insert command. Omit to use the default write concern. + */ + @embedNullable + Nullable!WriteConcern writeConcern; + + /** + Users can specify an arbitrary string to help trace the operation + through the database profiler, currentOp, and logs. + */ + @embedNullable + Nullable!string comment; +} + +/** + See_Also: $(LINK https://docs.mongodb.com/manual/reference/command/delete/) + + Standards: $(LINK https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/crud/crud.rst#insert-update-replace-delete-and-bulk-writes) +*/ +struct DeleteOptions { + /** + Collation allows users to specify language-specific rules for string + comparison, such as rules for letter-case and accent marks. + */ + @embedNullable @errorBefore(WireVersion.v34) + Nullable!Collation collation; + + /** + The index to use. Specify either the index name as a string or the index + key pattern. + + If specified, then the query system will only consider plans using the + hinted index. + */ + @embedNullable + Nullable!Bson hint; + + /** + A document that expresses the + $(LINK2 https://www.mongodb.com/docs/manual/reference/write-concern/,write concern) + of the insert command. Omit to use the default write concern. + */ + @embedNullable + Nullable!WriteConcern writeConcern; + + /** + Users can specify an arbitrary string to help trace the operation + through the database profiler, currentOp, and logs. + */ + @embedNullable + Nullable!string comment; +} + +struct BulkWriteResult { + /** + Number of documents inserted. + */ + long insertedCount; + + /** + The identifiers that were automatically generated, if not set. + */ + BsonObjectID[size_t] insertedIds; + + /** + Number of documents matched for update. + */ + long matchedCount; + + /** + Number of documents modified. + */ + long modifiedCount; + + /** + Number of documents deleted. + */ + long deletedCount; + + /** + Number of documents upserted. + */ + long upsertedCount; + + /** + Map of the index of the operation to the id of the upserted document. + */ + BsonObjectID[size_t] upsertedIds; +} + +struct InsertOneResult { + /** + The identifier that was automatically generated, if not set. + */ + BsonObjectID insertedId; +} + +struct InsertManyResult { + /** + The identifiers that were automatically generated, if not set. + */ + BsonObjectID[size_t] insertedIds; +} + +struct DeleteResult { + /** + The number of documents that were deleted. + */ + long deletedCount; +} + +struct UpdateResult { + /** + The number of documents that matched the filter. + */ + long matchedCount; + + /** + The number of documents that were modified. + */ + long modifiedCount; + + /** + The number of documents that were upserted. + * + NOT REQUIRED: Drivers may choose to not provide this property so long as + it is always possible to infer whether an upsert has taken place. Since + the "_id" of an upserted document could be null, a null "upsertedId" may + be ambiguous in some drivers. If so, this field can be used to indicate + whether an upsert has taken place. + */ + long upsertedCount; + + /** + The identifier of the inserted document if an upsert took place. + */ + Bson upsertedId; +} diff --git a/mongodb/vibe/db/mongo/sessionstore.d b/mongodb/vibe/db/mongo/sessionstore.d index b2647066ff..0f561df36b 100644 --- a/mongodb/vibe/db/mongo/sessionstore.d +++ b/mongodb/vibe/db/mongo/sessionstore.d @@ -85,7 +85,7 @@ final class MongoSessionStore : SessionStore { Session create() { auto s = createSessionInstance(); - m_sessions.insert(SessionEntry(s.id, Clock.currTime(UTC()))); + m_sessions.insertOne(SessionEntry(s.id, Clock.currTime(UTC()))); return s; } @@ -104,7 +104,9 @@ final class MongoSessionStore : SessionStore { Variant get(string id, string name, lazy Variant defaultVal) @trusted { auto f = name.escape; - auto r = m_sessions.findOne(["_id": id], [f: 1]); + FindOptions options; + options.projection = Bson([f: Bson(1)]); + auto r = m_sessions.findOne(["_id": id], options); if (r.isNull) return defaultVal; auto v = r.tryIndex(f); if (v.isNull) return defaultVal; @@ -114,7 +116,9 @@ final class MongoSessionStore : SessionStore { bool isKeySet(string id, string key) { auto f = key.escape; - auto r = m_sessions.findOne(["_id": id], [f: 1]); + FindOptions options; + options.projection = Bson([f: Bson(1)]); + auto r = m_sessions.findOne(["_id": id], options); if (r.isNull) return false; return !r.tryIndex(f).isNull; } diff --git a/mongodb/vibe/db/mongo/settings.d b/mongodb/vibe/db/mongo/settings.d index 33941e819f..f51d6cebc1 100644 --- a/mongodb/vibe/db/mongo/settings.d +++ b/mongodb/vibe/db/mongo/settings.d @@ -9,7 +9,7 @@ module vibe.db.mongo.settings; import vibe.core.log; import vibe.data.bson; -import vibe.db.mongo.flags : QueryFlags; +deprecated import vibe.db.mongo.flags : QueryFlags; import vibe.inet.webform; import std.conv : to; @@ -157,7 +157,6 @@ bool parseMongoDBUrl(out MongoClientSettings cfg, string url) default: logWarn("Unknown MongoDB option %s", option); break; case "appname": cfg.appName = value; break; - case "slaveok": bool v; if( setBool(v) && v ) cfg.defQueryFlags |= QueryFlags.SlaveOk; break; case "replicaset": cfg.replicaSet = value; warnNotImplemented(); break; case "safe": setBool(cfg.safe); break; case "fsync": setBool(cfg.fsync); break; @@ -208,7 +207,6 @@ unittest assert(cfg.database == ""); assert(cfg.hosts[0].name == "localhost"); assert(cfg.hosts[0].port == 27017); - assert(cfg.defQueryFlags == QueryFlags.None); assert(cfg.replicaSet == ""); assert(cfg.safe == false); assert(cfg.w == Bson.init); @@ -241,7 +239,7 @@ unittest assert(cfg.hosts[0].port == 27017); cfg = MongoClientSettings.init; - assert(parseMongoDBUrl(cfg, "mongodb://host1,host2,host3/?safe=true&w=2&wtimeoutMS=2000&slaveOk=true&ssl=true&sslverifycertificate=false")); + assert(parseMongoDBUrl(cfg, "mongodb://host1,host2,host3/?safe=true&w=2&wtimeoutMS=2000&ssl=true&sslverifycertificate=false")); assert(cfg.username == ""); //assert(cfg.password == ""); assert(cfg.digest == ""); @@ -256,7 +254,6 @@ unittest assert(cfg.safe == true); assert(cfg.w == Bson(2L)); assert(cfg.wTimeoutMS == 2000); - assert(cfg.defQueryFlags == QueryFlags.SlaveOk); assert(cfg.ssl == true); assert(cfg.sslverifycertificate == false); @@ -402,12 +399,7 @@ class MongoClientSettings */ string database; - /** - * Flags to use on all database query commands. The - * $(REF slaveOk, vibe,db,mongo,flags,QueryFlags) bit may be set using the - * "slaveok" query parameter inside the MongoDB URL. - */ - QueryFlags defQueryFlags = QueryFlags.None; + deprecated("unused since at least before v3.6") QueryFlags defQueryFlags = QueryFlags.None; /** * Specifies the name of the replica set, if the mongod is a member of a From cbef81f60a71f4159b0beea2d3ed14fa7435bd1d Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Sun, 25 Sep 2022 16:41:06 +0200 Subject: [PATCH 17/36] implement all MongoDB CRUD operations --- mongodb/vibe/db/mongo/collection.d | 202 ++++++++++++++++++++++++++++- mongodb/vibe/db/mongo/impl/crud.d | 49 ++++--- 2 files changed, 230 insertions(+), 21 deletions(-) diff --git a/mongodb/vibe/db/mongo/collection.d b/mongodb/vibe/db/mongo/collection.d index 0c6ffaf061..5bcb6fd0e2 100644 --- a/mongodb/vibe/db/mongo/collection.d +++ b/mongodb/vibe/db/mongo/collection.d @@ -18,12 +18,13 @@ import vibe.core.log; import vibe.db.mongo.client; import core.time; -import std.algorithm : countUntil, find; +import std.algorithm : among, countUntil, find; import std.array; import std.conv; import std.exception; +import std.meta : AliasSeq; import std.string; -import std.typecons : Tuple, tuple, Nullable; +import std.typecons : Nullable, tuple, Tuple; /** @@ -171,6 +172,203 @@ struct MongoCollection { return InsertManyResult(insertedIds); } + /** + Deletes at most one document matching the query `filter`. The returned + result identifies how many documents have been deleted. + + See_Also: $(LINK https://www.mongodb.com/docs/manual/reference/method/db.collection.deleteOne/#mongodb-method-db.collection.deleteOne) + + Standards: $(LINK https://www.mongodb.com/docs/manual/reference/command/delete/) + */ + DeleteResult deleteOne(T)(T filter, DeleteOptions options = DeleteOptions.init) + { + int limit = 1; + return deleteImpl([filter], options, (&limit)[0 .. 1]); + } + + /** + Deletes all documents matching the query `filter`. The returned result + identifies how many documents have been deleted. + + See_Also: $(LINK https://www.mongodb.com/docs/manual/reference/method/db.collection.deleteMany/#mongodb-method-db.collection.deleteMany) + + Standards: $(LINK https://www.mongodb.com/docs/manual/reference/command/delete/) + */ + DeleteResult deleteMany(T)(T filter, DeleteOptions options = DeleteOptions.init) + { + return deleteImpl([filter], options); + } + + /// Implementation helper. It's possible to set custom delete limits with + /// this method, otherwise it's identical to `deleteOne` and `deleteMany`. + DeleteResult deleteImpl(T)(T[] queries, DeleteOptions options = DeleteOptions.init, scope int[] limits = null) + { + assert(m_client !is null, "Querying uninitialized MongoCollection."); + + alias FieldsMovedIntoChildren = AliasSeq!("limit", "collation", "hint"); + + Bson cmd = Bson.emptyObject; // empty object because order is important + cmd["delete"] = Bson(m_name); + + MongoConnection conn = m_client.lockConnection(); + enforceWireVersionConstraints(options, conn.description.maxWireVersion); + auto optionsBson = serializeToBson(options); + foreach (string k, v; optionsBson.byKeyValue) + if (!k.among!FieldsMovedIntoChildren) + cmd[k] = v; + + Bson[] deletesBson = new Bson[queries.length]; + foreach (i, q; queries) + { + auto deleteBson = Bson.emptyObject; + deleteBson["q"] = serializeToBson(q); + foreach (string k, v; optionsBson.byKeyValue) + if (k.among!FieldsMovedIntoChildren) + deleteBson[k] = v; + if (i < limits.length) + deleteBson["limit"] = Bson(limits[i]); + deletesBson[i] = deleteBson; + } + cmd["deletes"] = Bson(deletesBson); + + auto n = database.runCommand(cmd, true)["n"].get!long; + return DeleteResult(n); + } + + /** + Replaces at most single document within the collection based on the filter. + + See_Also: $(LINK https://www.mongodb.com/docs/manual/reference/method/db.collection.replaceOne/#mongodb-method-db.collection.replaceOne) + + Standards: $(LINK https://www.mongodb.com/docs/manual/reference/command/update/) + */ + UpdateResult replaceOne(T, U)(T filter, U replacement, UpdateOptions options = UpdateOptions.init) + { + Bson opts = Bson.emptyObject; + opts["multi"] = Bson(false); + return updateImpl([filter], [replacement], [opts], options, true, false); + } + + /** + Updates at most single document within the collection based on the filter. + + See_Also: $(LINK https://www.mongodb.com/docs/manual/reference/method/db.collection.updateOne/#mongodb-method-db.collection.updateOne) + + Standards: $(LINK https://www.mongodb.com/docs/manual/reference/command/update/) + */ + UpdateResult updateOne(T, U)(T filter, U replacement, UpdateOptions options = UpdateOptions.init) + { + Bson opts = Bson.emptyObject; + opts["multi"] = Bson(false); + return updateImpl([filter], [replacement], [opts], options, false, true); + } + + /** + Updates all matching document within the collection based on the filter. + + See_Also: $(LINK https://www.mongodb.com/docs/manual/reference/method/db.collection.updateMany/#mongodb-method-db.collection.updateMany) + + Standards: $(LINK https://www.mongodb.com/docs/manual/reference/command/update/) + */ + UpdateResult updateMany(T, U)(T filter, U replacement, UpdateOptions options = UpdateOptions.init) + { + Bson opts = Bson.emptyObject; + opts["multi"] = Bson(true); + return updateImpl([filter], [replacement], [opts], options, false, true); + } + + /// Implementation helper. It's possible to set custom per-update object + /// options with this method, otherwise it's identical to `replaceOne`, + /// `updateOne` and `updateMany`. + UpdateResult updateImpl(T, U, O)(T[] queries, U[] documents, O[] perUpdateOptions, UpdateOptions options = UpdateOptions.init, + bool mustBeDocument = false, bool mustBeModification = false) + in(queries.length == documents.length && documents.length == perUpdateOptions.length, + "queries, documents and perUpdateOptions must have same length") + { + assert(m_client !is null, "Querying uninitialized MongoCollection."); + + alias FieldsMovedIntoChildren = AliasSeq!("arrayFilters", + "collation", + "hint", + "upsert"); + + Bson cmd = Bson.emptyObject; // empty object because order is important + cmd["update"] = Bson(m_name); + + MongoConnection conn = m_client.lockConnection(); + enforceWireVersionConstraints(options, conn.description.maxWireVersion); + auto optionsBson = serializeToBson(options); + foreach (string k, v; optionsBson.byKeyValue) + if (!k.among!FieldsMovedIntoChildren) + cmd[k] = v; + + Bson[] updatesBson = new Bson[queries.length]; + foreach (i, q; queries) + { + auto updateBson = Bson.emptyObject; + auto qbson = serializeToBson(q); + updateBson["q"] = qbson; + if (mustBeDocument) + { + if (qbson.type != Bson.Type.object) + assert(false, "Passed in non-document into a place where only replacements are expected. " + ~ "Maybe you want to call updateOne or updateMany instead?"); + + foreach (string k, v; qbson) + { + if (k.startsWith("$")) + assert(false, "Passed in atomic modifiers (" ~ k + ~ ") into a place where only replacements are expected. " + ~ "Maybe you want to call updateOne or updateMany instead?"); + debug break; // server checks that the rest is consistent (only $ or only non-$ allowed) + // however in debug mode we check the full document, as we can give better error messages to the dev + } + } + if (mustBeModification) + { + if (qbson.type == Bson.Type.object) + { + bool anyDollar = false; + foreach (string k, v; qbson) + { + if (k.startsWith("$")) + anyDollar = true; + debug break; // server checks that the rest is consistent (only $ or only non-$ allowed) + // however in debug mode we check the full document, as we can give better error messages to the dev + // also nice side effect: if this is an empty document, this also matches the assert(false) branch. + } + + if (!anyDollar) + assert(false, "Passed in a regular document into a place where only updates are expected. " + ~ "Maybe you want to call replaceOne instead? " + ~ "(this update call would otherwise replace the entire matched object with the passed in update object)"); + } + } + updateBson["u"] = serializeToBson(documents[i]); + foreach (string k, v; optionsBson.byKeyValue) + if (k.among!FieldsMovedIntoChildren) + updateBson[k] = v; + foreach (string k, v; perUpdateOptions[i].byKeyValue) + updateBson[k] = v; + updatesBson[i] = updateBson; + } + cmd["updates"] = Bson(updatesBson); + + auto res = database.runCommand(cmd, true); + auto ret = UpdateResult( + res["n"].get!long, + res["nModified"].get!long, + ); + auto upserted = res["upserted"].get!(Bson[]); + if (upserted.length) + { + ret.upsertedIds.length = upserted.length; + foreach (i, id; upserted) + ret.upsertedIds[i] = id.get!BsonObjectID; + } + return ret; + } + deprecated("Use the overload taking FindOptions instead, this method breaks in MongoDB 5.1 and onwards. Note: using a `$query` / `query` member to override the query arguments is no longer supported in the new overload.") MongoCursor!R find(R = Bson, T, U)(T query, U returnFieldSelector, QueryFlags flags = QueryFlags.None, int num_skip = 0, int num_docs_per_chunk = 0) { diff --git a/mongodb/vibe/db/mongo/impl/crud.d b/mongodb/vibe/db/mongo/impl/crud.d index 6319eae697..f88e5fc18e 100644 --- a/mongodb/vibe/db/mongo/impl/crud.d +++ b/mongodb/vibe/db/mongo/impl/crud.d @@ -232,6 +232,16 @@ struct FindOptions @embedNullable Nullable!Bson sort; + /** + If true, when an insert fails, return without performing the remaining + writes. If false, when a write fails, continue with the remaining writes, + if any. + + Defaults to true. + */ + @embedNullable + Nullable!bool ordered; + /** Specifies the read concern. Only compatible with a write stage. (e.g. `$out`, `$merge`) @@ -530,7 +540,8 @@ struct BulkWriteOptions { Defaults to true. */ - bool ordered = true; + @embedNullable + Nullable!bool ordered; /** If true, allows the write to opt-out of document level validation. @@ -613,7 +624,8 @@ struct InsertManyOptions { Defaults to true. */ - bool ordered = true; + @embedNullable + Nullable!bool ordered; /** A document that expresses the @@ -640,9 +652,6 @@ struct UpdateOptions { /** A set of filters specifying to which array elements an update should apply. - - This option is sent only if the caller explicitly provides a value. The - default is to not send a value. */ @embedNullable @errorBefore(WireVersion.v36) Nullable!(Bson[]) arrayFilters; @@ -653,7 +662,7 @@ struct UpdateOptions { For servers < 3.2, this option is ignored and not sent as document validation is not available. */ - @embedNullable + @embedNullable @since(WireVersion.v32) Nullable!bool bypassDocumentValidation; /** @@ -786,6 +795,17 @@ struct DeleteOptions { */ @embedNullable Nullable!string comment; + + /** + Map of parameter names and values. Values must be constant or closed + expressions that do not reference document fields. Parameters can then + be accessed as variables in an aggregate expression context + (e.g. `"$$var"`). + + This option is only supported by servers >= 5.0. Older servers >= 2.6 (and possibly earlier) will report an error for using this option. + */ + @embedNullable + Nullable!Bson let; } struct BulkWriteResult { @@ -858,18 +878,9 @@ struct UpdateResult { long modifiedCount; /** - The number of documents that were upserted. - * - NOT REQUIRED: Drivers may choose to not provide this property so long as - it is always possible to infer whether an upsert has taken place. Since - the "_id" of an upserted document could be null, a null "upsertedId" may - be ambiguous in some drivers. If so, this field can be used to indicate - whether an upsert has taken place. - */ - long upsertedCount; - - /** - The identifier of the inserted document if an upsert took place. + The identifier of the inserted document if an upsert took place. Can be + none if no upserts took place, can be multiple if using the updateImpl + helper. */ - Bson upsertedId; + BsonObjectID[] upsertedIds; } From 7be40c8b06e1d3f0a1ac92add4e6d2827c0ac270 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Sun, 25 Sep 2022 17:01:47 +0200 Subject: [PATCH 18/36] pass around $db everywhere --- mongodb/vibe/db/mongo/collection.d | 8 +++++--- mongodb/vibe/db/mongo/connection.d | 26 ++++++++++++++------------ mongodb/vibe/db/mongo/cursor.d | 7 +++++-- mongodb/vibe/db/mongo/database.d | 1 + 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/mongodb/vibe/db/mongo/collection.d b/mongodb/vibe/db/mongo/collection.d index 5bcb6fd0e2..3940759afc 100644 --- a/mongodb/vibe/db/mongo/collection.d +++ b/mongodb/vibe/db/mongo/collection.d @@ -18,7 +18,7 @@ import vibe.core.log; import vibe.db.mongo.client; import core.time; -import std.algorithm : among, countUntil, find; +import std.algorithm : among, countUntil, find, findSplit; import std.array; import std.conv; import std.exception; @@ -373,7 +373,7 @@ struct MongoCollection { MongoCursor!R find(R = Bson, T, U)(T query, U returnFieldSelector, QueryFlags flags = QueryFlags.None, int num_skip = 0, int num_docs_per_chunk = 0) { assert(m_client !is null, "Querying uninitialized MongoCollection."); - return MongoCursor!R(m_client, m_fullPath, flags, num_skip, num_docs_per_chunk, query, returnFieldSelector); + return MongoCursor!R(m_client, m_db.name, m_name, flags, num_skip, num_docs_per_chunk, query, returnFieldSelector); } /** @@ -385,7 +385,7 @@ struct MongoCollection { */ MongoCursor!R find(R = Bson, Q)(Q query, FindOptions options) { - return MongoCursor!R(m_client, m_fullPath, query, options); + return MongoCursor!R(m_client, m_db.name, m_name, query, options); } /// ditto @@ -668,6 +668,7 @@ struct MongoCollection { Bson cmd = Bson.emptyObject; // empty object because order is important cmd["aggregate"] = Bson(m_name); + cmd["$db"] = Bson(m_db.name); cmd["pipeline"] = serializeToBson(pipeline); MongoConnection conn = m_client.lockConnection(); enforceWireVersionConstraints(options, conn.description.maxWireVersion); @@ -1018,6 +1019,7 @@ struct MongoCollection { if (conn.description.satisfiesVersion(WireVersion.v30)) { Bson command = Bson.emptyObject; command["listIndexes"] = Bson(m_name); + command["$db"] = Bson(m_db.name); return MongoCursor!R(m_client, command); } else { throw new MongoDriverException("listIndexes not supported on MongoDB <3.0"); diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index 3cb89efa27..26242d97ae 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -18,15 +18,15 @@ import vibe.db.mongo.settings; import vibe.inet.webform; import vibe.stream.tls; -import std.algorithm : map, splitter; +import std.algorithm : findSplit, map, splitter; import std.array; import std.conv; import std.digest.md; import std.exception; import std.range; import std.string; -import std.typecons; import std.traits : hasIndirections; +import std.typecons; import core.time; @@ -349,6 +349,7 @@ final class MongoConnection { Bson runCommand(T, CommandFailException = MongoDriverException)(string database, Bson command, bool testOk = true, string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__) + in(database.length, "runCommand requires a database argument") { import std.array; @@ -363,8 +364,7 @@ final class MongoConnection { if (m_supportsOpMsg) { - if (database !is null) - command["$db"] = Bson(database); + command["$db"] = Bson(database); auto id = sendMsg(-1, 0, command); Appender!(Bson[])[string] docs; @@ -381,8 +381,6 @@ final class MongoConnection { } else { - if (database is null) - database = "$external"; auto id = send(OpCode.Query, -1, 0, database ~ ".$cmd", 0, -1, command, Bson(null)); recvReply!T(id, (cursor, flags, first_doc, num_docs) { @@ -412,7 +410,8 @@ final class MongoConnection { void getMore(string collection_name, int nret, long cursor_id, scope ReplyDelegate on_msg, scope DocDelegate!T on_doc) { scope(failure) disconnect(); - auto id = send(OpCode.GetMore, -1, cast(int)0, collection_name, nret, cursor_id); + auto parts = collection_name.findSplit("."); + auto id = send(OpCode.GetMore, -1, cast(int)0, parts[0], parts[2], nret, cursor_id); recvReply!T(id, on_msg, on_doc); } @@ -429,7 +428,7 @@ final class MongoConnection { * * Throws: $(LREF MongoDriverException) in case the command fails. */ - void getMore(long cursor_id, string collection_name, long nret, + void getMore(long cursor_id, string database, string collection_name, long nret, scope GetMoreHeaderDelegate on_header, scope GetMoreDocumentDelegate!T on_doc, Duration timeout = Duration.max, @@ -437,6 +436,7 @@ final class MongoConnection { { Bson command = Bson.emptyObject; command["getMore"] = Bson(cursor_id); + command["$db"] = Bson(database); command["collection"] = Bson(collection_name); command["batchSize"] = Bson(nret); if (timeout != Duration.max && timeout.total!"msecs" < int.max) @@ -479,7 +479,8 @@ final class MongoConnection { int num_docs; // array to store out-of-order items, to push them into the callback properly T[] compatibilitySort; - auto id = send(OpCode.GetMore, -1, cast(int)0, collection_name, nret, cursor_id); + string full_name = database ~ '.' ~ collection_name; + auto id = send(OpCode.GetMore, -1, cast(int)0, full_name, nret, cursor_id); recvReply!T(id, (long cursor, ReplyFlags flags, int first_doc, int num_docs) { enforce!MongoDriverException(!(flags & ReplyFlags.CursorNotFound), @@ -487,7 +488,7 @@ final class MongoConnection { enforce!MongoDriverException(!(flags & ReplyFlags.QueryFailure), formatErrorInfo("Query failed. Does the database exist?")); - on_header(cursor, collection_name, num_docs); + on_header(cursor, full_name, num_docs); }, (size_t idx, ref T doc) { if (cast(int)idx == nextId) { on_doc(doc); @@ -572,9 +573,10 @@ final class MongoConnection { if (m_supportsOpMsg) { Bson command = Bson.emptyObject; - command["killCursors"] = Bson(collection); + auto parts = collection.findSplit("."); + command["killCursors"] = Bson(parts[2]); command["cursors"] = cursors.serializeToBson; - runCommand!Bson(null, command); + runCommand!Bson(parts[0], command); } else { diff --git a/mongodb/vibe/db/mongo/cursor.d b/mongodb/vibe/db/mongo/cursor.d index 7b6f2ca5d7..7cc254b12a 100644 --- a/mongodb/vibe/db/mongo/cursor.d +++ b/mongodb/vibe/db/mongo/cursor.d @@ -41,10 +41,11 @@ struct MongoCursor(DocType = Bson) { m_data = new MongoGenericCursor!DocType(client, collection, cursor, existing_documents); } - this(Q)(MongoClient client, string collection, Q query, FindOptions options) + this(Q)(MongoClient client, string database, string collection, Q query, FindOptions options) { Bson command = Bson.emptyObject; command["find"] = Bson(collection); + command["$db"] = Bson(database); static if (is(Q == Bson)) command["filter"] = query; else @@ -405,6 +406,7 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { int m_refCount = 1; MongoClient m_client; Bson m_findQuery; + string m_database; string m_collection; long m_cursor; int m_batchSize; @@ -423,6 +425,7 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { m_findQuery = command; m_batchSize = batchSize; m_maxTimeMS = getMoreMaxTimeMS; + m_database = command["$db"].opt!string; } @property bool empty() @@ -438,7 +441,7 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { return true; auto conn = m_client.lockConnection(); - conn.getMore!DocType(m_cursor, m_collection, m_batchSize, &handleReply, &handleDocument, + conn.getMore!DocType(m_cursor, m_database, m_collection, m_batchSize, &handleReply, &handleDocument, m_maxTimeMS >= int.max ? Duration.max : m_maxTimeMS.msecs); return m_readDoc >= m_documents.length; } diff --git a/mongodb/vibe/db/mongo/database.d b/mongodb/vibe/db/mongo/database.d index 205bc9d514..08678797ab 100644 --- a/mongodb/vibe/db/mongo/database.d +++ b/mongodb/vibe/db/mongo/database.d @@ -158,6 +158,7 @@ struct MongoDatabase cmd = command_and_options; else cmd = command_and_options.serializeToBson; + cmd["$db"] = Bson(m_name); return MongoCursor!R(m_client, cmd, batchSize, getMoreMaxTimeMS); } From 77eb598a5ac6a4c98ef3cd18828d85ac174f9922 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Sun, 25 Sep 2022 17:15:36 +0200 Subject: [PATCH 19/36] verbose logging in MongoDB tests --- mongodb/vibe/db/mongo/connection.d | 41 ++++++++++++++++++++++---- tests/mongodb/_connection/dub.json | 3 +- tests/mongodb/_connection/source/app.d | 4 ++- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index 26242d97ae..167c7374de 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -7,6 +7,9 @@ */ module vibe.db.mongo.connection; +// /// prints ALL modern OP_MSG queries and legacy runCommand invocations to logDiagnostic +// debug = VibeVerboseMongo; + public import vibe.data.bson; import vibe.core.core : vibeVersionString; @@ -364,6 +367,9 @@ final class MongoConnection { if (m_supportsOpMsg) { + debug (VibeVerboseMongo) + logDiagnostic("runCommand: [db=%s] %s", database, command); + command["$db"] = Bson(database); auto id = sendMsg(-1, 0, command); @@ -381,6 +387,8 @@ final class MongoConnection { } else { + debug (VibeVerboseMongo) + logDiagnostic("runCommand(legacy): [db=%s] %s", database, command); auto id = send(OpCode.Query, -1, 0, database ~ ".$cmd", 0, -1, command, Bson(null)); recvReply!T(id, (cursor, flags, first_doc, num_docs) { @@ -453,6 +461,9 @@ final class MongoConnection { { enum needsDup = hasIndirections!T || is(T == Bson); + debug (VibeVerboseMongo) + logDiagnostic("getMore: [db=%s] %s", database, command); + auto id = sendMsg(-1, 0, command); recvMsg!needsDup(id, (flags, root) { if (root["ok"].get!double != 1.0) @@ -474,6 +485,9 @@ final class MongoConnection { } else { + debug (VibeVerboseMongo) + logDiagnostic("getMore(legacy): [db=%s] collection=%s, cursor=%s, nret=%s", database, collection_name, cursor_id, nret); + int brokenId = 0; int nextId = 0; int num_docs; @@ -532,6 +546,9 @@ final class MongoConnection { enum needsDup = hasIndirections!T || is(T == Bson); + debug (VibeVerboseMongo) + logDiagnostic("startFind: %s", command); + auto id = sendMsg(-1, 0, command); recvMsg!needsDup(id, (flags, root) { if (root["ok"].get!double != 1.0) @@ -680,10 +697,15 @@ final class MongoConnection { switch (payloadType) { case 0: gotSec0 = true; + scope Bson data; static if (dupBson) - on_sec0(flagBits, recvBsonDup()); + data = recvBsonDup(); else - on_sec0(flagBits, recvBson(bufsl)); + data = recvBson(bufsl); + + debug (VibeVerboseMongo) + logDiagnostic("recvData: sec0[flags=%x]: %s", flagBits, data); + on_sec0(flagBits, data); break; case 1: if (!gotSec0) @@ -694,10 +716,16 @@ final class MongoConnection { auto identifier = recvCString(); on_sec1_start(identifier, size); while (m_bytesRead - section_bytes_read < size) { + scope Bson data; static if (dupBson) - on_sec1_doc(identifier, recvBsonDup()); + data = recvBsonDup(); else - on_sec1_doc(identifier, recvBson(bufsl)); + data = recvBson(bufsl); + + debug (VibeVerboseMongo) + logDiagnostic("recvData: sec1[%s]: %s", identifier, data); + + on_sec1_doc(identifier, data); } break; default: @@ -709,6 +737,7 @@ final class MongoConnection { { uint crc = recvUInt(); // TODO: validate CRC + logDiagnostic("recvData: crc=%s (discarded)", crc); } assert(bytes_read + msglen == m_bytesRead, @@ -844,7 +873,7 @@ final class MongoConnection { } dst[0 .. 4] = toBsonData(len)[]; recv(dst[4 .. $]); - return Bson(Bson.Type.Object, cast(immutable)dst); + return Bson(Bson.Type.object, cast(immutable)dst); } private Bson recvBsonDup() @trusted { @@ -853,7 +882,7 @@ final class MongoConnection { ubyte[] dst = new ubyte[fromBsonData!uint(size)]; dst[0 .. 4] = size; recv(dst[4 .. $]); - return Bson(Bson.Type.Object, cast(immutable)dst); + return Bson(Bson.Type.object, cast(immutable)dst); } private void recv(ubyte[] dst) { enforce(m_stream); m_stream.read(dst); m_bytesRead += dst.length; } private const(char)[] recvCString() diff --git a/tests/mongodb/_connection/dub.json b/tests/mongodb/_connection/dub.json index d683b95dde..20c1c5a8c3 100644 --- a/tests/mongodb/_connection/dub.json +++ b/tests/mongodb/_connection/dub.json @@ -3,5 +3,6 @@ "description": "MongoDB connection tests", "dependencies": { "vibe-d:mongodb": {"path": "../../../"} - } + }, + "debugVersions": ["VibeVerboseMongo"] } diff --git a/tests/mongodb/_connection/source/app.d b/tests/mongodb/_connection/source/app.d index 49f3c490e6..7b73ea4cba 100644 --- a/tests/mongodb/_connection/source/app.d +++ b/tests/mongodb/_connection/source/app.d @@ -13,6 +13,8 @@ int main(string[] args) string username, password; ushort port; + setLogLevel(LogLevel.diagnostic); + if (args.length < 2) { logError("Usage: %s [port] (failconnect) (faildb) (failauth) (auth [username] [password])", @@ -107,7 +109,7 @@ int main(string[] args) try { logInfo(`Trying to insert {"_id": "%s", "hello": "world"}`, objID); - coll.insert(Bson(["_id": Bson(objID), "hello": Bson("world")])); + coll.insertOne(Bson(["_id": Bson(objID), "hello": Bson("world")])); } catch (MongoDriverException e) { From d34d10414083f7fb560642e1c3e0a9d821ac17d8 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Sun, 25 Sep 2022 17:24:29 +0200 Subject: [PATCH 20/36] avoid calling wrong mongo overloads --- mongodb/vibe/db/mongo/collection.d | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/mongodb/vibe/db/mongo/collection.d b/mongodb/vibe/db/mongo/collection.d index 3940759afc..548636e85b 100644 --- a/mongodb/vibe/db/mongo/collection.d +++ b/mongodb/vibe/db/mongo/collection.d @@ -100,7 +100,7 @@ struct MongoCollection { Throws: Exception if a DB communication error occurred. See_Also: $(LINK http://www.mongodb.org/display/DOCS/Inserting) */ - deprecated("Use the overload taking options, this method breaks in MongoDB 5.1 and onwards.") + deprecated("Use the insertOne or insertMany, this method breaks in MongoDB 5.1 and onwards.") void insert(T)(T document_or_documents, InsertFlags flags = InsertFlags.None) { assert(m_client !is null, "Inserting into uninitialized MongoCollection."); @@ -370,7 +370,7 @@ struct MongoCollection { } deprecated("Use the overload taking FindOptions instead, this method breaks in MongoDB 5.1 and onwards. Note: using a `$query` / `query` member to override the query arguments is no longer supported in the new overload.") - MongoCursor!R find(R = Bson, T, U)(T query, U returnFieldSelector, QueryFlags flags = QueryFlags.None, int num_skip = 0, int num_docs_per_chunk = 0) + MongoCursor!R find(R = Bson, T, U)(T query, U returnFieldSelector, QueryFlags flags, int num_skip = 0, int num_docs_per_chunk = 0) { assert(m_client !is null, "Querying uninitialized MongoCollection."); return MongoCursor!R(m_client, m_db.name, m_name, flags, num_skip, num_docs_per_chunk, query, returnFieldSelector); @@ -383,19 +383,16 @@ struct MongoCollection { See_Also: $(LINK http://www.mongodb.org/display/DOCS/Querying) */ - MongoCursor!R find(R = Bson, Q)(Q query, FindOptions options) + MongoCursor!R find(R = Bson, Q)(Q query, FindOptions options = FindOptions.init) { return MongoCursor!R(m_client, m_db.name, m_name, query, options); } - /// ditto - MongoCursor!R find(R = Bson, Q)(Q query) { return find!R(query, FindOptions.init); } - /// ditto MongoCursor!R find(R = Bson)() { return find!R(Bson.emptyObject, FindOptions.init); } deprecated("Use the overload taking FindOptions instead, this method breaks in MongoDB 5.1 and onwards. Note: using a `$query` / `query` member to override the query arguments is no longer supported in the new overload.") - auto findOne(R = Bson, T, U)(T query, U returnFieldSelector, QueryFlags flags = QueryFlags.None) + auto findOne(R = Bson, T, U)(T query, U returnFieldSelector, QueryFlags flags) { import std.traits; import std.typecons; @@ -466,7 +463,7 @@ struct MongoCollection { } /// ditto - deprecated("Use deleteOne or deleteMany taking DeleteOptions instead, this method breaks in MongoDB 5.1 and onwards.") + deprecated("Use deleteMany taking DeleteOptions instead, this method breaks in MongoDB 5.1 and onwards.") void remove()() { remove(Bson.emptyObject); } /** From 82dc01f1dccf027afd40cf4c477ae33d013a00a6 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Sun, 25 Sep 2022 19:37:31 +0200 Subject: [PATCH 21/36] fix Bson.toString to not mix up object order avoids confusing you when debugging MongoDB issues --- data/vibe/data/bson.d | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/data/vibe/data/bson.d b/data/vibe/data/bson.d index f0deef4c3b..320ce0f698 100644 --- a/data/vibe/data/bson.d +++ b/data/vibe/data/bson.d @@ -556,7 +556,42 @@ struct Bson { */ string toString() const { - return toJson().toString(); + auto ret = appender!string; + toString(ret); + return ret.data; + } + + void toString(R)(ref R range) + const { + switch (type) + { + case Bson.Type.object: + // keep ordering of objects + range.put("{"); + bool first = true; + foreach (k, v; this.byKeyValue) + { + if (!first) range.put(","); + first = false; + range.put(Json(k).toString()); + range.put(":"); + v.toString(range); + } + range.put("}"); + break; + case Bson.Type.array: + range.put("["); + foreach (i, v; this.byIndexValue) + { + if (i != 0) range.put(","); + v.toString(range); + } + range.put("]"); + break; + default: + range.put(toJson().toString()); + break; + } } import std.typecons : Nullable; From 9b8f0240cb6d0c9ddcf895f69fb6d7296986cd6b Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 30 Sep 2022 07:30:59 +0200 Subject: [PATCH 22/36] debug output in broken CI test --- tests/mongodb/_connection/source/app.d | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/mongodb/_connection/source/app.d b/tests/mongodb/_connection/source/app.d index 7b73ea4cba..817f34bf90 100644 --- a/tests/mongodb/_connection/source/app.d +++ b/tests/mongodb/_connection/source/app.d @@ -143,6 +143,10 @@ int main(string[] args) foreach (v; coll.find()) logInfo("\t%s", v); + logInfo("Filtering for target:", objID); + foreach (v; coll.find(["_id": objID])) + logInfo("\t%s", v); + auto v = coll.findOne(["_id": objID]); assert(!v.isNull, "Just-inserted entry is not added to the database"); assert(v["hello"].get!string == "world", From d7926a5e8a4b87d6d830f4ad3e4d3b084aed65a6 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 30 Sep 2022 08:00:02 +0200 Subject: [PATCH 23/36] fix Mongo tests --- mongodb/vibe/db/mongo/cursor.d | 3 ++- tests/mongodb/_connection/source/app.d | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mongodb/vibe/db/mongo/cursor.d b/mongodb/vibe/db/mongo/cursor.d index 7cc254b12a..53bce72bd5 100644 --- a/mongodb/vibe/db/mongo/cursor.d +++ b/mongodb/vibe/db/mongo/cursor.d @@ -448,7 +448,8 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { final @property long index() @safe { - return m_totalReceived + m_readDoc; + assert(m_totalReceived >= m_documents.length); + return m_totalReceived - m_documents.length + m_readDoc; } final @property DocType front() diff --git a/tests/mongodb/_connection/source/app.d b/tests/mongodb/_connection/source/app.d index 817f34bf90..c76d4eaff1 100644 --- a/tests/mongodb/_connection/source/app.d +++ b/tests/mongodb/_connection/source/app.d @@ -140,12 +140,12 @@ int main(string[] args) } logInfo("Everything in DB (target=%s):", objID); - foreach (v; coll.find()) - logInfo("\t%s", v); - - logInfo("Filtering for target:", objID); - foreach (v; coll.find(["_id": objID])) + size_t indexCheck; + foreach (v; coll.find().byPair) + { + assert(v[0] == indexCheck++); logInfo("\t%s", v); + } auto v = coll.findOne(["_id": objID]); assert(!v.isNull, "Just-inserted entry is not added to the database"); From 31d66303f96188b9c3b8c3cc721e98e2a848a309 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 30 Sep 2022 08:00:18 +0200 Subject: [PATCH 24/36] add ObjectID support to Bson.toString --- data/vibe/data/bson.d | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/data/vibe/data/bson.d b/data/vibe/data/bson.d index 320ce0f698..8503ef223d 100644 --- a/data/vibe/data/bson.d +++ b/data/vibe/data/bson.d @@ -565,6 +565,11 @@ struct Bson { const { switch (type) { + case Bson.Type.objectID: + range.put("ObjectID("); + range.put(get!BsonObjectID().toString()); + range.put(")"); + break; case Bson.Type.object: // keep ordering of objects range.put("{"); From b81e36ee7aa9cff5c0286ee0ae50f69536061b63 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 30 Sep 2022 08:04:32 +0200 Subject: [PATCH 25/36] use MongoDriverException in runCommand --- mongodb/vibe/db/mongo/database.d | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongodb/vibe/db/mongo/database.d b/mongodb/vibe/db/mongo/database.d index 08678797ab..8c4bedece4 100644 --- a/mongodb/vibe/db/mongo/database.d +++ b/mongodb/vibe/db/mongo/database.d @@ -140,7 +140,7 @@ struct MongoDatabase return runCommand(command_and_options, false, errorInfo, errorFile, errorLine); } /// ditto - Bson runCommand(T)(T command_and_options, bool checkOk, + Bson runCommand(T, ExceptionT = MongoDriverException)(T command_and_options, bool checkOk, string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__) { Bson cmd; @@ -148,7 +148,7 @@ struct MongoDatabase cmd = command_and_options; else cmd = command_and_options.serializeToBson; - return m_client.lockConnection().runCommand!(Bson, MongoException)(m_name, cmd, checkOk, errorInfo, errorFile, errorLine); + return m_client.lockConnection().runCommand!(Bson, ExceptionT)(m_name, cmd, checkOk, errorInfo, errorFile, errorLine); } /// ditto MongoCursor!R runListCommand(R = Bson, T)(T command_and_options, int batchSize = 0, long getMoreMaxTimeMS = long.max) From b19b8ca8225ab58f8a6a6e258c8c9880d9c83adb Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 30 Sep 2022 08:14:34 +0200 Subject: [PATCH 26/36] add some `@safe` annotations to fix CI --- mongodb/vibe/db/mongo/connection.d | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index 167c7374de..097ed67684 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -358,7 +358,7 @@ final class MongoConnection { scope (failure) disconnect(); - string formatErrorInfo(string msg) + string formatErrorInfo(string msg) @safe { return text(msg, " in ", errorInfo, " (", errorFile, ":", errorLine, ")"); } @@ -374,11 +374,11 @@ final class MongoConnection { auto id = sendMsg(-1, 0, command); Appender!(Bson[])[string] docs; - recvMsg!true(id, (flags, root) { + recvMsg!true(id, (flags, root) @safe { ret = root; - }, (ident, size) { + }, (ident, size) @safe { docs[ident] = appender!(Bson[]); - }, (ident, push) { + }, (ident, push) @safe { docs[ident].put(push); }); @@ -450,7 +450,7 @@ final class MongoConnection { if (timeout != Duration.max && timeout.total!"msecs" < int.max) command["maxTimeMS"] = Bson(cast(int)timeout.total!"msecs"); - string formatErrorInfo(string msg) + string formatErrorInfo(string msg) @safe { return text(msg, " in ", errorInfo, " (", errorFile, ":", errorLine, ")"); } @@ -465,7 +465,7 @@ final class MongoConnection { logDiagnostic("getMore: [db=%s] %s", database, command); auto id = sendMsg(-1, 0, command); - recvMsg!needsDup(id, (flags, root) { + recvMsg!needsDup(id, (flags, root) @safe { if (root["ok"].get!double != 1.0) throw new MongoDriverException(formatErrorInfo("getMore failed: " ~ root["errmsg"].opt!string("(no message)"))); @@ -479,7 +479,7 @@ final class MongoConnection { T doc = deserializeBson!T(push); on_doc(doc); } - }, (ident, size) {}, (ident, push) { + }, (ident, size) @safe {}, (ident, push) @safe { throw new MongoDriverException(formatErrorInfo("unexpected section type 1 in getMore response")); }); } @@ -535,7 +535,7 @@ final class MongoConnection { scope GetMoreDocumentDelegate!T on_doc, string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__) { - string formatErrorInfo(string msg) + string formatErrorInfo(string msg) @safe { return text(msg, " in ", errorInfo, " (", errorFile, ":", errorLine, ")"); } @@ -550,7 +550,7 @@ final class MongoConnection { logDiagnostic("startFind: %s", command); auto id = sendMsg(-1, 0, command); - recvMsg!needsDup(id, (flags, root) { + recvMsg!needsDup(id, (flags, root) @safe { if (root["ok"].get!double != 1.0) throw new MongoDriverException(formatErrorInfo("find failed: " ~ root["errmsg"].opt!string("(no message)"))); @@ -564,7 +564,7 @@ final class MongoConnection { T doc = deserializeBson!T(push); on_doc(doc); } - }, (ident, size) {}, (ident, push) { + }, (ident, size) @safe {}, (ident, push) @safe { throw new MongoDriverException(formatErrorInfo("unexpected section type 1 in find response")); }); } From f358b66dbcc30a20e199ac012bb6ad1eeb8c0530 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 30 Sep 2022 08:19:03 +0200 Subject: [PATCH 27/36] add some scope attributes to fix CI --- mongodb/vibe/db/mongo/connection.d | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index 097ed67684..6b9a79d84f 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -376,9 +376,9 @@ final class MongoConnection { Appender!(Bson[])[string] docs; recvMsg!true(id, (flags, root) @safe { ret = root; - }, (ident, size) @safe { + }, (scope ident, size) @safe { docs[ident] = appender!(Bson[]); - }, (ident, push) @safe { + }, (scope ident, push) @safe { docs[ident].put(push); }); @@ -479,7 +479,7 @@ final class MongoConnection { T doc = deserializeBson!T(push); on_doc(doc); } - }, (ident, size) @safe {}, (ident, push) @safe { + }, (scope ident, size) @safe {}, (scope ident, push) @safe { throw new MongoDriverException(formatErrorInfo("unexpected section type 1 in getMore response")); }); } @@ -564,7 +564,7 @@ final class MongoConnection { T doc = deserializeBson!T(push); on_doc(doc); } - }, (ident, size) @safe {}, (ident, push) @safe { + }, (scope ident, size) @safe {}, (scope ident, push) @safe { throw new MongoDriverException(formatErrorInfo("unexpected section type 1 in find response")); }); } From 04c692ed265944b0701e0309f5872fefa0f8f29d Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 30 Sep 2022 08:32:01 +0200 Subject: [PATCH 28/36] fix scoped recvBson overloads --- mongodb/vibe/db/mongo/connection.d | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index 6b9a79d84f..d512f04e67 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -701,7 +701,7 @@ final class MongoConnection { static if (dupBson) data = recvBsonDup(); else - data = recvBson(bufsl); + data = (() @trusted => recvBson(bufsl))(); debug (VibeVerboseMongo) logDiagnostic("recvData: sec0[flags=%x]: %s", flagBits, data); @@ -720,7 +720,7 @@ final class MongoConnection { static if (dupBson) data = recvBsonDup(); else - data = recvBson(bufsl); + data = (() @trusted => recvBson(bufsl))(); debug (VibeVerboseMongo) logDiagnostic("recvData: sec1[%s]: %s", identifier, data); From 90e96b369886db08405d2f4c8475e1a9fce8cf05 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 30 Sep 2022 08:53:17 +0200 Subject: [PATCH 29/36] add scope attribute when there might be scope as we simply discard these parameters, we declare those scope, as they might be scope in the type definition if the template parameter is true. Old DMD versions don't allow us to pass delegates here if we omit scope on scope parameters. --- mongodb/vibe/db/mongo/connection.d | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index d512f04e67..b9892215c2 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -479,7 +479,7 @@ final class MongoConnection { T doc = deserializeBson!T(push); on_doc(doc); } - }, (scope ident, size) @safe {}, (scope ident, push) @safe { + }, (scope ident, size) @safe {}, (scope ident, scope push) @safe { throw new MongoDriverException(formatErrorInfo("unexpected section type 1 in getMore response")); }); } @@ -564,7 +564,7 @@ final class MongoConnection { T doc = deserializeBson!T(push); on_doc(doc); } - }, (scope ident, size) @safe {}, (scope ident, push) @safe { + }, (scope ident, size) @safe {}, (scope ident, scope push) @safe { throw new MongoDriverException(formatErrorInfo("unexpected section type 1 in find response")); }); } From 865e1e97ddc39f7f5d1eb8ea54e92c99511a739f Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 30 Sep 2022 09:10:06 +0200 Subject: [PATCH 30/36] more scoping --- mongodb/vibe/db/mongo/connection.d | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index b9892215c2..a5401fc9f6 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -465,7 +465,7 @@ final class MongoConnection { logDiagnostic("getMore: [db=%s] %s", database, command); auto id = sendMsg(-1, 0, command); - recvMsg!needsDup(id, (flags, root) @safe { + recvMsg!needsDup(id, (flags, scope root) @safe { if (root["ok"].get!double != 1.0) throw new MongoDriverException(formatErrorInfo("getMore failed: " ~ root["errmsg"].opt!string("(no message)"))); @@ -550,7 +550,7 @@ final class MongoConnection { logDiagnostic("startFind: %s", command); auto id = sendMsg(-1, 0, command); - recvMsg!needsDup(id, (flags, root) @safe { + recvMsg!needsDup(id, (flags, scope root) @safe { if (root["ok"].get!double != 1.0) throw new MongoDriverException(formatErrorInfo("find failed: " ~ root["errmsg"].opt!string("(no message)"))); From 40d235f936252e4e3572cd5ce709d0fb0f21fa67 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 30 Sep 2022 09:24:16 +0200 Subject: [PATCH 31/36] Don't disconnect on all failed DB commands --- mongodb/vibe/db/mongo/connection.d | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index a5401fc9f6..e9ae1b116a 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -205,6 +205,8 @@ final class MongoConnection { throw new MongoDriverException(format("Failed to connect to MongoDB server at %s:%s.", m_settings.hosts[0].name, m_settings.hosts[0].port), __FILE__, __LINE__, e); } + scope (failure) disconnect(); + m_allowReconnect = false; scope (exit) m_allowReconnect = true; @@ -356,8 +358,6 @@ final class MongoConnection { { import std.array; - scope (failure) disconnect(); - string formatErrorInfo(string msg) @safe { return text(msg, " in ", errorInfo, " (", errorFile, ":", errorLine, ")"); @@ -933,6 +933,8 @@ final class MongoConnection { private void authenticate() { + scope (failure) disconnect(); + string cn = m_settings.getAuthDatabase; auto cmd = Bson(["getnonce": Bson(1)]); From 7b6ce691b8aff25624cdd9a2891d43cc12424a26 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Fri, 30 Sep 2022 09:47:38 +0200 Subject: [PATCH 32/36] CRC bits in sending MongoDB not yet implemented --- mongodb/vibe/db/mongo/connection.d | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index e9ae1b116a..acb574201a 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -833,6 +833,8 @@ final class MongoConnection { sendValue!int(response_to); sendValue!int(cast(int)OpCode.Msg); sendValue!uint(flagBits); + const bool hasCRC = (flagBits & (1 << 16)) != 0; + assert(!hasCRC, "sending with CRC bits not yet implemented"); sendValue!ubyte(0); sendValue(document); m_outRange.flush(); From ad81de393e02033e15647a713df863f8d5c75f70 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Wed, 19 Oct 2022 15:46:37 +0200 Subject: [PATCH 33/36] adjust to review --- mongodb/vibe/db/mongo/collection.d | 26 +++++++------- mongodb/vibe/db/mongo/connection.d | 58 +++++++++++++++++++++++++++--- mongodb/vibe/db/mongo/cursor.d | 41 +++++++++++++-------- mongodb/vibe/db/mongo/database.d | 56 ++++++++++++++++++++--------- mongodb/vibe/db/mongo/impl/crud.d | 10 ++++++ mongodb/vibe/db/mongo/impl/index.d | 7 ++++ mongodb/vibe/db/mongo/settings.d | 4 +-- 7 files changed, 151 insertions(+), 51 deletions(-) diff --git a/mongodb/vibe/db/mongo/collection.d b/mongodb/vibe/db/mongo/collection.d index 548636e85b..fd9f41b15f 100644 --- a/mongodb/vibe/db/mongo/collection.d +++ b/mongodb/vibe/db/mongo/collection.d @@ -139,7 +139,7 @@ struct MongoCollection { foreach (string k, v; serializeToBson(options).byKeyValue) cmd[k] = v; - database.runCommand(cmd, true); + database.runCommandChecked(cmd); return res; } @@ -168,7 +168,7 @@ struct MongoCollection { foreach (string k, v; serializeToBson(options).byKeyValue) cmd[k] = v; - database.runCommand(cmd, true); + database.runCommandChecked(cmd); return InsertManyResult(insertedIds); } @@ -231,7 +231,7 @@ struct MongoCollection { } cmd["deletes"] = Bson(deletesBson); - auto n = database.runCommand(cmd, true)["n"].get!long; + auto n = database.runCommandChecked(cmd)["n"].get!long; return DeleteResult(n); } @@ -354,7 +354,7 @@ struct MongoCollection { } cmd["updates"] = Bson(updatesBson); - auto res = database.runCommand(cmd, true); + auto res = database.runCommandChecked(cmd); auto ret = UpdateResult( res["n"].get!long, res["nModified"].get!long, @@ -493,7 +493,7 @@ struct MongoCollection { cmd.query = query; cmd.update = update; cmd.fields = returnFieldSelector; - auto ret = database.runCommand(cmd, true); + auto ret = database.runCommandChecked(cmd); return ret["value"]; } @@ -532,7 +532,7 @@ struct MongoCollection { cmd[key] = value; return 0; }); - auto ret = database.runCommand(cmd, true); + auto ret = database.runCommandChecked(cmd); return ret["value"]; } @@ -555,7 +555,7 @@ struct MongoCollection { Bson cmd = Bson.emptyObject; cmd["count"] = m_name; cmd["query"] = serializeToBson(query); - auto reply = database.runCommand(cmd, true); + auto reply = database.runCommandChecked(cmd); switch (reply["n"].type) with (Bson.Type) { default: assert(false, "Unsupported data type in BSON reply for COUNT"); case double_: return cast(ulong)reply["n"].get!double; // v2.x @@ -744,7 +744,7 @@ struct MongoCollection { import std.algorithm : map; - auto res = m_db.runCommand(cmd, true); + auto res = m_db.runCommandChecked(cmd); static if (is(R == Bson)) return res["values"].byValue; else return res["values"].byValue.map!(b => deserializeBson!R(b)); } @@ -824,7 +824,7 @@ struct MongoCollection { CMD cmd; cmd.dropIndexes = m_name; cmd.index = name; - database.runCommand(cmd, true); + database.runCommandChecked(cmd); } /// ditto @@ -872,7 +872,7 @@ struct MongoCollection { CMD cmd; cmd.dropIndexes = m_name; cmd.index = "*"; - database.runCommand(cmd, true); + database.runCommandChecked(cmd); } /// Unofficial API extension, more efficient multi-index removal on @@ -889,7 +889,7 @@ struct MongoCollection { CMD cmd; cmd.dropIndexes = m_name; cmd.index = names; - database.runCommand(cmd, true); + database.runCommandChecked(cmd); } else { foreach (name; names) dropIndex(name); @@ -988,7 +988,7 @@ struct MongoCollection { indexes ~= index; } cmd["indexes"] = Bson(indexes); - database.runCommand(cmd, true); + database.runCommandChecked(cmd); } else { foreach (model; models) { // trusted to support old compilers which think opt_dup has @@ -1050,7 +1050,7 @@ struct MongoCollection { CMD cmd; cmd.drop = m_name; - database.runCommand(cmd, true); + database.runCommandChecked(cmd); } } diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index acb574201a..7f8f0bc951 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -247,8 +247,7 @@ final class MongoConnection { handshake["client"]["application"] = Bson(["name": Bson(m_settings.appName)]); } - auto reply = runCommand!Bson("admin", handshake); - enforce!MongoAuthException(reply["ok"].get!double == 1.0, "Authentication failed."); + auto reply = runCommand!(Bson, MongoAuthException)("admin", handshake); m_description = deserializeBson!ServerDescription(reply); if (m_description.satisfiesVersion(WireVersion.v36)) @@ -352,8 +351,57 @@ final class MongoConnection { recvReply!T(id, on_msg, on_doc); } - Bson runCommand(T, CommandFailException = MongoDriverException)(string database, Bson command, bool testOk = true, - string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__) + /** + Runs the given Bson command (Bson object with the first entry in the map + being the command name) on the given database. + + Using `runCommand` checks that the command completed successfully by + checking that `result["ok"].get!double == 1.0`. Throws the + `CommandFailException` on failure. + + Using `runCommandUnchecked` will return the result as-is. Developers may + check the `result["ok"]` value themselves. (It's a double that needs to + be compared with 1.0 by default) + + Throws: + - `CommandFailException` (template argument) only in the + `runCommand` overload, when the command response is not ok. + - `MongoDriverException` when internal protocol errors occur. + */ + Bson runCommand(T, CommandFailException = MongoDriverException)( + string database, + Bson command, + string errorInfo = __FUNCTION__, + string errorFile = __FILE__, + size_t errorLine = __LINE__ + ) + in(database.length, "runCommand requires a database argument") + { + return runCommandImpl!(T, CommandFailException)( + database, command, true, errorInfo, errorFile, errorLine); + } + + Bson runCommandUnchecked(T, CommandFailException = MongoDriverException)( + string database, + Bson command, + string errorInfo = __FUNCTION__, + string errorFile = __FILE__, + size_t errorLine = __LINE__ + ) + in(database.length, "runCommand requires a database argument") + { + return runCommandImpl!(T, CommandFailException)( + database, command, false, errorInfo, errorFile, errorLine); + } + + private Bson runCommandImpl(T, CommandFailException)( + string database, + Bson command, + bool testOk = true, + string errorInfo = __FUNCTION__, + string errorFile = __FILE__, + size_t errorLine = __LINE__ + ) in(database.length, "runCommand requires a database argument") { import std.array; @@ -621,7 +669,7 @@ final class MongoConnection { _MongoErrorDescription ret; - auto error = runCommand!Bson(db, command_and_options); + auto error = runCommandUnchecked!Bson(db, command_and_options); try { ret = MongoErrorDescription( diff --git a/mongodb/vibe/db/mongo/cursor.d b/mongodb/vibe/db/mongo/cursor.d index 53bce72bd5..8fee0b9390 100644 --- a/mongodb/vibe/db/mongo/cursor.d +++ b/mongodb/vibe/db/mongo/cursor.d @@ -29,13 +29,15 @@ import core.time; struct MongoCursor(DocType = Bson) { private IMongoCursorData!DocType m_data; - package deprecated this(Q, S)(MongoClient client, string collection, QueryFlags flags, int nskip, int nret, Q query, S return_field_selector) + deprecated("Old (MongoDB <3.6) style cursor iteration no longer supported") + package this(Q, S)(MongoClient client, string collection, QueryFlags flags, int nskip, int nret, Q query, S return_field_selector) { // TODO: avoid memory allocation, if possible m_data = new MongoQueryCursor!(Q, DocType, S)(client, collection, flags, nskip, nret, query, return_field_selector); } - package deprecated this(MongoClient client, string collection, long cursor, DocType[] existing_documents) + deprecated("Old (MongoDB <3.6) style cursor iteration no longer supported") + package this(MongoClient client, string collection, long cursor, DocType[] existing_documents) { // TODO: avoid memory allocation, if possible m_data = new MongoGenericCursor!DocType(client, collection, cursor, existing_documents); @@ -274,16 +276,25 @@ struct MongoCursor(DocType = Bson) { } } +/// Actual iteration implementation details for MongoCursor. Abstracted using an +/// interface because we still have code for legacy (<3.6) MongoDB servers, +/// which may still used with the old legacy overloads. private interface IMongoCursorData(DocType) { - @property bool empty() @safe; - @property long index() @safe; - @property DocType front() @safe; + bool empty() @safe; /// Range implementation + long index() @safe; /// Range implementation + DocType front() @safe; /// Range implementation + void popFront() @safe; /// Range implementation + /// Before iterating, specify a MongoDB sort order void sort(Bson order) @safe; + /// Before iterating, specify maximum number of returned items void limit(long count) @safe; + /// Before iterating, skip the specified number of items (when sorted) void skip(long count) @safe; - void popFront() @safe; - void startIterating() @safe; + /// Kills the MongoDB cursor, further iteration attempts will result in + /// errors. Call this in the destructor. void killCursors() @safe; + /// Define an reference count property on the class, which is returned by + /// reference with this method. ref int refCount() @safe; } @@ -307,7 +318,7 @@ private deprecated abstract class LegacyMongoCursorData(DocType) : IMongoCursorD long m_limit = 0; } - final @property bool empty() + final bool empty() @safe { if (!m_iterationStarted) startIterating(); if (m_limit > 0 && index >= m_limit) { @@ -324,12 +335,12 @@ private deprecated abstract class LegacyMongoCursorData(DocType) : IMongoCursorD return m_currentDoc >= m_documents.length; } - final @property long index() + final long index() @safe { return m_offset + m_currentDoc; } - final @property DocType front() + final DocType front() @safe { if (!m_iterationStarted) startIterating(); assert(!empty(), "Cursor has no more data."); @@ -428,7 +439,7 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { m_database = command["$db"].opt!string; } - @property bool empty() + bool empty() @safe { if (!m_iterationStarted) startIterating(); if (m_queryLimit > 0 && index >= m_queryLimit) { @@ -446,13 +457,13 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { return m_readDoc >= m_documents.length; } - final @property long index() + final long index() @safe { assert(m_totalReceived >= m_documents.length); return m_totalReceived - m_documents.length + m_readDoc; } - final @property DocType front() + final DocType front() @safe { if (!m_iterationStarted) startIterating(); assert(!empty(), "Cursor has no more data."); @@ -484,7 +495,7 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { m_readDoc++; } - void startIterating() + private void startIterating() @safe { auto conn = m_client.lockConnection(); m_totalReceived = 0; @@ -540,7 +551,7 @@ private deprecated class MongoQueryCursor(Q, R, S) : LegacyMongoCursorData!R { m_returnFieldSelector = return_field_selector; } - override void startIterating() + void startIterating() @safe { auto conn = m_client.lockConnection(); diff --git a/mongodb/vibe/db/mongo/database.d b/mongodb/vibe/db/mongo/database.d index 8c4bedece4..b899ff979e 100644 --- a/mongodb/vibe/db/mongo/database.d +++ b/mongodb/vibe/db/mongo/database.d @@ -92,7 +92,7 @@ struct MongoDatabase } CMD cmd; cmd.getLog = mask; - return runCommand(cmd, true); + return runCommandChecked(cmd); } /** Performs a filesystem/disk sync of the database on the server. @@ -109,9 +109,15 @@ struct MongoDatabase } CMD cmd; cmd.async = async; - return runCommand(cmd, true); + return runCommandChecked(cmd); } + deprecated("use runCommandChecked or runCommandUnchecked instead") + Bson runCommand(T)(T command_and_options, + string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__) + { + return runCommandUnchecked(command_and_options, errorInfo, errorFile, errorLine); + } /** Generic means to run commands on the database. @@ -119,37 +125,55 @@ struct MongoDatabase of possible values for command_and_options. Note that some commands return a cursor instead of a single document. - In this case, use `runListCommand` instead of `runCommand` to be able - to properly iterate over the results. + In this case, use `runListCommand` instead of `runCommandChecked` or + `runCommandUnchecked` to be able to properly iterate over the results. + + Usually commands respond with a `double ok` field in them, the `Checked` + version of this function checks that they equal to `1.0`. The `Unchecked` + version of this function does not check that parameter. + + With cursor functions on `runListCommand` the error checking is well + defined. Params: command_and_options = Bson object containing the command to be executed as well as the command parameters as fields - checkOk = usually commands respond with a `double ok` field in them, - which is not checked if this parameter is false. Explicitly - specify this parameter to avoid issues with error checking. - Currently defaults to `false` (meaning don't check "ok" field), - omitting the argument may change to `true` in the future. Returns: The raw response of the MongoDB server */ - deprecated("use runCommand with explicit checkOk overload") - Bson runCommand(T)(T command_and_options, - string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__) + Bson runCommandChecked(T, ExceptionT = MongoDriverException)( + T command_and_options, + string errorInfo = __FUNCTION__, + string errorFile = __FILE__, + size_t errorLine = __LINE__ + ) { - return runCommand(command_and_options, false, errorInfo, errorFile, errorLine); + Bson cmd; + static if (is(T : Bson)) + cmd = command_and_options; + else + cmd = command_and_options.serializeToBson; + return m_client.lockConnection().runCommand!(Bson, ExceptionT)( + m_name, cmd, errorInfo, errorFile, errorLine); } + /// ditto - Bson runCommand(T, ExceptionT = MongoDriverException)(T command_and_options, bool checkOk, - string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__) + Bson runCommandUnchecked(T, ExceptionT = MongoDriverException)( + T command_and_options, + string errorInfo = __FUNCTION__, + string errorFile = __FILE__, + size_t errorLine = __LINE__ + ) { Bson cmd; static if (is(T : Bson)) cmd = command_and_options; else cmd = command_and_options.serializeToBson; - return m_client.lockConnection().runCommand!(Bson, ExceptionT)(m_name, cmd, checkOk, errorInfo, errorFile, errorLine); + return m_client.lockConnection().runCommandUnchecked!(Bson, ExceptionT)( + m_name, cmd, errorInfo, errorFile, errorLine); } + /// ditto MongoCursor!R runListCommand(R = Bson, T)(T command_and_options, int batchSize = 0, long getMoreMaxTimeMS = long.max) { diff --git a/mongodb/vibe/db/mongo/impl/crud.d b/mongodb/vibe/db/mongo/impl/crud.d index f88e5fc18e..67ab92e483 100644 --- a/mongodb/vibe/db/mongo/impl/crud.d +++ b/mongodb/vibe/db/mongo/impl/crud.d @@ -1,3 +1,10 @@ +/** + MongoDB CRUD API definitions. + + Copyright: © 2022 Jan Jurzitza + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Jan Jurzitza +*/ module vibe.db.mongo.impl.crud; import core.time; @@ -9,6 +16,7 @@ import std.typecons; @safe: +deprecated("Use UpdateOptions instead") enum UpdateFlags { none = 0, /// Normal update of a single document. upsert = 1<<0, /// Creates a document if none exists. @@ -19,6 +27,7 @@ enum UpdateFlags { MultiUpdate = multiUpdate /// Deprecated compatibility alias } +deprecated("Use InsertOneOptions or InsertManyOptions instead") enum InsertFlags { none = 0, /// Normal insert. continueOnError = 1<<0, /// For multiple inserted documents, continues inserting further documents after a failure. @@ -48,6 +57,7 @@ enum QueryFlags { Partial = partial /// Deprecated compatibility alias } +deprecated("Use DeleteOptions instead") enum DeleteFlags { none = 0, singleRemove = 1<<0, diff --git a/mongodb/vibe/db/mongo/impl/index.d b/mongodb/vibe/db/mongo/impl/index.d index 6c2aaa1da1..18fcc691f9 100644 --- a/mongodb/vibe/db/mongo/impl/index.d +++ b/mongodb/vibe/db/mongo/impl/index.d @@ -1,3 +1,10 @@ +/** + MongoDB index API definitions. + + Copyright: © 2020-2022 Jan Jurzitza + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Jan Jurzitza +*/ module vibe.db.mongo.impl.index; @safe: diff --git a/mongodb/vibe/db/mongo/settings.d b/mongodb/vibe/db/mongo/settings.d index f51d6cebc1..f2688a97d8 100644 --- a/mongodb/vibe/db/mongo/settings.d +++ b/mongodb/vibe/db/mongo/settings.d @@ -576,8 +576,8 @@ class MongoClientSettings * Resolves the database to run authentication commands on. * (authSource if set, otherwise the URI's database if set, otherwise "admin") */ - string getAuthDatabase() @safe @nogc nothrow pure - { + string getAuthDatabase() + @safe @nogc nothrow pure const return { if (authSource.length) return authSource; else if (database.length) From acba1f4a43a19b6e76b8b186b6d984bc891748d6 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Wed, 19 Oct 2022 15:54:36 +0200 Subject: [PATCH 34/36] use Duration everywhere --- mongodb/vibe/db/mongo/connection.d | 4 ++-- mongodb/vibe/db/mongo/cursor.d | 14 ++++++------ mongodb/vibe/db/mongo/database.d | 5 +++-- mongodb/vibe/db/mongo/settings.d | 35 +++++++++++++++++++++++++----- 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index 7f8f0bc951..53263735e5 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -178,8 +178,8 @@ final class MongoConnection { m_conn = connectTCP(m_settings.hosts[0].name, m_settings.hosts[0].port, null, 0, connectTimeout); m_conn.tcpNoDelay = true; - if (m_settings.socketTimeoutMS) - m_conn.readTimeout = m_settings.socketTimeoutMS.msecs; + if (m_settings.socketTimeout != Duration.zero) + m_conn.readTimeout = m_settings.socketTimeout; if (m_settings.ssl) { auto ctx = createTLSContext(TLSContextKind.client); if (!m_settings.sslverifycertificate) { diff --git a/mongodb/vibe/db/mongo/cursor.d b/mongodb/vibe/db/mongo/cursor.d index 8fee0b9390..3716bf320e 100644 --- a/mongodb/vibe/db/mongo/cursor.d +++ b/mongodb/vibe/db/mongo/cursor.d @@ -103,10 +103,10 @@ struct MongoCursor(DocType = Bson) { : long.max); } - this(MongoClient client, Bson command, int batchSize = 0, long getMoreMaxTimeMS = long.max) + this(MongoClient client, Bson command, int batchSize = 0, Duration getMoreMaxTime = Duration.max) { // TODO: avoid memory allocation, if possible - m_data = new MongoFindCursor!DocType(client, command, batchSize, getMoreMaxTimeMS); + m_data = new MongoFindCursor!DocType(client, command, batchSize, getMoreMaxTime); } this(this) @@ -421,7 +421,7 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { string m_collection; long m_cursor; int m_batchSize; - long m_maxTimeMS; + Duration m_maxTime; long m_totalReceived; size_t m_readDoc; size_t m_insertDoc; @@ -430,12 +430,12 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { long m_queryLimit; } - this(MongoClient client, Bson command, int batchSize = 0, long getMoreMaxTimeMS = long.max) + this(MongoClient client, Bson command, int batchSize = 0, Duration getMoreMaxTime = Duration.max) { m_client = client; m_findQuery = command; m_batchSize = batchSize; - m_maxTimeMS = getMoreMaxTimeMS; + m_maxTime = getMoreMaxTime; m_database = command["$db"].opt!string; } @@ -452,8 +452,8 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { return true; auto conn = m_client.lockConnection(); - conn.getMore!DocType(m_cursor, m_database, m_collection, m_batchSize, &handleReply, &handleDocument, - m_maxTimeMS >= int.max ? Duration.max : m_maxTimeMS.msecs); + conn.getMore!DocType(m_cursor, m_database, m_collection, m_batchSize, + &handleReply, &handleDocument, m_maxTime); return m_readDoc >= m_documents.length; } diff --git a/mongodb/vibe/db/mongo/database.d b/mongodb/vibe/db/mongo/database.d index b899ff979e..8f3e59d072 100644 --- a/mongodb/vibe/db/mongo/database.d +++ b/mongodb/vibe/db/mongo/database.d @@ -14,6 +14,7 @@ import vibe.db.mongo.client; import vibe.db.mongo.collection; import vibe.data.bson; +import core.time; /** Represents a single database accessible through a given MongoClient. */ @@ -175,7 +176,7 @@ struct MongoDatabase } /// ditto - MongoCursor!R runListCommand(R = Bson, T)(T command_and_options, int batchSize = 0, long getMoreMaxTimeMS = long.max) + MongoCursor!R runListCommand(R = Bson, T)(T command_and_options, int batchSize = 0, Duration getMoreMaxTime = Duration.max) { Bson cmd; static if (is(T : Bson)) @@ -184,6 +185,6 @@ struct MongoDatabase cmd = command_and_options.serializeToBson; cmd["$db"] = Bson(m_name); - return MongoCursor!R(m_client, cmd, batchSize, getMoreMaxTimeMS); + return MongoCursor!R(m_client, cmd, batchSize, getMoreMaxTime); } } diff --git a/mongodb/vibe/db/mongo/settings.d b/mongodb/vibe/db/mongo/settings.d index f2688a97d8..74418f4426 100644 --- a/mongodb/vibe/db/mongo/settings.d +++ b/mongodb/vibe/db/mongo/settings.d @@ -12,6 +12,7 @@ import vibe.data.bson; deprecated import vibe.db.mongo.flags : QueryFlags; import vibe.inet.webform; +import core.time; import std.conv : to; import std.digest : toHexString; import std.digest.md : md5Of; @@ -449,17 +450,41 @@ class MongoClientSettings bool journal; /** - * The time in milliseconds to attempt a connection before timing out. + * The time to attempt a connection before timing out. */ - long connectTimeoutMS = 10_000; + Duration connectTimeout = 10.seconds; + + /// ditto + long connectTimeoutMS() const @property + @safe { + return connectTimeout.total!"msecs"; + } + + /// ditto + void connectTimeoutMS(long ms) @property + @safe { + connectTimeout = ms.msecs; + } /** - * The time in milliseconds to attempt a send or receive on a socket before - * the attempt times out. + * The time to attempt a send or receive on a socket before the attempt + * times out. * * Bugs: Not implemented for sending */ - long socketTimeoutMS; + Duration socketTimeout = Duration.zero; + + /// ditto + long socketTimeoutMS() const @property + @safe { + return socketTimeout.total!"msecs"; + } + + /// ditto + void socketTimeoutMS(long ms) @property + @safe { + socketTimeout = ms.msecs; + } /** * Enables or disables TLS/SSL for the connection. From e226cb86bba5528be272e5c0b33c914db3cbb6b9 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Wed, 19 Oct 2022 15:57:14 +0200 Subject: [PATCH 35/36] raise MongoDB CI timeout minutes --- .github/workflows/mongo.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mongo.yml b/.github/workflows/mongo.yml index a4a14a8754..2633f1bf15 100644 --- a/.github/workflows/mongo.yml +++ b/.github/workflows/mongo.yml @@ -19,7 +19,7 @@ jobs: - '6.0' runs-on: ${{ matrix.os }} - timeout-minutes: 10 + timeout-minutes: 60 steps: - uses: actions/checkout@v3 @@ -50,4 +50,4 @@ jobs: VIBED_DRIVER: vibe-core PARTS: mongo run: | - ./run-ci.sh \ No newline at end of file + ./run-ci.sh From f788ebc82da5e1a7d12852970c5bbccf7c520906 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Wed, 19 Oct 2022 16:00:42 +0200 Subject: [PATCH 36/36] fix tests --- mongodb/vibe/db/mongo/collection.d | 6 +++--- mongodb/vibe/db/mongo/cursor.d | 6 +++--- mongodb/vibe/db/mongo/settings.d | 15 +++++++++++++-- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/mongodb/vibe/db/mongo/collection.d b/mongodb/vibe/db/mongo/collection.d index fd9f41b15f..425d002ed8 100644 --- a/mongodb/vibe/db/mongo/collection.d +++ b/mongodb/vibe/db/mongo/collection.d @@ -678,9 +678,9 @@ struct MongoCollection { } return MongoCursor!R(m_client, cmd, !options.batchSize.isNull ? options.batchSize.get : 0, - !options.maxAwaitTimeMS.isNull ? options.maxAwaitTimeMS.get - : !options.maxTimeMS.isNull ? options.maxTimeMS.get - : long.max); + !options.maxAwaitTimeMS.isNull ? options.maxAwaitTimeMS.get.msecs + : !options.maxTimeMS.isNull ? options.maxTimeMS.get.msecs + : Duration.max); } /// Example taken from the MongoDB documentation diff --git a/mongodb/vibe/db/mongo/cursor.d b/mongodb/vibe/db/mongo/cursor.d index 3716bf320e..0989a48073 100644 --- a/mongodb/vibe/db/mongo/cursor.d +++ b/mongodb/vibe/db/mongo/cursor.d @@ -98,9 +98,9 @@ struct MongoCursor(DocType = Bson) { this(client, command, options.batchSize.isNull ? 0 : options.batchSize.get, - !options.maxAwaitTimeMS.isNull ? options.maxAwaitTimeMS.get - : allowMaxTime && !options.maxTimeMS.isNull ? options.maxTimeMS.get - : long.max); + !options.maxAwaitTimeMS.isNull ? options.maxAwaitTimeMS.get.msecs + : allowMaxTime && !options.maxTimeMS.isNull ? options.maxTimeMS.get.msecs + : Duration.max); } this(MongoClient client, Bson command, int batchSize = 0, Duration getMoreMaxTime = Duration.max) diff --git a/mongodb/vibe/db/mongo/settings.d b/mongodb/vibe/db/mongo/settings.d index 74418f4426..6ed11db9f6 100644 --- a/mongodb/vibe/db/mongo/settings.d +++ b/mongodb/vibe/db/mongo/settings.d @@ -148,6 +148,17 @@ bool parseMongoDBUrl(out MongoClientSettings cfg, string url) } } + bool setMsecs(ref Duration dst) + { + try { + dst = to!long(value).msecs; + return true; + } catch( Exception e ){ + logError("Value for '%s' must be an integer but was '%s'.", option, value); + return false; + } + } + void warnNotImplemented() { logDiagnostic("MongoDB option %s not yet implemented.", option); @@ -162,8 +173,8 @@ bool parseMongoDBUrl(out MongoClientSettings cfg, string url) case "safe": setBool(cfg.safe); break; case "fsync": setBool(cfg.fsync); break; case "journal": setBool(cfg.journal); break; - case "connecttimeoutms": setLong(cfg.connectTimeoutMS); break; - case "sockettimeoutms": setLong(cfg.socketTimeoutMS); break; + case "connecttimeoutms": setMsecs(cfg.connectTimeout); break; + case "sockettimeoutms": setMsecs(cfg.socketTimeout); break; case "tls": setBool(cfg.ssl); break; case "ssl": setBool(cfg.ssl); break; case "sslverifycertificate": setBool(cfg.sslverifycertificate); break;