Skip to content

Commit f60f5cf

Browse files
committed
Cursor split + tests (#80)
* split cursor classes * little bit of cleanup * moved cursors test to documents/collections * unit tests for split cursors * integration tests for split cursors
1 parent 4f47fc9 commit f60f5cf

File tree

34 files changed

+613
-186
lines changed

34 files changed

+613
-186
lines changed

DEVGUIDE.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ The API for the test script is as the following:
6868
6. [-R | -no-report]
6969
7. [-c <http_client>]
7070
8. [-e <environment>]
71-
9. [-stargate]
71+
9. [-local]
7272
10. [-l | -logging]
7373
11. [-P | -skip-prelude]
7474
```
@@ -159,7 +159,7 @@ By default, `astra-db-ts` assumes you're running on Astra, but you can specify t
159159
flag. It should be one of `dse`, `hcd`, `cassandra`, or `other`. You can also provide `astra`, but it wouldn't really
160160
do anything. But I'm not the boss of you; you can make your own big-boy/girl/other decisions.
161161

162-
#### 9. Running the tests on Stargate (`[-stargate]`)
162+
#### 9. Running the tests on Stargate (`[-local]`)
163163

164164
If you're running the tests on a local Stargate instance, you can use this flag to set the `CLIENT_DB_URL` to
165165
`http://localhost:8080` and the `CLIENT_DB_TOKEN` to `cassandra:cassandra` without needing to modify your `.env` file.

scripts/repl.sh

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,19 @@ fi
1414
# Rebuild the client (without types or any extra processing for speed)
1515
sh scripts/build.sh -light || exit 2
1616

17-
if [ "$1" = "-stargate" ]; then
18-
export CLIENT_DB_ENVIRONMENT='dse'
19-
export CLIENT_DB_TOKEN='Cassandra:Y2Fzc2FuZHJh:Y2Fzc2FuZHJh'
20-
export CLIENT_DB_URL='http://localhost:8181'
21-
fi
17+
while [ $# -gt 0 ]; do
18+
case "$1" in
19+
"-local")
20+
export CLIENT_DB_ENVIRONMENT='dse'
21+
export CLIENT_DB_TOKEN='Cassandra:Y2Fzc2FuZHJh:Y2Fzc2FuZHJh'
22+
export CLIENT_DB_URL='http://localhost:8181'
23+
;;
24+
"-l" | "-logging")
25+
export LOG_ALL_TO_STDOUT=true
26+
;;
27+
esac
28+
shift
29+
done
2230

2331
# Start the REPL w/ some utility stuff and stuff
2432
node -i -e "
@@ -27,7 +35,7 @@ node -i -e "
2735
const $ = require('./dist');
2836
require('util').inspect.defaultOptions.depth = null;
2937
30-
let client = new $.DataAPIClient(process.env.CLIENT_DB_TOKEN, { environment: process.env.CLIENT_DB_ENVIRONMENT });
38+
let client = new $.DataAPIClient(process.env.CLIENT_DB_TOKEN, { environment: process.env.CLIENT_DB_ENVIRONMENT, logging: [{ events: 'all', emits: 'event' }] });
3139
let db = client.db(process.env.CLIENT_DB_URL);
3240
let dbAdmin = db.admin({ environment: process.env.CLIENT_DB_ENVIRONMENT });
3341
@@ -44,6 +52,12 @@ node -i -e "
4452
let coll = db.collection('test_coll');
4553
let table = db.table('test_table');
4654
55+
if (process.env.LOG_ALL_TO_STDOUT) {
56+
for (const event of ['commandSucceeded', 'adminCommandSucceeded', 'commandFailed', 'adminCommandFailed']) {
57+
client.on(event, (e) => console.dir(e, { depth: null }));
58+
}
59+
}
60+
4761
Object.defineProperty(this, 'cl', {
4862
get() {
4963
console.clear();

scripts/test.sh

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -81,19 +81,23 @@ while [ $# -gt 0 ]; do
8181
environment="$1"
8282
;;
8383
"-l" | "-logging")
84-
logging=1
84+
logging="!isGlobal"
85+
;;
86+
"-L" | "-logging-pred")
87+
shift
88+
logging="$1"
8589
;;
8690
"-P" | "-skip-prelude")
8791
skip_prelude=1
8892
;;
89-
"-stargate")
90-
stargate=1
93+
"-local")
94+
local=1
9195
;;
9296
*)
9397
echo "Invalid flag $1"
9498
echo ""
9599
echo "Usage:"
96-
echo "scripts/test.sh [-all | -light | -coverage] [-for] [-f/F <filter>]+ [-g/G <regex>]+ [-w/W <vectorize_whitelist>] [-b | -bail] [-R | -no-report] [-c <http_client>] [-e <environment>] [-stargate] [-l | -logging] [-P | -skip-prelude]"
100+
echo "scripts/test.sh [-all | -light | -coverage] [-for] [-f/F <filter>]+ [-g/G <regex>]+ [-w/W <vectorize_whitelist>] [-b | -bail] [-R | -no-report] [-c <http_client>] [-e <environment>] [-local] ([-l | -logging] | [-L | -logging-pred]) [-P | -skip-prelude]"
97101
echo "or"
98102
echo "scripts/test.sh [-lint] [-tc]"
99103
exit
@@ -168,11 +172,11 @@ if [ -n "$environment" ]; then
168172
export CLIENT_DB_ENVIRONMENT="$environment"
169173
fi
170174

171-
if [ -n "$stargate" ]; then
175+
if [ -n "$local" ]; then
172176
export USING_LOCAL_STARGATE=1
173177
fi
174178
if [ -n "$logging" ]; then
175-
export LOG_ALL_TO_STDOUT=1
179+
export LOGGING_PRED="$logging"
176180
fi
177181

178182
if [ -n "$skip_prelude" ]; then

src/administration/errors.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
// limitations under the License.
1414

1515
import { FetcherResponseInfo } from '@/src/lib/api';
16+
import { SomeDoc } from '@/src/documents';
1617

1718
/**
1819
* A representation of what went wrong when interacting with the DevOps API.
@@ -108,7 +109,7 @@ export class DevOpsAPIResponseError extends DevOpsAPIError {
108109
*
109110
* @internal
110111
*/
111-
constructor(resp: FetcherResponseInfo, data: Record<string, any> | undefined) {
112+
constructor(resp: FetcherResponseInfo, data: SomeDoc | undefined) {
112113
const errors = data?.errors ?? [];
113114
const maybeMsg = errors.find((e: any) => e.message)?.message;
114115

@@ -149,7 +150,7 @@ export class DevOpsUnexpectedStateError extends DevOpsAPIError {
149150
*
150151
* @internal
151152
*/
152-
constructor(message: string, expected: string[], data: Record<string, any> | undefined) {
153+
constructor(message: string, expected: string[], data: SomeDoc | undefined) {
153154
super(message);
154155
this.expected = expected;
155156
this.dbInfo = data;

src/documents/collections/collection.ts

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
import type { FindCursor } from '@/src/documents/cursor';
1615
import type {
1716
CollectionDeleteManyResult,
1817
CollectionDeleteOneOptions,
@@ -50,6 +49,8 @@ import { mkCollectionSerDes } from '@/src/documents/collections/ser-des';
5049
import { $CustomInspect } from '@/src/lib/constants';
5150
import { CollectionInsertManyError, TooManyDocumentsToCountError } from '@/src/documents';
5251
import JBI from 'json-bigint';
52+
import { CollectionFindCursor } from '@/src/documents/collections/cursor';
53+
import { withJbiNullProtoFix } from '@/src/lib/utils';
5354

5455
const jbi = JBI({ storeAsString: true });
5556

@@ -178,7 +179,7 @@ const jbi = JBI({ storeAsString: true });
178179
*/
179180
export class Collection<Schema extends SomeDoc = SomeDoc> {
180181
readonly #httpClient: DataAPIHttpClient;
181-
readonly #commands: CommandImpls<IdOf<Schema>>;
182+
readonly #commands: CommandImpls<Schema, IdOf<Schema>>;
182183
readonly #db: Db;
183184

184185
/**
@@ -209,11 +210,11 @@ export class Collection<Schema extends SomeDoc = SomeDoc> {
209210

210211
const hack: BigNumberHack = {
211212
parseWithBigNumbers: () => !!opts?.serdes?.enableBigNumbers,
212-
parser: jbi,
213+
parser: withJbiNullProtoFix(jbi),
213214
};
214215

215216
this.#httpClient = httpClient.forTableSlashCollectionOrWhateverWeWouldCallTheUnionOfTheseTypes(this.keyspace, this.name, opts, hack);
216-
this.#commands = new CommandImpls(this.name, this.#httpClient, mkCollectionSerDes(opts?.serdes));
217+
this.#commands = new CommandImpls(this, this.#httpClient, mkCollectionSerDes(opts?.serdes));
217218
this.#db = db;
218219

219220
Object.defineProperty(this, $CustomInspect, {
@@ -329,7 +330,7 @@ export class Collection<Schema extends SomeDoc = SomeDoc> {
329330
*
330331
* If any document does not contain an `_id` field, the server will generate an id for the document. The type of the id may be specified in {@link CollectionOptions.defaultId} at creation, otherwise it'll just be a UUID string. This generation will not mutate the documents.
331332
*
332-
* If any `_id` is provided which corresponds to a document that already exists in the collection, an {@link InsertManyError} is raised, and the insertion (partially) fails.
333+
* If any `_id` is provided which corresponds to a document that already exists in the collection, an {@link CollectionInsertManyError} is raised, and the insertion (partially) fails.
333334
*
334335
* If you prefer to upsert instead, see {@link Collection.replaceOne}.
335336
*
@@ -363,7 +364,7 @@ export class Collection<Schema extends SomeDoc = SomeDoc> {
363364
*
364365
* ##### `InsertManyError`
365366
*
366-
* If any 2XX insertion error occurs, the operation will throw an {@link InsertManyError} containing the partial result.
367+
* If any 2XX insertion error occurs, the operation will throw an {@link CollectionInsertManyError} containing the partial result.
367368
*
368369
* If a thrown exception is not due to an insertion error, e.g. a `5xx` error or network error, the operation will throw the underlying error.
369370
*
@@ -376,7 +377,7 @@ export class Collection<Schema extends SomeDoc = SomeDoc> {
376377
*
377378
* @throws InsertManyError - If the operation fails.
378379
*/
379-
public async insertMany(documents: MaybeId<Schema>[], options?: CollectionInsertManyOptions): Promise<CollectionInsertManyResult<Schema>> {
380+
public async insertMany(documents: readonly MaybeId<Schema>[], options?: CollectionInsertManyOptions): Promise<CollectionInsertManyResult<Schema>> {
380381
return this.#commands.insertMany(documents, options, CollectionInsertManyError);
381382
}
382383

@@ -791,7 +792,7 @@ export class Collection<Schema extends SomeDoc = SomeDoc> {
791792
* @see StrictSort
792793
* @see StrictProjection
793794
*/
794-
public find(filter: Filter<Schema>, options?: CollectionFindOptions & { projection?: never }): FindCursor<FoundDoc<Schema>, FoundDoc<Schema>>
795+
public find(filter: Filter<Schema>, options?: CollectionFindOptions & { projection?: never }): CollectionFindCursor<FoundDoc<Schema>, FoundDoc<Schema>>
795796

796797
/**
797798
* ##### Overview
@@ -939,10 +940,10 @@ export class Collection<Schema extends SomeDoc = SomeDoc> {
939940
* @see StrictSort
940941
* @see StrictProjection
941942
*/
942-
public find<TRaw extends SomeDoc = DeepPartial<Schema>>(filter: Filter<Schema>, options: CollectionFindOptions): FindCursor<FoundDoc<TRaw>, FoundDoc<TRaw>>
943+
public find<TRaw extends SomeDoc = DeepPartial<Schema>>(filter: Filter<Schema>, options: CollectionFindOptions): CollectionFindCursor<FoundDoc<TRaw>, FoundDoc<TRaw>>
943944

944-
public find(filter: Filter<Schema>, options?: CollectionFindOptions): FindCursor<SomeDoc> {
945-
return this.#commands.find(this.keyspace, filter, options);
945+
public find(filter: Filter<Schema>, options?: CollectionFindOptions): CollectionFindCursor<SomeDoc> {
946+
return this.#commands.find(filter, options, CollectionFindCursor);
946947
}
947948

948949
/**
@@ -1238,7 +1239,7 @@ export class Collection<Schema extends SomeDoc = SomeDoc> {
12381239
* @see StrictFilter
12391240
*/
12401241
public async distinct<Key extends string>(key: Key, filter: Filter<Schema>): Promise<Flatten<(SomeDoc & ToDotNotation<FoundDoc<Schema>>)[Key]>[]> {
1241-
return this.#commands.distinct(this.keyspace, key, filter);
1242+
return this.#commands.distinct(key, filter, CollectionFindCursor);
12421243
}
12431244

12441245
/**

src/documents/collections/cursor.ts

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// Copyright DataStax, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import { Collection, FindCursor, type SomeDoc } from '@/src/documents';
16+
17+
export class CollectionFindCursor<T, TRaw extends SomeDoc = SomeDoc> extends FindCursor<T, TRaw> {
18+
public override get dataSource(): Collection {
19+
return super.dataSource as Collection;
20+
}
21+
}

src/documents/collections/index.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
// limitations under the License.
1414
// noinspection DuplicatedCode
1515

16+
export * from './cursor';
1617
export * from './collection';
1718
export type * from './types';
1819

src/documents/collections/ser-des.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ const DefaultCollectionSerDesCfg = {
5454
}
5555

5656
if (key === '$vector' && DataAPIVector.isVectorLike(value)) {
57-
value = new DataAPIVector(value);
57+
value = new DataAPIVector(value, false);
5858
}
5959

6060
if ($SerializeForCollection in value) {

src/documents/commands/command-impls.ts

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import { DataAPIHttpClient } from '@/src/lib/api/clients';
1616
import { DataAPISerDes } from '@/src/lib/api/ser-des';
1717
import {
18+
Collection,
1819
CollectionDeleteManyError,
1920
CollectionInsertManyOptions,
2021
CollectionUpdateManyError,
@@ -35,7 +36,7 @@ import {
3536
GenericUpdateManyOptions,
3637
GenericUpdateOneOptions,
3738
GenericUpdateResult,
38-
SomeDoc,
39+
SomeDoc, Table,
3940
} from '@/src/documents';
4041
import { nullish, WithTimeout } from '@/src/lib';
4142
import { insertManyOrdered, insertManyUnordered } from '@/src/documents/commands/helpers/insertion';
@@ -45,15 +46,15 @@ import { normalizedSort } from '@/src/documents/utils';
4546
import { mkDistinctPathExtractor, pullSafeProjection4Distinct } from '@/src/documents/commands/helpers/distinct';
4647
import stableStringify from 'safe-stable-stringify';
4748

48-
export class CommandImpls<ID> {
49+
export class CommandImpls<Schema extends SomeDoc, ID> {
4950
readonly #httpClient: DataAPIHttpClient;
5051
readonly #serdes: DataAPISerDes;
51-
readonly #name: string;
52+
readonly #tOrC: Table<Schema> | Collection<Schema>;
5253

53-
constructor(name: string, httpClient: DataAPIHttpClient, serdes: DataAPISerDes) {
54+
constructor(tOrC: Table<Schema> | Collection<Schema>, httpClient: DataAPIHttpClient, serdes: DataAPISerDes) {
5455
this.#httpClient = httpClient;
5556
this.#serdes = serdes;
56-
this.#name = name;
57+
this.#tOrC = tOrC;
5758
}
5859

5960
public async insertOne(_document: SomeDoc, options: WithTimeout | nullish): Promise<GenericInsertOneResult<ID>> {
@@ -73,7 +74,7 @@ export class CommandImpls<ID> {
7374
};
7475
}
7576

76-
public async insertMany(docs: SomeDoc[], options: CollectionInsertManyOptions | nullish, err: new (descs: DataAPIDetailedErrorDescriptor[]) => DataAPIResponseError): Promise<GenericInsertManyResult<ID>> {
77+
public async insertMany(docs: readonly SomeDoc[], options: CollectionInsertManyOptions | nullish, err: new (descs: DataAPIDetailedErrorDescriptor[]) => DataAPIResponseError): Promise<GenericInsertManyResult<ID>> {
7778
const chunkSize = options?.chunkSize ?? 50;
7879
const timeoutManager = this.#httpClient.timeoutManager(options?.maxTimeMS);
7980

@@ -225,11 +226,11 @@ export class CommandImpls<ID> {
225226
};
226227
}
227228

228-
public find<Schema extends SomeDoc>(keyspace: string, filter: SomeDoc, options?: GenericFindOptions): FindCursor<Schema, Schema> {
229+
public find<Cursor extends FindCursor<SomeDoc>>(filter: SomeDoc, options: GenericFindOptions | undefined, cursor: new (...args: ConstructorParameters<typeof FindCursor<SomeDoc>>) => Cursor): Cursor {
229230
if (options?.sort) {
230231
options.sort = normalizedSort(options.sort);
231232
}
232-
return new FindCursor(keyspace, this.#name, this.#httpClient, this.#serdes, this.#serdes.serializeRecord(structuredClone(filter)), structuredClone(options));
233+
return new cursor(this.#tOrC as any, this.#serdes, this.#serdes.serializeRecord(structuredClone(filter)), structuredClone(options));
233234
}
234235

235236
public async findOne<Schema>(_filter: SomeDoc, options?: GenericFindOneOptions): Promise<Schema | null> {
@@ -304,9 +305,9 @@ export class CommandImpls<ID> {
304305
return resp.data?.document || null;
305306
}
306307

307-
public async distinct(keyspace: string, key: string, filter: SomeDoc): Promise<any[]> {
308+
public async distinct(key: string, filter: SomeDoc, mkCursor: new (...args: ConstructorParameters<typeof FindCursor<SomeDoc>>) => FindCursor<SomeDoc>): Promise<any[]> {
308309
const projection = pullSafeProjection4Distinct(key);
309-
const cursor = this.find(keyspace, filter, { projection: { _id: 0, [projection]: 1 } });
310+
const cursor = this.find(filter, { projection: { _id: 0, [projection]: 1 } }, mkCursor);
310311

311312
const seen = new Set<unknown>();
312313
const ret = [];

src/documents/commands/helpers/insertion.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import { GenericInsertManyDocumentResponse, SomeDoc, SomeId } from '@/src/docume
2626
export const insertManyOrdered = async <ID>(
2727
httpClient: DataAPIHttpClient,
2828
serdes: DataAPISerDes,
29-
documents: unknown[],
29+
documents: readonly unknown[],
3030
chunkSize: number,
3131
timeoutManager: TimeoutManager,
3232
err: new (descs: DataAPIDetailedErrorDescriptor[]) => DataAPIResponseError,
@@ -57,7 +57,7 @@ export const insertManyOrdered = async <ID>(
5757
export const insertManyUnordered = async <ID>(
5858
httpClient: DataAPIHttpClient,
5959
serdes: DataAPISerDes,
60-
documents: unknown[],
60+
documents: readonly unknown[],
6161
concurrency: number,
6262
chunkSize: number,
6363
timeoutManager: TimeoutManager,
@@ -111,7 +111,7 @@ export const insertManyUnordered = async <ID>(
111111
const insertMany = async <ID>(
112112
httpClient: DataAPIHttpClient,
113113
serdes: DataAPISerDes,
114-
documents: unknown[],
114+
documents: readonly unknown[],
115115
ordered: boolean,
116116
timeoutManager: TimeoutManager,
117117
): Promise<[GenericInsertManyDocumentResponse<ID>[], ID[], DataAPIDetailedErrorDescriptor | undefined]> => {

0 commit comments

Comments
 (0)