From 24a6a12833a77affbe938e2ed4103415ccb35eb8 Mon Sep 17 00:00:00 2001 From: dogquery <> Date: Fri, 3 Jul 2020 23:25:30 +0900 Subject: [PATCH 01/36] initial commit. --- src/api.js | 216 ++++++++++++++++++++++--------- test/test_aggregate_functions.js | 17 +++ 2 files changed, 171 insertions(+), 62 deletions(-) create mode 100644 test/test_aggregate_functions.js diff --git a/src/api.js b/src/api.js index 854bf32e..b5f14f12 100644 --- a/src/api.js +++ b/src/api.js @@ -1131,81 +1131,90 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { return sqlite3_changes(this.db); }; - /** Register a custom function with SQLite - @example Register a simple function - db.create_function("addOne", function (x) {return x+1;}) - db.exec("SELECT addOne(1)") // = 2 + var extract_blob = function extract_blob(ptr) { + var size = sqlite3_value_bytes(ptr); + var blob_ptr = sqlite3_value_blob(ptr); + var blob_arg = new Uint8Array(size); + for (var j = 0; j < size; j += 1) { + blob_arg[j] = HEAP8[blob_ptr + j]; + } + return blob_arg; + }; - @param {string} name the name of the function as referenced in - SQL statements. - @param {function} func the actual function to be executed. - @return {Database} The database object. Useful for method chaining - */ + var parseFunctionArguments = function parseFunctionArguments(argc, argv) { + var args = []; + for (var i = 0; i < argc; i += 1) { + var value_ptr = getValue(argv + (4 * i), "i32"); + var value_type = sqlite3_value_type(value_ptr); + var arg; + if ( + value_type === SQLITE_INTEGER + || value_type === SQLITE_FLOAT + ) { + arg = sqlite3_value_double(value_ptr); + } else if (value_type === SQLITE_TEXT) { + arg = sqlite3_value_text(value_ptr); + } else if (value_type === SQLITE_BLOB) { + arg = extract_blob(value_ptr); + } else arg = null; + args.push(arg); + } + return args; + }; + var setFunctionResult = function setFunctionResult(cx, result) { + switch (typeof result) { + case "boolean": + sqlite3_result_int(cx, result ? 1 : 0); + break; + case "number": + sqlite3_result_double(cx, result); + break; + case "string": + sqlite3_result_text(cx, result, -1, -1); + break; + case "object": + if (result === null) { + sqlite3_result_null(cx); + } else if (result.length != null) { + var blobptr = allocate(result, "i8", ALLOC_NORMAL); + sqlite3_result_blob(cx, blobptr, result.length, -1); + _free(blobptr); + } else { + sqlite3_result_error(cx, ( + "Wrong API use : tried to return a value " + + "of an unknown type (" + result + ")." + ), -1); + } + break; + default: + sqlite3_result_null(cx); + } + }; + + /** Register a custom function with SQLite + @example Register a simple function + db.create_function("addOne", function (x) {return x+1;}) + db.exec("SELECT addOne(1)") // = 2 + + @param {string} name the name of the function as referenced in + SQL statements. + @param {function} func the actual function to be executed. + @return {Database} The database object. Useful for method chaining + */ Database.prototype["create_function"] = function create_function( name, func ) { function wrapped_func(cx, argc, argv) { + var args = parseFunctionArguments(argc, argv); var result; - function extract_blob(ptr) { - var size = sqlite3_value_bytes(ptr); - var blob_ptr = sqlite3_value_blob(ptr); - var blob_arg = new Uint8Array(size); - for (var j = 0; j < size; j += 1) { - blob_arg[j] = HEAP8[blob_ptr + j]; - } - return blob_arg; - } - var args = []; - for (var i = 0; i < argc; i += 1) { - var value_ptr = getValue(argv + (4 * i), "i32"); - var value_type = sqlite3_value_type(value_ptr); - var arg; - if ( - value_type === SQLITE_INTEGER - || value_type === SQLITE_FLOAT - ) { - arg = sqlite3_value_double(value_ptr); - } else if (value_type === SQLITE_TEXT) { - arg = sqlite3_value_text(value_ptr); - } else if (value_type === SQLITE_BLOB) { - arg = extract_blob(value_ptr); - } else arg = null; - args.push(arg); - } try { result = func.apply(null, args); } catch (error) { sqlite3_result_error(cx, error, -1); return; } - switch (typeof result) { - case "boolean": - sqlite3_result_int(cx, result ? 1 : 0); - break; - case "number": - sqlite3_result_double(cx, result); - break; - case "string": - sqlite3_result_text(cx, result, -1, -1); - break; - case "object": - if (result === null) { - sqlite3_result_null(cx); - } else if (result.length != null) { - var blobptr = allocate(result, ALLOC_NORMAL); - sqlite3_result_blob(cx, blobptr, result.length, -1); - _free(blobptr); - } else { - sqlite3_result_error(cx, ( - "Wrong API use : tried to return a value " - + "of an unknown type (" + result + ")." - ), -1); - } - break; - default: - sqlite3_result_null(cx); - } + setFunctionResult(cx, result); } if (Object.prototype.hasOwnProperty.call(this.functions, name)) { removeFunction(this.functions[name]); @@ -1229,6 +1238,89 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { return this; }; + /** Register a custom aggregate with SQLite + @example Register a aggregate function + db.create_aggregate( + "js_sum", + function () { return { sum: 0 }; }, + function (state, value) { state.sum+=value; }, + function (state) { return state.sum; } + ); + db.exec("CREATE TABLE test (col); INSERT INTO test VALUES (1), (2)"); + db.exec("SELECT js_sum(col) FROM test"); // = 3 + + @param {string} name the name of the aggregate as referenced in + SQL statements. + @param {function} init the actual function to be executed on initialize. + @param {function} step the actual function to be executed on step by step. + @param {function} finalize the actual function to be executed on finalize. + @return {Database} The database object. Useful for method chaining + */ + Database.prototype["create_aggregate"] = function create_aggregate( + name, + init, + step, + finalize + ) { + var state; + function wrapped_step(cx, argc, argv) { + if (!state) { + state = init(); + } + var args = parseFunctionArguments(argc, argv); + var mergedArgs = [state].concat(args); + try { + step.apply(null, mergedArgs); + } catch (error) { + sqlite3_result_error(cx, error, -1); + } + } + function wrapped_finalize(cx) { + var result; + try { + result = finalize.apply(null, [state]); + } catch (error) { + sqlite3_result_error(cx, error, -1); + state = null; + return; + } + setFunctionResult(cx, result); + state = null; + } + + if (Object.prototype.hasOwnProperty.call(this.functions, name)) { + removeFunction(this.functions[name]); + delete this.functions[name]; + } + if (Object.prototype.hasOwnProperty.call( + this.functions, + name + "__finalize" + )) { + removeFunction(this.functions[name + "__finalize"]); + delete this.functions[name + "__finalize"]; + } + // The signature of the wrapped function is : + // void wrapped(sqlite3_context *db, int argc, sqlite3_value **argv) + var step_ptr = addFunction(wrapped_step, "viii"); + // The signature of the wrapped function is : + // void wrapped(sqlite3_context *db) + var finalize_ptr = addFunction(wrapped_finalize, "vi"); + this.functions[name] = step_ptr; + this.functions[name + "__finalize"] = finalize_ptr; + this.handleError(sqlite3_create_function_v2( + this.db, + name, + step.length - 1, + SQLITE_UTF8, + 0, + 0, + step_ptr, + finalize_ptr, + 0 + )); + return this; + }; + // export Database to Module Module.Database = Database; }; diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js new file mode 100644 index 00000000..5a0ed16e --- /dev/null +++ b/test/test_aggregate_functions.js @@ -0,0 +1,17 @@ +exports.test = function (SQL, assert) { + var db = new SQL.Database(); + + db.create_aggregate( + "js_sum", + function () { return { sum: 0 }; }, + function (state, value) { state.sum += value; }, + function (state) { return state.sum; } + ); + + db.exec("CREATE TABLE test (col);"); + db.exec("INSERT INTO test VALUES (1), (2), (3);"); + var result = db.exec("SELECT js_sum(col) FROM test;"); + assert.equal(result[0].values[0][0], 6, "Simple aggregate function."); + + // TODO: Add test cases.. +} \ No newline at end of file From fad9ba66470428c2d64b857181b2770b0543b005 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Sun, 4 Sep 2022 21:24:55 -0400 Subject: [PATCH 02/36] documentation --- README.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/README.md b/README.md index bfc130b4..b779eabb 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,45 @@ db.create_function("add_js", add); // Run a query in which the function is used db.run("INSERT INTO hello VALUES (add_js(7, 3), add_js('Hello ', 'world'));"); // Inserts 10 and 'Hello world' +// You can create aggregation functions, by passing a name and three functions +// to `db.create_aggregate`: +// +// - an init function. This function receives no arguments and will be called +// when the aggregate begins. Returns a state object that will be passed to the +// other two functions if you need to track state. +// - a step function. This function receives as a first argument the state +// object created in init, as well as the values received in the step. It +// will be called on every value to be aggregated. Does not return anything. +// - a finalizer. This function receives one argument, the state object, and +// returns the final value of the aggregate +// +// Here is an example aggregation function, `json_agg`, which will collect all +// input values and return them as a JSON array: +db.create_aggregate( + "json_agg", + function() { + // This is the init function, which returns a state object: + return { + values: [] + }; + }, + function(state, val) { + // This is the step function, which will store each value it receives in + // the values array of the state object + state.values.push(val); + }, + function(state) { + // This is the finalize function, which converts the received values from + // the state object into a JSON array and returns that + return JSON.stringify(state.values); + } +); + +// Now if you run this query: +var result = db.exec("SELECT json_agg(somecol) FROM atable;"); + +// result will be a json-encoded string representing each value of `somecol` in `atable`. + // Export the database to an Uint8Array containing the SQLite database file const binaryArray = db.export(); ``` From d191caa34cbfc39e85b10612429cbf7b59d77b5b Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Sun, 4 Sep 2022 21:29:03 -0400 Subject: [PATCH 03/36] remove no-longer-valid type --- src/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api.js b/src/api.js index b5f14f12..0eef62b6 100644 --- a/src/api.js +++ b/src/api.js @@ -1176,7 +1176,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { if (result === null) { sqlite3_result_null(cx); } else if (result.length != null) { - var blobptr = allocate(result, "i8", ALLOC_NORMAL); + var blobptr = allocate(result, ALLOC_NORMAL); sqlite3_result_blob(cx, blobptr, result.length, -1); _free(blobptr); } else { From 0d937a7c67368204fa16f88fa1c3f9aa5dd1b240 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Sun, 4 Sep 2022 21:29:52 -0400 Subject: [PATCH 04/36] close over state initialization for performance --- src/api.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/api.js b/src/api.js index 0eef62b6..1fdf68c0 100644 --- a/src/api.js +++ b/src/api.js @@ -1262,11 +1262,8 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { step, finalize ) { - var state; + var state = init(); function wrapped_step(cx, argc, argv) { - if (!state) { - state = init(); - } var args = parseFunctionArguments(argc, argv); var mergedArgs = [state].concat(args); try { From 8fd3f8a20a002007a6e457051eab3e6bae2fc8fa Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Sun, 4 Sep 2022 21:30:07 -0400 Subject: [PATCH 05/36] link documentation in comment --- src/api.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/api.js b/src/api.js index 1fdf68c0..a0a4e525 100644 --- a/src/api.js +++ b/src/api.js @@ -1304,6 +1304,13 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var finalize_ptr = addFunction(wrapped_finalize, "vi"); this.functions[name] = step_ptr; this.functions[name + "__finalize"] = finalize_ptr; + + // passing null to the sixth parameter defines this as an aggregate + // function + // + // > An aggregate SQL function requires an implementation of xStep and + // > xFinal and NULL pointer must be passed for xFunc. + // - http://www.sqlite.org/c3ref/create_function.html this.handleError(sqlite3_create_function_v2( this.db, name, From ba733ba041c04c38ba4b1cb08098172e74c964ce Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Sun, 4 Sep 2022 21:30:38 -0400 Subject: [PATCH 06/36] more testing --- test/test_aggregate_functions.js | 46 +++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index 5a0ed16e..30c7b475 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -2,7 +2,7 @@ exports.test = function (SQL, assert) { var db = new SQL.Database(); db.create_aggregate( - "js_sum", + "sum", function () { return { sum: 0 }; }, function (state, value) { state.sum += value; }, function (state) { return state.sum; } @@ -10,8 +10,46 @@ exports.test = function (SQL, assert) { db.exec("CREATE TABLE test (col);"); db.exec("INSERT INTO test VALUES (1), (2), (3);"); - var result = db.exec("SELECT js_sum(col) FROM test;"); + var result = db.exec("SELECT sum(col) FROM test;"); assert.equal(result[0].values[0][0], 6, "Simple aggregate function."); - // TODO: Add test cases.. -} \ No newline at end of file + db.create_aggregate( + "percentile", + function () { return { vals: [], pctile: null }; }, // init + function (state, value, pctile) { + state.vals.push(value); + }, + function (state) { + return percentile(state.vals, state.pctile); + } + ); + var result = db.exec("SELECT percentile(col, 20) FROM test;"); + assert.equal(result[0].values[0][0], 1, "Aggregate function with two args"); + + db.create_aggregate( + "json_agg", + function() { return { vals: [] }; }, + function(state, val) { state.vals.push(val); }, + function(state) { return JSON.stringify(state.vals); } + ); + + db.exec("CREATE TABLE test2 (col, col2);"); + db.exec("INSERT INTO test2 values ('four score', 12), ('and seven', 7), ('years ago', 1);"); + var result = db.exec("SELECT json_agg(col) FROM test2;"); + assert.deepEqual(JSON.parse(result[0].values[0]), ["four score", "and seven", "years ago"], "Aggregate function that returns JSON"); +} + +// helper function to calculate a percentile from an array. Will modify the +// array in-place. +function percentile(arr, p) { + arr.sort(); + const pos = (arr.length - 1) * (p / 100); + const base = Math.floor(pos); + const rest = pos - base; + if (arr[base + 1] !== undefined) { + return arr[base] + rest * (arr[base + 1] - arr[base]); + } else { + return arr[base]; + } +}; + From 9e6b46290a73a38f7ccb0bb7a14df30796920a26 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Mon, 5 Sep 2022 20:07:54 -0400 Subject: [PATCH 07/36] run tests if they're main --- test/test_aggregate_functions.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index 30c7b475..243a2d1f 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -53,3 +53,18 @@ function percentile(arr, p) { } }; +if (module == require.main) { + const target_file = process.argv[2]; + const sql_loader = require('./load_sql_lib'); + sql_loader(target_file).then((sql)=>{ + require('test').run({ + 'test functions': function(assert, done){ + exports.test(sql, assert, done); + } + }); + }) + .catch((e)=>{ + console.error(e); + assert.fail(e); + }); +} From 573afa71e8b2fe6daca1db327851e642cac95772 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Mon, 5 Sep 2022 22:17:40 -0400 Subject: [PATCH 08/36] accept a single arg --- src/api.js | 24 +++++++++++++---------- test/test_aggregate_functions.js | 33 +++++++++++++++++--------------- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/api.js b/src/api.js index a0a4e525..0a3110b8 100644 --- a/src/api.js +++ b/src/api.js @@ -1251,23 +1251,27 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { @param {string} name the name of the aggregate as referenced in SQL statements. - @param {function} init the actual function to be executed on initialize. - @param {function} step the actual function to be executed on step by step. - @param {function} finalize the actual function to be executed on finalize. + @param {object} Aggregate function containing three functions @return {Database} The database object. Useful for method chaining */ Database.prototype["create_aggregate"] = function create_aggregate( name, - init, - step, - finalize + aggregateFunctions ) { - var state = init(); + if (!aggregateFunctions.hasOwnProperty("init") || + !aggregateFunctions.hasOwnProperty("step") || + !aggregateFunctions.hasOwnProperty("finalize")) + throw "An aggregate function must have init, step and finalize properties"; + + var state; function wrapped_step(cx, argc, argv) { + if (!state) { + state = aggregateFunctions["init"].apply(null); + } var args = parseFunctionArguments(argc, argv); var mergedArgs = [state].concat(args); try { - step.apply(null, mergedArgs); + aggregateFunctions["step"].apply(null, mergedArgs); } catch (error) { sqlite3_result_error(cx, error, -1); } @@ -1275,7 +1279,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { function wrapped_finalize(cx) { var result; try { - result = finalize.apply(null, [state]); + result = aggregateFunctions["finalize"].apply(null, [state]); } catch (error) { sqlite3_result_error(cx, error, -1); state = null; @@ -1314,7 +1318,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { this.handleError(sqlite3_create_function_v2( this.db, name, - step.length - 1, + aggregateFunctions["step"].length - 1, SQLITE_UTF8, 0, 0, diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index 243a2d1f..a7ec77a3 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -2,10 +2,11 @@ exports.test = function (SQL, assert) { var db = new SQL.Database(); db.create_aggregate( - "sum", - function () { return { sum: 0 }; }, - function (state, value) { state.sum += value; }, - function (state) { return state.sum; } + "sum", { + init: function () { return { sum: 0 }; }, + step: function (state, value) { state.sum += value; }, + finalize: function (state) { return state.sum; } + } ); db.exec("CREATE TABLE test (col);"); @@ -14,23 +15,25 @@ exports.test = function (SQL, assert) { assert.equal(result[0].values[0][0], 6, "Simple aggregate function."); db.create_aggregate( - "percentile", - function () { return { vals: [], pctile: null }; }, // init - function (state, value, pctile) { - state.vals.push(value); - }, - function (state) { - return percentile(state.vals, state.pctile); + "percentile", { + init: function () { return { vals: [], pctile: null }; }, // init + step: function (state, value, pctile) { + state.vals.push(value); + }, + finalize: function (state) { + return percentile(state.vals, state.pctile); + } } ); var result = db.exec("SELECT percentile(col, 20) FROM test;"); assert.equal(result[0].values[0][0], 1, "Aggregate function with two args"); db.create_aggregate( - "json_agg", - function() { return { vals: [] }; }, - function(state, val) { state.vals.push(val); }, - function(state) { return JSON.stringify(state.vals); } + "json_agg", { + init: function() { return { vals: [] }; }, + step: function(state, val) { state.vals.push(val); }, + finalize: function(state) { return JSON.stringify(state.vals); } + } ); db.exec("CREATE TABLE test2 (col, col2);"); From a3abdcb6eb36d55a36942f27761dc629d4a662bb Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Mon, 5 Sep 2022 23:41:37 -0400 Subject: [PATCH 09/36] this kind of works but I'm abandoning this branch Basically it seems that the sqlite extension pattern of 'allocate a struct and stick it in the context pointer' is not going to work for us here. I wonder if using the id of the pointer returned by sqlite3_aggregate_context would be enough? Since no two functions could use the same pointer, per https://www.sqlite.org/c3ref/aggregate_context.html ? --- src/api.js | 10 ++++++++++ src/exported_functions.json | 1 + 2 files changed, 11 insertions(+) diff --git a/src/api.js b/src/api.js index 0a3110b8..9cc1c72d 100644 --- a/src/api.js +++ b/src/api.js @@ -69,6 +69,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var SQLITE_FLOAT = 2; var SQLITE_TEXT = 3; var SQLITE_BLOB = 4; + var SQLITE_NULL = 5; // var - Encodings, used for registering functions. var SQLITE_UTF8 = 1; // var - cwrap function @@ -225,6 +226,14 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { "", ["number", "string", "number"] ); + + // https://www.sqlite.org/c3ref/aggregate_context.html + // void *sqlite3_aggregate_context(sqlite3_context*, int nBytes) + var sqlite3_aggregate_context = cwrap( + "sqlite3_aggregate_context", + "number", + ["number", "number"] + ); var registerExtensionFunctions = cwrap( "RegisterExtensionFunctions", "number", @@ -1265,6 +1274,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var state; function wrapped_step(cx, argc, argv) { + var p = sqlite3_aggregate_context(cx, 999); if (!state) { state = aggregateFunctions["init"].apply(null); } diff --git a/src/exported_functions.json b/src/exported_functions.json index b93b07d2..324017ae 100644 --- a/src/exported_functions.json +++ b/src/exported_functions.json @@ -41,5 +41,6 @@ "_sqlite3_result_int", "_sqlite3_result_int64", "_sqlite3_result_error", +"_sqlite3_aggregate_context", "_RegisterExtensionFunctions" ] From 9daf01f942ba656c0bfbbc3eafbe254f6aa9da67 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Tue, 6 Sep 2022 00:01:41 -0400 Subject: [PATCH 10/36] a middle road sqlite3_agg_context solution --- src/api.js | 51 ++++++++++++++++++++++++-------- test/test_aggregate_functions.js | 6 ++-- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/src/api.js b/src/api.js index 9cc1c72d..5d79dfd7 100644 --- a/src/api.js +++ b/src/api.js @@ -69,7 +69,6 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var SQLITE_FLOAT = 2; var SQLITE_TEXT = 3; var SQLITE_BLOB = 4; - var SQLITE_NULL = 5; // var - Encodings, used for registering functions. var SQLITE_UTF8 = 1; // var - cwrap function @@ -1267,36 +1266,64 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { name, aggregateFunctions ) { - if (!aggregateFunctions.hasOwnProperty("init") || - !aggregateFunctions.hasOwnProperty("step") || - !aggregateFunctions.hasOwnProperty("finalize")) - throw "An aggregate function must have init, step and finalize properties"; + if (!Object.hasOwnProperty.call(aggregateFunctions, "step") + || !Object.hasOwnProperty.call( + aggregateFunctions, "finalize" + ) + ) { + throw "An aggregate function must have step and finalize " + + "properties"; + } + + // state is a state array; we'll use the pointer p to serve as the + // key for where we hold our state so that multiple invocations of + // this function never step on each other + var state = {}; - var state; function wrapped_step(cx, argc, argv) { - var p = sqlite3_aggregate_context(cx, 999); - if (!state) { - state = aggregateFunctions["init"].apply(null); + // The first time the sqlite3_aggregate_context(C,N) routine is + // called for a particular aggregate function, SQLite allocates N + // bytes of memory, zeroes out that memory, and returns a pointer + // to the new memory. + // + // We're going to use that pointer as a key to our state array, + // since using sqlite3_aggregate_context as it's meant to be used + // through webassembly seems to be very difficult. Just allocate + // one byte. + var p = sqlite3_aggregate_context(cx, 1); + + // If this is the first invocation of wrapped_step, state[p] + if (!state[p]) { + if (Object.hasOwnProperty.call(aggregateFunctions, "init")) { + state[p] = aggregateFunctions["init"].apply(null); + } else { + state[p] = []; + } } + var args = parseFunctionArguments(argc, argv); - var mergedArgs = [state].concat(args); + var mergedArgs = [state[p]].concat(args); try { aggregateFunctions["step"].apply(null, mergedArgs); } catch (error) { sqlite3_result_error(cx, error, -1); } } + function wrapped_finalize(cx) { var result; + var p = sqlite3_aggregate_context(cx, 1); try { - result = aggregateFunctions["finalize"].apply(null, [state]); + result = aggregateFunctions["finalize"].apply(null, [state[p]]); } catch (error) { sqlite3_result_error(cx, error, -1); state = null; return; } setFunctionResult(cx, result); - state = null; + + // clear the state for this invocation + delete state[p]; } if (Object.prototype.hasOwnProperty.call(this.functions, name)) { diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index a7ec77a3..aa4467aa 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -30,15 +30,15 @@ exports.test = function (SQL, assert) { db.create_aggregate( "json_agg", { - init: function() { return { vals: [] }; }, - step: function(state, val) { state.vals.push(val); }, - finalize: function(state) { return JSON.stringify(state.vals); } + step: function(state, val) { state.push(val); }, + finalize: function(state) { return JSON.stringify(state); } } ); db.exec("CREATE TABLE test2 (col, col2);"); db.exec("INSERT INTO test2 values ('four score', 12), ('and seven', 7), ('years ago', 1);"); var result = db.exec("SELECT json_agg(col) FROM test2;"); + console.log("output: ", result[0].values); assert.deepEqual(JSON.parse(result[0].values[0]), ["four score", "and seven", "years ago"], "Aggregate function that returns JSON"); } From ec5c72b059c2b3323c9712ab0df26eba33d63450 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Tue, 6 Sep 2022 09:42:50 -0400 Subject: [PATCH 11/36] try out auto-updating state --- src/api.js | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/api.js b/src/api.js index 5d79dfd7..0443a61e 100644 --- a/src/api.js +++ b/src/api.js @@ -1267,24 +1267,23 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { aggregateFunctions ) { if (!Object.hasOwnProperty.call(aggregateFunctions, "step") - || !Object.hasOwnProperty.call( - aggregateFunctions, "finalize" - ) ) { - throw "An aggregate function must have step and finalize " - + "properties"; + throw "An aggregate function must have a step property"; } - // state is a state array; we'll use the pointer p to serve as the + aggregateFunctions["init"] ||= (() => null) + aggregateFunctions["finalize"] ||= ((state) => state) + + // state is a state object; we'll use the pointer p to serve as the // key for where we hold our state so that multiple invocations of // this function never step on each other var state = {}; function wrapped_step(cx, argc, argv) { - // The first time the sqlite3_aggregate_context(C,N) routine is - // called for a particular aggregate function, SQLite allocates N - // bytes of memory, zeroes out that memory, and returns a pointer - // to the new memory. + // > The first time the sqlite3_aggregate_context(C,N) routine is + // > called for a particular aggregate function, SQLite allocates N + // > bytes of memory, zeroes out that memory, and returns a pointer + // > to the new memory. // // We're going to use that pointer as a key to our state array, // since using sqlite3_aggregate_context as it's meant to be used @@ -1292,19 +1291,15 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // one byte. var p = sqlite3_aggregate_context(cx, 1); - // If this is the first invocation of wrapped_step, state[p] + // If this is the first invocation of wrapped_step, call `init` if (!state[p]) { - if (Object.hasOwnProperty.call(aggregateFunctions, "init")) { - state[p] = aggregateFunctions["init"].apply(null); - } else { - state[p] = []; - } + state[p] = aggregateFunctions["init"].apply(null); } var args = parseFunctionArguments(argc, argv); var mergedArgs = [state[p]].concat(args); try { - aggregateFunctions["step"].apply(null, mergedArgs); + state[p] = aggregateFunctions["step"].apply(null, mergedArgs); } catch (error) { sqlite3_result_error(cx, error, -1); } @@ -1340,6 +1335,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // The signature of the wrapped function is : // void wrapped(sqlite3_context *db, int argc, sqlite3_value **argv) var step_ptr = addFunction(wrapped_step, "viii"); + // The signature of the wrapped function is : // void wrapped(sqlite3_context *db) var finalize_ptr = addFunction(wrapped_finalize, "vi"); From a927950821a5de68d673432f69e644ac22d4c300 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Tue, 6 Sep 2022 09:43:08 -0400 Subject: [PATCH 12/36] improve quantile test, add multiple agg test --- test/test_aggregate_functions.js | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index aa4467aa..09971184 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -1,11 +1,13 @@ exports.test = function (SQL, assert) { + function assertFloat(got, expected, message="", sigma=0.001) { + assert.ok(got > expected - sigma && got < expected + sigma, message); + } + var db = new SQL.Database(); db.create_aggregate( "sum", { - init: function () { return { sum: 0 }; }, - step: function (state, value) { state.sum += value; }, - finalize: function (state) { return state.sum; } + step: function (state, value) { return (state || 0) + value; }, } ); @@ -18,28 +20,32 @@ exports.test = function (SQL, assert) { "percentile", { init: function () { return { vals: [], pctile: null }; }, // init step: function (state, value, pctile) { + state.pctile = pctile; state.vals.push(value); + return state; }, finalize: function (state) { return percentile(state.vals, state.pctile); } } ); - var result = db.exec("SELECT percentile(col, 20) FROM test;"); - assert.equal(result[0].values[0][0], 1, "Aggregate function with two args"); + result = db.exec("SELECT percentile(col, 80) FROM test;"); + assertFloat(result[0].values[0][0], 2.6, "Aggregate function with two args"); db.create_aggregate( "json_agg", { - step: function(state, val) { state.push(val); }, + step: function(state, val) { state = (state || []); state.push(val); return state; }, finalize: function(state) { return JSON.stringify(state); } } ); db.exec("CREATE TABLE test2 (col, col2);"); db.exec("INSERT INTO test2 values ('four score', 12), ('and seven', 7), ('years ago', 1);"); - var result = db.exec("SELECT json_agg(col) FROM test2;"); - console.log("output: ", result[0].values); + result = db.exec("SELECT json_agg(col) FROM test2;"); assert.deepEqual(JSON.parse(result[0].values[0]), ["four score", "and seven", "years ago"], "Aggregate function that returns JSON"); + + result = db.exec("SELECT json_agg(col), json_agg(col2) FROM test2;"); + assert.deepEqual(result[0].values[0].map(JSON.parse), [["four score", "and seven", "years ago"], [12, 7, 1]], "Multiple aggregations at once"); } // helper function to calculate a percentile from an array. Will modify the From e643bd9cfa8baa78c56922c05e886e9fa35bec7d Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Tue, 6 Sep 2022 09:52:02 -0400 Subject: [PATCH 13/36] add a null to the test --- test/test_aggregate_functions.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index 09971184..eecd3156 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -12,7 +12,7 @@ exports.test = function (SQL, assert) { ); db.exec("CREATE TABLE test (col);"); - db.exec("INSERT INTO test VALUES (1), (2), (3);"); + db.exec("INSERT INTO test VALUES (1), (2), (3), (null);"); var result = db.exec("SELECT sum(col) FROM test;"); assert.equal(result[0].values[0][0], 6, "Simple aggregate function."); @@ -20,8 +20,10 @@ exports.test = function (SQL, assert) { "percentile", { init: function () { return { vals: [], pctile: null }; }, // init step: function (state, value, pctile) { - state.pctile = pctile; - state.vals.push(value); + if (value && !isNaN(value)) { + state.pctile = pctile; + state.vals.push(value); + } return state; }, finalize: function (state) { From 2cbdb0e058164ec14e6e521e5c057f86060abaea Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Tue, 6 Sep 2022 09:58:48 -0400 Subject: [PATCH 14/36] acorn fails to parse ||=, whatever --- src/api.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/api.js b/src/api.js index 0443a61e..f299745e 100644 --- a/src/api.js +++ b/src/api.js @@ -1268,11 +1268,12 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { ) { if (!Object.hasOwnProperty.call(aggregateFunctions, "step") ) { - throw "An aggregate function must have a step property"; + throw "An aggregate function must have a step function"; } - aggregateFunctions["init"] ||= (() => null) - aggregateFunctions["finalize"] ||= ((state) => state) + aggregateFunctions["init"] = aggregateFunctions["init"] || (() => null); + aggregateFunctions["finalize"] = aggregateFunctions["finalize"] + || ((state) => state); // state is a state object; we'll use the pointer p to serve as the // key for where we hold our state so that multiple invocations of From b9ccd48f515503a4ff79e4c6d7afd28b89e3a2f8 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Tue, 6 Sep 2022 10:12:08 -0400 Subject: [PATCH 15/36] make eslint happy --- src/api.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/api.js b/src/api.js index f299745e..b54c9ef2 100644 --- a/src/api.js +++ b/src/api.js @@ -1271,9 +1271,13 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { throw "An aggregate function must have a step function"; } - aggregateFunctions["init"] = aggregateFunctions["init"] || (() => null); + // Default initializer and finalizer + function init() { return null; } + function finalize(state) { return state; } + + aggregateFunctions["init"] = aggregateFunctions["init"] || init; aggregateFunctions["finalize"] = aggregateFunctions["finalize"] - || ((state) => state); + || finalize; // state is a state object; we'll use the pointer p to serve as the // key for where we hold our state so that multiple invocations of From ac548d429b16f618c0a6e1f1d81783062ff5cf48 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 08:24:37 -0400 Subject: [PATCH 16/36] make initial_value an argument --- src/api.js | 8 +++----- test/test_aggregate_functions.js | 17 +++++++++-------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/api.js b/src/api.js index b54c9ef2..ecb77aec 100644 --- a/src/api.js +++ b/src/api.js @@ -1264,6 +1264,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { */ Database.prototype["create_aggregate"] = function create_aggregate( name, + initial_value, aggregateFunctions ) { if (!Object.hasOwnProperty.call(aggregateFunctions, "step") @@ -1271,11 +1272,8 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { throw "An aggregate function must have a step function"; } - // Default initializer and finalizer - function init() { return null; } + // Default finalizer function finalize(state) { return state; } - - aggregateFunctions["init"] = aggregateFunctions["init"] || init; aggregateFunctions["finalize"] = aggregateFunctions["finalize"] || finalize; @@ -1298,7 +1296,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // If this is the first invocation of wrapped_step, call `init` if (!state[p]) { - state[p] = aggregateFunctions["init"].apply(null); + state[p] = initial_value; } var args = parseFunctionArguments(argc, argv); diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index eecd3156..f9b4ff2d 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -6,8 +6,8 @@ exports.test = function (SQL, assert) { var db = new SQL.Database(); db.create_aggregate( - "sum", { - step: function (state, value) { return (state || 0) + value; }, + "sum", 0, { + step: function (state, value) { return state + value; }, } ); @@ -17,10 +17,11 @@ exports.test = function (SQL, assert) { assert.equal(result[0].values[0][0], 6, "Simple aggregate function."); db.create_aggregate( - "percentile", { - init: function () { return { vals: [], pctile: null }; }, // init + "percentile", { vals: [], pctile: null }, + { step: function (state, value, pctile) { - if (value && !isNaN(value)) { + var typ = typeof value; + if (typ == "number" || typ == "bigint") { state.pctile = pctile; state.vals.push(value); } @@ -35,9 +36,9 @@ exports.test = function (SQL, assert) { assertFloat(result[0].values[0][0], 2.6, "Aggregate function with two args"); db.create_aggregate( - "json_agg", { - step: function(state, val) { state = (state || []); state.push(val); return state; }, - finalize: function(state) { return JSON.stringify(state); } + "json_agg", [], { + step: (state, val) => [...state, val], + finalize: (state) => JSON.stringify(state), } ); From bf22aa164c469338b9c7e5d3db05b931ead5b55d Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 08:42:36 -0400 Subject: [PATCH 17/36] test step and finalize exceptions --- test/test_aggregate_functions.js | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index f9b4ff2d..87cafe90 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -45,10 +45,34 @@ exports.test = function (SQL, assert) { db.exec("CREATE TABLE test2 (col, col2);"); db.exec("INSERT INTO test2 values ('four score', 12), ('and seven', 7), ('years ago', 1);"); result = db.exec("SELECT json_agg(col) FROM test2;"); - assert.deepEqual(JSON.parse(result[0].values[0]), ["four score", "and seven", "years ago"], "Aggregate function that returns JSON"); + assert.deepEqual( + JSON.parse(result[0].values[0]), + ["four score", "and seven", "years ago"], + "Aggregate function that returns JSON" + ); result = db.exec("SELECT json_agg(col), json_agg(col2) FROM test2;"); - assert.deepEqual(result[0].values[0].map(JSON.parse), [["four score", "and seven", "years ago"], [12, 7, 1]], "Multiple aggregations at once"); + assert.deepEqual( + result[0].values[0].map(JSON.parse), + [["four score", "and seven", "years ago"], [12, 7, 1]], + "Multiple aggregations at once" + ); + + db.create_aggregate("throws_step", 0, {step: (state, value) => { throw "bananas" }}) + assert.throws( + () => db.exec("SELECT throws_step(col) FROM test;"), + "Error: bananas", + "Handles exception in a step function" + ); + + db.create_aggregate("throws_finalize", 0, { + step: (state, value) => state + value, + finalize: (state) => { throw "shoes" }}) + assert.throws( + () => db.exec("SELECT throws_finalize(col) FROM test;"), + "Error: shoes", + "Handles exception in a finalize function" + ); } // helper function to calculate a percentile from an array. Will modify the From 55858e9fbd80127da39a7503522bd5616698a06f Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 08:55:32 -0400 Subject: [PATCH 18/36] add memory leak test --- test/test_aggregate_redefinition.js | 80 +++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 test/test_aggregate_redefinition.js diff --git a/test/test_aggregate_redefinition.js b/test/test_aggregate_redefinition.js new file mode 100644 index 00000000..9ed47dfa --- /dev/null +++ b/test/test_aggregate_redefinition.js @@ -0,0 +1,80 @@ +exports.test = function(sql, assert) { + // Test 1: Create a database, Register single function, close database, repeat 1000 times + for (var i = 1; i <= 1000; i++) + { + let lastStep = i == 1000; + let db = new sql.Database(); + try { + db.create_aggregate("TestFunction"+i, 0, {step: (state, value) => i}) + } catch(e) { + assert.ok( + false, + "Test 1: Recreate database "+i+"th times and register aggregate" + +" function failed with exception:"+e + ); + db.close(); + break; + } + var result = db.exec("SELECT TestFunction"+i+"(1)"); + var result_str = result[0]["values"][0][0]; + if(result_str != i || lastStep) + { + assert.equal( + result_str, + i, + "Test 1: Recreate database "+i+"th times and register aggregate function" + ); + db.close(); + break; + } + db.close(); + } + + // Test 2: Create a database, Register same function 1000 times, close database + { + let db = new sql.Database(); + for (var i = 1; i <= 1000; i++) + { + let lastStep = i == 1000; + try { + db.create_aggregate("TestFunction", 0, {step: (state, value) => i}) + } catch(e) { + assert.ok( + false, + "Test 2: Reregister aggregate function "+i+"th times failed with" + +" exception:"+e + ); + break; + } + var result = db.exec("SELECT TestFunction(1)"); + var result_str = result[0]["values"][0][0]; + if(result_str != i || lastStep) + { + assert.equal( + result_str, + i, + "Test 2: Reregister function "+i+"th times" + ); + break; + } + } + db.close(); + } +}; + + +if (module == require.main) { + const target_file = process.argv[2]; + const sql_loader = require('./load_sql_lib'); + sql_loader(target_file).then((sql)=>{ + require('test').run({ + 'test creating multiple functions': function(assert){ + exports.test(sql, assert); + } + }); + }) + .catch((e)=>{ + console.error(e); + assert.fail(e); + }); +} From 9a0c185d30697537d9c774ba262326f82f33faa0 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 09:03:00 -0400 Subject: [PATCH 19/36] update docs to current interface --- README.md | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index b779eabb..67c32274 100644 --- a/README.md +++ b/README.md @@ -75,37 +75,25 @@ db.create_function("add_js", add); // Run a query in which the function is used db.run("INSERT INTO hello VALUES (add_js(7, 3), add_js('Hello ', 'world'));"); // Inserts 10 and 'Hello world' -// You can create aggregation functions, by passing a name and three functions -// to `db.create_aggregate`: +// You can create custom aggregation functions, by passing a name, an initial +// value, and two functions to `db.create_aggregate`: // -// - an init function. This function receives no arguments and will be called -// when the aggregate begins. Returns a state object that will be passed to the -// other two functions if you need to track state. // - a step function. This function receives as a first argument the state // object created in init, as well as the values received in the step. It -// will be called on every value to be aggregated. Does not return anything. +// will be called on every value to be aggregated, and its return value +// will be used as the state for the next iteration. // - a finalizer. This function receives one argument, the state object, and -// returns the final value of the aggregate +// returns the final value of the aggregate. It can be omitted, in which case +// the value of the `state` variable will be used. // // Here is an example aggregation function, `json_agg`, which will collect all // input values and return them as a JSON array: db.create_aggregate( "json_agg", - function() { - // This is the init function, which returns a state object: - return { - values: [] - }; - }, - function(state, val) { - // This is the step function, which will store each value it receives in - // the values array of the state object - state.values.push(val); - }, - function(state) { - // This is the finalize function, which converts the received values from - // the state object into a JSON array and returns that - return JSON.stringify(state.values); + [], + { + step: (state, val) => state.push(val), + finalize: (state) => JSON.stringify(state), } ); From 2445107948ccc348551e729f75eaf0b7fd1b530e Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 09:16:58 -0400 Subject: [PATCH 20/36] delete state in exception handlers --- src/api.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/api.js b/src/api.js index ecb77aec..cc2abcfb 100644 --- a/src/api.js +++ b/src/api.js @@ -1295,6 +1295,10 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var p = sqlite3_aggregate_context(cx, 1); // If this is the first invocation of wrapped_step, call `init` + // + // Make sure that every path through the step and finalize + // functions deletes the value state[p] when it's done so we don't + // leak memory and possibly stomp the init value of future calls if (!state[p]) { state[p] = initial_value; } @@ -1304,6 +1308,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { try { state[p] = aggregateFunctions["step"].apply(null, mergedArgs); } catch (error) { + delete state[p]; sqlite3_result_error(cx, error, -1); } } @@ -1314,13 +1319,14 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { try { result = aggregateFunctions["finalize"].apply(null, [state[p]]); } catch (error) { + delete state[p]; sqlite3_result_error(cx, error, -1); state = null; return; } + setFunctionResult(cx, result); - // clear the state for this invocation delete state[p]; } From 5b62cf60b82238c16f8af451b70423ae0065e50a Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 09:19:58 -0400 Subject: [PATCH 21/36] remove null state --- src/api.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/api.js b/src/api.js index cc2abcfb..93e905db 100644 --- a/src/api.js +++ b/src/api.js @@ -1321,7 +1321,6 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { } catch (error) { delete state[p]; sqlite3_result_error(cx, error, -1); - state = null; return; } From 062f147e605d4c65093cba8613baca15f674fde0 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 10:51:04 -0400 Subject: [PATCH 22/36] return init function and document object --- src/api.js | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/api.js b/src/api.js index 93e905db..5dbf4613 100644 --- a/src/api.js +++ b/src/api.js @@ -1259,21 +1259,33 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { @param {string} name the name of the aggregate as referenced in SQL statements. - @param {object} Aggregate function containing three functions + @param {object} Aggregate function containing at least a step function. + Valid keys for this object are: + - init: a function receiving no arguments and returning an initial + value for the aggregate function. The initial value will be + null if this key is omitted. + - step (required): a function receiving the current state and one to + many values and returning an updated state value. + Will receive the value from init for the first step. + - finalize: a function returning the final value of the aggregate + function. If omitted, the value returned by the last step + wil be used as the final value. @return {Database} The database object. Useful for method chaining */ Database.prototype["create_aggregate"] = function create_aggregate( name, - initial_value, aggregateFunctions ) { if (!Object.hasOwnProperty.call(aggregateFunctions, "step") ) { - throw "An aggregate function must have a step function"; + throw "An aggregate function must have a step function in " + name; } - // Default finalizer + // Default initializer and finalizer + function init() { return null; } function finalize(state) { return state; } + + aggregateFunctions["init"] = aggregateFunctions["init"] || init; aggregateFunctions["finalize"] = aggregateFunctions["finalize"] || finalize; @@ -1299,8 +1311,8 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // Make sure that every path through the step and finalize // functions deletes the value state[p] when it's done so we don't // leak memory and possibly stomp the init value of future calls - if (!state[p]) { - state[p] = initial_value; + if (!Object.hasOwnProperty.call(state, p)) { + state[p] = aggregateFunctions["init"].apply(null); } var args = parseFunctionArguments(argc, argv); From 7aff1aeda4deee6ef8889833295b8b5a7e5e9d28 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 10:51:16 -0400 Subject: [PATCH 23/36] more tests and update back to init function --- test/test_aggregate_functions.js | 54 +++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index 87cafe90..59029b93 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -6,8 +6,8 @@ exports.test = function (SQL, assert) { var db = new SQL.Database(); db.create_aggregate( - "sum", 0, { - step: function (state, value) { return state + value; }, + "sum", { + step: function (state, value) { return (state || 0) + value; }, } ); @@ -17,11 +17,12 @@ exports.test = function (SQL, assert) { assert.equal(result[0].values[0][0], 6, "Simple aggregate function."); db.create_aggregate( - "percentile", { vals: [], pctile: null }, + "percentile", { + init: function() { return { vals: [], pctile: null }}, step: function (state, value, pctile) { var typ = typeof value; - if (typ == "number" || typ == "bigint") { + if (typ == "number" || typ == "bigint") { // ignore nulls state.pctile = pctile; state.vals.push(value); } @@ -36,7 +37,8 @@ exports.test = function (SQL, assert) { assertFloat(result[0].values[0][0], 2.6, "Aggregate function with two args"); db.create_aggregate( - "json_agg", [], { + "json_agg", { + init: () => [], step: (state, val) => [...state, val], finalize: (state) => JSON.stringify(state), } @@ -58,21 +60,51 @@ exports.test = function (SQL, assert) { "Multiple aggregations at once" ); - db.create_aggregate("throws_step", 0, {step: (state, value) => { throw "bananas" }}) + db.create_aggregate("is_even", { + init: () => true, + step: state => !state + }); + result = db.exec("SELECT is_even() FROM (VALUES (1),(2),(0));"); + assert.deepEqual( + result[0].values[0][0], + 0, // this gets convert from "false" to an int by sqlite + "Aggregate functions respect falsy values" + ); + + db.create_aggregate("sum_non_zero", { + init: () => 0, + step: (state, value) => { + if (!value) throw "bananas"; + return state + value + } + }) assert.throws( - () => db.exec("SELECT throws_step(col) FROM test;"), + () => db.exec("SELECT sum_non_zero(column1) FROM (VALUES (1),(2),(0));"), "Error: bananas", "Handles exception in a step function" ); + assert.deepEqual( + db.exec("SELECT sum_non_zero(column1) FROM (VALUES (1),(2));")[0].values[0][0], + 3, + "Aggregate functions work after an exception has been thrown in step" + ); - db.create_aggregate("throws_finalize", 0, { - step: (state, value) => state + value, - finalize: (state) => { throw "shoes" }}) + db.create_aggregate("throws_finalize", { + step: (state, value) => (state || 0) + value, + finalize: (state) => { + if (!state) throw "shoes" + return state; + }}) assert.throws( - () => db.exec("SELECT throws_finalize(col) FROM test;"), + () => db.exec("SELECT throws_finalize(column1) FROM (VALUES (0));"), "Error: shoes", "Handles exception in a finalize function" ); + assert.deepEqual( + db.exec("SELECT throws_finalize(column1) FROM (VALUES (1),(2));")[0].values[0][0], + 3, + "Aggregate functions work after an exception has been thrown in finalize" + ); } // helper function to calculate a percentile from an array. Will modify the From 67f85e5ca600ffa1055bc7f2dbff82dceaf0baaf Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 11:05:55 -0400 Subject: [PATCH 24/36] update redefinition test for new interface --- test/test_aggregate_redefinition.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_aggregate_redefinition.js b/test/test_aggregate_redefinition.js index 9ed47dfa..05779720 100644 --- a/test/test_aggregate_redefinition.js +++ b/test/test_aggregate_redefinition.js @@ -5,7 +5,7 @@ exports.test = function(sql, assert) { let lastStep = i == 1000; let db = new sql.Database(); try { - db.create_aggregate("TestFunction"+i, 0, {step: (state, value) => i}) + db.create_aggregate("TestFunction"+i, {step: (state, value) => i}) } catch(e) { assert.ok( false, @@ -37,7 +37,7 @@ exports.test = function(sql, assert) { { let lastStep = i == 1000; try { - db.create_aggregate("TestFunction", 0, {step: (state, value) => i}) + db.create_aggregate("TestFunction", {step: (state, value) => i}) } catch(e) { assert.ok( false, From b8692d439cae6a2f5e84178b744b5a8497b3bbb2 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 11:59:48 -0400 Subject: [PATCH 25/36] update README to match fixed signature --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 67c32274..d4fb2c1a 100644 --- a/README.md +++ b/README.md @@ -75,9 +75,11 @@ db.create_function("add_js", add); // Run a query in which the function is used db.run("INSERT INTO hello VALUES (add_js(7, 3), add_js('Hello ', 'world'));"); // Inserts 10 and 'Hello world' -// You can create custom aggregation functions, by passing a name, an initial -// value, and two functions to `db.create_aggregate`: +// You can create custom aggregation functions, by passing a name +// and a set of functions to `db.create_aggregate`: // +// - a initialization function. This function receives no arguments and returns +// the initial value for the aggregation function // - a step function. This function receives as a first argument the state // object created in init, as well as the values received in the step. It // will be called on every value to be aggregated, and its return value @@ -90,8 +92,8 @@ db.run("INSERT INTO hello VALUES (add_js(7, 3), add_js('Hello ', 'world'));"); / // input values and return them as a JSON array: db.create_aggregate( "json_agg", - [], { + init: () => [], step: (state, val) => state.push(val), finalize: (state) => JSON.stringify(state), } From b41e5cf7e6bf6b7afac76daea450091c944d6cf3 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 12:07:55 -0400 Subject: [PATCH 26/36] more consistent test formatting --- test/test_aggregate_functions.js | 52 ++++++++++++++------------------ 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/test/test_aggregate_functions.js b/test/test_aggregate_functions.js index 59029b93..d28c775f 100644 --- a/test/test_aggregate_functions.js +++ b/test/test_aggregate_functions.js @@ -5,44 +5,37 @@ exports.test = function (SQL, assert) { var db = new SQL.Database(); - db.create_aggregate( - "sum", { - step: function (state, value) { return (state || 0) + value; }, - } - ); + db.create_aggregate("sum", { + step: function (state, value) { return (state || 0) + value; }, + }); db.exec("CREATE TABLE test (col);"); db.exec("INSERT INTO test VALUES (1), (2), (3), (null);"); var result = db.exec("SELECT sum(col) FROM test;"); assert.equal(result[0].values[0][0], 6, "Simple aggregate function."); - db.create_aggregate( - "percentile", - { - init: function() { return { vals: [], pctile: null }}, - step: function (state, value, pctile) { - var typ = typeof value; - if (typ == "number" || typ == "bigint") { // ignore nulls - state.pctile = pctile; - state.vals.push(value); - } - return state; - }, - finalize: function (state) { - return percentile(state.vals, state.pctile); + db.create_aggregate("percentile", { + init: function() { return { vals: [], pctile: null }}, + step: function (state, value, pctile) { + var typ = typeof value; + if (typ == "number" || typ == "bigint") { // ignore nulls + state.pctile = pctile; + state.vals.push(value); } + return state; + }, + finalize: function (state) { + return percentile(state.vals, state.pctile); } - ); + }); result = db.exec("SELECT percentile(col, 80) FROM test;"); assertFloat(result[0].values[0][0], 2.6, "Aggregate function with two args"); - db.create_aggregate( - "json_agg", { - init: () => [], - step: (state, val) => [...state, val], - finalize: (state) => JSON.stringify(state), - } - ); + db.create_aggregate("json_agg", { + init: () => [], + step: (state, val) => [...state, val], + finalize: (state) => JSON.stringify(state), + }); db.exec("CREATE TABLE test2 (col, col2);"); db.exec("INSERT INTO test2 values ('four score', 12), ('and seven', 7), ('years ago', 1);"); @@ -77,7 +70,7 @@ exports.test = function (SQL, assert) { if (!value) throw "bananas"; return state + value } - }) + }); assert.throws( () => db.exec("SELECT sum_non_zero(column1) FROM (VALUES (1),(2),(0));"), "Error: bananas", @@ -94,7 +87,8 @@ exports.test = function (SQL, assert) { finalize: (state) => { if (!state) throw "shoes" return state; - }}) + } + }); assert.throws( () => db.exec("SELECT throws_finalize(column1) FROM (VALUES (0));"), "Error: shoes", From d257bbab67e3344a2339988742649764c5da9b70 Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 12:39:51 -0400 Subject: [PATCH 27/36] Update README.md Co-authored-by: Ophir LOJKINE --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d4fb2c1a..1d0068e9 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ db.create_aggregate( "json_agg", { init: () => [], - step: (state, val) => state.push(val), + step: (state, val) => [...state, val], finalize: (state) => JSON.stringify(state), } ); From e82c2862acfd1665642fb60c66adbde3af86717c Mon Sep 17 00:00:00 2001 From: Bill Mill Date: Wed, 7 Sep 2022 12:43:24 -0400 Subject: [PATCH 28/36] clarify what exactly the result will contain --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 1d0068e9..71e70416 100644 --- a/README.md +++ b/README.md @@ -101,8 +101,7 @@ db.create_aggregate( // Now if you run this query: var result = db.exec("SELECT json_agg(somecol) FROM atable;"); - -// result will be a json-encoded string representing each value of `somecol` in `atable`. +console.log("You'll get a json-encoded list of values: ", result[0].values[0]) // Export the database to an Uint8Array containing the SQLite database file const binaryArray = db.export(); From b65457cbaae8d4f78716adb2055580ef3adb519b Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Thu, 8 Sep 2022 01:31:22 +0200 Subject: [PATCH 29/36] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 71e70416..9f50b830 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ db.run("INSERT INTO hello VALUES (add_js(7, 3), add_js('Hello ', 'world'));"); / // You can create custom aggregation functions, by passing a name // and a set of functions to `db.create_aggregate`: // -// - a initialization function. This function receives no arguments and returns +// - an initialization function. This function receives no arguments and returns // the initial value for the aggregation function // - a step function. This function receives as a first argument the state // object created in init, as well as the values received in the step. It From 8d2c2e0d053498c262f13efdd0fd32a894992b88 Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Thu, 8 Sep 2022 01:31:52 +0200 Subject: [PATCH 30/36] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f50b830..344788c6 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ db.run("INSERT INTO hello VALUES (add_js(7, 3), add_js('Hello ', 'world'));"); / // You can create custom aggregation functions, by passing a name // and a set of functions to `db.create_aggregate`: // -// - an initialization function. This function receives no arguments and returns +// - an initialization function. This function receives no argument and returns // the initial value for the aggregation function // - a step function. This function receives as a first argument the state // object created in init, as well as the values received in the step. It From f8f4a7c8740e72d39e16f74315952686c2d9ac45 Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Thu, 8 Sep 2022 01:37:25 +0200 Subject: [PATCH 31/36] Update README.md --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 344788c6..aa444607 100644 --- a/README.md +++ b/README.md @@ -78,15 +78,15 @@ db.run("INSERT INTO hello VALUES (add_js(7, 3), add_js('Hello ', 'world'));"); / // You can create custom aggregation functions, by passing a name // and a set of functions to `db.create_aggregate`: // -// - an initialization function. This function receives no argument and returns -// the initial value for the aggregation function -// - a step function. This function receives as a first argument the state -// object created in init, as well as the values received in the step. It -// will be called on every value to be aggregated, and its return value -// will be used as the state for the next iteration. -// - a finalizer. This function receives one argument, the state object, and +// - an `init` function. This function receives no argument and returns +// the initial value for the state of the aggregate function. +// - a `step` function. This function takes two arguments +// - the current state of the aggregation +// - a new value to aggregate to the state +// It should return a new value for the state. +// - a `finalize` function. This function receives a state object, and // returns the final value of the aggregate. It can be omitted, in which case -// the value of the `state` variable will be used. +// the final value of the state will be returned directly by the aggregate function. // // Here is an example aggregation function, `json_agg`, which will collect all // input values and return them as a JSON array: From bdaa1b6d45b0699bdc863bc74d892a919e25eefa Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Thu, 8 Sep 2022 01:41:02 +0200 Subject: [PATCH 32/36] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index aa444607..00a5da28 100644 --- a/README.md +++ b/README.md @@ -99,9 +99,9 @@ db.create_aggregate( } ); -// Now if you run this query: -var result = db.exec("SELECT json_agg(somecol) FROM atable;"); -console.log("You'll get a json-encoded list of values: ", result[0].values[0]) +```suggestion +db.exec("SELECT json_agg(column1) FROM (VALUES ('hello'), ('world'))"); +// -> The result of the query is the string '["hello","world"]' // Export the database to an Uint8Array containing the SQLite database file const binaryArray = db.export(); From e86d7ff81127fa764b42b54f2ea0cc9e4688ad3a Mon Sep 17 00:00:00 2001 From: Ophir LOJKINE Date: Thu, 8 Sep 2022 01:41:20 +0200 Subject: [PATCH 33/36] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 00a5da28..4e98ee5e 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,6 @@ db.create_aggregate( } ); -```suggestion db.exec("SELECT json_agg(column1) FROM (VALUES ('hello'), ('world'))"); // -> The result of the query is the string '["hello","world"]' From 423fc3615557333bba30c442bb3abd8daeddc875 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Thu, 8 Sep 2022 00:18:00 +0000 Subject: [PATCH 34/36] Improve documentation and type annotations --- src/api.js | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/src/api.js b/src/api.js index 5dbf4613..cb7052b1 100644 --- a/src/api.js +++ b/src/api.js @@ -1200,7 +1200,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { }; /** Register a custom function with SQLite - @example Register a simple function + @example Register a simple function db.create_function("addOne", function (x) {return x+1;}) db.exec("SELECT addOne(1)") // = 2 @@ -1247,30 +1247,33 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { }; /** Register a custom aggregate with SQLite - @example Register a aggregate function - db.create_aggregate( - "js_sum", - function () { return { sum: 0 }; }, - function (state, value) { state.sum+=value; }, - function (state) { return state.sum; } - ); - db.exec("CREATE TABLE test (col); INSERT INTO test VALUES (1), (2)"); - db.exec("SELECT js_sum(col) FROM test"); // = 3 + @example Register a custom sum function + db.create_aggregate("js_sum", { + init: () => 0, + step: (state, value) => state + value, + finalize: state => state + }); + db.exec("SELECT js_sum(column1) FROM (VALUES (1), (2))"); // = 3 @param {string} name the name of the aggregate as referenced in SQL statements. - @param {object} Aggregate function containing at least a step function. - Valid keys for this object are: - - init: a function receiving no arguments and returning an initial - value for the aggregate function. The initial value will be - null if this key is omitted. - - step (required): a function receiving the current state and one to - many values and returning an updated state value. - Will receive the value from init for the first step. - - finalize: a function returning the final value of the aggregate - function. If omitted, the value returned by the last step - wil be used as the final value. + @param {object} aggregateFunctions + object containing at least a step function. + @param {function(): T} [aggregateFunctions.init = ()=>null] + a function receiving no arguments and returning an initial + value for the aggregate function. The initial value will be + null if this key is omitted. + @param {function(T, any) : T} aggregateFunctions.step + a function receiving the current state and a value to aggregate + and returning a new state. + Will receive the value from init for the first step. + @param {function(T): any} [aggregateFunctions.finalize = (state)=>state] + a function returning the result of the aggregate function + given its final state. + If omitted, the value returned by the last step + will be used as the final value. @return {Database} The database object. Useful for method chaining + @template T */ Database.prototype["create_aggregate"] = function create_aggregate( name, From f8e7bd3012fbf00444c96f2d1a99b3865b0e39d4 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Thu, 8 Sep 2022 00:38:14 +0000 Subject: [PATCH 35/36] ignore documentation in eslintrc --- .eslintrc.js | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc.js b/.eslintrc.js index f730a261..02e6047c 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -16,6 +16,7 @@ module.exports = { ignorePatterns: [ "/dist/", "/examples/", + "/documentation/", "/node_modules/", "/out/", "/src/shell-post.js", From 799ebcd1b70b4ac1f2c097e23cf6a1232c2b39a4 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Thu, 8 Sep 2022 00:39:08 +0000 Subject: [PATCH 36/36] reduce code size --- src/api.js | 44 ++++++++++++++++++-------------------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/src/api.js b/src/api.js index cb7052b1..ec8c2fe7 100644 --- a/src/api.js +++ b/src/api.js @@ -1279,18 +1279,16 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { name, aggregateFunctions ) { - if (!Object.hasOwnProperty.call(aggregateFunctions, "step") - ) { - throw "An aggregate function must have a step function in " + name; - } - // Default initializer and finalizer - function init() { return null; } - function finalize(state) { return state; } + var init = aggregateFunctions["init"] + || function init() { return null; }; + var finalize = aggregateFunctions["finalize"] + || function finalize(state) { return state; }; + var step = aggregateFunctions["step"]; - aggregateFunctions["init"] = aggregateFunctions["init"] || init; - aggregateFunctions["finalize"] = aggregateFunctions["finalize"] - || finalize; + if (!step) { + throw "An aggregate function must have a step function in " + name; + } // state is a state object; we'll use the pointer p to serve as the // key for where we hold our state so that multiple invocations of @@ -1314,14 +1312,12 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // Make sure that every path through the step and finalize // functions deletes the value state[p] when it's done so we don't // leak memory and possibly stomp the init value of future calls - if (!Object.hasOwnProperty.call(state, p)) { - state[p] = aggregateFunctions["init"].apply(null); - } + if (!Object.hasOwnProperty.call(state, p)) state[p] = init(); var args = parseFunctionArguments(argc, argv); var mergedArgs = [state[p]].concat(args); try { - state[p] = aggregateFunctions["step"].apply(null, mergedArgs); + state[p] = step.apply(null, mergedArgs); } catch (error) { delete state[p]; sqlite3_result_error(cx, error, -1); @@ -1332,28 +1328,24 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var result; var p = sqlite3_aggregate_context(cx, 1); try { - result = aggregateFunctions["finalize"].apply(null, [state[p]]); + result = finalize(state[p]); } catch (error) { delete state[p]; sqlite3_result_error(cx, error, -1); return; } - setFunctionResult(cx, result); - delete state[p]; } - if (Object.prototype.hasOwnProperty.call(this.functions, name)) { + if (Object.hasOwnProperty.call(this.functions, name)) { removeFunction(this.functions[name]); delete this.functions[name]; } - if (Object.prototype.hasOwnProperty.call( - this.functions, - name + "__finalize" - )) { - removeFunction(this.functions[name + "__finalize"]); - delete this.functions[name + "__finalize"]; + var finalize_name = name + "__finalize"; + if (Object.hasOwnProperty.call(this.functions, finalize_name)) { + removeFunction(this.functions[finalize_name]); + delete this.functions[finalize_name]; } // The signature of the wrapped function is : // void wrapped(sqlite3_context *db, int argc, sqlite3_value **argv) @@ -1363,7 +1355,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // void wrapped(sqlite3_context *db) var finalize_ptr = addFunction(wrapped_finalize, "vi"); this.functions[name] = step_ptr; - this.functions[name + "__finalize"] = finalize_ptr; + this.functions[finalize_name] = finalize_ptr; // passing null to the sixth parameter defines this as an aggregate // function @@ -1374,7 +1366,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { this.handleError(sqlite3_create_function_v2( this.db, name, - aggregateFunctions["step"].length - 1, + step.length - 1, SQLITE_UTF8, 0, 0, pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy