From 6adb6c689e5fc93ed3972fa15c27944f408542c1 Mon Sep 17 00:00:00 2001 From: Benevolent General Kroll Who cannot spell <82620104+general-kroll-4-life@users.noreply.github.com> Date: Mon, 28 Apr 2025 22:22:23 +1000 Subject: [PATCH 01/10] rewritten-testing-pagination (#544) Summary: - No functional change. - Test with pattern supporting pagination on `xml` -> `json` transformed reponses. - Added robot test `Select Paginated Projection From Transformed XML Response Body`. - Amended robot test `Select Star From Transformed XML Response Body` to cover `postgres`. --- .../flask/aws/root_path_cfg.json | 81 +++++++++++++++++++ .../templates/describe_volumes_02_page_01.xml | 33 ++++++++ .../templates/describe_volumes_02_page_02.xml | 33 ++++++++ .../templates/describe_volumes_02_page_03.xml | 32 ++++++++ .../registry/src/aws/v0.1.0/services/ec2.yaml | 7 ++ .../stackql_mocked_from_cmd_line.robot | 45 ++++++++++- 6 files changed, 229 insertions(+), 2 deletions(-) create mode 100644 test/python/stackql_test_tooling/flask/aws/templates/describe_volumes_02_page_01.xml create mode 100644 test/python/stackql_test_tooling/flask/aws/templates/describe_volumes_02_page_02.xml create mode 100644 test/python/stackql_test_tooling/flask/aws/templates/describe_volumes_02_page_03.xml diff --git a/test/python/stackql_test_tooling/flask/aws/root_path_cfg.json b/test/python/stackql_test_tooling/flask/aws/root_path_cfg.json index 78eb40fc..81c7847a 100644 --- a/test/python/stackql_test_tooling/flask/aws/root_path_cfg.json +++ b/test/python/stackql_test_tooling/flask/aws/root_path_cfg.json @@ -317,6 +317,87 @@ "Content-Type": ["text/xml"] } }, + "POST:/:14alt3": { + "method": "POST", + "path": "/", + "headers": { + "Authorization": [ + "^.*eu-south-*.*SignedHeaders=.*content-type;host;x-amz-date.*$" + ] + }, + "body_conditions": { + "type": "PARAMETERS", + "parameters": { + "Action": [ + "^DescribeVolumes$" + ], + "Version": [ + "^2016\\-11\\-15$" + ], + "NextToken": [ + "^token02$" + ] + } + }, + "template": "describe_volumes_02_page_03.xml", + "status": 200, + "response_headers": { + "Content-Type": ["text/xml"] + } + }, + "POST:/:14alt2": { + "method": "POST", + "path": "/", + "headers": { + "Authorization": [ + "^.*eu-south-*.*SignedHeaders=.*content-type;host;x-amz-date.*$" + ] + }, + "body_conditions": { + "type": "PARAMETERS", + "parameters": { + "Action": [ + "^DescribeVolumes$" + ], + "Version": [ + "^2016\\-11\\-15$" + ], + "NextToken": [ + "^token01$" + ] + } + }, + "template": "describe_volumes_02_page_02.xml", + "status": 200, + "response_headers": { + "Content-Type": ["text/xml"] + } + }, + "POST:/:14alt": { + "method": "POST", + "path": "/", + "headers": { + "Authorization": [ + "^.*eu-south-*.*SignedHeaders=.*content-type;host;x-amz-date.*$" + ] + }, + "body_conditions": { + "type": "PARAMETERS", + "parameters": { + "Action": [ + "^DescribeVolumes$" + ], + "Version": [ + "^2016\\-11\\-15$" + ] + } + }, + "template": "describe_volumes_02_page_01.xml", + "status": 200, + "response_headers": { + "Content-Type": ["text/xml"] + } + }, "POST:/:14": { "method": "POST", "path": "/", diff --git a/test/python/stackql_test_tooling/flask/aws/templates/describe_volumes_02_page_01.xml b/test/python/stackql_test_tooling/flask/aws/templates/describe_volumes_02_page_01.xml new file mode 100644 index 00000000..d50787d3 --- /dev/null +++ b/test/python/stackql_test_tooling/flask/aws/templates/describe_volumes_02_page_01.xml @@ -0,0 +1,33 @@ + + + 00000000-0000-0000-aaaa-000000000001 + token01 + + + vol-20100000000000000 + 10 + + us-east-1a + available + 2022-05-02T23:09:30.171Z + + gp2 + 100 + false + false + + + vol-20200000000000000 + 8 + + us-east-1a + available + 2022-05-11T04:45:40.627Z + + gp2 + 100 + false + false + + + \ No newline at end of file diff --git a/test/python/stackql_test_tooling/flask/aws/templates/describe_volumes_02_page_02.xml b/test/python/stackql_test_tooling/flask/aws/templates/describe_volumes_02_page_02.xml new file mode 100644 index 00000000..a7a9bd18 --- /dev/null +++ b/test/python/stackql_test_tooling/flask/aws/templates/describe_volumes_02_page_02.xml @@ -0,0 +1,33 @@ + + + 00000000-0000-0000-aaaa-000000000002 + token02 + + + vol-20300000000000000 + 10 + + us-east-1a + available + 2022-05-02T23:09:30.171Z + + gp2 + 100 + false + false + + + vol-20400000000000000 + 8 + + us-east-1a + available + 2022-05-11T04:45:40.627Z + + gp2 + 100 + false + false + + + \ No newline at end of file diff --git a/test/python/stackql_test_tooling/flask/aws/templates/describe_volumes_02_page_03.xml b/test/python/stackql_test_tooling/flask/aws/templates/describe_volumes_02_page_03.xml new file mode 100644 index 00000000..482d1bce --- /dev/null +++ b/test/python/stackql_test_tooling/flask/aws/templates/describe_volumes_02_page_03.xml @@ -0,0 +1,32 @@ + + + 00000000-0000-0000-aaaa-000000000003 + + + vol-20500000000000000 + 10 + + us-east-1a + available + 2022-05-02T23:09:30.171Z + + gp2 + 100 + false + false + + + vol-20600000000000000 + 8 + + us-east-1a + available + 2022-05-11T04:45:40.627Z + + gp2 + 100 + false + false + + + \ No newline at end of file diff --git a/test/registry/src/aws/v0.1.0/services/ec2.yaml b/test/registry/src/aws/v0.1.0/services/ec2.yaml index 75a3c178..098879c7 100644 --- a/test/registry/src/aws/v0.1.0/services/ec2.yaml +++ b/test/registry/src/aws/v0.1.0/services/ec2.yaml @@ -45740,6 +45740,13 @@ components: algorithm: AWSCanonical requestTranslate: algorithm: get_query_to_post_form_utf_8 + pagination: + requestToken: + key: NextToken + location: query + responseToken: + key: $.next_page_token + location: body operation: $ref: '#/paths/~1?Action=DescribeVolumes&Version=2016-11-15/get' response: diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index 3d655162..5bbf5cf4 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -7676,10 +7676,9 @@ Local Execution Openssl x509 Select ... stderr=${CURDIR}/tmp/Local-Execution-Openssl-x509-Select-stderr.tmp Select Star From Transformed XML Response Body - Pass Execution If "${SQL_BACKEND}" == "postgres_tcp" TODO: FIX THIS... Skipping postgres for speed, simple enough to support bool there. ${inputStr} = Catenate ... select * from aws.ec2.volumes_presented where region = 'ap-southeast-2' order by volume_id; - ${outputStr} = Catenate SEPARATOR=\n + ${outputStrSQLite} = Catenate SEPARATOR=\n ... |-------------------|--------------------------|-----------|----------------------|----------------|------|-------------|-----------|-----------------------|-------------| ... |${SPACE}availability_zone${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}create_time${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}encrypted${SPACE}|${SPACE}multi_attach_enabled${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}size${SPACE}|${SPACE}snapshot_id${SPACE}|${SPACE}${SPACE}status${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}volume_type${SPACE}| ... |-------------------|--------------------------|-----------|----------------------|----------------|------|-------------|-----------|-----------------------|-------------| @@ -7687,6 +7686,15 @@ Select Star From Transformed XML Response Body ... |-------------------|--------------------------|-----------|----------------------|----------------|------|-------------|-----------|-----------------------|-------------| ... |${SPACE}ap-southeast-1a${SPACE}${SPACE}${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}0${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}0${SPACE}|${SPACE}ap-southeast-2${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}available${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}gp2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| ... |-------------------|--------------------------|-----------|----------------------|----------------|------|-------------|-----------|-----------------------|-------------| + ${outputStrPostgres} = Catenate SEPARATOR=\n + ... |-------------------|--------------------------|-----------|----------------------|----------------|------|-------------|-----------|-----------------------|-------------| + ... |${SPACE}availability_zone${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}create_time${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}encrypted${SPACE}|${SPACE}multi_attach_enabled${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}size${SPACE}|${SPACE}snapshot_id${SPACE}|${SPACE}${SPACE}status${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}volume_type${SPACE}| + ... |-------------------|--------------------------|-----------|----------------------|----------------|------|-------------|-----------|-----------------------|-------------| + ... |${SPACE}ap-southeast-1a${SPACE}${SPACE}${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}false${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}false${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}ap-southeast-2${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}available${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}gp2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------|--------------------------|-----------|----------------------|----------------|------|-------------|-----------|-----------------------|-------------| + ... |${SPACE}ap-southeast-1a${SPACE}${SPACE}${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}false${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}false${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}ap-southeast-2${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}available${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}gp2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------|--------------------------|-----------|----------------------|----------------|------|-------------|-----------|-----------------------|-------------| + ${outputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${outputStrPostgres} ${outputStrSQLite} Should Stackql Exec Inline Equal Both Streams ... ${STACKQL_EXE} ... ${OKTA_SECRET_STR} @@ -7764,3 +7772,36 @@ Describe Transformed XML Response Body ... ${EMPTY} ... stdout=${CURDIR}/tmp/Describe-Transformed-XML-Response-Body.tmp ... stderr=${CURDIR}/tmp/Describe-Transformed-XML-Response-Body-stderr.tmp + +Select Paginated Projection From Transformed XML Response Body + ${inputStr} = Catenate + ... select volume_id, create_time, region, size from aws.ec2.volumes_presented where region = 'eu-south-2' order by volume_id asc; + ${outputStr} = Catenate SEPARATOR=\n + ... |-----------------------|--------------------------|------------|------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}create_time${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}|${SPACE}size${SPACE}| + ... |-----------------------|--------------------------|------------|------| + ... |${SPACE}vol-20100000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|------| + ... |${SPACE}vol-20200000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|------| + ... |${SPACE}vol-20300000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|------| + ... |${SPACE}vol-20400000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|------| + ... |${SPACE}vol-20500000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|------| + ... |${SPACE}vol-20600000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|------| + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${EMPTY} + ... stdout=${CURDIR}/tmp/Select-Paginated-Projection-From-Transformed-XML-Response-Body.tmp + ... stderr=${CURDIR}/tmp/Select-Paginated-Projection-From-Transformed-XML-Response-Body-stderr.tmp From 865fe1d86dba8ac6d1cbe2c270da157e3ea2b029 Mon Sep 17 00:00:00 2001 From: Benevolent General Kroll Who cannot spell <82620104+general-kroll-4-life@users.noreply.github.com> Date: Tue, 29 Apr 2025 00:25:27 +1000 Subject: [PATCH 02/10] xml-transform-testing-exotic (#545) Summary: - Expanded `xml` tranform testing to cover joins. - Expanded `xml` tranform testing to cover materialized views. - Expanded `xml` tranform testing to cover views. - Added robot test `Select Join Paginated Projection From Transformed XML Response Body`. - Added robot test `Select View of Join Paginated Projection From Transformed XML Response Body`. - Added robot test `Select Materialized View of Join Paginated Projection From Transformed XML Response Body`. --- .../stackql_mocked_from_cmd_line.robot | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index 5bbf5cf4..1da11785 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -7805,3 +7805,144 @@ Select Paginated Projection From Transformed XML Response Body ... ${EMPTY} ... stdout=${CURDIR}/tmp/Select-Paginated-Projection-From-Transformed-XML-Response-Body.tmp ... stderr=${CURDIR}/tmp/Select-Paginated-Projection-From-Transformed-XML-Response-Body-stderr.tmp + +Select Join Paginated Projection From Transformed XML Response Body + ${inputStr} = Catenate + ... select lhs.volume_id, lhs.create_time, lhs.region, rhs.region as rhs_region, lhs.size from aws.ec2.volumes_presented lhs inner join aws.ec2.volumes_presented rhs on lhs.size = rhs.size where lhs.region = 'eu-south-2' and rhs.region in ('ap-southeast-1', 'us-east-1') order by lhs.volume_id, rhs.region asc; + ${outputStr} = Catenate SEPARATOR=\n + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}create_time${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}rhs_region${SPACE}${SPACE}${SPACE}|${SPACE}size${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20100000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20100000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20200000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20200000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20300000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20300000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20400000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20400000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20500000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20500000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20600000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20600000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${EMPTY} + ... stdout=${CURDIR}/tmp/Select-Join-Paginated-Projection-From-Transformed-XML-Response-Body.tmp + ... stderr=${CURDIR}/tmp/Select-Join-Paginated-Projection-From-Transformed-XML-Response-Body-stderr.tmp + +Select View of Join Paginated Projection From Transformed XML Response Body + ${inputStr} = Catenate + ... create or replace view xml_v_01 as select lhs.volume_id, lhs.create_time, lhs.region, rhs.region as rhs_region, lhs.size from aws.ec2.volumes_presented lhs inner join aws.ec2.volumes_presented rhs on lhs.size = rhs.size where lhs.region = 'eu-south-2' and rhs.region in ('ap-southeast-1', 'us-east-1') order by lhs.volume_id, rhs.region asc; + ... select volume_id, create_time, region, rhs_region, size from xml_v_01 order by volume_id, rhs_region asc; + ${stdErrStr} = Catenate SEPARATOR=\n + ... DDL Execution Completed + ${outputStr} = Catenate SEPARATOR=\n + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}create_time${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}rhs_region${SPACE}${SPACE}${SPACE}|${SPACE}size${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20100000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20100000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20200000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20200000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20300000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20300000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20400000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20400000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20500000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20500000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20600000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20600000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${stdErrStr} + ... stdout=${CURDIR}/tmp/Select-View-of-Join-Paginated-Projection-From-Transformed-XML-Response-Body.tmp + ... stderr=${CURDIR}/tmp/Select-View-of-Join-Paginated-Projection-From-Transformed-XML-Response-Body-stderr.tmp + +Select Materialized View of Join Paginated Projection From Transformed XML Response Body + ${inputStr} = Catenate + ... create or replace materialized view xml_mv_01 as select lhs.volume_id, lhs.create_time, lhs.region, rhs.region as rhs_region, lhs.size from aws.ec2.volumes_presented lhs inner join aws.ec2.volumes_presented rhs on lhs.size = rhs.size where lhs.region = 'eu-south-2' and rhs.region in ('ap-southeast-1', 'us-east-1') order by lhs.volume_id, rhs.region asc; + ... select volume_id, create_time, region, rhs_region, size from xml_mv_01 order by volume_id, rhs_region asc; + ${stdErrStr} = Catenate SEPARATOR=\n + ... DDL Execution Completed + ${outputStr} = Catenate SEPARATOR=\n + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}create_time${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}rhs_region${SPACE}${SPACE}${SPACE}|${SPACE}size${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20100000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20100000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20200000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20200000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20300000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20300000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20400000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20400000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20500000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20500000000000000${SPACE}|${SPACE}2022-05-02T23:09:30.171Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}10${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20600000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}ap-southeast-1${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + ... |${SPACE}vol-20600000000000000${SPACE}|${SPACE}2022-05-11T04:45:40.627Z${SPACE}|${SPACE}eu-south-2${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}8${SPACE}| + ... |-----------------------|--------------------------|------------|----------------|------| + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${stdErrStr} + ... stdout=${CURDIR}/tmp/Select-Materialized-View-of-Join-Paginated-Projection-From-Transformed-XML-Response-Body.tmp + ... stderr=${CURDIR}/tmp/Select-Materialized-View-of-Join-Paginated-Projection-From-Transformed-XML-Response-Body-stderr.tmp From d81d9302a16964dfa568cdb97e45a8cfd4482396 Mon Sep 17 00:00:00 2001 From: Benevolent General Kroll Who cannot spell <82620104+general-kroll-4-life@users.noreply.github.com> Date: Thu, 1 May 2025 17:54:36 +1000 Subject: [PATCH 03/10] improve-testing-package (#546) Summary: - Integrate `poetry` into virtual environment, in testing package build workflow. - Default `cwd` for flask app aggregator better supports testing mobility. --- cicd/util/01-build-robot-lib.sh | 38 ++++++++++++------- .../web_service_keywords.py | 6 ++- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/cicd/util/01-build-robot-lib.sh b/cicd/util/01-build-robot-lib.sh index 3ae45d1b..76740a05 100755 --- a/cicd/util/01-build-robot-lib.sh +++ b/cicd/util/01-build-robot-lib.sh @@ -1,15 +1,12 @@ #! /usr/bin/env bash -poetryExe="$(which poetry)" -rv="$?" -if [ $rv -ne 0 ]; then - >&2 echo "Poetry is not installed. Please install it first." - exit 1 -fi -if [ "$poetryExe" = "" ]; then - >&2 echo "No poetry executable found in PATH. Please install it first." - exit 1 -fi +checkPoetry () { + if ! command -v poetry &> /dev/null + then + >&2 echo "Poetry is not installed. Please install it first." + exit 1 + fi +} CURDIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" @@ -17,13 +14,27 @@ REPOSITORY_ROOT="$(realpath ${CURDIR}/../..)" PACKAGE_ROOT="${REPOSITORY_ROOT}/test" -venv_path="${REPOSITORY_ROOT}/.venv" - - rm -f ${PACKAGE_ROOT}/dist/*.whl || true +if [ ! -d "${PACKAGE_ROOT}/.venv" ]; then + >&2 echo "No existing virtual environment, creating one..." + >&2 echo "Creating virtual environment in ${PACKAGE_ROOT}/.venv" + python -m venv "${PACKAGE_ROOT}/.venv" + >&2 echo "Virtual environment created." + >&2 echo "Installing poetry into virtual environment." + ${PACKAGE_ROOT}/.venv/bin/pip install -U pip setuptools + ${PACKAGE_ROOT}/.venv/bin/pip install poetry + >&2 echo "Poetry installed into virtual environment." +else + >&2 echo "Using existing virtual environment in ${PACKAGE_ROOT}/.venv" +fi + cd "${PACKAGE_ROOT}" +source ${PACKAGE_ROOT}/.venv/bin/activate + +checkPoetry + poetry install poetry build @@ -38,7 +49,6 @@ else fi -# >&2 echo "Artifact built successfully: ${expectedRobotLibArtifact}" diff --git a/test/python/stackql_test_tooling/web_service_keywords.py b/test/python/stackql_test_tooling/web_service_keywords.py index e29646b7..af3ca295 100644 --- a/test/python/stackql_test_tooling/web_service_keywords.py +++ b/test/python/stackql_test_tooling/web_service_keywords.py @@ -6,14 +6,16 @@ import os +from pathlib import Path, PurePosixPath + from typing import Optional @library class web_service_keywords(Process): + _THIS_DIR: str = PurePosixPath(Path(os.path.dirname(os.path.abspath(__file__)))).as_posix() - - _DEFAULT_APP_ROOT: str = 'test/python/stackql_test_tooling/flask' + _DEFAULT_APP_ROOT: str = os.path.join(_THIS_DIR, 'flask') _DEFAULT_TLS_KEY_PATH: str = 'test/server/mtls/credentials/pg_server_key.pem' From a9cc164758a0f29c3887fc96cd5b265012073c8b Mon Sep 17 00:00:00 2001 From: Benevolent General Kroll Who cannot spell <82620104+general-kroll-4-life@users.noreply.github.com> Date: Sun, 4 May 2025 15:48:45 +1000 Subject: [PATCH 04/10] consume-improved-response-transform-interface (#547) Summary: - Consume new reponse transform interface. - Support for `json` response transforms. - Support for `text` response transforms. - Added robot test `Select Paginated Star From Transformed JSON Response Body`. - Added robot test `Select Paginated Projection From Transformed JSON Response Body`. - Added robot test `Select Join of Paginated Projection From Transformed JSON and XML Response Bodies`. - Added robot test `Select View of Join of Paginated Projection From Transformed JSON and XML Response Bodies`. - Added robot test `Select Materialized View of Join of Paginated Projection From Transformed JSON and XML Response Bodies`. --- cicd/scripts/context.sh | 8 + cicd/scripts/testing-env.sh | 28 +++ go.mod | 2 +- go.sum | 4 +- .../execution/mono_valent_execution.go | 36 +++- .../stackql_test_tooling/flask/README.md | 16 +- .../stackql_test_tooling/flask/azure/app.py | 63 ++++-- .../flask/azure/expectations.json | 37 ++++ ...irtual-networks-list-all-paginated-01.json | 54 +++++ ...irtual-networks-list-all-paginated-02.json | 53 +++++ .../src/azure/v0.1.0/services/network.yaml | 123 +++++++++++ .../stackql_mocked_from_cmd_line.robot | 191 ++++++++++++++++++ 12 files changed, 582 insertions(+), 33 deletions(-) create mode 100755 cicd/scripts/context.sh create mode 100755 cicd/scripts/testing-env.sh create mode 100644 test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-paginated-01.json create mode 100644 test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-paginated-02.json diff --git a/cicd/scripts/context.sh b/cicd/scripts/context.sh new file mode 100755 index 00000000..c003a005 --- /dev/null +++ b/cicd/scripts/context.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export REPOSITORY_ROOT="$(realpath ${CUR_DIR}/../..)" + + + diff --git a/cicd/scripts/testing-env.sh b/cicd/scripts/testing-env.sh new file mode 100755 index 00000000..e824c297 --- /dev/null +++ b/cicd/scripts/testing-env.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${CUR_DIR}/context.sh" + +export REPOSITORY_ROOT="${REPOSITORY_ROOT}" +export workspaceFolder="${REPOSITORY_ROOT}" + +export OKTA_SECRET_KEY='some-dummy-api-key' +export GITHUB_SECRET_KEY='some-dummy-github-key' +export K8S_SECRET_KEY='some-k8s-token' +export AZ_ACCESS_TOKEN='dummy_azure_token' +export SUMO_CREDS='somesumologictoken' +export DIGITALOCEAN_TOKEN='somedigitaloceantoken' +export DUMMY_DIGITALOCEAN_USERNAME='myusername' +export DUMMY_DIGITALOCEAN_PASSWORD='mypassword' + + +googleCredentialsFilePath="${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json" + +export stackqlMockedRegistryStr="{ \"url\": \"file://${workspaceFolder}/test/registry-mocked\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked\", \"verifyConfig\": { \"nopVerify\": true } }" + +export stackqlAuthStr="{\"google\": {\"credentialsfilepath\": \"${googleCredentialsFilePath}\", \"type\": \"service_account\"}, \"okta\": {\"credentialsenvvar\": \"OKTA_SECRET_KEY\", \"type\": \"api_key\"}, \"aws\": {\"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"/Users/admin/stackql/stackql-devel/test/robot/functional/../../../test/assets/credentials/dummy/aws/functional-test-dummy-aws-key.txt\", \"keyID\": \"NON_SECRET\"}, \"github\": {\"type\": \"basic\", \"credentialsenvvar\": \"GITHUB_SECRET_KEY\"}, \"k8s\": {\"credentialsenvvar\": \"K8S_SECRET_KEY\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \"}, \"azure\": {\"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsenvvar\": \"AZ_ACCESS_TOKEN\"}, \"sumologic\": {\"type\": \"basic\", \"credentialsenvvar\": \"SUMO_CREDS\"}, \"digitalocean\": {\"type\": \"bearer\", \"username\": \"myusername\", \"password\": \"mypassword\"}}" + + + + diff --git a/go.mod b/go.mod index aacf50a3..f07b802e 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.10.1 - github.com/stackql/any-sdk v0.1.3-alpha13 + github.com/stackql/any-sdk v0.1.3-beta02 github.com/stackql/go-suffix-map v0.0.1-alpha01 github.com/stackql/psql-wire v0.1.1-beta23 github.com/stackql/stackql-parser v0.0.14-alpha05 diff --git a/go.sum b/go.sum index 84db37c3..4c60470e 100644 --- a/go.sum +++ b/go.sum @@ -484,8 +484,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= -github.com/stackql/any-sdk v0.1.3-alpha13 h1:IJLnZojR7JU6f3cixL0U8KdJva+qociPb7Uu3Vctq9w= -github.com/stackql/any-sdk v0.1.3-alpha13/go.mod h1:AKS/g28y7m4SWL/YW8veE9MCNy8XJgaicVibemVE9e8= +github.com/stackql/any-sdk v0.1.3-beta02 h1:0jSwyYFddjAN++U+yNNLF7SMLPIIaRhv522UzeWDf2E= +github.com/stackql/any-sdk v0.1.3-beta02/go.mod h1:AKS/g28y7m4SWL/YW8veE9MCNy8XJgaicVibemVE9e8= github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8= github.com/stackql/go-suffix-map v0.0.1-alpha01/go.mod h1:QAi+SKukOyf4dBtWy8UMy+hsXXV+yyEE4vmBkji2V7g= github.com/stackql/psql-wire v0.1.1-beta23 h1:1ayYMjZArfDcIMyEOKnm+Bp1zRCISw8pguvTFuUhhVQ= diff --git a/internal/stackql/execution/mono_valent_execution.go b/internal/stackql/execution/mono_valent_execution.go index 6b1585d4..620096e5 100644 --- a/internal/stackql/execution/mono_valent_execution.go +++ b/internal/stackql/execution/mono_valent_execution.go @@ -1,7 +1,6 @@ package execution import ( - "bytes" "encoding/json" "fmt" "io" @@ -1330,19 +1329,34 @@ func (mv *monoValentExecution) GetExecutor() (func(pc primitive.IPrimitiveCtx) i expectedResponse, isExpectedResponse := m.GetResponse() if isExpectedResponse { responseTransform, responseTransformExists := expectedResponse.GetTransform() - if responseTransformExists && responseTransform.GetType() == "golang_template_v0.1.0" { + if responseTransformExists { input := stdoutStr - tmpl := responseTransform.GetBody() - inStream := stream_transform.NewTextReader(bytes.NewBufferString(input)) - outStream := bytes.NewBuffer(nil) - tfm, setupErr := stream_transform.NewTemplateStreamTransformer(tmpl, inStream, outStream) - if setupErr != nil { - return internaldto.NewErroneousExecutorOutput(fmt.Errorf("template stream transform error: %w", setupErr)) + streamTransformerFactory := stream_transform.NewStreamTransformerFactory( + responseTransform.GetType(), + responseTransform.GetBody(), + ) + if !streamTransformerFactory.IsTransformable() { + return internaldto.NewErroneousExecutorOutput( + fmt.Errorf("unsupported template type: %s", responseTransform.GetType()), + ) } - if tfErr := tfm.Transform(); tfErr != nil { - return internaldto.NewErroneousExecutorOutput(fmt.Errorf("failed to transform: %w", tfErr)) + tfm, getTfmErr := streamTransformerFactory.GetTransformer(input) + if getTfmErr != nil { + return internaldto.NewErroneousExecutorOutput( + fmt.Errorf("failed to transform: %w", getTfmErr)) } - outputStr := outStream.String() + transformError := tfm.Transform() + if transformError != nil { + return internaldto.NewErroneousExecutorOutput( + fmt.Errorf("failed to transform: %w", transformError)) + } + outStream := tfm.GetOutStream() + outputBytes, readErr := io.ReadAll(outStream) + if readErr != nil { + return internaldto.NewErroneousExecutorOutput( + fmt.Errorf("failed to read transformed stream: %w", readErr)) + } + outputStr := string(outputBytes) stdoutStr = outputStr } } diff --git a/test/python/stackql_test_tooling/flask/README.md b/test/python/stackql_test_tooling/flask/README.md index 0c8103ef..d1a7e65c 100644 --- a/test/python/stackql_test_tooling/flask/README.md +++ b/test/python/stackql_test_tooling/flask/README.md @@ -81,9 +81,9 @@ flask --app=${HOME}/stackql/stackql-devel/test/python/stackql_test_tooling/flask With embedded `sqlite` (default), from the root of this repository: ```bash -export workspaceFolder="$(pwd)" +source cicd/scripts/testing-env.sh -stackql --registry="{ \"url\": \"file://${workspaceFolder}/test/registry-mocked\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked\", \"verifyConfig\": { \"nopVerify\": true } }" --tls.allowInsecure shell +stackql --registry="${stackqlMockedRegistryStr}" --auth="${stackqlAuthStr}" --tls.allowInsecure shell ``` With `postgres`, from the root of this repository: @@ -91,7 +91,15 @@ With `postgres`, from the root of this repository: ```bash docker compose -f docker-compose-externals.yml up postgres_stackql -d -export workspaceFolder="$(pwd)" +source cicd/scripts/testing-env.sh -stackql --registry="{ \"url\": \"file://${workspaceFolder}/test/registry-mocked\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked\", \"verifyConfig\": { \"nopVerify\": true } }" --tls.allowInsecure --sqlBackend="{ \"dbEngine\": \"postgres_tcp\", \"sqlDialect\": \"postgres\", \"dsn\": \"postgres://stackql:stackql@127.0.0.1:7432/stackql\" }" shell +stackql --registry="${stackqlMockedRegistryStr}" --tls.allowInsecure --sqlBackend="{ \"dbEngine\": \"postgres_tcp\", \"sqlDialect\": \"postgres\", \"dsn\": \"postgres://stackql:stackql@127.0.0.1:7432/stackql\" }" shell ``` + +## Sources of Mock Data + +There are some decent examples in vendor documentation, eg: + +- [Azure vendor documenation](https://learn.microsoft.com/en-us/rest/api/azure/). + + diff --git a/test/python/stackql_test_tooling/flask/azure/app.py b/test/python/stackql_test_tooling/flask/azure/app.py index 0e736432..1be1af25 100644 --- a/test/python/stackql_test_tooling/flask/azure/app.py +++ b/test/python/stackql_test_tooling/flask/azure/app.py @@ -10,13 +10,44 @@ app.template_folder = os.path.join(os.path.dirname(__file__), "templates") # Configure logging -logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") +logging.basicConfig(level=logging.DEBUG, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") logger = logging.getLogger(__name__) @app.before_request def log_request_info(): logger.info(f"Request: {request.method} {request.path}\n - Query: {request.args}\n - Headers: {request.headers}\n - Body: {request.get_data()}") + +def _extract_req_adornments(req: Request) -> dict: + """ + Extracts the request adornments from the request object. + """ + req_adornments = {} + if req.headers.get('Authorization'): + auth_header = req.headers.get('Authorization') + if auth_header.startswith('Basic '): + auth_value = auth_header.split(' ')[1] + decoded_value = base64.b64decode(auth_value).decode('utf-8') + username, password = decoded_value.split(':', 1) + req_adornments['username'] = username + req_adornments['password'] = password + elif auth_header.startswith('Bearer '): + token = auth_header.split(' ')[1] + req_adornments['token'] = token + logger.debug(f"Host url: {request.host_url}") + host_components = request.host_url.split(':') + logger.debug(f"Host components: {host_components}") + if len(host_components) == 3: + req_adornments['scheme'] = host_components[0] + req_adornments['host_name'] = host_components[1].lstrip('/') + req_adornments['port'] = int(host_components[2].strip('/')) + elif len(host_components) == 2: + req_adornments['scheme'] = host_components[0] + req_adornments['host_name'] = host_components[1].lstrip('/') + + logger.debug(f"Request adornments:\n {json.dumps(req_adornments, indent=2)}\n\n") + + return req_adornments class GetMatcherConfig: _ROOT_PATH_CFG: List[dict] = {} @@ -45,6 +76,7 @@ def _match_json_strict(self, lhs: dict, rhs: dict) -> bool: def _match_json_by_key(self, lhs: dict, rhs: dict) -> bool: for key, value in rhs.items(): + logger.debug(f"Matching key: {key} from {json.dumps(lhs)} with value: {value}") if key not in lhs: return False if isinstance(value, dict): @@ -53,6 +85,7 @@ def _match_json_by_key(self, lhs: dict, rhs: dict) -> bool: elif isinstance(value, list): for item in value: if not self._match_string(lhs[key], item): + logger.debug(f"Matching item {item} in list {lhs[key]} failed") return False elif isinstance(value, str): if not self._match_string(lhs[key], value): @@ -138,7 +171,7 @@ def match_route(self, req: Request) -> dict: for i in range(len(self._ROOT_PATH_CFG)): route_name: str = f"route_{i}" cfg: dict = self._ROOT_PATH_CFG[i] - logger.debug(f"Evaluating route: {route_name}") + logger.debug(f"Evaluating route: {route_name} with config: \n{json.dumps(cfg, indent=2, sort_keys=True)}\n\n") is_method_match: bool = self._is_method_match(req, cfg) if not is_method_match: @@ -200,7 +233,7 @@ def match_route(self, req: Request) -> dict: @app.route('/subscriptions/000000-0000-0000-0000-000000000022/resourceGroups//providers/Microsoft.KeyVault/vaults/stackql-testing-keyvault/keys/', methods=['GET']) def keys_list_01(resourceGroupName): template_name = "keys-list-01.json" - response = make_response(render_template(template_name, request=request)) + response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request))) response.headers.update({"Content-Type": "application/json"}) response.status_code = 200 return response @@ -208,7 +241,7 @@ def keys_list_01(resourceGroupName): @app.route('/subscriptions/000000-0000-0000-0000-000000000022/resourceGroups//providers/Microsoft.KeyVault/vaults/stackql-alt-keyvault/keys/', methods=['GET']) def keys_list_02(resourceGroupName): template_name = "keys-list-02.json" - response = make_response(render_template(template_name, request=request)) + response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request))) response.headers.update({"Content-Type": "application/json"}) response.status_code = 200 return response @@ -216,7 +249,7 @@ def keys_list_02(resourceGroupName): @app.route('/subscriptions/000000-0000-0000-0000-000000000022/resourceGroups//providers/Microsoft.KeyVault/vaults/stackql-testing-keyvault/keys/dummy-key-01/', methods=['GET']) def key_detail_01(resourceGroupName): template_name = "key-detail-01.json" - response = make_response(render_template(template_name, request=request)) + response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request))) response.headers.update({"Content-Type": "application/json"}) response.status_code = 200 return response @@ -224,7 +257,7 @@ def key_detail_01(resourceGroupName): @app.route('/subscriptions/000000-0000-0000-0000-000000000022/resourceGroups//providers/Microsoft.KeyVault/vaults/stackql-testing-keyvault/keys/dummy-key-02/', methods=['GET']) def key_detail_02(resourceGroupName): template_name = "key-detail-02.json" - response = make_response(render_template(template_name, request=request)) + response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request))) response.headers.update({"Content-Type": "application/json"}) response.status_code = 200 return response @@ -232,7 +265,7 @@ def key_detail_02(resourceGroupName): @app.route('/subscriptions/000000-0000-0000-0000-000000000022/resourceGroups//providers/Microsoft.KeyVault/vaults/stackql-testing-keyvault/keys/alt-dummy-key-02/', methods=['GET']) def key_detail_03(resourceGroupName): template_name = "key-detail-03.json" - response = make_response(render_template(template_name, request=request)) + response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request))) response.headers.update({"Content-Type": "application/json"}) response.status_code = 200 return response @@ -240,7 +273,7 @@ def key_detail_03(resourceGroupName): @app.route('/subscriptions/000000-0000-0000-0000-000000000022/resourceGroups//providers/Microsoft.KeyVault/vaults/stackql-alt-keyvault/keys/alt-dummy-key-01/', methods=['GET']) def key_detail_04(resourceGroupName): template_name = "key-detail-04.json" - response = make_response(render_template(template_name, request=request)) + response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request))) response.headers.update({"Content-Type": "application/json"}) response.status_code = 200 return response @@ -248,7 +281,7 @@ def key_detail_04(resourceGroupName): @app.route('/subscriptions/000000-0000-0000-0000-000000000022/resourceGroups//providers/Microsoft.KeyVault/vaults/stackql-alt-keyvault/keys/alt-dummy-key-02/', methods=['GET']) def key_detail_05(resourceGroupName): template_name = "key-detail-05.json" - response = make_response(render_template(template_name, request=request)) + response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request))) response.headers.update({"Content-Type": "application/json"}) response.status_code = 200 return response @@ -256,7 +289,7 @@ def key_detail_05(resourceGroupName): @app.route('/subscriptions/000000-0000-0000-0000-000000000011/resourceGroups//providers/Microsoft.KeyVault/vaults/stackql-testing-keyvault/keys/dummy-key-01/', methods=['GET']) def key_detail_06(resourceGroupName): template_name = "key-detail-06.json" - response = make_response(render_template(template_name, request=request)) + response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request))) response.headers.update({"Content-Type": "application/json"}) response.status_code = 200 return response @@ -264,7 +297,7 @@ def key_detail_06(resourceGroupName): @app.route('/subscriptions/000000-0000-0000-0000-000000000011/resourceGroups//providers/Microsoft.KeyVault/vaults/stackql-testing-keyvault/keys/', methods=['GET']) def keys_list_03(resourceGroupName): template_name = "keys-list-03.json" - response = make_response(render_template(template_name, request=request)) + response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request))) response.headers.update({"Content-Type": "application/json"}) response.status_code = 200 return response @@ -272,7 +305,7 @@ def keys_list_03(resourceGroupName): @app.route('/subscriptions//providers/Microsoft.Compute/sshPublicKeys/', methods=['GET']) def ssh_public_keys_list(subscriptionId): template_name = "ssh-public-keys-list-01.json" - response = make_response(render_template(template_name, request=request)) + response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request))) response.headers.update({"Content-Type": "application/json"}) response.status_code = 200 return response @@ -280,7 +313,7 @@ def ssh_public_keys_list(subscriptionId): @app.route('/subscriptions//resourceGroups//providers/Microsoft.Compute/virtualMachines/', methods=['GET']) def virtual_machines_list(subscriptionId, resourceGroupId): template_name = "virtual-machines-list-01.json" - response = make_response(render_template(template_name, request=request)) + response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request))) response.headers.update({"Content-Type": "application/json"}) response.status_code = 200 return response @@ -295,11 +328,11 @@ def generic_handler(request: Request): route_cfg: dict = cfg_obj.match_route(request) template_name = route_cfg.get("httpResponse", {}).get("template", "") if not template_name: - rv = make_response(render_template('nil-response.json', request=request)) + rv = make_response(render_template('nil-response.json', request=request, **_extract_req_adornments(request))) rv.status_code = 404 return rv logger.info(f"routing to template: {template_name}") - response = make_response(render_template(template_name, request=request)) + response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request))) response.headers.update(route_cfg.get("httpResponse", {}).get("headers", {})) response.status_code = route_cfg.get("httpResponse", {}).get("status", 200) return response diff --git a/test/python/stackql_test_tooling/flask/azure/expectations.json b/test/python/stackql_test_tooling/flask/azure/expectations.json index 8134eaae..3bb35fcb 100644 --- a/test/python/stackql_test_tooling/flask/azure/expectations.json +++ b/test/python/stackql_test_tooling/flask/azure/expectations.json @@ -1,4 +1,41 @@ [ + { + "httpRequest": { + "headers": { + "Accept": [ "application/json" ] + }, + "method": "GET", + "path": "/subscriptions/subid/providers/Microsoft.Network/virtualNetworks/", + "queryStringParameters" : { + "api-version" : [ "2023-11-01" ], + "$skiptoken": [ "0011" ] + } + }, + "httpResponse": { + "template": "virtual-networks-list-all-paginated-02.json", + "headers": { + "Content-Type": "application/json; charset=utf-8" + } + } + }, + { + "httpRequest": { + "headers": { + "Accept": [ "application/json" ] + }, + "method": "GET", + "path": "/subscriptions/subid/providers/Microsoft.Network/virtualNetworks/", + "queryStringParameters" : { + "api-version" : [ "2023-11-01" ] + } + }, + "httpResponse": { + "template": "virtual-networks-list-all-paginated-01.json", + "headers": { + "Content-Type": "application/json; charset=utf-8" + } + } + }, { "httpRequest": { "headers": { diff --git a/test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-paginated-01.json b/test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-paginated-01.json new file mode 100644 index 00000000..9be2b511 --- /dev/null +++ b/test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-paginated-01.json @@ -0,0 +1,54 @@ +{ + "value": [ + { + "id": "/subscriptions/subid/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1", + "name": "vnet1", + "type": "Microsoft.Network/virtualNetworks", + "location": "westus", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/8" + ] + }, + "dhcpOptions": { + "dnsServers": [] + }, + "subnets": [ + { + "id": "/subscriptions/subid/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-1", + "name": "test-1", + "properties": { + "addressPrefix": "10.0.0.0/24", + "provisioningState": "Succeeded" + } + } + ], + "virtualNetworkPeerings": [], + "provisioningState": "Succeeded" + } + }, + { + "id": "/subscriptions/subid/resourceGroups/rg2/providers/Microsoft.Network/virtualNetworks/vnet2", + "name": "vnet2", + "type": "Microsoft.Network/virtualNetworks", + "location": "westus", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/16" + ] + }, + "dhcpOptions": { + "dnsServers": [ + "8.8.8.8" + ] + }, + "subnets": [], + "virtualNetworkPeerings": [], + "provisioningState": "Succeeded" + } + } + ], + "nextLink": "{{ scheme }}://{{ host_name }}{% if port is defined %}:{{ port }}{% endif %}/subscriptions/subid/providers/Microsoft.Network/virtualNetworks/?api-version=2023-11-01&$skiptoken=0011" +} \ No newline at end of file diff --git a/test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-paginated-02.json b/test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-paginated-02.json new file mode 100644 index 00000000..3f0d5999 --- /dev/null +++ b/test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-paginated-02.json @@ -0,0 +1,53 @@ +{ + "value": [ + { + "id": "/subscriptions/subid/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1", + "name": "vnet3", + "type": "Microsoft.Network/virtualNetworks", + "location": "australiasoutheast", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/8" + ] + }, + "dhcpOptions": { + "dnsServers": [] + }, + "subnets": [ + { + "id": "/subscriptions/subid/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-1", + "name": "test-1", + "properties": { + "addressPrefix": "10.0.0.0/24", + "provisioningState": "Succeeded" + } + } + ], + "virtualNetworkPeerings": [], + "provisioningState": "Succeeded" + } + }, + { + "id": "/subscriptions/subid/resourceGroups/rg2/providers/Microsoft.Network/virtualNetworks/vnet2", + "name": "vnet4", + "type": "Microsoft.Network/virtualNetworks", + "location": "australiasoutheast", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/16" + ] + }, + "dhcpOptions": { + "dnsServers": [ + "8.8.8.8" + ] + }, + "subnets": [], + "virtualNetworkPeerings": [], + "provisioningState": "Succeeded" + } + } + ] +} \ No newline at end of file diff --git a/test/registry/src/azure/v0.1.0/services/network.yaml b/test/registry/src/azure/v0.1.0/services/network.yaml index 8d185feb..b5ac8a93 100644 --- a/test/registry/src/azure/v0.1.0/services/network.yaml +++ b/test/registry/src/azure/v0.1.0/services/network.yaml @@ -197,6 +197,54 @@ components: schema: type: string schemas: + VNetListingTransformed: + type: object + properties: + id: + type: string + description: Resource ID. + name: + type: string + description: Resource name. + type: + type: string + description: Resource type. + location: + type: string + description: Resource location. + provisioning_state: + type: string + subnets: + type: array + items: + type: object + properties: + id: + type: string + description: Resource ID. + name: + type: string + description: Resource name. + address_prefixes: + type: array + items: + type: string + description: Address prefixes. + description: Resource type. + provisioning_state: + type: string + VNetListingsTransformed: + type: object + properties: + line_items: + type: array + items: + $ref: '#/components/schemas/VNetListingTransformed' + description: List of virtual networks. + next_link: + type: string + description: The URL to get the next set of results. + description: Response for ListVNet API service call. NetworkManagerConnection: description: The Network Manager Connection resource properties: @@ -25324,6 +25372,80 @@ components: mediaType: application/json openAPIDocKey: '200' objectKey: $.value + # list_all: + # operation: + # $ref: '#/paths/~1subscriptions~1{subscriptionId}~1providers~1Microsoft.Network~1virtualNetworks~1?api-version=2023-11-01/get' + # response: + # mediaType: application/json + # openAPIDocKey: '200' + # objectKey: $.value + list_all_presented: + config: + pagination: + requestToken: + key: '' + location: request + responseToken: + key: $.next_link + location: body + operation: + $ref: '#/paths/~1subscriptions~1{subscriptionId}~1providers~1Microsoft.Network~1virtualNetworks~1?api-version=2023-11-01/get' + response: + mediaType: application/json + openAPIDocKey: '200' + overrideMediaType: application/json + objectKey: $.line_items + schema_override: + $ref: '#/components/schemas/VNetListingsTransformed' + transform: + body: > + { + {{- with index . "nextLink" }} + "next_link": {{printf "%q" .}}, + {{- end }} + "line_items": [ + {{- $items := index . "value" -}} + {{- if eq (printf "%T" $items) "map[string]interface {}" }} + {{template "vnet" $items}} + {{- else }} + {{- range $i, $v := $items }} + {{- if $i}},{{end}} + {{template "vnet" $v}} + {{- end }} + {{- end }} + ] + } + {{define "subnet"}} + { + "id": {{printf "%q" (index . "id")}}, + "name": {{printf "%q" (index . "name")}}, + "address_prefixes": {{with index . "properties" "addressPrefixes"}}{{printf "%q" .}}{{else}}null{{end}}, + "provisioning_state": {{with index . "properties" "provisioningState"}}{{printf "%q" .}}{{else}}null{{end}} + } + {{end}} + {{define "vnet"}} + { + "id": {{printf "%q" (index . "id")}}, + "name": {{printf "%q" (index . "name")}}, + "type": {{with index . "type"}}{{printf "%q" .}}{{else}}null{{end}}, + "location": {{printf "%q" (index . "location")}}, + "address_prefixes": {{with index . "properties" "addressSpace" "addressPrefixes"}}{{printf "%q" .}}{{else}}[]{{end}}, + "subnets": [ + {{- with index . "properties" "subnets" }} + {{- if eq (printf "%T" .) "map[string]interface {}" }} + {{template "subnet" .}} + {{- else if eq (printf "%T" .) "[]interface {}" }} + {{- range $i, $v := . }} + {{- if $i}},{{end}} + {{template "subnet" $v}} + {{- end }} + {{- end }} + {{- end }} + ], + "provisioning_state": {{with index . "properties" "provisioningState"}}{{printf "%q" .}}{{else}}null{{end}} + } + {{end}} + type: 'golang_template_json_v0.1.0' check_ip_address_availability: operation: $ref: '#/paths/~1subscriptions~1{subscriptionId}~1resourceGroups~1{resourceGroupName}~1providers~1Microsoft.Network~1virtualNetworks~1{virtualNetworkName}~1CheckIPAddressAvailability~1?api-version=2023-11-01/get' @@ -25334,6 +25456,7 @@ components: select: - $ref: '#/components/x-stackQL-resources/virtual_networks/methods/get' - $ref: '#/components/x-stackQL-resources/virtual_networks/methods/list' + - $ref: '#/components/x-stackQL-resources/virtual_networks/methods/list_all_presented' insert: - $ref: '#/components/x-stackQL-resources/virtual_networks/methods/create_or_update' update: [] diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index 1da11785..9d0ddbf8 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -7946,3 +7946,194 @@ Select Materialized View of Join Paginated Projection From Transformed XML Respo ... ${stdErrStr} ... stdout=${CURDIR}/tmp/Select-Materialized-View-of-Join-Paginated-Projection-From-Transformed-XML-Response-Body.tmp ... stderr=${CURDIR}/tmp/Select-Materialized-View-of-Join-Paginated-Projection-From-Transformed-XML-Response-Body-stderr.tmp + +Select Paginated Star From Transformed JSON Response Body + [Documentation] Based upon https://learn.microsoft.com/en-us/rest/api/virtualnetwork/virtual-networks/list-all?view=rest-virtualnetwork-2024-05-01&tabs=HTTP + ${inputStr} = Catenate + ... select * from azure.network.virtual_networks where subscriptionId = 'subid' order by name asc; + ${outputStr} = Catenate SEPARATOR=\n + ... |-------------------------------------------------------------------------------------------|--------------------|-------|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}name${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}subnets${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}type${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------------------------------------------------------------------|--------------------|-------|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/subid/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vnet1${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}\[{"address_prefixes":null,"id":"/subscriptions/subid/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-1","name":"test-1","provisioning_state":"Succeeded"}]${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |-------------------------------------------------------------------------------------------|--------------------|-------|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/subid/resourceGroups/rg2/providers/Microsoft.Network/virtualNetworks/vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vnet2${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}\[]${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |-------------------------------------------------------------------------------------------|--------------------|-------|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/subid/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}vnet3${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}\[{"address_prefixes":null,"id":"/subscriptions/subid/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-1","name":"test-1","provisioning_state":"Succeeded"}]${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |-------------------------------------------------------------------------------------------|--------------------|-------|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/subid/resourceGroups/rg2/providers/Microsoft.Network/virtualNetworks/vnet2${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}vnet4${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}\[]${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |-------------------------------------------------------------------------------------------|--------------------|-------|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------| + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${EMPTY} + ... stdout=${CURDIR}/tmp/Select-Paginated-Star-From-Transformed-JSON-Response-Body.tmp + ... stderr=${CURDIR}/tmp/Select-Paginated-Star-From-Transformed-JSON-Response-Body-stderr.tmp + +Select Paginated Projection From Transformed JSON Response Body + [Documentation] Based upon https://learn.microsoft.com/en-us/rest/api/virtualnetwork/virtual-networks/list-all?view=rest-virtualnetwork-2024-05-01&tabs=HTTP + ${inputStr} = Catenate + ... select name, location, provisioning_state from azure.network.virtual_networks where subscriptionId = 'subid' order by name asc; + ${outputStr} = Catenate SEPARATOR=\n + ... |-------|--------------------|--------------------| + ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}| + ... |-------|--------------------|--------------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------| + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${EMPTY} + ... stdout=${CURDIR}/tmp/Select-Paginated-Projection-From-Transformed-JSON-Response-Body.tmp + ... stderr=${CURDIR}/tmp/Select-Paginated-Projection-From-Transformed-JSON-Response-Body-stderr.tmp + +Select Join of Paginated Projection From Transformed JSON and XML Response Bodies + ${inputStr} = Catenate + ... select lhs.name, lhs.location, lhs.provisioning_state, rhs.volume_id, rhs.region + ... from azure.network.virtual_networks lhs inner join aws.ec2.volumes_presented rhs + ... on lhs.location = case when rhs.region = 'us-east-1' then 'westus' when rhs.region = 'ap-southeast-1' then 'australiasoutheast' else '__unknown__' end + ... where lhs.subscriptionId = 'subid' and rhs.region in ('us-east-1', 'ap-southeast-1', 'eu-south-2') + ... order by name asc, volume_id asc; + ${outputStr} = Catenate SEPARATOR=\n + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${EMPTY} + ... stdout=${CURDIR}/tmp/Select-Join-of-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies.tmp + ... stderr=${CURDIR}/tmp/Select-Join-of-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies-stderr.tmp + +Select View of Join of Paginated Projection From Transformed JSON and XML Response Bodies + ${inputStr} = Catenate + ... create or replace materialized view join_v_01 as + ... select lhs.name, lhs.location, lhs.provisioning_state, rhs.volume_id, rhs.region + ... from azure.network.virtual_networks lhs inner join aws.ec2.volumes_presented rhs + ... on lhs.location = case when rhs.region = 'us-east-1' then 'westus' when rhs.region = 'ap-southeast-1' then 'australiasoutheast' else '__unknown__' end + ... where lhs.subscriptionId = 'subid' and rhs.region in ('us-east-1', 'ap-southeast-1', 'eu-south-2') + ... order by name asc, volume_id asc; + ... select name, location, provisioning_state, volume_id, region from join_v_01 order by name asc, volume_id asc; + ${stdErrStr} = Catenate SEPARATOR=\n + ... DDL Execution Completed + ${outputStr} = Catenate SEPARATOR=\n + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${stdErrStr} + ... stdout=${CURDIR}/tmp/Select-View-of-Join-of-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies.tmp + ... stderr=${CURDIR}/tmp/Select-View-of-Join-of-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies-stderr.tmp + +Select Materialized View of Join of Paginated Projection From Transformed JSON and XML Response Bodies + ${inputStr} = Catenate + ... create or replace materialized view join_mv_01 as + ... select lhs.name, lhs.location, lhs.provisioning_state, rhs.volume_id, rhs.region + ... from azure.network.virtual_networks lhs inner join aws.ec2.volumes_presented rhs + ... on lhs.location = case when rhs.region = 'us-east-1' then 'westus' when rhs.region = 'ap-southeast-1' then 'australiasoutheast' else '__unknown__' end + ... where lhs.subscriptionId = 'subid' and rhs.region in ('us-east-1', 'ap-southeast-1', 'eu-south-2') + ... order by name asc, volume_id asc; + ... select name, location, provisioning_state, volume_id, region from join_mv_01 order by name asc, volume_id asc; + ${stdErrStr} = Catenate SEPARATOR=\n + ... DDL Execution Completed + ${outputStr} = Catenate SEPARATOR=\n + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|----------------| + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${stdErrStr} + ... stdout=${CURDIR}/tmp/Select-Materialized-View-of-Join-of-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies.tmp + ... stderr=${CURDIR}/tmp/Select-Materialized-View-of-Join-of-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies-stderr.tmp From 49cb90e715af206294e7e626634cf2d817ae6824 Mon Sep 17 00:00:00 2001 From: Benevolent General Kroll Who cannot spell <82620104+general-kroll-4-life@users.noreply.github.com> Date: Sun, 4 May 2025 20:26:13 +1000 Subject: [PATCH 05/10] flatten-json-results (#548) Summary: - Flatten `json` result sets. - Obviate `json_each` table-valued functions. - Added robot test `Select Paginated Star From Flattened Transformed JSON Response Body`. - Added robot test `Select Paginated Projection From Flattened Transformed JSON Response Body`. - Added robot test `Select Join of Flattened Paginated Projection From Transformed JSON and XML Response Bodies`. - Added robot test `Select View of Join of Flattened Paginated Projection From Transformed JSON and XML Response Bodies`. - Added robot test `Select Materialized View of Join of Flattened Paginated Projection From Transformed JSON and XML Response Bodies`. --- .../flask/azure/expectations.json | 37 ++ ...ual-networks-list-all-02-paginated-01.json | 62 ++++ ...ual-networks-list-all-02-paginated-02.json | 62 ++++ .../src/azure/v0.1.0/services/network.yaml | 130 +++++++ .../stackql_mocked_from_cmd_line.robot | 342 ++++++++++++++++++ 5 files changed, 633 insertions(+) create mode 100644 test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-02-paginated-01.json create mode 100644 test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-02-paginated-02.json diff --git a/test/python/stackql_test_tooling/flask/azure/expectations.json b/test/python/stackql_test_tooling/flask/azure/expectations.json index 3bb35fcb..74c40c89 100644 --- a/test/python/stackql_test_tooling/flask/azure/expectations.json +++ b/test/python/stackql_test_tooling/flask/azure/expectations.json @@ -1,4 +1,41 @@ [ + { + "httpRequest": { + "headers": { + "Accept": [ "application/json" ] + }, + "method": "GET", + "path": "/subscriptions/1111/providers/Microsoft.Network/virtualNetworks/", + "queryStringParameters" : { + "api-version" : [ "2023-11-01" ], + "$skiptoken": [ "0011" ] + } + }, + "httpResponse": { + "template": "virtual-networks-list-all-02-paginated-02.json", + "headers": { + "Content-Type": "application/json; charset=utf-8" + } + } + }, + { + "httpRequest": { + "headers": { + "Accept": [ "application/json" ] + }, + "method": "GET", + "path": "/subscriptions/1111/providers/Microsoft.Network/virtualNetworks/", + "queryStringParameters" : { + "api-version" : [ "2023-11-01" ] + } + }, + "httpResponse": { + "template": "virtual-networks-list-all-02-paginated-01.json", + "headers": { + "Content-Type": "application/json; charset=utf-8" + } + } + }, { "httpRequest": { "headers": { diff --git a/test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-02-paginated-01.json b/test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-02-paginated-01.json new file mode 100644 index 00000000..1209707c --- /dev/null +++ b/test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-02-paginated-01.json @@ -0,0 +1,62 @@ +{ + "value": [ + { + "id": "/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1", + "name": "vnet1", + "type": "Microsoft.Network/virtualNetworks", + "location": "westus", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/8" + ] + }, + "dhcpOptions": { + "dnsServers": [] + }, + "subnets": [ + { + "id": "/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-1", + "name": "test-1", + "properties": { + "addressPrefix": "10.0.0.0/24", + "provisioningState": "Succeeded" + } + }, + { + "id": "/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-2", + "name": "test-2", + "properties": { + "addressPrefix": "10.0.1.0/24", + "provisioningState": "Succeeded" + } + } + ], + "virtualNetworkPeerings": [], + "provisioningState": "Succeeded" + } + }, + { + "id": "/subscriptions/1111/resourceGroups/rg2/providers/Microsoft.Network/virtualNetworks/vnet2", + "name": "vnet2", + "type": "Microsoft.Network/virtualNetworks", + "location": "westus", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/16" + ] + }, + "dhcpOptions": { + "dnsServers": [ + "8.8.8.8" + ] + }, + "subnets": [], + "virtualNetworkPeerings": [], + "provisioningState": "Succeeded" + } + } + ], + "nextLink": "{{ scheme }}://{{ host_name }}{% if port is defined %}:{{ port }}{% endif %}/subscriptions/1111/providers/Microsoft.Network/virtualNetworks/?api-version=2023-11-01&$skiptoken=0011" +} \ No newline at end of file diff --git a/test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-02-paginated-02.json b/test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-02-paginated-02.json new file mode 100644 index 00000000..3ecb32e0 --- /dev/null +++ b/test/python/stackql_test_tooling/flask/azure/templates/virtual-networks-list-all-02-paginated-02.json @@ -0,0 +1,62 @@ +{ + "value": [ + { + "id": "/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1", + "name": "vnet3", + "type": "Microsoft.Network/virtualNetworks", + "location": "australiasoutheast", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/8" + ] + }, + "dhcpOptions": { + "dnsServers": [] + }, + "subnets": [ + { + "id": "/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet3/subnets/test-1", + "name": "test-1", + "properties": { + "addressPrefix": "10.0.0.0/24", + "provisioningState": "Succeeded" + } + }, + + { + "id": "/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet3/subnets/test-2", + "name": "test-2", + "properties": { + "addressPrefix": "10.0.1.0/24", + "provisioningState": "Succeeded" + } + } + ], + "virtualNetworkPeerings": [], + "provisioningState": "Succeeded" + } + }, + { + "id": "/subscriptions/1111/resourceGroups/rg2/providers/Microsoft.Network/virtualNetworks/vnet2", + "name": "vnet4", + "type": "Microsoft.Network/virtualNetworks", + "location": "australiasoutheast", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/16" + ] + }, + "dhcpOptions": { + "dnsServers": [ + "8.8.8.8" + ] + }, + "subnets": [], + "virtualNetworkPeerings": [], + "provisioningState": "Succeeded" + } + } + ] +} \ No newline at end of file diff --git a/test/registry/src/azure/v0.1.0/services/network.yaml b/test/registry/src/azure/v0.1.0/services/network.yaml index b5ac8a93..b704bc26 100644 --- a/test/registry/src/azure/v0.1.0/services/network.yaml +++ b/test/registry/src/azure/v0.1.0/services/network.yaml @@ -197,6 +197,31 @@ components: schema: type: string schemas: + VNetListingFlattened: + type: object + properties: + id: + type: string + description: Resource ID. + name: + type: string + description: Resource name. + type: + type: string + description: Resource type. + location: + type: string + description: Resource location. + provisioning_state: + type: string + subnet_id: + type: string + subnet_name: + type: string + subnet_address_prefix: + type: string + subnet_provisioning_state: + type: string VNetListingTransformed: type: object properties: @@ -245,6 +270,18 @@ components: type: string description: The URL to get the next set of results. description: Response for ListVNet API service call. + VNetListingsFlattened: + type: object + properties: + line_items: + type: array + items: + $ref: '#/components/schemas/VNetListingFlattened' + description: List of virtual networks. + next_link: + type: string + description: The URL to get the next set of results. + description: Response for ListVNet API service call. NetworkManagerConnection: description: The Network Manager Connection resource properties: @@ -25342,6 +25379,99 @@ components: insert: [] update: [] delete: [] + virtual_networks_flattened: + id: azure.network.virtual_networks + name: virtual_networks + title: virtual_networks + methods: + list_all_flattened: + config: + pagination: + requestToken: + key: '' + location: request + responseToken: + key: $.next_link + location: body + operation: + $ref: '#/paths/~1subscriptions~1{subscriptionId}~1providers~1Microsoft.Network~1virtualNetworks~1?api-version=2023-11-01/get' + response: + mediaType: application/json + openAPIDocKey: '200' + overrideMediaType: application/json + objectKey: $.line_items + schema_override: + $ref: '#/components/schemas/VNetListingsFlattened' + transform: + body: > + { + {{- with index . "nextLink" }} + "next_link": {{printf "%q" .}}, + {{- end }} + "line_items": [ + {{- $items := index . "value" -}} + {{- $first := true -}} + {{- if eq (printf "%T" $items) "map[string]interface {}" }} + {{template "rows" $items}} + {{- else }} + {{- range $vnet := $items }} + {{- if not $first}},{{end}} + {{template "rows" $vnet}} + {{- $first = false }} + {{- end }} + {{- end }} + ] + } + {{define "rows"}} + {{- $vnet := . }} + {{- $subnets := index $vnet "properties" "subnets" }} + {{- if eq (printf "%T" $subnets) "map[string]interface {}" }} + {{template "row" $vnet}} + {{- else if eq (printf "%T" $subnets) "[]interface {}" }} + {{- if eq (len $subnets) 0 }} + {{template "row" $vnet}} + {{- else }} + {{- range $i, $s := $subnets }} + {{- if $i}},{{end}} + { + "id": {{printf "%q" (index $vnet "id")}}, + "name": {{printf "%q" (index $vnet "name")}}, + "type": {{with index $vnet "type"}}{{printf "%q" .}}{{else}}null{{end}}, + "location": {{printf "%q" (index $vnet "location")}}, + "address_prefixes": {{with index $vnet "properties" "addressSpace" "addressPrefixes"}}{{printf "%q" .}}{{else}}[]{{end}}, + "provisioning_state": {{with index $vnet "properties" "provisioningState"}}{{printf "%q" .}}{{else}}null{{end}}, + "subnet_id": {{with $s}}{{printf "%q" (index . "id")}}{{else}}null{{end}}, + "subnet_name": {{with $s}}{{printf "%q" (index . "name")}}{{else}}null{{end}}, + "subnet_address_prefix": {{with $s}}{{with index . "properties" "addressPrefix"}}{{printf "%q" .}}{{else}}null{{end}}{{else}}null{{end}}, + "subnet_provisioning_state": {{with $s}}{{with index . "properties" "provisioningState"}}{{printf "%q" .}}{{else}}null{{end}}{{else}}null{{end}} + } + {{- end }} + {{- end }} + {{- else }} + {{template "row" $vnet}} + {{- end }} + {{end}} + {{define "row"}} + { + "id": {{printf "%q" (index . "id")}}, + "name": {{printf "%q" (index . "name")}}, + "type": {{with index . "type"}}{{printf "%q" .}}{{else}}null{{end}}, + "location": {{printf "%q" (index . "location")}}, + "address_prefixes": {{with index . "properties" "addressSpace" "addressPrefixes"}}{{printf "%q" .}}{{else}}[]{{end}}, + "provisioning_state": {{with index . "properties" "provisioningState"}}{{printf "%q" .}}{{else}}null{{end}}, + "subnet_id": null, + "subnet_name": null, + "subnet_address_prefix": null, + "subnet_provisioning_state": null + } + {{end}} + type: 'golang_template_json_v0.1.0' + sqlVerbs: + select: + - $ref: '#/components/x-stackQL-resources/virtual_networks_flattened/methods/list_all_flattened' + insert: [] + update: [] + delete: [] virtual_networks: id: azure.network.virtual_networks name: virtual_networks diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index 9d0ddbf8..a064b87b 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -8137,3 +8137,345 @@ Select Materialized View of Join of Paginated Projection From Transformed JSON a ... ${stdErrStr} ... stdout=${CURDIR}/tmp/Select-Materialized-View-of-Join-of-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies.tmp ... stderr=${CURDIR}/tmp/Select-Materialized-View-of-Join-of-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies-stderr.tmp + +Select Paginated Star From Flattened Transformed JSON Response Body + ${inputStr} = Catenate + ... select * from azure.network.virtual_networks_flattened where subscriptionId = '1111' order by name asc; + ${outputStrSQLite} = Catenate SEPARATOR=\n + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}name${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}subnet_address_prefix${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}subnet_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}subnet_name${SPACE}|${SPACE}subnet_provisioning_state${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}type${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vnet1${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.0.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-1${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vnet1${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.1.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-2${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/1111/resourceGroups/rg2/providers/Microsoft.Network/virtualNetworks/vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vnet2${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}vnet3${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.0.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet3/subnets/test-1${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}vnet3${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.1.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet3/subnets/test-2${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/1111/resourceGroups/rg2/providers/Microsoft.Network/virtualNetworks/vnet2${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}vnet4${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ${outputStrPostgres} = Catenate SEPARATOR=\n + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}name${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}subnet_address_prefix${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}subnet_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}subnet_name${SPACE}|${SPACE}subnet_provisioning_state${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}type${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vnet1${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.0.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-1${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vnet1${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.1.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-2${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/1111/resourceGroups/rg2/providers/Microsoft.Network/virtualNetworks/vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vnet2${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}vnet3${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.0.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet3/subnets/test-1${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}vnet3${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.1.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet3/subnets/test-2${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ... |${SPACE}/subscriptions/1111/resourceGroups/rg2/providers/Microsoft.Network/virtualNetworks/vnet2${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}vnet4${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Microsoft.Network/virtualNetworks${SPACE}| + ... |------------------------------------------------------------------------------------------|--------------------|-------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------|-----------------------------------| + ${outputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${outputStrPostgres} ${outputStrSQLite} + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${EMPTY} + ... stdout=${CURDIR}/tmp/Select-Paginated-Star-From-Flattened-Transformed-JSON-Response-Body.tmp + ... stderr=${CURDIR}/tmp/Select-Paginated-Star-From-Flattened-Transformed-JSON-Response-Body-stderr.tmp + +Select Paginated Projection From Flattened Transformed JSON Response Body + ${inputStr} = Catenate + ... select name, location, provisioning_state, subnet_address_prefix, subnet_id, subnet_name, subnet_provisioning_state from azure.network.virtual_networks_flattened where subscriptionId = '1111' order by name asc; + ${outputStrSQLite} = Catenate SEPARATOR=\n + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}subnet_address_prefix${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}subnet_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}subnet_name${SPACE}|${SPACE}subnet_provisioning_state${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.0.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-1${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.1.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-2${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.0.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet3/subnets/test-1${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.1.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet3/subnets/test-2${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ${outputStrPostgres} = Catenate SEPARATOR=\n + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}subnet_address_prefix${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}subnet_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}subnet_name${SPACE}|${SPACE}subnet_provisioning_state${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.0.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-1${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.1.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-2${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.0.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet3/subnets/test-1${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}10.0.1.0/24${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}/subscriptions/1111/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet3/subnets/test-2${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-----------------------|---------------------------------------------------------------------------------------------------------|-------------|---------------------------| + ${outputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${outputStrPostgres} ${outputStrSQLite} + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${EMPTY} + ... stdout=${CURDIR}/tmp/Select-Paginated-Projection-From-Flattened-Transformed-JSON-Response-Body.tmp + ... stderr=${CURDIR}/tmp/Select-Paginated-Projection-From-Flattened-Transformed-JSON-Response-Body-stderr.tmp + +Select Join of Flattened Paginated Projection From Transformed JSON and XML Response Bodies + ${inputStr} = Catenate + ... select lhs.name, lhs.location, lhs.provisioning_state, lhs.subnet_name, rhs.volume_id, rhs.region + ... from azure.network.virtual_networks_flattened lhs inner join aws.ec2.volumes_presented rhs + ... on lhs.location = case when rhs.region = 'us-east-1' then 'westus' when rhs.region = 'ap-southeast-1' then 'australiasoutheast' else '__unknown__' end + ... where lhs.subscriptionId = '1111' and rhs.region in ('us-east-1', 'ap-southeast-1', 'eu-south-2') + ... order by name asc, subnet_name asc, volume_id asc; + ${outputStrSQLite} = Catenate SEPARATOR=\n + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}subnet_name${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ${outputStrPostgres} = Catenate SEPARATOR=\n + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}subnet_name${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ${outputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${outputStrPostgres} ${outputStrSQLite} + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${EMPTY} + ... stdout=${CURDIR}/tmp/Select-Join-of-Flattened-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies.tmp + ... stderr=${CURDIR}/tmp/Select-Join-of-Flattened-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies-stderr.tmp + +Select View of Join of Flattened Paginated Projection From Transformed JSON and XML Response Bodies + ${inputStr} = Catenate + ... create or replace materialized view join_fv_01 as + ... select lhs.name, lhs.location, lhs.provisioning_state, lhs.subnet_name, rhs.volume_id, rhs.region + ... from azure.network.virtual_networks_flattened lhs inner join aws.ec2.volumes_presented rhs + ... on lhs.location = case when rhs.region = 'us-east-1' then 'westus' when rhs.region = 'ap-southeast-1' then 'australiasoutheast' else '__unknown__' end + ... where lhs.subscriptionId = '1111' and rhs.region in ('us-east-1', 'ap-southeast-1', 'eu-south-2') + ... order by name asc, subnet_name asc, volume_id asc; + ... select name, location, provisioning_state, subnet_name, volume_id, region from join_fv_01 order by name asc, subnet_name asc, volume_id asc; + ${stdErrStr} = Catenate SEPARATOR=\n + ... DDL Execution Completed + ${outputStrSQLite} = Catenate SEPARATOR=\n + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}subnet_name${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ${outputStrPostgres} = Catenate SEPARATOR=\n + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}subnet_name${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ${outputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${outputStrPostgres} ${outputStrSQLite} + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${stdErrStr} + ... stdout=${CURDIR}/tmp/Select-View-of-Join-of-Flattened-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies.tmp + ... stderr=${CURDIR}/tmp/Select-View-of-Join-of-Flattened-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies-stderr.tmp + +Select Materialized View of Join of Flattened Paginated Projection From Transformed JSON and XML Response Bodies + ${inputStr} = Catenate + ... create or replace materialized view join_fmv_01 as + ... select lhs.name, lhs.location, lhs.provisioning_state, lhs.subnet_name, rhs.volume_id, rhs.region + ... from azure.network.virtual_networks_flattened lhs inner join aws.ec2.volumes_presented rhs + ... on lhs.location = case when rhs.region = 'us-east-1' then 'westus' when rhs.region = 'ap-southeast-1' then 'australiasoutheast' else '__unknown__' end + ... where lhs.subscriptionId = '1111' and rhs.region in ('us-east-1', 'ap-southeast-1', 'eu-south-2') + ... order by name asc, subnet_name asc, volume_id asc; + ... select name, location, provisioning_state, subnet_name, volume_id, region from join_fmv_01 order by name asc, subnet_name asc, volume_id asc; + ${stdErrStr} = Catenate SEPARATOR=\n + ... DDL Execution Completed + ${outputStrSQLite} = Catenate SEPARATOR=\n + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}subnet_name${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}null${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ${outputStrPostgres} = Catenate SEPARATOR=\n + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}name${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}location${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}provisioning_state${SPACE}|${SPACE}subnet_name${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}volume_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet1${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet2${SPACE}|${SPACE}westus${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet3${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}test-2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00100000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ... |${SPACE}vnet4${SPACE}|${SPACE}australiasoutheast${SPACE}|${SPACE}Succeeded${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}vol-00200000000000000${SPACE}|${SPACE}ap-southeast-1${SPACE}| + ... |-------|--------------------|--------------------|-------------|-----------------------|----------------| + ${outputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${outputStrPostgres} ${outputStrSQLite} + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${stdErrStr} + ... stdout=${CURDIR}/tmp/Select-Materialized-View-of-Join-of-Flattened-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies.tmp + ... stderr=${CURDIR}/tmp/Select-Materialized-View-of-Join-of-Flattened-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies-stderr.tmp From f06ad23d953971e128c3a97dbfb83c7e6853dbb6 Mon Sep 17 00:00:00 2001 From: Benevolent General Kroll Who cannot spell <82620104+general-kroll-4-life@users.noreply.github.com> Date: Sun, 8 Jun 2025 19:16:08 +1000 Subject: [PATCH 06/10] support-dev-efforts (#549) --- Dockerfile | 2 +- ...GC_cache_concurrency.md => ACID_and_GC.md} | 8 ++++++ .../stackql_test_tooling/flask/README.md | 28 ++++++++++++------- 3 files changed, 27 insertions(+), 11 deletions(-) rename docs/{GC_cache_concurrency.md => ACID_and_GC.md} (96%) diff --git a/Dockerfile b/Dockerfile index 84f0e6c3..5f18321f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -162,7 +162,7 @@ ENV PATH="${APP_DIR}:${PATH}" COPY --from=integration ${TEST_ROOT_DIR}/build/stackql ${APP_DIR}/ RUN apt-get update \ - && apt-get install -y ca-certificates openssl \ + && apt-get install -y ca-certificates openssl netcat-traditional \ && update-ca-certificates EXPOSE ${STACKQL_PG_PORT}/tcp diff --git a/docs/GC_cache_concurrency.md b/docs/ACID_and_GC.md similarity index 96% rename from docs/GC_cache_concurrency.md rename to docs/ACID_and_GC.md index 4d61c088..a61d17f5 100644 --- a/docs/GC_cache_concurrency.md +++ b/docs/ACID_and_GC.md @@ -111,3 +111,11 @@ Please **watch this space** on all items which are **TBD**-inclusive. - List active Txns. - Halt / Allow new Txns. - Cancel Txns (filtered / unfiltered). + +## ACID + + +### When RDBMS ACID fails + +Despite all efforts to enforce ACID, there are times when it can fail and require admin intervention, even for local disk backed RDBMs, [per `postgres` corruption documentation](https://wiki.postgresql.org/wiki/Corruption). + diff --git a/test/python/stackql_test_tooling/flask/README.md b/test/python/stackql_test_tooling/flask/README.md index d1a7e65c..706a0d9b 100644 --- a/test/python/stackql_test_tooling/flask/README.md +++ b/test/python/stackql_test_tooling/flask/README.md @@ -15,64 +15,72 @@ pgrep -f flask | xargs kill -9 ### To Run +In order to get the environmental variables required, you can go to the repository root ans then `source cicd/scripts/context.sh`, or set manually; hopefully self-explanatory. + GCP mocks: ```bash -flask --app=${HOME}/stackql/stackql-devel/test/python/stackql_test_tooling/flask/gcp/app run --cert=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_cert.pem --key=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1080 +flask --app=${REPOSITORY_ROOT}/test/python/stackql_test_tooling/flask/gcp/app run --cert=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_cert.pem --key=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1080 ``` Azure mocks: ```bash -flask --app=${HOME}/stackql/stackql-devel/test/python/stackql_test_tooling/flask/azure/app run --cert=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_cert.pem --key=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1095 +flask --app=${REPOSITORY_ROOT}/test/python/stackql_test_tooling/flask/azure/app run --cert=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_cert.pem --key=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1095 ``` Okta mocks: ```bash -flask --app=${HOME}/stackql/stackql-devel/test/python/stackql_test_tooling/flask/okta/app run --cert=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_cert.pem --key=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1090 +flask --app=${REPOSITORY_ROOT}/test/python/stackql_test_tooling/flask/okta/app run --cert=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_cert.pem --key=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1090 ``` AWS mocks: ```bash -flask --app=${HOME}/stackql/stackql-devel/test/python/stackql_test_tooling/flask/aws/app run --cert=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_cert.pem --key=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1091 +flask --app=${REPOSITORY_ROOT}/test/python/stackql_test_tooling/flask/aws/app run --cert=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_cert.pem --key=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1091 ``` Github mocks: ```bash -flask --app=${HOME}/stackql/stackql-devel/test/python/stackql_test_tooling/flask/github/app run --cert=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_cert.pem --key=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1093 +flask --app=${REPOSITORY_ROOT}/test/python/stackql_test_tooling/flask/github/app run --cert=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_cert.pem --key=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1093 ``` Sumologic mocks: ```bash -flask --app=${HOME}/stackql/stackql-devel/test/python/stackql_test_tooling/flask/okta/app run --cert=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_cert.pem --key=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1096 +flask --app=${REPOSITORY_ROOT}/test/python/stackql_test_tooling/flask/okta/app run --cert=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_cert.pem --key=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1096 ``` Digitalocean mocks: ```bash -flask --app=${HOME}/stackql/stackql-devel/test/python/stackql_test_tooling/flask/digitalocean/app run --cert=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_cert.pem --key=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1097 +flask --app=${REPOSITORY_ROOT}/test/python/stackql_test_tooling/flask/digitalocean/app run --cert=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_cert.pem --key=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1097 ``` `googleadmin` mocks: ```bash -flask --app=${HOME}/stackql/stackql-devel/test/python/stackql_test_tooling/flask/googleadmin/app run --cert=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_cert.pem --key=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1098 +flask --app=${REPOSITORY_ROOT}/test/python/stackql_test_tooling/flask/googleadmin/app run --cert=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_cert.pem --key=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1098 ``` stackql auth testing mocks: ```bash -flask --app=${HOME}/stackql/stackql-devel/test/python/stackql_test_tooling/flask/static_auth/app run --cert=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_cert.pem --key=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1170 +flask --app=${REPOSITORY_ROOT}/test/python/stackql_test_tooling/flask/static_auth/app run --cert=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_cert.pem --key=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1170 ``` Token server mocks: ```bash -flask --app=${HOME}/stackql/stackql-devel/test/python/stackql_test_tooling/flask/oauth2/token_srv run --cert=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_cert.pem --key=${HOME}/stackql/stackql-devel/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 2091 +flask --app=${REPOSITORY_ROOT}/test/python/stackql_test_tooling/flask/oauth2/token_srv run --cert=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_cert.pem --key=${REPOSITORY_ROOT}/test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 2091 +``` + +Registry mocks: + +```bash +flask --app=${REPOSITORY_ROOT}/test/python/stackql_test_tooling/flask/registry/app run --host 0.0.0.0 --port 1094 ``` From 18933eee1b5bc67f02cf284b090b26a65546ecd9 Mon Sep 17 00:00:00 2001 From: Benevolent General Kroll Who cannot spell <82620104+general-kroll-4-life@users.noreply.github.com> Date: Sun, 6 Jul 2025 11:42:13 +1000 Subject: [PATCH 07/10] insert-returning (#552) Summary: - Support for `INSERT RETURNING`. - Documented expected behaviour. - Working for synchronously created objects **only**. - Added robot test `Insert Returning Simple Projection`. --- .vscode/launch.json | 1 + docs/developer_guide.md | 10 +++ go.mod | 2 +- go.sum | 4 +- .../builder_input/builder_input.go | 48 ++++++++----- internal/stackql/parserutil/parser_util.go | 20 ++++++ internal/stackql/planbuilder/plan_builder.go | 34 ++++++++- internal/stackql/primitivebuilder/exec.go | 12 ++-- .../primitivebuilder/insert_or_update.go | 10 +-- .../single_acquire_and_select.go | 28 ++++++-- .../sql_data_source_single_select_acquire.go | 2 - internal/stackql/primitivegenerator/select.go | 13 ++-- .../primitivegenerator/statement_analyzer.go | 72 +++++++++++++++++-- .../primitivegenerator/unary_selection.go | 14 +++- internal/stackql/sql_system/postgres.go | 2 +- internal/stackql/taxonomy/hierarchy.go | 2 + .../show-methods-google-storage-buckets.csv | 6 +- .../stackql_test_tooling/flask/gcp/app.py | 14 ++++ .../buckets-insert-generic.jinja.json | 29 ++++++++ .../v0.1.2/services/storage-v1.yaml | 11 +++ .../stackql_mocked_from_cmd_line.robot | 27 +++++++ 21 files changed, 305 insertions(+), 56 deletions(-) create mode 100644 test/python/stackql_test_tooling/flask/gcp/templates/buckets-insert-generic.jinja.json diff --git a/.vscode/launch.json b/.vscode/launch.json index 733357eb..a2b92513 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -175,6 +175,7 @@ "select BackupId, BackupState from aws.cloudhsm.backups where region = 'rubbish-region' order by BackupId;", "select rings.projectsId as project, rings.locationsId as locale, split_part(rings.name, '/', -1) as key_ring_name, split_part(keys.name, '/', -1) as key_name, json_extract(keys.\"versionTemplate\", '$.algorithm') as key_algorithm, json_extract(keys.\"versionTemplate\", '$.protectionLevel') as key_protection_level from google.cloudkms.key_rings rings inner join google.cloudkms.crypto_keys keys on keys.keyRingsId = split_part(rings.name, '/', -1) and keys.projectsId = rings.projectsId and keys.locationsId = rings.locationsId where rings.projectsId in ('testing-project', 'testing-project-two', 'testing-project-three') and rings.locationsId in ('global', 'australia-southeast1', 'australia-southeast2') order by project, locale, key_name ;", "delete from aws.cloud_control.resources where region = 'ap-southeast-1' and data__TypeName = 'AWS::Logs::LogGroup' and data__Identifier = 'LogGroupResourceExampleThird' ;", + "insert into google.storage.buckets( project, data__name) select 'testing-project', 'silly-bucket' returning projectNumber;", ], "default": "show providers;" }, diff --git a/docs/developer_guide.md b/docs/developer_guide.md index f635d8ef..a792d847 100644 --- a/docs/developer_guide.md +++ b/docs/developer_guide.md @@ -268,3 +268,13 @@ time ./stackql exec --cpuprofile=./select-disks-improved-05.profile --auth='{ "g ## AWS HTTP request signing https://docs.aws.amazon.com/sdk-for-go/api/aws/signer/v4/ + +## Selection from non-select DML + +`INSERT RETURNING` can function in two mechanisms: + +- Synchronous responses, such as [`google.storage.buckets`](https://cloud.google.com/storage/docs/json_api/v1/buckets/insert). The returning clause is a projection on the immediately available reponse body. +- Asynchronous responses, such as [`google.compute.instances`](https://cloud.google.com/compute/docs/reference/rest/v1/instances/insert). The returning clause is a projection on the reponse body **after** the await flow has concluded. + +Future use cases for `UPDATE RETURNING`, `REPLACE RETURNING` and `DELETE RETURNING` will function the same observable fashion. + diff --git a/go.mod b/go.mod index f07b802e..30c1ed87 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/stackql/any-sdk v0.1.3-beta02 github.com/stackql/go-suffix-map v0.0.1-alpha01 github.com/stackql/psql-wire v0.1.1-beta23 - github.com/stackql/stackql-parser v0.0.14-alpha05 + github.com/stackql/stackql-parser v0.0.15-alpha06 github.com/stretchr/testify v1.10.0 golang.org/x/sync v0.10.0 gonum.org/v1/gonum v0.11.0 diff --git a/go.sum b/go.sum index 4c60470e..1642c53e 100644 --- a/go.sum +++ b/go.sum @@ -494,8 +494,8 @@ github.com/stackql/readline v0.0.2-alpha05 h1:ID4QzGdplFBsrSnTuz8pvKzWw96JbrJg8f github.com/stackql/readline v0.0.2-alpha05/go.mod h1:OFAYOdXk/X4+5GYiDXFfaGrk+bCN6Qv0SYY5HNzD2E0= github.com/stackql/stackql-go-sqlite3 v1.0.3-stackql h1:j0yt6T5thZuz5+HIr81PXz2AClAtCko0vzr5tm8C1g8= github.com/stackql/stackql-go-sqlite3 v1.0.3-stackql/go.mod h1:HemqCrcMK2xyhMMMt6oZ7ERDtoSmyyDsw5LBBcTZ+Rk= -github.com/stackql/stackql-parser v0.0.14-alpha05 h1:DLcsaeTypH5p1T+g/9NqaGYdC9uIQ+7pZetnkjHi0G0= -github.com/stackql/stackql-parser v0.0.14-alpha05/go.mod h1:iyB47SvRS+Fvpn7joF7mHAkeiWSq83TbUhglRmLzPLQ= +github.com/stackql/stackql-parser v0.0.15-alpha06 h1:bdaudybbEmrR9m88CO1HpX/eKda2+gjiy4Kf6ZjwPoQ= +github.com/stackql/stackql-parser v0.0.15-alpha06/go.mod h1:iyB47SvRS+Fvpn7joF7mHAkeiWSq83TbUhglRmLzPLQ= github.com/stackql/stackql-provider-registry v0.0.1-rc06 h1:MgroWOr0bSqjSTDGnXB0UoZGFXpW3SRtN0EFkzB8Rpo= github.com/stackql/stackql-provider-registry v0.0.1-rc06/go.mod h1:87rVxnS2aRASK20lBQgoYA0o7FSJTZBGGRaWFR7IDm4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/internal/stackql/internal_data_transfer/builder_input/builder_input.go b/internal/stackql/internal_data_transfer/builder_input/builder_input.go index 34b5695b..4fa6ee51 100644 --- a/internal/stackql/internal_data_transfer/builder_input/builder_input.go +++ b/internal/stackql/internal_data_transfer/builder_input/builder_input.go @@ -9,6 +9,7 @@ import ( "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/primitivegraph" "github.com/stackql/stackql/internal/stackql/provider" + "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" "github.com/stackql/stackql/internal/stackql/tablemetadata" ) @@ -51,27 +52,30 @@ type BuilderInput interface { SetIsTargetPhysicalTable(isPhysical bool) SetTxnCtrlCtrs(internaldto.TxnControlCounters) GetTxnCtrlCtrs() (internaldto.TxnControlCounters, bool) + GetTableInsertionContainer() (tableinsertioncontainer.TableInsertionContainer, bool) + SetTableInsertionContainer(tableinsertioncontainer.TableInsertionContainer) } type builderInput struct { - graphHolder primitivegraph.PrimitiveGraphHolder - handlerCtx handler.HandlerContext - paramMap map[int]map[string]interface{} - tbl tablemetadata.ExtendedTableMetadata - dependencyNode primitivegraph.PrimitiveNode - commentDirectives sqlparser.CommentDirectives - isAwait bool - verb string - inputAlias string - isUndo bool - node sqlparser.SQLNode - paramMapStream streaming.MapStream - httpPrepStream anysdk.HttpPreparatorStream - op anysdk.OperationStore - prov provider.IProvider - annotatedAst annotatedast.AnnotatedAst - isTargetPhysical bool - txnCtrlCtrs internaldto.TxnControlCounters + graphHolder primitivegraph.PrimitiveGraphHolder + handlerCtx handler.HandlerContext + paramMap map[int]map[string]interface{} + tbl tablemetadata.ExtendedTableMetadata + dependencyNode primitivegraph.PrimitiveNode + commentDirectives sqlparser.CommentDirectives + isAwait bool + verb string + inputAlias string + isUndo bool + node sqlparser.SQLNode + paramMapStream streaming.MapStream + httpPrepStream anysdk.HttpPreparatorStream + op anysdk.OperationStore + prov provider.IProvider + annotatedAst annotatedast.AnnotatedAst + isTargetPhysical bool + txnCtrlCtrs internaldto.TxnControlCounters + tableInsertionContainer tableinsertioncontainer.TableInsertionContainer } func NewBuilderInput( @@ -88,6 +92,14 @@ func NewBuilderInput( } } +func (bi *builderInput) GetTableInsertionContainer() (tableinsertioncontainer.TableInsertionContainer, bool) { + return bi.tableInsertionContainer, bi.tableInsertionContainer != nil +} + +func (bi *builderInput) SetTableInsertionContainer(ti tableinsertioncontainer.TableInsertionContainer) { + bi.tableInsertionContainer = ti +} + func (bi *builderInput) SetTxnCtrlCtrs(tcc internaldto.TxnControlCounters) { bi.txnCtrlCtrs = tcc } diff --git a/internal/stackql/parserutil/parser_util.go b/internal/stackql/parserutil/parser_util.go index f16e08cb..838a3267 100644 --- a/internal/stackql/parserutil/parser_util.go +++ b/internal/stackql/parserutil/parser_util.go @@ -88,6 +88,26 @@ func ExtractSelectColumnNames(selStmt *sqlparser.Select, formatter sqlparser.Nod return colNames, err } +func ExtractInsertReturningColumnNames( + insertStmt *sqlparser.Insert, + formatter sqlparser.NodeFormatter, +) ([]ColumnHandle, error) { + var colNames []ColumnHandle + var err error + for _, node := range insertStmt.SelectExprs { + switch node := node.(type) { + case *sqlparser.AliasedExpr: + cn, cErr := inferColNameFromExpr(node.Expr, formatter, node.As.GetRawVal()) + if cErr != nil { + return nil, cErr + } + colNames = append(colNames, cn) + case *sqlparser.StarExpr: + } + } + return colNames, err +} + func ExtractInsertColumnNames(insertStmt *sqlparser.Insert) ([]string, error) { var colNames []string var err error diff --git a/internal/stackql/planbuilder/plan_builder.go b/internal/stackql/planbuilder/plan_builder.go index fc4f4e43..e9dfb79d 100644 --- a/internal/stackql/planbuilder/plan_builder.go +++ b/internal/stackql/planbuilder/plan_builder.go @@ -24,6 +24,7 @@ import ( "github.com/stackql/stackql/internal/stackql/primitivebuilder" "github.com/stackql/stackql/internal/stackql/primitivegenerator" "github.com/stackql/stackql/internal/stackql/primitivegraph" + "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" "github.com/stackql/stackql/internal/stackql/tablemetadata" "github.com/stackql/stackql/internal/stackql/util" @@ -919,9 +920,36 @@ func (pgb *standardPlanGraphBuilder) handleInsert(pbi planbuilderinput.PlanBuild if isPhysicalTable { bldrInput.SetIsTargetPhysicalTable(true) } - bldr := primitivebuilder.NewInsertOrUpdate( - bldrInput, - ) + var bldr primitivebuilder.Builder + if len(node.SelectExprs) > 0 { + // Two cases: + // 1. Synchronous. Equivalent to select. + // 2. Asynchronous. Whole other story. + // Synchronous only for now... + tableMeta, tableMetaExists := bldrInput.GetTableMetadata() + if !tableMetaExists { + return fmt.Errorf("could not obtain table metadata for node '%s'", node.Action) + } + rc, rcErr := tableinsertioncontainer.NewTableInsertionContainer( + tableMeta, + handlerCtx.GetSQLEngine(), + handlerCtx.GetTxnCounterMgr(), + ) + if rcErr != nil { + return rcErr + } + bldrInput.SetTableInsertionContainer(rc) + bldr = primitivebuilder.NewSingleAcquireAndSelect( + bldrInput, + primitiveGenerator.GetPrimitiveComposer().GetInsertPreparedStatementCtx(), + primitiveGenerator.GetPrimitiveComposer().GetSelectPreparedStatementCtx(), + nil, + ) + } else { + bldr = primitivebuilder.NewInsertOrUpdate( + bldrInput, + ) + } err = bldr.Build() if err != nil { return err diff --git a/internal/stackql/primitivebuilder/exec.go b/internal/stackql/primitivebuilder/exec.go index e5adf915..a8698543 100644 --- a/internal/stackql/primitivebuilder/exec.go +++ b/internal/stackql/primitivebuilder/exec.go @@ -6,6 +6,7 @@ import ( "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/execution" "github.com/stackql/stackql/internal/stackql/handler" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/primitive_context" "github.com/stackql/stackql/internal/stackql/primitive" @@ -92,11 +93,14 @@ func (ss *Exec) Build() error { return analysisErr } methodAnalysisOutput.GetInsertTabulation() - deFactoSelectBuilder := NewSingleAcquireAndSelect( + bldrInput := builder_input.NewBuilderInput( ss.graph, - ss.tcc, - ss.handlerCtx.Clone(), - nil, + handlerCtx.Clone(), + tbl, + ) + bldrInput.SetTxnCtrlCtrs(ss.tcc) + deFactoSelectBuilder := NewSingleAcquireAndSelect( + bldrInput, nil, nil, nil, diff --git a/internal/stackql/primitivebuilder/insert_or_update.go b/internal/stackql/primitivebuilder/insert_or_update.go index dc86860b..b90bbd6d 100644 --- a/internal/stackql/primitivebuilder/insert_or_update.go +++ b/internal/stackql/primitivebuilder/insert_or_update.go @@ -8,7 +8,7 @@ import ( "github.com/stackql/stackql/internal/stackql/primitivegraph" ) -type InsertOrUpdate struct { +type insertOrUpdate struct { bldrInput builder_input.BuilderInput root primitivegraph.PrimitiveNode } @@ -16,20 +16,20 @@ type InsertOrUpdate struct { func NewInsertOrUpdate( bldrInput builder_input.BuilderInput, ) Builder { - return &InsertOrUpdate{ + return &insertOrUpdate{ bldrInput: bldrInput, } } -func (ss *InsertOrUpdate) GetRoot() primitivegraph.PrimitiveNode { +func (ss *insertOrUpdate) GetRoot() primitivegraph.PrimitiveNode { return ss.root } -func (ss *InsertOrUpdate) GetTail() primitivegraph.PrimitiveNode { +func (ss *insertOrUpdate) GetTail() primitivegraph.PrimitiveNode { return ss.root } -func (ss *InsertOrUpdate) Build() error { +func (ss *insertOrUpdate) Build() error { node, nodeExists := ss.bldrInput.GetParserNode() if !nodeExists { return fmt.Errorf("mutation executor: node does not exist") diff --git a/internal/stackql/primitivebuilder/single_acquire_and_select.go b/internal/stackql/primitivebuilder/single_acquire_and_select.go index f9836a9e..0935343a 100644 --- a/internal/stackql/primitivebuilder/single_acquire_and_select.go +++ b/internal/stackql/primitivebuilder/single_acquire_and_select.go @@ -3,8 +3,7 @@ package primitivebuilder import ( "github.com/stackql/any-sdk/pkg/streaming" "github.com/stackql/stackql/internal/stackql/drm" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" "github.com/stackql/stackql/internal/stackql/primitivegraph" "github.com/stackql/stackql/internal/stackql/tableinsertioncontainer" ) @@ -13,17 +12,24 @@ type SingleAcquireAndSelect struct { graph primitivegraph.PrimitiveGraphHolder acquireBuilder Builder selectBuilder Builder + bldrInput builder_input.BuilderInput + root primitivegraph.PrimitiveNode } func NewSingleAcquireAndSelect( - graph primitivegraph.PrimitiveGraphHolder, - txnControlCounters internaldto.TxnControlCounters, //nolint:revive // future proofing - handlerCtx handler.HandlerContext, - insertContainer tableinsertioncontainer.TableInsertionContainer, + // graph primitivegraph.PrimitiveGraphHolder, + // txnControlCounters internaldto.TxnControlCounters, //nolint:revive // future proofing + // handlerCtx handler.HandlerContext, + // insertContainer tableinsertioncontainer.TableInsertionContainer, + bldrInput builder_input.BuilderInput, insertCtx drm.PreparedStatementCtx, selectCtx drm.PreparedStatementCtx, rowSort func(map[string]map[string]interface{}) []string, ) Builder { + graph, _ := bldrInput.GetGraphHolder() + // txnControlCounters, _ := bldrInput.GetTxnCtrlCtrs() + handlerCtx, _ := bldrInput.GetHandlerContext() + insertContainer, _ := bldrInput.GetTableInsertionContainer() return &SingleAcquireAndSelect{ graph: graph, acquireBuilder: NewSingleSelectAcquire( @@ -38,6 +44,7 @@ func NewSingleAcquireAndSelect( []tableinsertioncontainer.TableInsertionContainer{insertContainer}, rowSort, streaming.NewNopMapStream()), + bldrInput: bldrInput, } } @@ -60,5 +67,14 @@ func (ss *SingleAcquireAndSelect) Build() error { } graph := ss.graph graph.NewDependency(ss.acquireBuilder.GetTail(), ss.selectBuilder.GetRoot(), 1.0) + rootNode := ss.acquireBuilder.GetRoot() + ss.root = rootNode + dependencyNode, dependencyNodeExists := ss.bldrInput.GetDependencyNode() + if dependencyNodeExists { + //nolint:errcheck // TODO: fix this + rootNode.SetInputAlias("", dependencyNode.ID()) + ss.graph.NewDependency(dependencyNode, rootNode, 1.0) + // ss.root = dependencyNode // dont think this is needed + } return nil } diff --git a/internal/stackql/primitivebuilder/sql_data_source_single_select_acquire.go b/internal/stackql/primitivebuilder/sql_data_source_single_select_acquire.go index f3d843f5..e504bdc6 100644 --- a/internal/stackql/primitivebuilder/sql_data_source_single_select_acquire.go +++ b/internal/stackql/primitivebuilder/sql_data_source_single_select_acquire.go @@ -108,8 +108,6 @@ func (ss *sqlDataSourceSingleSelectAcquire) Build() error { if err != nil { return err } - // targetTableName := annotationCtx.GetHIDs().GetStackQLTableName() - // inputQuery := fmt.Sprintf(`INSERT INTO %s ( %s ) VALUES ( ?, )`, targetTableName, projectionStr, tableName) //nolint:revive // no big deal ex := func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { // ss.tableMeta.GetP diff --git a/internal/stackql/primitivegenerator/select.go b/internal/stackql/primitivegenerator/select.go index 61053c81..ccf73a37 100644 --- a/internal/stackql/primitivegenerator/select.go +++ b/internal/stackql/primitivegenerator/select.go @@ -10,6 +10,7 @@ import ( "github.com/stackql/stackql/internal/stackql/astindirect" "github.com/stackql/stackql/internal/stackql/astvisit" "github.com/stackql/stackql/internal/stackql/dependencyplanner" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" "github.com/stackql/stackql/internal/stackql/parserutil" "github.com/stackql/stackql/internal/stackql/planbuilderinput" "github.com/stackql/stackql/internal/stackql/primitivebuilder" @@ -251,12 +252,16 @@ func (pb *standardPrimitiveGenerator) analyzeSelect(pbi planbuilderinput.PlanBui return indirectErr } annotatedAST.SetSelectIndirect(node, selIndirect) + bldrInput := builder_input.NewBuilderInput( + pChild.GetPrimitiveComposer().GetGraphHolder(), + handlerCtx, + tbl, + ) + bldrInput.SetTxnCtrlCtrs(pChild.GetPrimitiveComposer().GetTxnCtrlCtrs()) + bldrInput.SetTableInsertionContainer(insertionContainer) pChild.GetPrimitiveComposer().SetBuilder( primitivebuilder.NewSingleAcquireAndSelect( - pChild.GetPrimitiveComposer().GetGraphHolder(), - pChild.GetPrimitiveComposer().GetTxnCtrlCtrs(), - handlerCtx, - insertionContainer, + bldrInput, pChild.GetPrimitiveComposer().GetInsertPreparedStatementCtx(), pChild.GetPrimitiveComposer().GetSelectPreparedStatementCtx(), nil)) diff --git a/internal/stackql/primitivegenerator/statement_analyzer.go b/internal/stackql/primitivegenerator/statement_analyzer.go index 8fb8d762..b9e0617d 100644 --- a/internal/stackql/primitivegenerator/statement_analyzer.go +++ b/internal/stackql/primitivegenerator/statement_analyzer.go @@ -704,14 +704,20 @@ func (pb *standardPrimitiveGenerator) analyzeExec(pbi planbuilderinput.PlanBuild return indirectErr } annotatedAST.SetExecIndirect(node, selIndirect) + bldrInput := builder_input.NewBuilderInput( + pb.PrimitiveComposer.GetGraphHolder(), + handlerCtx, + tbl, + ) + bldrInput.SetTxnCtrlCtrs(pb.PrimitiveComposer.GetTxnCtrlCtrs()) + bldrInput.SetTableInsertionContainer(insertionContainer) pb.PrimitiveComposer.SetBuilder( primitivebuilder.NewSingleAcquireAndSelect( - pb.PrimitiveComposer.GetGraphHolder(), - pb.PrimitiveComposer.GetTxnCtrlCtrs(), - handlerCtx, - insertionContainer, + bldrInput, pb.PrimitiveComposer.GetInsertPreparedStatementCtx(), - pb.PrimitiveComposer.GetSelectPreparedStatementCtx(), nil)) + pb.PrimitiveComposer.GetSelectPreparedStatementCtx(), + nil, + )) return nil } @@ -996,7 +1002,7 @@ func (pb *standardPrimitiveGenerator) buildRequestContext( return err } -//nolint:gocognit,funlen // TODO: review +//nolint:gocognit,funlen,gocyclo,cyclop // TODO: review func (pb *standardPrimitiveGenerator) AnalyzeInsert(pbi planbuilderinput.PlanBuilderInput) error { handlerCtx := pbi.GetHandlerCtx() annotatedAST := pbi.GetAnnotatedAST() @@ -1088,11 +1094,65 @@ func (pb *standardPrimitiveGenerator) AnalyzeInsert(pbi planbuilderinput.PlanBui if err != nil { return err } + pb.parseComments(node.Comments) + if tbl.IsPhysicalTable() { + return nil + } + svc, err := tbl.GetService() + if err != nil { + return err + } + _, isOpenapi := svc.(anysdk.OpenAPIService) + if !isOpenapi { + err = pb.buildRequestContext(node, tbl, nil, insertValOnlyRows) + if err != nil { + return err + } + return nil + } + + if pb.PrimitiveComposer.IsAwait() && !method.IsAwaitable() { + return fmt.Errorf("method %s is not awaitable", method.GetName()) + } + if pb.PrimitiveComposer.IsAwait() && !method.IsAwaitable() { + return fmt.Errorf("method %s is not awaitable", method.GetName()) + } + analysisInput := anysdk.NewMethodAnalysisInput( + method, + svc, + true, + []anysdk.ColumnDescriptor{}, + ) + analyser := anysdk.NewMethodAnalyzer() + methodAnalysisOutput, analysisErr := analyser.AnalyzeUnaryAction(analysisInput) + if analysisErr != nil { + return analysisErr + } err = pb.buildRequestContext(node, tbl, nil, insertValOnlyRows) if err != nil { return err } + columnHandles := []parserutil.ColumnHandle{} + if len(node.SelectExprs) > 0 { + columnHandles, err = parserutil.ExtractInsertReturningColumnNames(node, handlerCtx.GetASTFormatter()) + if err != nil { + return err + } + } + err = pb.analyzeUnaryAction( + pbi, + handlerCtx, + node, + nil, + tbl, + columnHandles, + methodAnalysisOutput, + ) + if err != nil { + return err + } + pb.PrimitiveComposer.SetTable(node, tbl) return nil } diff --git a/internal/stackql/primitivegenerator/unary_selection.go b/internal/stackql/primitivegenerator/unary_selection.go index 63751bda..849b4b4e 100644 --- a/internal/stackql/primitivegenerator/unary_selection.go +++ b/internal/stackql/primitivegenerator/unary_selection.go @@ -2,6 +2,7 @@ package primitivegenerator import ( "fmt" + "strings" "github.com/stackql/any-sdk/anysdk" "github.com/stackql/stackql/internal/stackql/astvisit" @@ -206,7 +207,18 @@ func (pb *standardPrimitiveGenerator) analyzeUnaryAction( if itemObjS != nil { itemSchemaName = itemObjS.GetName() } - hIDs := internaldto.NewHeirarchyIdentifiers(rawhIDs.GetProviderStr(), rawhIDs.GetServiceStr(), itemSchemaName, "") + publishedMethodKey := "" + switch node := node.(type) { + case *sqlparser.Insert: + publishedMethodKey = node.Action + case *sqlparser.Update: + publishedMethodKey = node.Action + case *sqlparser.Delete: + publishedMethodKey = "delete" + default: + } + hIDs := internaldto.NewHeirarchyIdentifiers( + rawhIDs.GetProviderStr(), rawhIDs.GetServiceStr(), itemSchemaName, strings.ToLower(publishedMethodKey)) // annotatedInsertTabulation := util.NewAnnotatedTabulation(insertTabulation, hIDs, inputTableName, "") diff --git a/internal/stackql/sql_system/postgres.go b/internal/stackql/sql_system/postgres.go index 276fedc1..8eb35b3e 100644 --- a/internal/stackql/sql_system/postgres.go +++ b/internal/stackql/sql_system/postgres.go @@ -1237,7 +1237,7 @@ func (eng *postgresSystem) generateSelectDML( quotedColNames, ", ", ), - eng.tableCatalog, + eng.tableSchema, tableName, aliasStr, ), diff --git a/internal/stackql/taxonomy/hierarchy.go b/internal/stackql/taxonomy/hierarchy.go index 968f909c..da6ea4f5 100644 --- a/internal/stackql/taxonomy/hierarchy.go +++ b/internal/stackql/taxonomy/hierarchy.go @@ -136,6 +136,8 @@ func GetTableNameFromStatement(node sqlparser.SQLNode, formatter sqlparser.NodeF } case *sqlparser.Exec: return n.MethodName.GetRawVal() + case *sqlparser.Insert: + return n.Table.GetRawVal() case *sqlparser.Delete: if len(n.TableExprs) != 1 { return astformat.String(n, formatter) diff --git a/test/assets/expected/show/show-methods-google-storage-buckets.csv b/test/assets/expected/show/show-methods-google-storage-buckets.csv index 7a4e343a..5466ae26 100644 --- a/test/assets/expected/show/show-methods-google-storage-buckets.csv +++ b/test/assets/expected/show/show-methods-google-storage-buckets.csv @@ -3,9 +3,9 @@ get,bucket,SELECT list,project,SELECT insert,"project, data__name",INSERT delete,bucket,DELETE +patch,bucket,UPDATE +update,"bucket, data__acl",UPDATE getIamPolicy,bucket,EXEC lockRetentionPolicy,"bucket, ifMetagenerationMatch",EXEC -patch,bucket,EXEC setIamPolicy,"bucket, data__bindings",EXEC -testIamPermissions,"bucket, permissions",EXEC -update,"bucket, data__acl",EXEC \ No newline at end of file +testIamPermissions,"bucket, permissions",EXEC \ No newline at end of file diff --git a/test/python/stackql_test_tooling/flask/gcp/app.py b/test/python/stackql_test_tooling/flask/gcp/app.py index 8bf6f32c..708bb8cd 100644 --- a/test/python/stackql_test_tooling/flask/gcp/app.py +++ b/test/python/stackql_test_tooling/flask/gcp/app.py @@ -18,6 +18,20 @@ def v1_storage_buckets_list(): return render_template('buckets-list.json'), 200, {'Content-Type': 'application/json'} return '{"msg": "Project Not Found"}', 404, {'Content-Type': 'application/json'} +@app.route('/storage/v1/b', methods=['POST']) +def v1_storage_buckets_insert(): + # Validate the incoming query + body = request.get_json() + if not body or 'name' not in body: + return '{"msg": "Invalid request body"}', 400, {'Content-Type': 'application/json'} + bucket_name = body['name'] + project_name = request.args.get('project') + if not project_name: + return '{"msg": "Invalid request: project not supplied"}', 400, {'Content-Type': 'application/json'} + if project_name == 'testing-project': + return render_template('buckets-insert-generic.jinja.json', bucket_name=bucket_name), 200, {'Content-Type': 'application/json'} + return '{"msg": "Disallowed"}', 401, {'Content-Type': 'application/json'} + @app.route('/v1/projects/testing-project-three/locations/global/keyRings/testing-three/cryptoKeys', methods=['GET']) def v1_projects_testing_project_three_locations_global_keyRings_testing_three_cryptoKeys(): return render_template('route_1_template.json'), 200, {'Content-Type': 'application/json'} diff --git a/test/python/stackql_test_tooling/flask/gcp/templates/buckets-insert-generic.jinja.json b/test/python/stackql_test_tooling/flask/gcp/templates/buckets-insert-generic.jinja.json new file mode 100644 index 00000000..5572c61c --- /dev/null +++ b/test/python/stackql_test_tooling/flask/gcp/templates/buckets-insert-generic.jinja.json @@ -0,0 +1,29 @@ +{ + "kind": "storage#bucket", + "selfLink": "https://www.googleapis.com/storage/v1/b/{{ bucket_name }}", + "id": "{{ bucket_name }}", + "name": "{{ bucket_name }}", + "projectNumber": "100000000001", + "generation": "1000000000000000001", + "metageneration": "1", + "location": "US", + "storageClass": "STANDARD", + "etag": "CAE=", + "timeCreated": "2025-07-03T00:03:44.250Z", + "updated": "2025-07-03T00:03:44.250Z", + "softDeletePolicy": { + "retentionDurationSeconds": "604800", + "effectiveTime": "2025-07-03T00:03:44.250Z" + }, + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "uniformBucketLevelAccess": { + "enabled": false + }, + "publicAccessPrevention": "inherited" + }, + "locationType": "multi-region", + "rpo": "DEFAULT" +} diff --git a/test/registry/src/googleapis.com/v0.1.2/services/storage-v1.yaml b/test/registry/src/googleapis.com/v0.1.2/services/storage-v1.yaml index ab0388b4..7ae5f9b5 100644 --- a/test/registry/src/googleapis.com/v0.1.2/services/storage-v1.yaml +++ b/test/registry/src/googleapis.com/v0.1.2/services/storage-v1.yaml @@ -1327,6 +1327,17 @@ components: openAPIDocKey: '200' name: buckets title: buckets + sqlVerbs: + select: + - $ref: '#/components/x-stackQL-resources/buckets/methods/get' + - $ref: '#/components/x-stackQL-resources/buckets/methods/list' + insert: + - $ref: '#/components/x-stackQL-resources/buckets/methods/insert' + update: + - $ref: '#/components/x-stackQL-resources/buckets/methods/patch' + - $ref: '#/components/x-stackQL-resources/buckets/methods/update' + delete: + - $ref: '#/components/x-stackQL-resources/buckets/methods/delete' channels: id: google.storage.channels methods: diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index a064b87b..7633f207 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -8479,3 +8479,30 @@ Select Materialized View of Join of Flattened Paginated Projection From Transfor ... ${stdErrStr} ... stdout=${CURDIR}/tmp/Select-Materialized-View-of-Join-of-Flattened-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies.tmp ... stderr=${CURDIR}/tmp/Select-Materialized-View-of-Join-of-Flattened-Paginated-Projection-From-Transformed-JSON-and-XML-Response-Bodies-stderr.tmp + +Insert Returning Simple Projection + [Documentation] Insert a row into a table and return projected new object values. For synchronously created objects. + ${inputStrSQLite} = Catenate + ... insert into google.storage.buckets( project, data__name) select 'testing-project', 'silly-bucket' returning projectNumber, name, location, json_extract(iamConfiguration, '$.publicAccessPrevention') ic; + ${inputStrPostgres} = Catenate + ... insert into google.storage.buckets( project, data__name) select 'testing-project', 'silly-bucket' returning projectNumber, name, location, json_extract_path_text(iamConfiguration, 'publicAccessPrevention') ic; + ${outputStr} = Catenate SEPARATOR=\n + ... |---------------|--------------|----------|-----------| + ... |${SPACE}projectNumber${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}location${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}ic${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |---------------|--------------|----------|-----------| + ... |${SPACE}${SPACE}100000000001${SPACE}|${SPACE}silly-bucket${SPACE}|${SPACE}US${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}inherited${SPACE}| + ... |---------------|--------------|----------|-----------| + ${inputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${inputStrPostgres} ${inputStrSQLite} + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${EMPTY} + ... stdout=${CURDIR}/tmp/Insert-Returning-Simple-Projection.tmp + ... stderr=${CURDIR}/tmp/Insert-Returning-Simple-Projection-stderr.tmp From 6723f79bc364c809b102c28848b86ad39cc2ffd0 Mon Sep 17 00:00:00 2001 From: Jeffrey Aven Date: Wed, 9 Jul 2025 17:39:46 +1000 Subject: [PATCH 08/10] added aws_json_equal sqlite ext function (#553) * added aws_json_equal sqlite ext function * added aws_json_equal sqlite ext function * fixed robot test --- go.mod | 4 +- go.sum | 4 +- .../stackql_mocked_from_cmd_line.robot | 44 +++++++++++++++++++ 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 30c1ed87..7adfcc70 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/jackc/pgx/v5 v5.0.4 github.com/lib/pq v1.10.4 github.com/magiconair/properties v1.8.6 - github.com/mattn/go-sqlite3 v1.0.3-stackql + github.com/mattn/go-sqlite3 v1.0.4-stackql github.com/olekukonko/tablewriter v0.0.0-20180130162743-b8a9be070da4 github.com/sirupsen/logrus v1.9.0 github.com/snowflakedb/gosnowflake v1.6.16 @@ -131,4 +131,4 @@ require ( replace github.com/chzyer/readline => github.com/stackql/readline v0.0.2-alpha05 -replace github.com/mattn/go-sqlite3 => github.com/stackql/stackql-go-sqlite3 v1.0.3-stackql +replace github.com/mattn/go-sqlite3 => github.com/stackql/stackql-go-sqlite3 v1.0.4-stackql diff --git a/go.sum b/go.sum index 1642c53e..01ecdef2 100644 --- a/go.sum +++ b/go.sum @@ -492,8 +492,8 @@ github.com/stackql/psql-wire v0.1.1-beta23 h1:1ayYMjZArfDcIMyEOKnm+Bp1zRCISw8pgu github.com/stackql/psql-wire v0.1.1-beta23/go.mod h1:a44Wd8kDC3irFLpGutarKDBqhJ/aqXlj1aMzO5bVJYg= github.com/stackql/readline v0.0.2-alpha05 h1:ID4QzGdplFBsrSnTuz8pvKzWw96JbrJg8fsLry2UriU= github.com/stackql/readline v0.0.2-alpha05/go.mod h1:OFAYOdXk/X4+5GYiDXFfaGrk+bCN6Qv0SYY5HNzD2E0= -github.com/stackql/stackql-go-sqlite3 v1.0.3-stackql h1:j0yt6T5thZuz5+HIr81PXz2AClAtCko0vzr5tm8C1g8= -github.com/stackql/stackql-go-sqlite3 v1.0.3-stackql/go.mod h1:HemqCrcMK2xyhMMMt6oZ7ERDtoSmyyDsw5LBBcTZ+Rk= +github.com/stackql/stackql-go-sqlite3 v1.0.4-stackql h1:fp70Vdw+PCVEoPrAhkyqPuAlrIiHT79mght/0rlR4oY= +github.com/stackql/stackql-go-sqlite3 v1.0.4-stackql/go.mod h1:HemqCrcMK2xyhMMMt6oZ7ERDtoSmyyDsw5LBBcTZ+Rk= github.com/stackql/stackql-parser v0.0.15-alpha06 h1:bdaudybbEmrR9m88CO1HpX/eKda2+gjiy4Kf6ZjwPoQ= github.com/stackql/stackql-parser v0.0.15-alpha06/go.mod h1:iyB47SvRS+Fvpn7joF7mHAkeiWSq83TbUhglRmLzPLQ= github.com/stackql/stackql-provider-registry v0.0.1-rc06 h1:MgroWOr0bSqjSTDGnXB0UoZGFXpW3SRtN0EFkzB8Rpo= diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index 7633f207..0a22b913 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -6726,6 +6726,50 @@ Run JSON_EQUAL Tests ... stdout=${CURDIR}/tmp/JSON_EQUAL_test_output.tmp ... stderr=${CURDIR}/tmp/JSON_EQUAL_test_stderr.tmp +Run AWS_POLICY_EQUAL Tests + Pass Execution If "${SQL_BACKEND}" == "postgres_tcp" TODO: FIX THIS... Skipping postgres backend test due to unsupported function aws_policy_equal + ${inputStr} = Catenate + ... SELECT + ... aws_policy_equal( + ... '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}', + ... '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}' + ... ) AS identical_policy_match, + ... aws_policy_equal( + ... '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject"],"Resource":"*"}]}', + ... '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:PutObject","s3:GetObject"],"Resource":"*"}]}' + ... ) AS unordered_action_match, + ... aws_policy_equal( + ... '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":"*"}]}', + ... '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}' + ... ) AS array_string_match, + ... aws_policy_equal( + ... '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"arn:aws:iam::123456789012:role/role1"},"Action":"s3:*","Resource":"*"}]}', + ... '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"arn:aws:iam::123456789012:role/role1"},"Action":"s3:*","Resource":"*"}]}' + ... ) AS principal_match, + ... aws_policy_equal( + ... '{"Version":"2012-10-17","Statement":[{"Condition":{"StringEquals":{"sts:ExternalId":"0000"}},"Action":"sts:AssumeRole","Effect":"Allow","Principal":{"AWS":"arn:aws:iam::414351767826:role/role-name"}}]}', + ... '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"arn:aws:iam::414351767826:role/role-name"},"Action":"sts:AssumeRole","Condition":{"StringEquals":{"sts:ExternalId":"0000"}}}]}' + ... ) AS condition_reordering_match; + ${outputStr} = Catenate SEPARATOR=\n + ... |------------------------|------------------------|--------------------|-----------------|----------------------------| + ... |${SPACE}identical_policy_match${SPACE}|${SPACE}unordered_action_match${SPACE}|${SPACE}array_string_match${SPACE}|${SPACE}principal_match${SPACE}|${SPACE}condition_reordering_match${SPACE}| + ... |------------------------|------------------------|--------------------|-----------------|----------------------------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}1${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}1${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}1${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}1${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}1${SPACE}| + ... |------------------------|------------------------|--------------------|-----------------|----------------------------| + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${EMPTY} + ... stdout=${CURDIR}/tmp/AWS_POLICY_EQUAL_test_output.tmp + ... stderr=${CURDIR}/tmp/AWS_POLICY_EQUAL_test_stderr.tmp + Sum on Materialized View as Exemplified By Okta Apps ${sqliteInputStr} = Catenate ... create or replace materialized view okta_apps as From 6e37779a730dcdf52995c181c7027271d062b5fd Mon Sep 17 00:00:00 2001 From: Benevolent General Kroll Who cannot spell <82620104+general-kroll-4-life@users.noreply.github.com> Date: Wed, 9 Jul 2025 18:17:49 +1000 Subject: [PATCH 09/10] insert-returning-asynchronous (#554) Summary: - Support for asynchronous `INSERT RETURNINNG` semantics. - Update docs. - Flask app uplift. - Added robot test `Insert Async Returning Simple Projection`. - Async progress messages now routed to `stderr`, prior case was `stdout`. --- .github/workflows/build.yml | 2 + .vscode/launch.json | 1 + docs/developer_guide.md | 2 +- go.mod | 2 +- go.sum | 4 +- .../async_compose.go | 13 ++- .../asyncmonitor.go | 21 +++- .../dependencyplanner/dependencyplanner.go | 1 + .../dependent_simple_integration_test.go | 4 +- .../stackql/driver/driver_integration_test.go | 4 +- .../execution/mono_valent_execution.go | 96 ++++++++++++++++++- .../builder_input/builder_input.go | 28 ++++++ internal/stackql/planbuilder/plan_builder.go | 49 ++++++++-- internal/stackql/primitivebuilder/delete.go | 4 +- internal/stackql/primitivebuilder/exec.go | 5 +- .../primitivebuilder/generic_http_reversal.go | 15 ++- .../generic_http_stream_input.go | 15 ++- .../primitivebuilder/insert_or_update.go | 7 +- .../primitivebuilder/mono_valent_builder.go | 4 +- .../single_acquire_and_select.go | 4 +- .../primitivebuilder/single_select_acquire.go | 2 + .../primitivegenerator/statement_analyzer.go | 3 +- internal/test/stackqltestutil/helper.go | 22 +++++ stackql/main_integration_test.go | 8 +- .../stackql_test_tooling/flask/gcp/app.py | 67 ++++++++++++- .../gcp/templates/global-operation.jinja.json | 15 +++ .../networks-insert-generic-mature.jinja.json | 14 +++ .../v0.1.2/resources/compute-v1.yaml | 3 + .../stackql_mocked_from_cmd_line.robot | 27 ++++++ 29 files changed, 399 insertions(+), 43 deletions(-) rename internal/stackql/{primitivebuilder => asynccompose}/async_compose.go (71%) rename internal/stackql/{primitivebuilder => asynccompose}/asyncmonitor.go (91%) create mode 100644 test/python/stackql_test_tooling/flask/gcp/templates/global-operation.jinja.json create mode 100644 test/python/stackql_test_tooling/flask/gcp/templates/networks-insert-generic-mature.jinja.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b8922540..03a57bf0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1461,6 +1461,8 @@ jobs: dockertest: + env: + IS_DOCKER: 'true' name: Docker Test needs: - dockerbuild diff --git a/.vscode/launch.json b/.vscode/launch.json index a2b92513..e81b7264 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -176,6 +176,7 @@ "select rings.projectsId as project, rings.locationsId as locale, split_part(rings.name, '/', -1) as key_ring_name, split_part(keys.name, '/', -1) as key_name, json_extract(keys.\"versionTemplate\", '$.algorithm') as key_algorithm, json_extract(keys.\"versionTemplate\", '$.protectionLevel') as key_protection_level from google.cloudkms.key_rings rings inner join google.cloudkms.crypto_keys keys on keys.keyRingsId = split_part(rings.name, '/', -1) and keys.projectsId = rings.projectsId and keys.locationsId = rings.locationsId where rings.projectsId in ('testing-project', 'testing-project-two', 'testing-project-three') and rings.locationsId in ('global', 'australia-southeast1', 'australia-southeast2') order by project, locale, key_name ;", "delete from aws.cloud_control.resources where region = 'ap-southeast-1' and data__TypeName = 'AWS::Logs::LogGroup' and data__Identifier = 'LogGroupResourceExampleThird' ;", "insert into google.storage.buckets( project, data__name) select 'testing-project', 'silly-bucket' returning projectNumber;", + "insert /*+ AWAIT */ into google.compute.networks(project, data__name, data__autoCreateSubnetworks) select 'mutable-project', 'auto-test-01', false returning creationTimestamp, name;", ], "default": "show providers;" }, diff --git a/docs/developer_guide.md b/docs/developer_guide.md index a792d847..90f3b5ed 100644 --- a/docs/developer_guide.md +++ b/docs/developer_guide.md @@ -274,7 +274,7 @@ https://docs.aws.amazon.com/sdk-for-go/api/aws/signer/v4/ `INSERT RETURNING` can function in two mechanisms: - Synchronous responses, such as [`google.storage.buckets`](https://cloud.google.com/storage/docs/json_api/v1/buckets/insert). The returning clause is a projection on the immediately available reponse body. -- Asynchronous responses, such as [`google.compute.instances`](https://cloud.google.com/compute/docs/reference/rest/v1/instances/insert). The returning clause is a projection on the reponse body **after** the await flow has concluded. +- Asynchronous responses, such as [`google.compute.instances`](https://cloud.google.com/compute/docs/reference/rest/v1/instances/insert) and [`google.compute.networks`](https://cloud.google.com/compute/docs/reference/rest/v1/networks/insert). The returning clause is a projection on the reponse body **after** the await flow has concluded. Future use cases for `UPDATE RETURNING`, `REPLACE RETURNING` and `DELETE RETURNING` will function the same observable fashion. diff --git a/go.mod b/go.mod index 7adfcc70..45192424 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.10.1 - github.com/stackql/any-sdk v0.1.3-beta02 + github.com/stackql/any-sdk v0.1.4-alpha06 github.com/stackql/go-suffix-map v0.0.1-alpha01 github.com/stackql/psql-wire v0.1.1-beta23 github.com/stackql/stackql-parser v0.0.15-alpha06 diff --git a/go.sum b/go.sum index 01ecdef2..4902a5d8 100644 --- a/go.sum +++ b/go.sum @@ -484,8 +484,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= -github.com/stackql/any-sdk v0.1.3-beta02 h1:0jSwyYFddjAN++U+yNNLF7SMLPIIaRhv522UzeWDf2E= -github.com/stackql/any-sdk v0.1.3-beta02/go.mod h1:AKS/g28y7m4SWL/YW8veE9MCNy8XJgaicVibemVE9e8= +github.com/stackql/any-sdk v0.1.4-alpha06 h1:QJPf3ehPrRqmYZR+TmD897AsmsaOamHErLhaE5B9v/w= +github.com/stackql/any-sdk v0.1.4-alpha06/go.mod h1:AKS/g28y7m4SWL/YW8veE9MCNy8XJgaicVibemVE9e8= github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8= github.com/stackql/go-suffix-map v0.0.1-alpha01/go.mod h1:QAi+SKukOyf4dBtWy8UMy+hsXXV+yyEE4vmBkji2V7g= github.com/stackql/psql-wire v0.1.1-beta23 h1:1ayYMjZArfDcIMyEOKnm+Bp1zRCISw8pguvTFuUhhVQ= diff --git a/internal/stackql/primitivebuilder/async_compose.go b/internal/stackql/asynccompose/async_compose.go similarity index 71% rename from internal/stackql/primitivebuilder/async_compose.go rename to internal/stackql/asynccompose/async_compose.go index 3f0c0017..4cd5adbb 100644 --- a/internal/stackql/primitivebuilder/async_compose.go +++ b/internal/stackql/asynccompose/async_compose.go @@ -1,22 +1,26 @@ -package primitivebuilder +package asynccompose import ( "github.com/stackql/any-sdk/anysdk" "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/primitive" "github.com/stackql/stackql/internal/stackql/provider" ) -func composeAsyncMonitor( +func ComposeAsyncMonitor( handlerCtx handler.HandlerContext, precursor primitive.IPrimitive, prov provider.IProvider, method anysdk.OperationStore, commentDirectives sqlparser.CommentDirectives, + isReturning bool, + insertCtx drm.PreparedStatementCtx, + drmCfg drm.Config, ) (primitive.IPrimitive, error) { - asm, err := NewAsyncMonitor(handlerCtx, prov, method) + asm, err := NewAsyncMonitor(handlerCtx, prov, method, isReturning) if err != nil { return nil, err } @@ -31,7 +35,8 @@ func composeAsyncMonitor( handlerCtx.GetOutfile(), handlerCtx.GetOutErrFile(), ) - primitive, err := asm.GetMonitorPrimitive(prov, method, precursor, pl, commentDirectives) + primitive, err := asm.GetMonitorPrimitive( + prov, method, precursor, pl, commentDirectives, isReturning, insertCtx, drmCfg) if err != nil { return nil, err } diff --git a/internal/stackql/primitivebuilder/asyncmonitor.go b/internal/stackql/asynccompose/asyncmonitor.go similarity index 91% rename from internal/stackql/primitivebuilder/asyncmonitor.go rename to internal/stackql/asynccompose/asyncmonitor.go index 4752b53a..85ae73cc 100644 --- a/internal/stackql/primitivebuilder/asyncmonitor.go +++ b/internal/stackql/asynccompose/asyncmonitor.go @@ -1,4 +1,4 @@ -package primitivebuilder +package asynccompose import ( "fmt" @@ -6,6 +6,7 @@ import ( "github.com/stackql/any-sdk/anysdk" "github.com/stackql/stackql/internal/stackql/acid/binlog" + "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/execution" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" @@ -22,6 +23,9 @@ type IAsyncMonitor interface { precursor primitive.IPrimitive, initialCtx primitive.IPrimitiveCtx, comments sqlparser.CommentDirectives, + isReturning bool, + insertCtx drm.PreparedStatementCtx, + drmCfg drm.Config, ) (primitive.IPrimitive, error) } @@ -120,11 +124,12 @@ func NewAsyncMonitor( handlerCtx handler.HandlerContext, prov provider.IProvider, op anysdk.OperationStore, + isReturning bool, ) (IAsyncMonitor, error) { //nolint:gocritic //TODO: refactor switch prov.GetProviderString() { case "google": - return newGoogleAsyncMonitor(handlerCtx, prov, op, prov.GetVersion()) + return newGoogleAsyncMonitor(handlerCtx, prov, op, prov.GetVersion(), isReturning) } return nil, fmt.Errorf( "async operation monitor for provider = '%s', api version = '%s' currently not supported", @@ -136,6 +141,7 @@ func newGoogleAsyncMonitor( prov provider.IProvider, op anysdk.OperationStore, version string, //nolint:unparam // TODO: refactor + isReturning bool, //nolint:unparam,revive // TODO: refactor ) (IAsyncMonitor, error) { //nolint:gocritic //TODO: refactor switch version { @@ -160,11 +166,14 @@ func (gm *DefaultGoogleAsyncMonitor) GetMonitorPrimitive( precursor primitive.IPrimitive, initialCtx primitive.IPrimitiveCtx, comments sqlparser.CommentDirectives, + isReturning bool, + insertCtx drm.PreparedStatementCtx, + drmCfg drm.Config, ) (primitive.IPrimitive, error) { //nolint:gocritic,staticcheck //TODO: refactor switch strings.ToLower(prov.GetVersion()) { default: - return gm.getV1Monitor(prov, op, precursor, initialCtx, comments) + return gm.getV1Monitor(prov, op, precursor, initialCtx, comments, isReturning, insertCtx, drmCfg) } } @@ -174,6 +183,9 @@ func (gm *DefaultGoogleAsyncMonitor) getV1Monitor( precursor primitive.IPrimitive, initialCtx primitive.IPrimitiveCtx, comments sqlparser.CommentDirectives, + isReturning bool, + insertCtx drm.PreparedStatementCtx, + drmCfg drm.Config, ) (primitive.IPrimitive, error) { provider, providerErr := prov.GetProvider() if providerErr != nil { @@ -186,6 +198,9 @@ func (gm *DefaultGoogleAsyncMonitor) getV1Monitor( precursor, initialCtx, comments, + isReturning, + insertCtx, + drmCfg, ) if exPrepErr != nil { return nil, exPrepErr diff --git a/internal/stackql/dependencyplanner/dependencyplanner.go b/internal/stackql/dependencyplanner/dependencyplanner.go index 46e1b114..560d2713 100644 --- a/internal/stackql/dependencyplanner/dependencyplanner.go +++ b/internal/stackql/dependencyplanner/dependencyplanner.go @@ -485,6 +485,7 @@ func (dp *standardDependencyPlanner) orchestrate( insPsc, nil, outStream, + false, // returning hardcoded to false for now ) } dp.execSlice = append(dp.execSlice, builder) diff --git a/internal/stackql/driver/dependent_simple_integration_test.go b/internal/stackql/driver/dependent_simple_integration_test.go index 1b9c4261..3a44d081 100644 --- a/internal/stackql/driver/dependent_simple_integration_test.go +++ b/internal/stackql/driver/dependent_simple_integration_test.go @@ -38,7 +38,7 @@ func TestSimpleInsertDependentGoogleComputeDiskAsync(t *testing.T) { if err != nil { t.Fatalf("Test failed: %v", err) } - handlerCtx, err := entryutil.BuildHandlerContext(*runtimeCtx, rdr, lrucache.NewLRUCache(int64(runtimeCtx.QueryCacheSize)), inputBundle.WithStdOut(outFile), true) + handlerCtx, err := entryutil.BuildHandlerContext(*runtimeCtx, rdr, lrucache.NewLRUCache(int64(runtimeCtx.QueryCacheSize)), inputBundle.WithStdErr(outFile), true) if err != nil { t.Fatalf("Test failed: %v", err) } @@ -76,7 +76,7 @@ func TestSimpleInsertDependentGoogleComputeDiskAsyncReversed(t *testing.T) { if err != nil { t.Fatalf("Test failed: %v", err) } - handlerCtx, err := entryutil.BuildHandlerContext(*runtimeCtx, rdr, lrucache.NewLRUCache(int64(runtimeCtx.QueryCacheSize)), inputBundle.WithStdOut(outFile), true) + handlerCtx, err := entryutil.BuildHandlerContext(*runtimeCtx, rdr, lrucache.NewLRUCache(int64(runtimeCtx.QueryCacheSize)), inputBundle.WithStdErr(outFile), true) if err != nil { t.Fatalf("Test failed: %v", err) } diff --git a/internal/stackql/driver/driver_integration_test.go b/internal/stackql/driver/driver_integration_test.go index 636fdfb8..355e3ffd 100644 --- a/internal/stackql/driver/driver_integration_test.go +++ b/internal/stackql/driver/driver_integration_test.go @@ -180,7 +180,7 @@ func TestSimpleInsertGoogleComputeNetworkAsync(t *testing.T) { } testSubject := func(t *testing.T, outFile *bufio.Writer) { - handlerCtx, err := entryutil.BuildHandlerContext(*runtimeCtx, strings.NewReader(""), lrucache.NewLRUCache(int64(runtimeCtx.QueryCacheSize)), inputBundle.WithStdOut(outFile), true) + handlerCtx, err := entryutil.BuildHandlerContext(*runtimeCtx, strings.NewReader(""), lrucache.NewLRUCache(int64(runtimeCtx.QueryCacheSize)), inputBundle.WithStdErr(outFile), true) if err != nil { t.Fatalf("Test failed: %v", err) } @@ -226,7 +226,7 @@ func TestK8sTheHardWayAsync(t *testing.T) { runtimeCtx.InfilePath = k8sthwRenderedFile runtimeCtx.CSVHeadersDisable = true - handlerCtx, err := entryutil.BuildHandlerContext(*runtimeCtx, strings.NewReader(""), lrucache.NewLRUCache(int64(runtimeCtx.QueryCacheSize)), inputBundle.WithStdOut(outFile), true) + handlerCtx, err := entryutil.BuildHandlerContext(*runtimeCtx, strings.NewReader(""), lrucache.NewLRUCache(int64(runtimeCtx.QueryCacheSize)), inputBundle.WithStdErr(outFile), true) if err != nil { t.Fatalf("Test failed: %v", err) } diff --git a/internal/stackql/execution/mono_valent_execution.go b/internal/stackql/execution/mono_valent_execution.go index 620096e5..fb3a0798 100644 --- a/internal/stackql/execution/mono_valent_execution.go +++ b/internal/stackql/execution/mono_valent_execution.go @@ -1130,6 +1130,7 @@ func (sp *standardProcessor) Process() ProcessorResponse { if httpResponseErr != nil { return newHTTPProcessorResponse(nil, reversalStream, false, httpResponseErr) } + // TODO: add async monitor here processed, resErr := method.ProcessResponse(httpResponse) if resErr != nil { if isSkipResponse && isMutation && httpResponse.StatusCode < 300 { @@ -1444,6 +1445,32 @@ func (mv *monoValentExecution) GetExecutor() (func(pc primitive.IPrimitiveCtx) i return ex, nil } +func shimProcessHTTP( + url string, + rtCtx dto.RuntimeCtx, + authCtx *dto.AuthCtx, + provider anysdk.Provider, + m anysdk.OperationStore, + outErrFile io.Writer, +) (*http.Response, error) { + req, monitorReqErr := anysdk.GetMonitorRequest(url) + if monitorReqErr != nil { + return nil, monitorReqErr + } + cc := anysdk.NewAnySdkClientConfigurator(rtCtx, provider.GetName()) + anySdkResponse, apiErr := anysdk.CallFromSignature( + cc, rtCtx, authCtx, authCtx.Type, false, outErrFile, provider, anysdk.NewAnySdkOpStoreDesignation(m), req) + + if apiErr != nil { + return nil, apiErr + } + httpResponse, httpResponseErr := anySdkResponse.GetHttpResponse() + if httpResponseErr != nil { + return nil, httpResponseErr + } + return httpResponse, nil +} + //nolint:funlen,gocognit // acceptable for now func GetMonitorExecutor( handlerCtx handler.HandlerContext, @@ -1452,6 +1479,9 @@ func GetMonitorExecutor( precursor primitive.IPrimitive, initialCtx primitive.IPrimitiveCtx, comments sqlparser.CommentDirectives, + isReturning bool, + insertCtx drm.PreparedStatementCtx, + drmCfg drm.Config, ) (primitive.IPrimitive, error) { m := op // tableName, err := mv.tableMeta.GetTableName() @@ -1471,6 +1501,8 @@ func GetMonitorExecutor( elapsedSeconds: 0, pollIntervalSeconds: MonitorPollIntervalSeconds, comments: comments, + insertCtx: insertCtx, + drmCfg: drmCfg, } if comments != nil { asyncPrim.noStatus = comments.IsSet("NOSTATUS") @@ -1498,7 +1530,64 @@ func GetMonitorExecutor( operationDescriptor := getOpDescriptor(body) endTime, endTimeOk := body["endTime"] + prStr := provider.GetName() + //nolint:nestif // acceptable for now if endTimeOk && endTime != "" { + targetLink, targetLinkOK := body["targetLink"] + if targetLinkOK && isReturning { + authCtx, authErr := pc.GetAuthContext(prStr) + if authErr != nil { + return internaldto.NewExecutorOutput(nil, nil, nil, nil, authErr) + } + if authCtx == nil { + return internaldto.NewExecutorOutput(nil, nil, nil, nil, fmt.Errorf("cannot execute monitor: no auth context")) + } + targetLinkStr, targetLinkStrOk := targetLink.(string) + if !targetLinkStrOk { + return internaldto.NewExecutorOutput( + nil, + nil, + nil, + nil, + fmt.Errorf("cannot execute monitor: 'targetLink' is not a string"), + ) + } + httpResponse, httpResponseErr := shimProcessHTTP( + targetLinkStr, + rtCtx, + authCtx, + provider, + m, + outErrFile, + ) + if httpResponseErr != nil { + return internaldto.NewExecutorOutput(nil, nil, nil, nil, httpResponseErr) + } + + if httpResponse != nil && httpResponse.Body != nil { + defer httpResponse.Body.Close() + } + target, targetErr := m.DeprecatedProcessResponse(httpResponse) + handlerCtx.LogHTTPResponseMap(target) + if targetErr != nil { + return internaldto.NewExecutorOutput(nil, nil, nil, nil, targetErr) + } + // TODO: insert into table here + if isReturning { + if asyncPrim.insertCtx != nil { + _, rErr := asyncPrim.drmCfg.ExecuteInsertDML( + handlerCtx.GetSQLEngine(), + asyncPrim.insertCtx, + target, + "", // TODO: figure out how on earth to compute this encoding + ) + if rErr != nil { + return internaldto.NewExecutorOutput(nil, nil, nil, nil, rErr) + } + } + } + return prepareResultSet(&asyncPrim, pc, target, operationDescriptor) + } return prepareResultSet(&asyncPrim, pc, body, operationDescriptor) } url, ok := body["selfLink"] @@ -1511,7 +1600,6 @@ func GetMonitorExecutor( fmt.Errorf("cannot execute monitor: no 'selfLink' property present"), ) } - prStr := provider.GetName() authCtx, authErr := pc.GetAuthContext(prStr) if authErr != nil { return internaldto.NewExecutorOutput(nil, nil, nil, nil, authErr) @@ -1523,7 +1611,7 @@ func GetMonitorExecutor( asyncPrim.elapsedSeconds += asyncPrim.pollIntervalSeconds if !asyncPrim.noStatus { //nolint:errcheck //TODO: handle error - pc.GetWriter().Write( + pc.GetErrWriter().Write( []byte( fmt.Sprintf( "%s in progress, %d seconds elapsed", @@ -1649,6 +1737,8 @@ type asyncHTTPMonitorPrimitive struct { noStatus bool id int64 comments sqlparser.CommentDirectives + insertCtx drm.PreparedStatementCtx + drmCfg drm.Config } func (pr *asyncHTTPMonitorPrimitive) SetTxnID(_ int) { @@ -1762,7 +1852,7 @@ func prepareResultSet( } if !prim.noStatus { //nolint:errcheck //TODO: handle error - pc.GetWriter().Write([]byte(fmt.Sprintf("%s complete", operationDescriptor) + fmt.Sprintln(""))) + pc.GetErrWriter().Write([]byte(fmt.Sprintf("%s complete", operationDescriptor) + fmt.Sprintln(""))) } return util.PrepareResultSet(payload) } diff --git a/internal/stackql/internal_data_transfer/builder_input/builder_input.go b/internal/stackql/internal_data_transfer/builder_input/builder_input.go index 4fa6ee51..145cb9af 100644 --- a/internal/stackql/internal_data_transfer/builder_input/builder_input.go +++ b/internal/stackql/internal_data_transfer/builder_input/builder_input.go @@ -5,6 +5,7 @@ import ( "github.com/stackql/any-sdk/pkg/streaming" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast" + "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/primitivegraph" @@ -31,6 +32,8 @@ type BuilderInput interface { GetOperationStore() (anysdk.OperationStore, bool) SetOperationStore(op anysdk.OperationStore) IsAwait() bool + IsReturning() bool + SetIsReturning(bool) GetVerb() string GetInputAlias() string IsUndo() bool @@ -54,6 +57,8 @@ type BuilderInput interface { GetTxnCtrlCtrs() (internaldto.TxnControlCounters, bool) GetTableInsertionContainer() (tableinsertioncontainer.TableInsertionContainer, bool) SetTableInsertionContainer(tableinsertioncontainer.TableInsertionContainer) + SetInsertCtx(insertCtx drm.PreparedStatementCtx) + GetInsertCtx() (drm.PreparedStatementCtx, bool) } type builderInput struct { @@ -64,6 +69,7 @@ type builderInput struct { dependencyNode primitivegraph.PrimitiveNode commentDirectives sqlparser.CommentDirectives isAwait bool + isReturning bool verb string inputAlias string isUndo bool @@ -76,6 +82,7 @@ type builderInput struct { isTargetPhysical bool txnCtrlCtrs internaldto.TxnControlCounters tableInsertionContainer tableinsertioncontainer.TableInsertionContainer + insertCtx drm.PreparedStatementCtx } func NewBuilderInput( @@ -92,6 +99,25 @@ func NewBuilderInput( } } +func (bi *builderInput) SetInsertCtx(insertCtx drm.PreparedStatementCtx) { + bi.insertCtx = insertCtx +} + +func (bi *builderInput) GetInsertCtx() (drm.PreparedStatementCtx, bool) { + if bi.insertCtx == nil { + return nil, false + } + return bi.insertCtx, true +} + +func (bi *builderInput) IsReturning() bool { + return bi.isReturning +} + +func (bi *builderInput) SetIsReturning(isReturning bool) { + bi.isReturning = isReturning +} + func (bi *builderInput) GetTableInsertionContainer() (tableinsertioncontainer.TableInsertionContainer, bool) { return bi.tableInsertionContainer, bi.tableInsertionContainer != nil } @@ -260,5 +286,7 @@ func (bi *builderInput) Clone() BuilderInput { isTargetPhysical: bi.isTargetPhysical, annotatedAst: bi.annotatedAst, txnCtrlCtrs: bi.txnCtrlCtrs, + isReturning: bi.isReturning, + insertCtx: bi.insertCtx, } } diff --git a/internal/stackql/planbuilder/plan_builder.go b/internal/stackql/planbuilder/plan_builder.go index e9dfb79d..9ca38730 100644 --- a/internal/stackql/planbuilder/plan_builder.go +++ b/internal/stackql/planbuilder/plan_builder.go @@ -10,6 +10,7 @@ import ( "github.com/stackql/any-sdk/anysdk" "github.com/stackql/any-sdk/pkg/logging" + "github.com/stackql/any-sdk/pkg/streaming" "github.com/stackql/stackql/internal/stackql/acid/txn_context" "github.com/stackql/stackql/internal/stackql/astanalysis/routeanalysis" "github.com/stackql/stackql/internal/stackql/handler" @@ -912,7 +913,8 @@ func (pgb *standardPlanGraphBuilder) handleInsert(pbi planbuilderinput.PlanBuild ) bldrInput.SetDependencyNode(selectPrimitiveNode) bldrInput.SetCommentDirectives(primitiveGenerator.GetPrimitiveComposer().GetCommentDirectives()) - bldrInput.SetIsAwait(primitiveGenerator.GetPrimitiveComposer().IsAwait()) + isAwait := primitiveGenerator.GetPrimitiveComposer().IsAwait() + bldrInput.SetIsAwait(isAwait) bldrInput.SetParserNode(node) bldrInput.SetAnnotatedAST(pbi.GetAnnotatedAST()) bldrInput.SetTxnCtrlCtrs(pbi.GetTxnCtrlCtrs()) @@ -925,7 +927,6 @@ func (pgb *standardPlanGraphBuilder) handleInsert(pbi planbuilderinput.PlanBuild // Two cases: // 1. Synchronous. Equivalent to select. // 2. Asynchronous. Whole other story. - // Synchronous only for now... tableMeta, tableMetaExists := bldrInput.GetTableMetadata() if !tableMetaExists { return fmt.Errorf("could not obtain table metadata for node '%s'", node.Action) @@ -939,12 +940,44 @@ func (pgb *standardPlanGraphBuilder) handleInsert(pbi planbuilderinput.PlanBuild return rcErr } bldrInput.SetTableInsertionContainer(rc) - bldr = primitivebuilder.NewSingleAcquireAndSelect( - bldrInput, - primitiveGenerator.GetPrimitiveComposer().GetInsertPreparedStatementCtx(), - primitiveGenerator.GetPrimitiveComposer().GetSelectPreparedStatementCtx(), - nil, - ) + bldrInput.SetIsReturning(true) + if !isAwait { + bldr = primitivebuilder.NewSingleAcquireAndSelect( + bldrInput, + primitiveGenerator.GetPrimitiveComposer().GetInsertPreparedStatementCtx(), + primitiveGenerator.GetPrimitiveComposer().GetSelectPreparedStatementCtx(), + nil, + ) + } else { + bldrInput.SetIsAwait(true) + bldrInput.SetIsReturning(true) + bldrInput.SetInsertCtx(primitiveGenerator.GetPrimitiveComposer().GetInsertPreparedStatementCtx()) + lhsBldr := primitivebuilder.NewInsertOrUpdate( + bldrInput, + ) + newBldrInput := builder_input.NewBuilderInput( + pgb.planGraphHolder, + handlerCtx, + tbl, + ) + newBldrInput.SetParserNode(node) + newBldrInput.SetAnnotatedAST(pbi.GetAnnotatedAST()) + newBldrInput.SetTxnCtrlCtrs(pbi.GetTxnCtrlCtrs()) + newBldrInput.SetTableInsertionContainer(rc) + newBldrInput.SetDependencyNode(selectPrimitiveNode) + newBldrInput.SetIsAwait(isAwait) + rhsBldr := primitivebuilder.NewSingleSelect( + pgb.planGraphHolder, handlerCtx, primitiveGenerator.GetPrimitiveComposer().GetSelectPreparedStatementCtx(), + []tableinsertioncontainer.TableInsertionContainer{rc}, + nil, + streaming.NewNopMapStream(), + ) + bldr = primitivebuilder.NewDependencySubDAGBuilder( + pgb.planGraphHolder, + []primitivebuilder.Builder{lhsBldr}, + rhsBldr, + ) + } } else { bldr = primitivebuilder.NewInsertOrUpdate( bldrInput, diff --git a/internal/stackql/primitivebuilder/delete.go b/internal/stackql/primitivebuilder/delete.go index 212141b9..62fea2bb 100644 --- a/internal/stackql/primitivebuilder/delete.go +++ b/internal/stackql/primitivebuilder/delete.go @@ -3,6 +3,7 @@ package primitivebuilder import ( "github.com/stackql/any-sdk/pkg/streaming" "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/asynccompose" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/execution" "github.com/stackql/stackql/internal/stackql/handler" @@ -93,7 +94,8 @@ func (ss *Delete) Build() error { primitive_context.NewPrimitiveContext(), ) if ss.isAwait { - deletePrimitive, err = composeAsyncMonitor(handlerCtx, deletePrimitive, prov, method, nil) + deletePrimitive, err = asynccompose.ComposeAsyncMonitor( + handlerCtx, deletePrimitive, prov, method, nil, false, nil, nil) // isReturning hardcoded to false for now } if err != nil { return err diff --git a/internal/stackql/primitivebuilder/exec.go b/internal/stackql/primitivebuilder/exec.go index a8698543..f8919982 100644 --- a/internal/stackql/primitivebuilder/exec.go +++ b/internal/stackql/primitivebuilder/exec.go @@ -3,6 +3,7 @@ package primitivebuilder import ( "github.com/stackql/any-sdk/anysdk" "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/asynccompose" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/execution" "github.com/stackql/stackql/internal/stackql/handler" @@ -174,7 +175,9 @@ func (ss *Exec) Build() error { ss.graph.CreatePrimitiveNode(execPrimitive) return nil } - pr, err := composeAsyncMonitor(handlerCtx, execPrimitive, prov, m, nil) + pr, err := asynccompose.ComposeAsyncMonitor( + handlerCtx, execPrimitive, prov, m, + nil, false, nil, nil) // returning hardcoded to false for now if err != nil { return err } diff --git a/internal/stackql/primitivebuilder/generic_http_reversal.go b/internal/stackql/primitivebuilder/generic_http_reversal.go index 04dd737c..05b84224 100644 --- a/internal/stackql/primitivebuilder/generic_http_reversal.go +++ b/internal/stackql/primitivebuilder/generic_http_reversal.go @@ -6,6 +6,7 @@ import ( "github.com/stackql/any-sdk/anysdk" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/acid/binlog" + "github.com/stackql/stackql/internal/stackql/asynccompose" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/execution" "github.com/stackql/stackql/internal/stackql/handler" @@ -21,10 +22,12 @@ type genericHTTPReversal struct { graphHolder primitivegraph.PrimitiveGraphHolder handlerCtx handler.HandlerContext drmCfg drm.Config + insertCtx drm.PreparedStatementCtx root primitivegraph.PrimitiveNode op anysdk.OperationStore commentDirectives sqlparser.CommentDirectives isAwait bool + isReturning bool verb string // may be "insert" or "update" inputAlias string isUndo bool @@ -56,7 +59,7 @@ func newGenericHTTPReversal( return nil, fmt.Errorf("provider is required") } - return &genericHTTPReversal{ + rv := &genericHTTPReversal{ prov: prov, graphHolder: graphHolder, handlerCtx: handlerCtx, @@ -67,7 +70,12 @@ func newGenericHTTPReversal( verb: builderInput.GetVerb(), inputAlias: builderInput.GetInputAlias(), isUndo: builderInput.IsUndo(), - }, nil + } + insertCtx, insertCtxExists := builderInput.GetInsertCtx() + if insertCtxExists { + rv.insertCtx = insertCtx + } + return rv, nil } func (gh *genericHTTPReversal) GetRoot() primitivegraph.PrimitiveNode { @@ -206,7 +214,8 @@ func (gh *genericHTTPReversal) Build() error { if err != nil { return internaldto.NewErroneousExecutorOutput(err) } - execPrim, execErr := composeAsyncMonitor(handlerCtx, dependentInsertPrimitive, prov, m, commentDirectives) + execPrim, execErr := asynccompose.ComposeAsyncMonitor( + handlerCtx, dependentInsertPrimitive, prov, m, commentDirectives, gh.isReturning, gh.insertCtx, gh.drmCfg) if execErr != nil { return internaldto.NewErroneousExecutorOutput(execErr) } diff --git a/internal/stackql/primitivebuilder/generic_http_stream_input.go b/internal/stackql/primitivebuilder/generic_http_stream_input.go index 5bb42525..87f95e22 100644 --- a/internal/stackql/primitivebuilder/generic_http_stream_input.go +++ b/internal/stackql/primitivebuilder/generic_http_stream_input.go @@ -10,6 +10,7 @@ import ( "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/acid/binlog" + "github.com/stackql/stackql/internal/stackql/asynccompose" "github.com/stackql/stackql/internal/stackql/drm" "github.com/stackql/stackql/internal/stackql/execution" "github.com/stackql/stackql/internal/stackql/handler" @@ -27,11 +28,13 @@ type genericHTTPStreamInput struct { handlerCtx handler.HandlerContext drmCfg drm.Config root primitivegraph.PrimitiveNode + tail primitivegraph.PrimitiveNode tbl tablemetadata.ExtendedTableMetadata commentDirectives sqlparser.CommentDirectives dependencyNode primitivegraph.PrimitiveNode parserNode sqlparser.SQLNode isAwait bool + isReturning bool verb string // may be "insert" or "update" inputAlias string isUndo bool @@ -39,6 +42,7 @@ type genericHTTPStreamInput struct { reversalStream anysdk.HttpPreparatorStream reversalBuilder Builder rollbackType constants.RollbackType + insertCtx drm.PreparedStatementCtx } func newGenericHTTPStreamInput( @@ -62,6 +66,7 @@ func newGenericHTTPStreamInput( return nil, fmt.Errorf("dependency node is required") } parserNode, _ := builderInput.GetParserNode() + insertCtx, _ := builderInput.GetInsertCtx() return &genericHTTPStreamInput{ graphHolder: graphHolder, handlerCtx: handlerCtx, @@ -70,12 +75,14 @@ func newGenericHTTPStreamInput( commentDirectives: commentDirectives, dependencyNode: dependencyNode, isAwait: builderInput.IsAwait(), + isReturning: builderInput.IsReturning(), verb: builderInput.GetVerb(), inputAlias: builderInput.GetInputAlias(), isUndo: builderInput.IsUndo(), parserNode: parserNode, reversalStream: anysdk.NewHttpPreparatorStream(), rollbackType: handlerCtx.GetRollbackType(), + insertCtx: insertCtx, }, nil } @@ -88,6 +95,9 @@ func (gh *genericHTTPStreamInput) GetRoot() primitivegraph.PrimitiveNode { } func (gh *genericHTTPStreamInput) GetTail() primitivegraph.PrimitiveNode { + if gh.tail != nil { + return gh.tail + } return gh.root } @@ -179,6 +189,7 @@ func (gh *genericHTTPStreamInput) Build() error { reverseInput.SetHTTPPreparatorStream(gh.reversalStream) reverseInput.SetOperationStore(inverseOpStore) reverseInput.SetProvider(prov) + reverseInput.SetInsertCtx(gh.insertCtx) gh.reversalBuilder, reversalBuildInitErr = newGenericHTTPReversal(reverseInput) if reversalBuildInitErr != nil { return reversalBuildInitErr @@ -357,7 +368,8 @@ func (gh *genericHTTPStreamInput) Build() error { if err != nil { return internaldto.NewErroneousExecutorOutput(err) } - execPrim, execErr := composeAsyncMonitor(handlerCtx, dependentInsertPrimitive, prov, m, commentDirectives) + execPrim, execErr := asynccompose.ComposeAsyncMonitor( + handlerCtx, dependentInsertPrimitive, prov, m, commentDirectives, gh.isReturning, gh.insertCtx, gh.drmCfg) if execErr != nil { return internaldto.NewErroneousExecutorOutput(execErr) } @@ -386,6 +398,7 @@ func (gh *genericHTTPStreamInput) Build() error { actionNode := graphHolder.CreatePrimitiveNode(actionPrimitive) graphHolder.NewDependency(gh.dependencyNode, actionNode, 1.0) gh.root = gh.dependencyNode + gh.tail = actionNode return nil } diff --git a/internal/stackql/primitivebuilder/insert_or_update.go b/internal/stackql/primitivebuilder/insert_or_update.go index b90bbd6d..ba7e9510 100644 --- a/internal/stackql/primitivebuilder/insert_or_update.go +++ b/internal/stackql/primitivebuilder/insert_or_update.go @@ -11,6 +11,7 @@ import ( type insertOrUpdate struct { bldrInput builder_input.BuilderInput root primitivegraph.PrimitiveNode + tail primitivegraph.PrimitiveNode } func NewInsertOrUpdate( @@ -26,7 +27,7 @@ func (ss *insertOrUpdate) GetRoot() primitivegraph.PrimitiveNode { } func (ss *insertOrUpdate) GetTail() primitivegraph.PrimitiveNode { - return ss.root + return ss.tail } func (ss *insertOrUpdate) Build() error { @@ -38,6 +39,9 @@ func (ss *insertOrUpdate) Build() error { switch node := node.(type) { case *sqlparser.Insert: mutableInput.SetVerb("insert") + if len(node.SelectExprs) > 0 { + mutableInput.SetIsReturning(true) + } case *sqlparser.Update: mutableInput.SetVerb("update") default: @@ -82,6 +86,7 @@ func (ss *insertOrUpdate) Build() error { return genericBldrErr } ss.root = genericBldr.GetRoot() + ss.tail = genericBldr.GetTail() return nil } diff --git a/internal/stackql/primitivebuilder/mono_valent_builder.go b/internal/stackql/primitivebuilder/mono_valent_builder.go index 57ccfa9b..4ee987c1 100644 --- a/internal/stackql/primitivebuilder/mono_valent_builder.go +++ b/internal/stackql/primitivebuilder/mono_valent_builder.go @@ -31,6 +31,7 @@ type monoValentBuilder struct { root primitivegraph.PrimitiveNode stream streaming.MapStream isReadOnly bool //nolint:unused // TODO: build out + isAwait bool //nolint:unused // TODO: build out monoValentExecutorFactory execution.MonoValentExecutorFactory } @@ -44,6 +45,7 @@ func newMonoValentBuilder( stream streaming.MapStream, isSkipResponse bool, isMutation bool, + isAwait bool, ) Builder { var tcc internaldto.TxnControlCounters if insertCtx != nil { @@ -72,7 +74,7 @@ func newMonoValentBuilder( stream, isSkipResponse, isMutation, - false, + isAwait, ), } } diff --git a/internal/stackql/primitivebuilder/single_acquire_and_select.go b/internal/stackql/primitivebuilder/single_acquire_and_select.go index 0935343a..90dca5ae 100644 --- a/internal/stackql/primitivebuilder/single_acquire_and_select.go +++ b/internal/stackql/primitivebuilder/single_acquire_and_select.go @@ -38,7 +38,9 @@ func NewSingleAcquireAndSelect( insertContainer, insertCtx, rowSort, - nil), + nil, + bldrInput.IsAwait(), + ), selectBuilder: NewSingleSelect( graph, handlerCtx, selectCtx, []tableinsertioncontainer.TableInsertionContainer{insertContainer}, diff --git a/internal/stackql/primitivebuilder/single_select_acquire.go b/internal/stackql/primitivebuilder/single_select_acquire.go index 2df924c4..b82e8af5 100644 --- a/internal/stackql/primitivebuilder/single_select_acquire.go +++ b/internal/stackql/primitivebuilder/single_select_acquire.go @@ -15,6 +15,7 @@ func NewSingleSelectAcquire( insertCtx drm.PreparedStatementCtx, rowSort func(map[string]map[string]interface{}) []string, stream streaming.MapStream, + isAwait bool, ) Builder { tableMeta := insertionContainer.GetTableMetadata() _, isGraphQL := tableMeta.GetGraphQL() @@ -39,5 +40,6 @@ func NewSingleSelectAcquire( stream, false, false, + isAwait, ) } diff --git a/internal/stackql/primitivegenerator/statement_analyzer.go b/internal/stackql/primitivegenerator/statement_analyzer.go index b9e0617d..f2e62cb0 100644 --- a/internal/stackql/primitivegenerator/statement_analyzer.go +++ b/internal/stackql/primitivegenerator/statement_analyzer.go @@ -695,7 +695,7 @@ func (pb *standardPrimitiveGenerator) analyzeExec(pbi planbuilderinput.PlanBuild handlerCtx, insertionContainer, pb.PrimitiveComposer.GetInsertPreparedStatementCtx(), - nil, nil)) + nil, nil, false)) // returning hardcoded to false for now return nil } selIndirect, indirectErr := astindirect.NewParserExecIndirect( @@ -1125,6 +1125,7 @@ func (pb *standardPrimitiveGenerator) AnalyzeInsert(pbi planbuilderinput.PlanBui []anysdk.ColumnDescriptor{}, ) analyser := anysdk.NewMethodAnalyzer() + // TODO: this ought to cater for async methodAnalysisOutput, analysisErr := analyser.AnalyzeUnaryAction(analysisInput) if analysisErr != nil { return analysisErr diff --git a/internal/test/stackqltestutil/helper.go b/internal/test/stackqltestutil/helper.go index e71eb9b9..b3128568 100644 --- a/internal/test/stackqltestutil/helper.go +++ b/internal/test/stackqltestutil/helper.go @@ -33,6 +33,28 @@ func RunStdOutTestAgainstFiles(t *testing.T, testSubject func(*testing.T), possi checkPossibleMatchFiles(t, out, possibleExpectedOutputFiles) } +func RunStdErrTestAgainstFiles(t *testing.T, testSubject func(*testing.T), possibleExpectedOutputFiles []string) { + old := os.Stderr // keep backup of the real stderr + r, w, _ := os.Pipe() + os.Stderr = w + outC := make(chan string) + + testSubject(t) + + // copy the output in a separate goroutine so printing can't block indefinitely + go func() { + var buf bytes.Buffer + io.Copy(&buf, r) //nolint:errcheck // ok for testing + outC <- buf.String() + }() + w.Close() + os.Stderr = old // restoring the real stderr + out := <-outC + t.Logf("outC = %s", out) + + checkPossibleMatchFiles(t, out, possibleExpectedOutputFiles) +} + func checkPossibleMatchFiles(t *testing.T, subject string, possibleExpectedOutputFiles []string) { hasMatchedExpected := false for _, expectedOpFile := range possibleExpectedOutputFiles { diff --git a/stackql/main_integration_test.go b/stackql/main_integration_test.go index c0640800..70c20c65 100644 --- a/stackql/main_integration_test.go +++ b/stackql/main_integration_test.go @@ -79,7 +79,7 @@ func TestK8STemplatedE2eSuccess(t *testing.T) { os.Args = args - stackqltestutil.RunStdOutTestAgainstFiles(t, execStuff, []string{testobjects.ExpectedK8STheHardWayAsyncFile}) + stackqltestutil.RunStdErrTestAgainstFiles(t, execStuff, []string{testobjects.ExpectedK8STheHardWayAsyncFile}) } func TestInsertAwaitExecSuccess(t *testing.T) { @@ -105,7 +105,7 @@ func TestInsertAwaitExecSuccess(t *testing.T) { os.Args = args - stackqltestutil.RunStdOutTestAgainstFiles(t, execStuff, []string{testobjects.ExpectedComputeNetworkInsertAsyncFile}) + stackqltestutil.RunStdErrTestAgainstFiles(t, execStuff, []string{testobjects.ExpectedComputeNetworkInsertAsyncFile}) } func TestDeleteAwaitSuccess(t *testing.T) { @@ -130,7 +130,7 @@ func TestDeleteAwaitSuccess(t *testing.T) { os.Args = args - stackqltestutil.RunStdOutTestAgainstFiles(t, execStuff, []string{testobjects.ExpectedComputeNetworkDeleteAsyncFile}) + stackqltestutil.RunStdErrTestAgainstFiles(t, execStuff, []string{testobjects.ExpectedComputeNetworkDeleteAsyncFile}) } func TestDeleteAwaitExecSuccess(t *testing.T) { @@ -156,7 +156,7 @@ func TestDeleteAwaitExecSuccess(t *testing.T) { os.Args = args - stackqltestutil.RunStdOutTestAgainstFiles(t, execStuff, []string{testobjects.ExpectedComputeNetworkDeleteAsyncFile}) + stackqltestutil.RunStdErrTestAgainstFiles(t, execStuff, []string{testobjects.ExpectedComputeNetworkDeleteAsyncFile}) } func execStuff(t *testing.T) { diff --git a/test/python/stackql_test_tooling/flask/gcp/app.py b/test/python/stackql_test_tooling/flask/gcp/app.py index 708bb8cd..d703d833 100644 --- a/test/python/stackql_test_tooling/flask/gcp/app.py +++ b/test/python/stackql_test_tooling/flask/gcp/app.py @@ -2,8 +2,12 @@ import logging from flask import Flask, render_template, request, jsonify +import os + app = Flask(__name__) +_IS_DOCKER = True if os.getenv('IS_DOCKER', 'false').lower() == 'true' else False + # Configure logging logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") logger = logging.getLogger(__name__) @@ -32,6 +36,66 @@ def v1_storage_buckets_insert(): return render_template('buckets-insert-generic.jinja.json', bucket_name=bucket_name), 200, {'Content-Type': 'application/json'} return '{"msg": "Disallowed"}', 401, {'Content-Type': 'application/json'} +@app.route('/compute/v1/projects/testing-project/global/networks', methods=['GET']) +def projects_testing_project_global_networks(): + return render_template('route_27_template.json'), 200, {'Content-Type': 'application/json'} + +@app.route('/compute/v1/projects//global/networks', methods=['POST']) +def compute_networks_insert(project_name: str): + # Validate the incoming query + body = request.get_json() + operation_id = '1000000000001' + operation_name = 'operation-100000000001-10000000001-10000001-10000001' + network_name = body['name'] + host_name = 'host.docker.internal' if _IS_DOCKER else 'localhost' + target_link = f'https://{host_name}:1080/compute/v1/projects/{ project_name }/global/networks/{ network_name }' + if not body or 'name' not in body: + return '{"msg": "Invalid request body"}', 400, {'Content-Type': 'application/json'} + if not project_name: + return '{"msg": "Invalid request: project not supplied"}', 400, {'Content-Type': 'application/json'} + if project_name == 'mutable-project' and network_name == 'auto-test-01': + return render_template( + 'global-operation.jinja.json', + target_link=target_link, + operation_id=operation_id, + operation_name=operation_name, + project_name=project_name, + host_name=host_name, + kind='compute#operation', + operation_type='insert', + progress=0, + ), 200, {'Content-Type': 'application/json'} + return '{"msg": "Disallowed"}', 401, {'Content-Type': 'application/json'} + +@app.route('/compute/v1/projects//global/operations/', methods=['GET']) +def projects_testing_project_global_operation_detail(project_name: str, operation_name: str): + if project_name == 'mutable-project' and 'operation-100000000001-10000000001-10000001-10000001': + operation_id = '1000000000001' + network_name = 'auto-test-01' + host_name = 'host.docker.internal' if _IS_DOCKER else 'localhost' + target_link = f'https://{host_name}:1080/compute/v1/projects/{ project_name }/global/networks/{ network_name }' + return render_template( + 'global-operation.jinja.json', + target_link=target_link, + operation_id=operation_id, + operation_name=operation_name, + project_name=project_name, + host_name=host_name, + kind='compute#operation', + operation_type='insert', + progress=100, + end_time='2025-07-05T19:43:34.491-07:00', + ), 200, {'Content-Type': 'application/json'} + return '{"msg": "Disallowed"}', 401, {'Content-Type': 'application/json'} + +@app.route('/compute/v1/projects//global/networks/', methods=['GET']) +def projects_testing_project_global_network_detail(project_name: str, network_name: str): + return render_template( + 'networks-insert-generic-mature.jinja.json', + project_name=project_name, + network_name=network_name + ), 200, {'Content-Type': 'application/json'} + @app.route('/v1/projects/testing-project-three/locations/global/keyRings/testing-three/cryptoKeys', methods=['GET']) def v1_projects_testing_project_three_locations_global_keyRings_testing_three_cryptoKeys(): return render_template('route_1_template.json'), 200, {'Content-Type': 'application/json'} @@ -188,9 +252,6 @@ def projects_testing_project_zones_australia_southeast1_a_disks(): def projects_testing_project_zones_australia_southeast1_b_disks(): return render_template('route_26_template.json'), 200, {'Content-Type': 'application/json'} -@app.route('/compute/v1/projects/testing-project/global/networks', methods=['GET']) -def projects_testing_project_global_networks(): - return render_template('route_27_template.json'), 200, {'Content-Type': 'application/json'} @app.route('/compute/v1/projects/testing-project/regions/australia-southeast1/subnetworks', methods=['GET']) def projects_testing_project_regions_australia_southeast1_subnetworks(): diff --git a/test/python/stackql_test_tooling/flask/gcp/templates/global-operation.jinja.json b/test/python/stackql_test_tooling/flask/gcp/templates/global-operation.jinja.json new file mode 100644 index 00000000..143f6c52 --- /dev/null +++ b/test/python/stackql_test_tooling/flask/gcp/templates/global-operation.jinja.json @@ -0,0 +1,15 @@ +{ + "kind": "{{ kind if kind else 'compute#operation' }}", + "id": "{{ operation_id if operation_id else '1000000000001' }}", + "name": "{{ operation_name }}", + "operationType": "insert", + "targetLink": "{{ target_link }}", + "targetId": "{{ target_id if target_id else '2000000000002' }}", + "status": "{{ 'DONE' if progress and progress > 99 else 'RUNNING' }}", + "user": "krimmer@ryukit.com", + "progress": {{ progress }}, + "insertTime": "2025-07-05T19:42:34.488-07:00", + "startTime": "2025-07-05T19:42:34.491-07:00", + {% if end_time %}"endTime": "{{ end_time }}",{% endif %} + "selfLink": "https://{{ host_name }}:1080/compute/v1/projects/{{ project_name }}/global/operations/{{ operation_name }}" +} \ No newline at end of file diff --git a/test/python/stackql_test_tooling/flask/gcp/templates/networks-insert-generic-mature.jinja.json b/test/python/stackql_test_tooling/flask/gcp/templates/networks-insert-generic-mature.jinja.json new file mode 100644 index 00000000..e95d0453 --- /dev/null +++ b/test/python/stackql_test_tooling/flask/gcp/templates/networks-insert-generic-mature.jinja.json @@ -0,0 +1,14 @@ +{ + "kind": "compute#network", + "id": "{{ network_id if network_id else '1000000000000001' }}", + "creationTimestamp": "2025-07-05T19:42:34.483-07:00", + "name": "{{ network_name }}", + "selfLink": "https://www.googleapis.com/compute/v1/projects/{{ project_name }}/global/networks/{{ network_name }}", + "selfLinkWithId": "https://www.googleapis.com/compute/v1/projects/{{ project_name }}/global/networks/{{ network_id if network_id else '1000000000000001' }}", + "autoCreateSubnetworks": false, + "routingConfig": { + "routingMode": "REGIONAL", + "bgpBestPathSelectionMode": "LEGACY" + }, + "networkFirewallPolicyEnforcementOrder": "AFTER_CLASSIC_FIREWALL" +} diff --git a/test/registry/src/googleapis.com/v0.1.2/resources/compute-v1.yaml b/test/registry/src/googleapis.com/v0.1.2/resources/compute-v1.yaml index 9adeed6c..74377774 100644 --- a/test/registry/src/googleapis.com/v0.1.2/resources/compute-v1.yaml +++ b/test/registry/src/googleapis.com/v0.1.2/resources/compute-v1.yaml @@ -2130,6 +2130,9 @@ resources: response: mediaType: application/json openAPIDocKey: '200' + asyncOverrideMediaType: application/json + async_schema_override: + $ref: '#/components/schemas/Network' list: operation: $ref: 'googleapis.com/v0.1.2/services-split/compute/compute-v1.yaml#/paths/~1projects~1{project}~1global~1networks/get' diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index 0a22b913..16b49eaf 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -8550,3 +8550,30 @@ Insert Returning Simple Projection ... ${EMPTY} ... stdout=${CURDIR}/tmp/Insert-Returning-Simple-Projection.tmp ... stderr=${CURDIR}/tmp/Insert-Returning-Simple-Projection-stderr.tmp + +Insert Async Returning Simple Projection + [Documentation] Insert a row into a table and return projected new object values. For **asynchronously** created objects. + ${inputStr} = Catenate + ... insert /*+ AWAIT */ into google.compute.networks(project, data__name, data__autoCreateSubnetworks) select 'mutable-project', 'auto-test-01', false returning creationTimestamp, name; + ${outputStr} = Catenate SEPARATOR=\n + ... |-------------------------------|--------------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}creationTimestamp${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |-------------------------------|--------------| + ... |${SPACE}2025-07-05T19:42:34.483-07:00${SPACE}|${SPACE}auto-test-01${SPACE}| + ... |-------------------------------|--------------| + ${stdErrStr} = Catenate SEPARATOR=\n + ... compute#operation: insert in progress, 10 seconds elapsed + ... compute#operation: insert complete + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${stdErrStr} + ... stdout=${CURDIR}/tmp/Insert-Async-Returning-Simple-Projection.tmp + ... stderr=${CURDIR}/tmp/Insert-Async-Returning-Simple-Projection-stderr.tmp From 414a233f964965ed2a6e74c195077e2ec5c44f3a Mon Sep 17 00:00:00 2001 From: Derek Kim <69182529+derek10cloud@users.noreply.github.com> Date: Wed, 9 Jul 2025 17:56:05 -0400 Subject: [PATCH 10/10] support overwrite with registry pull (#550) * support overwrite with registry pull * fix: Format error * skip all of the breakable build and push stuff in fork mode * fix formatting error * robot-test-coverage-registry-overwrite Summary: - Changed existing mocked registry entries for point of difference, in otder to support meaningful testing. - Force drop registry cache entry for a given provider on `registry pull`. - Added robot test `Registry Pull Overwrite Works in Both Directions`. * update-any-sdk Summary: - Update `anny-sdk` version. - Attempt support for windows provider delete. * update-any-sdk Summary: - Update `anny-sdk` version. - Fix file locking issue with signature check. - Attempt support for windows provider delete. * update-any-sdk Summary: - Update `anny-sdk` version. - Eager close on provider file itself, inside `any-sdk`. * update-any-sdk Summary: - Update `anny-sdk` version. - Eager close on provider file itself, inside `any-sdk`. --------- Co-authored-by: General Kroll Co-authored-by: Benevolent General Kroll Who cannot spell <82620104+general-kroll-4-life@users.noreply.github.com> --- .github/workflows/build.yml | 1 + .vscode/launch.json | 4 +- go.mod | 2 +- go.sum | 4 +- internal/stackql/handler/handler.go | 18 ++++ internal/stackql/planbuilder/plan_builder.go | 77 ++++++++++++------ .../flask/registry/expectations.json | 13 +++ .../registry/templates/google-v0.1.0.tgz | Bin 0 -> 1706 bytes .../templates/google-v0.1.1-alpha01.tgz | Bin 10240 -> 1676 bytes .../stackql_mocked_from_cmd_line.robot | 58 +++++++++++++ 10 files changed, 148 insertions(+), 29 deletions(-) create mode 100644 test/python/stackql_test_tooling/flask/registry/templates/google-v0.1.0.tgz diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 03a57bf0..bbf65fb7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1537,6 +1537,7 @@ jobs: name: digests-${{ env.PLATFORM_PAIR }} - name: Prepare digest information + if: env.BUILD_IMAGE_REQUIRED == 'true' working-directory: ${{ runner.temp }}/digests run: | platform=${{ matrix.platform }} diff --git a/.vscode/launch.json b/.vscode/launch.json index e81b7264..a89c1e3a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -177,6 +177,7 @@ "delete from aws.cloud_control.resources where region = 'ap-southeast-1' and data__TypeName = 'AWS::Logs::LogGroup' and data__Identifier = 'LogGroupResourceExampleThird' ;", "insert into google.storage.buckets( project, data__name) select 'testing-project', 'silly-bucket' returning projectNumber;", "insert /*+ AWAIT */ into google.compute.networks(project, data__name, data__autoCreateSubnetworks) select 'mutable-project', 'auto-test-01', false returning creationTimestamp, name;", + "registry pull google 'v0.1.2'; show resources in google.storage; registry pull google 'v0.1.1-alpha01'; show resources in google.storage; registry pull google 'v0.1.0'; show resources in google.storage;", ], "default": "show providers;" }, @@ -195,7 +196,8 @@ "{ \"url\": \"https://cdn.statically.io/gh/stackql/stackql-provider-registry/main/providers\", \"localDocRoot\": \"${workspaceFolder}/test/registry\" }", "{ \"url\": \"https://cdn.statically.io/gh/stackql/stackql-provider-registry/dev/providers\" }", "{ \"url\": \"https://registry-dev.stackql.app/providers\" }", - "{ \"url\": \"https://registry.stackql.app/providers\" }" + "{ \"url\": \"https://registry.stackql.app/providers\" }", + "{\"url\": \"http://localhost:1094/gh/stackql/stackql-provider-registry/main/providers\", \"verifyConfig\": {\"nopVerify\": true}}", ], "default": "{ \"url\": \"file://${workspaceFolder}/test/registry\", \"localDocRoot\": \"${workspaceFolder}/test/registry\", \"verifyConfig\": { \"nopVerify\": true } }" }, diff --git a/go.mod b/go.mod index 45192424..e88885ea 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.10.1 - github.com/stackql/any-sdk v0.1.4-alpha06 + github.com/stackql/any-sdk v0.1.4-alpha11 github.com/stackql/go-suffix-map v0.0.1-alpha01 github.com/stackql/psql-wire v0.1.1-beta23 github.com/stackql/stackql-parser v0.0.15-alpha06 diff --git a/go.sum b/go.sum index 4902a5d8..51440bd5 100644 --- a/go.sum +++ b/go.sum @@ -484,8 +484,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= -github.com/stackql/any-sdk v0.1.4-alpha06 h1:QJPf3ehPrRqmYZR+TmD897AsmsaOamHErLhaE5B9v/w= -github.com/stackql/any-sdk v0.1.4-alpha06/go.mod h1:AKS/g28y7m4SWL/YW8veE9MCNy8XJgaicVibemVE9e8= +github.com/stackql/any-sdk v0.1.4-alpha11 h1:WeFvLcml37sq+HbVk3hiXuImQ578SE9LM/K1MzqFovo= +github.com/stackql/any-sdk v0.1.4-alpha11/go.mod h1:AKS/g28y7m4SWL/YW8veE9MCNy8XJgaicVibemVE9e8= github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8= github.com/stackql/go-suffix-map v0.0.1-alpha01/go.mod h1:QAi+SKukOyf4dBtWy8UMy+hsXXV+yyEE4vmBkji2V7g= github.com/stackql/psql-wire v0.1.1-beta23 h1:1ayYMjZArfDcIMyEOKnm+Bp1zRCISw8pguvTFuUhhVQ= diff --git a/internal/stackql/handler/handler.go b/internal/stackql/handler/handler.go index 4d201c6a..2f3e0be1 100644 --- a/internal/stackql/handler/handler.go +++ b/internal/stackql/handler/handler.go @@ -48,6 +48,7 @@ type HandlerContext interface { //nolint:revive // don't mind stuttering this on GetAuthContext(providerName string) (*dto.AuthCtx, error) GetDBMSInternalRouter() dbmsinternal.Router GetProvider(providerName string) (provider.IProvider, error) + DeleteProvider(providerName string) error GetSupportedProviders(extended bool) (map[string]map[string]interface{}, error) LogHTTPResponseMap(target interface{}) // @@ -286,6 +287,23 @@ func (hc *standardHandlerContext) GetASTFormatter() sqlparser.NodeFormatter { return hc.formatter } +func (hc *standardHandlerContext) DeleteProvider(providerName string) error { + hc.providersMapMutex.Lock() + defer hc.providersMapMutex.Unlock() + if providerName == "googleapis.com" { + providerName = "google" + } + if providerName == "" { + return fmt.Errorf("provider name cannot be empty") + } + _, ok := hc.providers[providerName] + if len(hc.providers) == 0 || !ok { + return fmt.Errorf("provider '%s' not found", providerName) + } + delete(hc.providers, providerName) + return nil +} + func (hc *standardHandlerContext) GetProvider(providerName string) (provider.IProvider, error) { var err error if providerName == "" { diff --git a/internal/stackql/planbuilder/plan_builder.go b/internal/stackql/planbuilder/plan_builder.go index 9ca38730..4cd6159c 100644 --- a/internal/stackql/planbuilder/plan_builder.go +++ b/internal/stackql/planbuilder/plan_builder.go @@ -647,9 +647,7 @@ func (pgb *standardPlanGraphBuilder) handleDelete(pbi planbuilderinput.PlanBuild } //nolint:gocognit // acceptable -func (pgb *standardPlanGraphBuilder) handleRegistry( - pbi planbuilderinput.PlanBuilderInput, -) error { +func (pgb *standardPlanGraphBuilder) handleRegistry(pbi planbuilderinput.PlanBuilderInput) error { handlerCtx := pbi.GetHandlerCtx() node, ok := pbi.GetRegistry() if !ok { @@ -665,29 +663,10 @@ func (pgb *standardPlanGraphBuilder) handleRegistry( return err } pr := primitive.NewLocalPrimitive( - //nolint:revive // acceptable for now - func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput { + func(_ primitive.IPrimitiveCtx) internaldto.ExecutorOutput { switch at := strings.ToLower(node.ActionType); at { case "pull": - providerVersion := node.ProviderVersion - if providerVersion == "" { - providerVersion, err = reg.GetLatestPublishedVersion(node.ProviderId) - } - if err != nil { - return internaldto.NewErroneousExecutorOutput(err) - } - err = reg.PullAndPersistProviderArchive(node.ProviderId, providerVersion) - if err != nil { - return internaldto.NewErroneousExecutorOutput(err) - } - return util.PrepareResultSet( - internaldto.NewPrepareResultSetPlusRawDTO( - nil, nil, nil, nil, nil, - internaldto.NewBackendMessages([]string{fmt.Sprintf( - "%s provider, version '%s' successfully installed", - node.ProviderId, providerVersion)}), - nil, - pbi.GetHandlerCtx().GetTypingConfig())) + return pgb.handleRegistryPull(reg, node, pbi) case "list": var colz []string var provz map[string]anysdk.ProviderDescription @@ -737,10 +716,58 @@ func (pgb *standardPlanGraphBuilder) handleRegistry( }, ) pgb.planGraphHolder.CreatePrimitiveNode(pr) - return nil } +// handleRegistryPull is a helper function that pulls a provider version from the registry and persists it locally. +func (pgb *standardPlanGraphBuilder) handleRegistryPull( + reg anysdk.RegistryAPI, + node *sqlparser.Registry, + pbi planbuilderinput.PlanBuilderInput, +) internaldto.ExecutorOutput { + providerVersion := node.ProviderVersion + var err error + if providerVersion == "" { + providerVersion, err = reg.GetLatestPublishedVersion(node.ProviderId) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + } + // Get all existing versions from local filesystem + localProviders := reg.ListLocallyAvailableProviders() + // Handle special case for google provider + providerID := node.ProviderId + if providerID == "google" { + providerID = "googleapis.com" + } + if providerDesc, exists := localProviders[providerID]; exists { + // Remove all existing versions + for _, version := range providerDesc.Versions { + if err = reg.RemoveProviderVersion(providerID, version); err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + } + } + // Clear provider cache before pulling new version + if err = reg.ClearProviderCache(providerID); err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + handlerCtx := pbi.GetHandlerCtx() + _ = handlerCtx.DeleteProvider(providerID) + // Pull and persist the requested version + if err = reg.PullAndPersistProviderArchive(node.ProviderId, providerVersion); err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + return util.PrepareResultSet( + internaldto.NewPrepareResultSetPlusRawDTO( + nil, nil, nil, nil, nil, + internaldto.NewBackendMessages([]string{fmt.Sprintf( + "%s provider, version '%s' successfully installed", + node.ProviderId, providerVersion)}), + nil, + pbi.GetHandlerCtx().GetTypingConfig())) +} + func (pgb *standardPlanGraphBuilder) handlePurge(pbi planbuilderinput.PlanBuilderInput) error { handlerCtx := pbi.GetHandlerCtx() node, ok := pbi.GetPurge() diff --git a/test/python/stackql_test_tooling/flask/registry/expectations.json b/test/python/stackql_test_tooling/flask/registry/expectations.json index f77942a3..15b191ba 100644 --- a/test/python/stackql_test_tooling/flask/registry/expectations.json +++ b/test/python/stackql_test_tooling/flask/registry/expectations.json @@ -37,5 +37,18 @@ "status": 200, "file": "google-v0.1.1-alpha01.tgz" } + }, + { + "httpRequest": { + "method": "GET", + "path": "/gh/stackql/stackql-provider-registry/main/providers/dist/googleapis.com/v0.1.0.tgz" + }, + "httpResponse": { + "headers": { + "Content-Type": "application/gzip" + }, + "status": 200, + "file": "google-v0.1.0.tgz" + } } ] \ No newline at end of file diff --git a/test/python/stackql_test_tooling/flask/registry/templates/google-v0.1.0.tgz b/test/python/stackql_test_tooling/flask/registry/templates/google-v0.1.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..0d10bd98023932161ff787ff8b64d1bc84d30754 GIT binary patch literal 1706 zcmV;b237eViwFSlC~jx~1MM2iZressa&s`;i;q1nkO00^eCTCVKv6q(QKU_r)K1Rs zid@kflU!zZDN6_v_|)&}r*-y0ij?J79XmSkc?h`!E+a3&% z#zTX>Zr@-F$RBvUUccKN_ygqmUBB0Bp~1acWg!<5b1(&m2~Dq{1G35Fiu=}&5`QHy zHT&=E9~~b_#>su?fOri1{mb^>?|WXq-2Yy82>ag~`hyno?!%a6_{R4ip#3xt{Z?xe zzOJ_nZvDNQblrX~z*-^8d_V?m1ufDNp$Oue+M$ID;Nq=mm4D1w+`ffF4K&a|0}V9L zz>UDR*Y)o|uNe(A(7+c6sy-l#cPu_PxNYyRS^TKNvuQE1c*o*%gWHyF&EiKEuUoun zF|v5a;&X#-(Q4a=kG9p^wofSSO^cDmJGXMVfd;-A_|BkB)&GBOUHbgrKm!f@kD>i& z|7ibdt9&510IJs_`21yQ{wWwQ83fkM4~9Rq7Iets9gEKmZiayd8fc(_hXninFXL=V zL&BXoPHy_X=EiXK`#;a`^|u%A|At-U`TgNw*u4LH2%zBzMT|u;*+_9h%8Mc)jtHO9 zfQZosLP9c*Bci?{6cQ0|nn}vi5z?=Y1;S~FIFX!^DO_btl2lS0$8!{89+4jrO_E&V zaZFH%B}Q!gmIP8bMj#adn-V^MPSev7a$KBI;aZ*U)OSIHtWzGs${QY!(210p7`g7* z*_mS;P|Qxi5?5FU+>%SlT9Vya$Yw#FfZ13%W-KX$hu~GrGQx$`VdPsOz$o0ef~ChG z+tspYz*1qNh0x3&uCi0hpb1D2uBKNv=bSS+LUIHVjliShX}NfJ#F<1vY?V;yxR^jDKVR+?VPIGGU63CNQ?H*0o3 zim*HZk|OEKJ+nor+`7W>sjuskW|dUNpl(ljrt=R7(Czs`9S1CC{J6W_o&D4eGdthSI$%gdQ)9`5 z@dV3q!oQ%pIHF6#*ptQJtfu5&Yz%+4gZSsT|N^$tqRSu5{{(l^=*r5MGiG6%p<#N6H0rj0{=1$jQ5$*2iXDWpy#G7BHzR z7bGq%pAD8wd4W&im@B8bj)pQzT^Wsw&_ITVRI%U~!O=ZCfr?CcRaMEfB~XyV4yT4a zBB_lUg7g9tjk*X9d5Weu&Pm<9Wv0BGHC*EclQsNyn6(W zz)f%(&K_j7q$(QB2E#%7WPa~>Z2J!Vqf*{oiXr(h=G>fEpQoV+%lgDvwL94r! zC$Ax%R=cuy3>!k@7*4M_&s69G1GNbPse><7CzH#k3S28G3GZZhE>@n;OPrrlq0{6I z6suRpnmg^!jlFW9kePa*ZygpMRJ6TX_#&=%CjYM<6f;z1P}7r8l}(K>a75~3*79ss zzq?BNr8X$2=XoB$94a8sO8YicKhfp6bE6iw_#tQmj-h6&*5zHTU=JMQEEY>Q7uUfF zPE%4L2+bjTsVAsQgh17RPGmx}SY=nS*omyXwn-$KA2}Lmpn+S#zsw~jq5wDm0DId= AdH?_b literal 0 HcmV?d00001 diff --git a/test/python/stackql_test_tooling/flask/registry/templates/google-v0.1.1-alpha01.tgz b/test/python/stackql_test_tooling/flask/registry/templates/google-v0.1.1-alpha01.tgz index 56b799220cc0d6ca98a1a395e4c1116757807b64..953d0860b8c827cae27c794affa241d4375c0ee1 100644 GIT binary patch literal 1676 zcmV;726OoziwFSVC~jx~1ML_~Z`(H58!!y>X{Q}_BG3Z4%zF9N7$^oe9oB4lElG#% z6wneavynuVq#PSX3+%dIv7fN}{@gy2k}Ro7oGfX(qW%EGCiT7F^4zgp%kAJOongnd zTkirK#~JqfNPiE^x8wE9cMizyJKb*2^9GKK9Jkx;_gbj`PPA!2CM4#d3JzkLT!RN_ z)9E$Ob34lKj|ArR_*-WwXLA}5ZY^;f-H#0L$Dr4{I{py!GX6cM5Ah##onZ?(_apv# z_{8Ji`r?}}Tc5XD``AZ^C+IhngSu;d1s@MSZ{VZOf8UJd`1JHR|EVbcg3s3tl+RZ8 z-})@JaGFNMim;HGAVGj7d2n>15dVCb%+&WE|Ip5z?A*Ym(5U|-ym&@1l)rtCePC_< zcL(+Q_Z+7;XraaZn0P&W;`QHkP%LRo9=pT7=ky+JJFW!*8Fn8*u-b!dw72_Ye7y5) z_tzh+1(uRqSC;EzN{=%NkkBoJVtEtg7{Kch9MO`n-jj= zqsjRQSvF3ouydLA+_gc1tYaR-&2t`&&`ioyjBNYj;=(cpC}P!VY>^vamsE;ZgLh}e zoB3G`YGY-X(WGD=fK?Go2^YBx!#`&Ngu;FySaJruZOx1NED^?A2rb;fWqN8JGzJdB z*7&LfgUpw@tg;&=rd8zOhVV%4B>YOHUS$gY1Ksf}7@#U^0 zvBrq-ghaVv9d#1=r^%n3o4!donG((k@RK|Ptw>$)tBI(MQHJVZx!oq;LtNW8? zA*qZ&?2fWf=OG}VpH2=B(D-OqV}KPxXDJx?q08-m2UofJK4skI{lDLLp#Q@v;%(ml z-v`{V)b1o>&HV2VU2ir2hi3mDx|{j`J^m27FM24Rvj_N1Bo*hlw=5nM6M19shO+i zarLqo9a|6V6w8^g-?)=E?kgkUwGAw`D%hYem?=n|LremSPnRjo5A{MqeJw>#@fsc| zkiax~%KV?n5=26NZnen*{FLAbZnf%JbrHGMu0WG1QygGqgL`w+%{2YdY-8%8#(%|B zKU;L<1(l%B2rXGAvxy?P=M4?qLrM=C?RGc;kIf0%VQ~!ZMPqosqmxXDL|x0N>kVrA zIe~BREyhDT=PT^V(gqr}I)NdvIGpCY=Q^)yY)!Ui6{s!(%4gvGorr=5j!8&`tmatF z*=m(yAna1{1t$?^JnKLPk&uCVbxL%=-#A)I>I;3+`KAdX7P3nuUB1grIE0NRVIFlw zYkP~z>C9&l<7eK`TWotlns0ZD4k!}R#Aq^QJjSxz@81$#RA^%rd$?Mg)sFmq?!(9V zve=!D01V#O0%eC{4tRk zhn$4&f^{x~2auc*BvRtjDJ4PPwybue&Frq0;=Ex&z=_P!Dm^m4>C^=)I}n>9+~5xl z5uPeT$_rCQ2Ccm0dpm;OUpNdhAR8`9Lii7&9yg_TIysZu3Q5d z9#BPsBLt;;F@ugwcvVzMv?Y+AL4*@S9+D*Y8iMo!6^*zMiabGc9A%_#{>uzfsbN2z z?$#=F*kE69@>6BB)+d&s6Awf!+j;)XtY` zlF4ROfoUZs;hhvO#ijLmh4FJLbecSeW_4t=xl?>@Y?TF#%=82O=(w<;BJF5pi&$?= z_FpY1cBsmtrZ=G~nOb3>MCxl+bGNELy-fO*Iw+{;dFI0wDj?5F{n~1t==R*X(TH39 z4b+A*)MVAZJk|nsz%b4t(RjEz`!k#*q=FEbB0JO@)D=viYrrHju36o(>qv^#xl=Dp WvH5Fb0~^@DE#Y76fRjN0Bme+oSU=qW literal 10240 zcmeIu`6JVf0{~z$XS|lLBuvAG-bkUBTw}2zx5(Jbn={vIZps-(juCIi@FsUM%$55n zazred+sM5+Lb>MH`#*et{ycxeb0vj`4KLO1!p53N7v56oK&ou=3;JwyH#NGJ z)MkzR`5BVw{y6-Bu!BG7y&LQ{ENE)39bL0A!ETI^_w@*L6zWH>mINN6c8|I=5XYHF=A?4I?JR{w++xHiu zgGDnE?1Kn6nQOFRjyHR0G@LOrOtalsr+xOMWf>dUxSbtGbDx?5?^GUp-q7w^{LnKH zIXwr<5l3@RPp<~$5D@h&vDEkII=Wh7*8ZSD~> zDxM?#uw2>9n#L5a?RcJemn;6#x8c@yaFa|sD+zp{u(M-oiEK@Ys6Y?7lGM$_jpj6AWOdQ`9R0`w&{NK(3-E?hIZw>qxjyH*h>*5!Vo zVpV4uXOub|3VW)GPC~;}3Di;JwJOY+sOKZO@2)25a!~VK)oGQ3XNk+4V+tYxDWL5TR@l}PhRkpq9xo@A?hfRiUC`J{XXR9` zleSsb6iD&DyvEim)+V8)lU31t{$P$?H%mQ<=6e#7Ll)(Fy|v1_Lg|L94qa!S7^Fq` zA*FMx+;1g435}|$T7lnf`oPJPX7wYt&v{HyH^FVUUs(dr9%S;$XcVaM?y6GPeJbE*_3FcZ^RHRx>~RM(Nl2+P%o&=I4Q||rzQMc7KfaXp>~DA@Fw3C zQE-<;cvq{oGWT<{w)S?EJD4b%NG>-uAD}EcuEq4Yl)ySO@G)ZJwMR|=epjlueAWSe zm-oeMH`AD0nzEwsoNfpScQxVqE@c2xS%0q{n`Fv6X5=^70Cw}DAW7h&k^i7CnXVGO#TAGl;7!cSfj+q zx~Tc4o04E+v05@VY7ABLSce2f*#ve{ZNfpsB5!QCbNWVWUoG2v3`j}fKc=K zu_pctx&g=x!z6U?KZhlj81K8j?jG!MKh4}?MY?Pm$?m1LHl!0>pVaNlh_(3bd$+B8 zT;2YuzL=AXEvn7!d~InwM1~WAm3utp_Z$ZwIfgYngBqHi zv(HW)|03-#_I#WYJaU8N^Dy{ecL~z{?cx1>!aVY!D00tz&m`fx=$a|m5E4-)OT%FJ*|>l z5`706RTcXEiZz(axQMUzMYYWy-E_*YUeEH@u(cqrG)C*%hJ*D_{f5=4(6q)+f(UO5 zJkzd8>7ee3A}+PoLO~l#YU3riSUGRkN^6erFhry%xBe1x%g1cg_(5(ak?e9{Lg~AO zJ&+&K>AUwSrz}>iU-?0YT>dT#QPsQH6-T3yJ3Ja_PJ4v8ES$?kj%MT8-`dsvqt|o=@_rB)r*}}H`!oNJ zw^3P$YxZB0ZGT<++OhP pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy