Nest with group_by: repeated-record fix (#2090)

* add failing test

* move tests to "right"place in file

* maybe throw instead of silently fail?

* Revert "maybe throw instead of silently fail?"

This reverts commit e50f66f.

* try and isolate the problem

* lint

* see what  what fails

* find where it fails

* fix anyvalueturtle

* more precision in queryfield taxonomy

* fix test

* fix bq anyvalueturtle

* maybe trino any value turtle?

* maybe iSScalar more correct places?

* clean up simple fields

* make "scalar" mean "!aggregate,!calc"

* get trino to allow nested group by record

* fix mysql and postgres for nesting

* still wokring on mysql compound

* more mysql fixing

* normalize to "basic" for "atomic non compound"

* to a todo rename "atomic" to "basic"

Author: mtoy-googly-moogly

package.json

@@ -42,6 +42,7 @@
     "ci-snowflake": "MALLOY_DATABASE=snowflake JEST_SILENT_REPORTER_SHOW_PATHS=true jest --config jest.snowflake.config.ts --reporters jest-silent-reporter --reporters summary",
     "ci-trino": "MALLOY_DATABASE=trino JEST_SILENT_REPORTER_SHOW_PATHS=true jest --config jest.trino.config.ts --reporters jest-silent-reporter --reporters summary",
     "ci-presto": "MALLOY_DATABASE=presto JEST_SILENT_REPORTER_SHOW_PATHS=true jest --config jest.presto.config.ts --reporters jest-silent-reporter --reporters summary",
+    "ci-postgres": "MALLOY_DATABASE=postgres JEST_SILENT_REPORTER_SHOW_PATHS=true jest --config jest.postgres.config.ts --reporters jest-silent-reporter --reporters summary",
     "test-silent": "JEST_SILENT_REPORTER_SHOW_PATHS=true jest --runInBand --reporters jest-silent-reporter --no-color",
     "test-deps": "npm run build && npx jest -t dependencies",
     "third-party-licenses": "ts-node scripts/third_party_licenses",

packages/malloy-db-trino/src/trino_connection.ts

@@ -44,7 +44,7 @@ import type {
 import {
   TrinoDialect,
   mkFieldDef,
-  isScalarArray,
+  isBasicArray,
   TinyParser,
   isRepeatedRecord,
   sqlKey,
@@ -314,7 +314,7 @@ export abstract class TrinoPrestoConnection
       return this.convertRow(colSchema.fields, rawRow);
     } else if (isRepeatedRecord(colSchema)) {
       return this.convertNest(colSchema.fields, rawRow) as QueryValue;
-    } else if (isScalarArray(colSchema)) {
+    } else if (isBasicArray(colSchema)) {
       const elType = colSchema.elementTypeDef;
       let theArray = this.unpackArray([], rawRow);
       if (elType.type === 'array') {

packages/malloy-render/src/component/bar-chart/generate-bar_chart-vega-spec.ts

@@ -822,7 +822,7 @@ export function generateBarChartVegaSpec(
             );
 
         const value = rec.y;
-        return field.isAtomic()
+        return field.isBasic()
           ? renderNumericField(field, value)
           : String(value);
       };

packages/malloy-render/src/component/bar-chart/get-bar_chart-settings.ts

@@ -103,7 +103,7 @@ export function getBarChartSettings(
   });
 
   const dimensions = explore.fields.filter(
-    f => f.isAtomic() && f.wasDimension()
+    f => f.isBasic() && f.wasDimension()
   );
 
   // If still no x or y, attempt to pick the best choice

packages/malloy-render/src/component/chart-layout-settings.ts

@@ -132,10 +132,10 @@ export function getChartLayoutSettings(
     const maxAxisVal = yScale.domain().at(1);
     const minAxisVal = yScale.domain().at(0);
     const l = locale();
-    const formattedMin = yField.isAtomic()
+    const formattedMin = yField.isBasic()
       ? renderNumericField(yField, minAxisVal)
       : l.format(',')(minAxisVal);
-    const formattedMax = yField.isAtomic()
+    const formattedMax = yField.isBasic()
       ? renderNumericField(yField, maxAxisVal)
       : l.format(',')(maxAxisVal);
     // const formattedMin = l.format(',')(minAxisVal);

packages/malloy-render/src/component/chart/chart.tsx

@@ -160,7 +160,7 @@ export function Chart(props: ChartProps) {
   // Pass relevant brushes from store into the vega view
   createEffect(() => {
     const fieldRefIds = field.fields.map(f =>
-      f.isAtomic() ? f.referenceId : null
+      f.isBasic() ? f.referenceId : null
     );
     const relevantBrushes = metadata.store.store.brushes.filter(brush =>
       fieldRefIds.includes(brush.fieldRefId)

packages/malloy-render/src/component/dashboard/dashboard.tsx

@@ -102,7 +102,7 @@ export function Dashboard(props: {
 
   const dimensions = () =>
     field.fields.filter(f => {
-      return !f.isHidden() && f.isAtomic() && f.wasDimension();
+      return !f.isHidden() && f.isBasic() && f.wasDimension();
     });
 
   const nonDimensions = () => {
@@ -111,9 +111,9 @@ export function Dashboard(props: {
 
     for (const f of field.fields) {
       if (f.isHidden()) continue;
-      if (f.isAtomic() && f.wasCalculation()) {
+      if (f.isBasic() && f.wasCalculation()) {
         measureFields.push(f);
-      } else if (!f.isAtomic() || !f.wasDimension()) otherFields.push(f);
+      } else if (!f.isBasic() || !f.wasDimension()) otherFields.push(f);
     }
     return [...measureFields, ...otherFields];
   };

packages/malloy-render/src/component/line-chart/generate-line_chart-vega-spec.ts

@@ -787,7 +787,7 @@ export function generateLineChartVegaSpec(explore: NestField): VegaChartProps {
             );
 
         const value = rec.y;
-        return field.isAtomic()
+        return field.isBasic()
           ? renderNumericField(field, value)
           : String(value);
       };

packages/malloy-render/src/component/line-chart/get-line_chart-settings.ts

@@ -103,7 +103,7 @@ export function getLineChartSettings(
   });
 
   const dimensions = explore.fields.filter(
-    f => f.isAtomic() && f.wasDimension()
+    f => f.isBasic() && f.wasDimension()
   );
 
   // If still no x or y, attempt to pick the best choice

packages/malloy-render/src/component/render-image.tsx

@@ -3,7 +3,7 @@ import type {RendererProps} from './types';
 export function renderImage(props: RendererProps) {
   const imgTag = props.tag.tag('image');
   if (!imgTag) throw new Error('Missing tag for Image renderer');
-  if (!props.dataColumn.field.isAtomic())
+  if (!props.dataColumn.field.isBasic())
     throw new Error('Image renderer: Field must be AtomicField');
   if (!props.dataColumn.isString() && !props.dataColumn.isNull())
     throw new Error('Image renderer: DataColumn must be StringCell');