Updates to MalloyTranslator to enable model caching (#2091)

* First pass at allowing translator to accept pre-transalted documents

* Simplify

* Move `queryList` into `ModelDef`

* Change `translationResponse.translated` to `.modelDef`

* Add dependencies to `ModelDef`

* Internally, call sql blocks `sqlSources`

* Add util method to get a child translator

* Keep `fromSources` correctly in sync

* Remove Malloy file that I didn't mean to check in

* Fix hand-written models

* Don't error if a child translator is not present -- that just means its parent was cached and therefore never created

* Add `newlyTranslatedDependencies` util

* Remove unused import

* Add a test for newlyTranslatedDependencies

Author: christopherswenson

packages/malloy/src/lang/ast/statements/import-statement.ts

@@ -97,8 +97,9 @@ export class ImportStatement
         'Cannot import without translation context'
       );
     } else if (this.fullURL) {
+      const pretranslated = trans.root.pretranslatedModels.get(this.fullURL);
       const src = trans.root.importZone.getEntry(this.fullURL);
-      if (src.status === 'present') {
+      if (pretranslated || src.status === 'present') {
         const importable = trans.getChildExports(this.fullURL);
         if (this.notEmpty()) {
           // just import the named objects

packages/malloy/src/lang/ast/types/document-compile-result.ts

@@ -21,13 +21,11 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-import {ModelDef, Query, SQLSourceDef} from '../../../model/malloy_types';
+import {ModelDef} from '../../../model/malloy_types';
 
 import {ModelDataRequest} from '../../translate-response';
 
 export interface DocumentCompileResult {
   modelDef: ModelDef;
-  queryList: Query[];
-  sqlBlocks: SQLSourceDef[];
   needs: ModelDataRequest;
 }

packages/malloy/src/lang/ast/types/malloy-element.ts

@@ -30,7 +30,6 @@ import {
   ModelAnnotation,
   NamedModelObject,
   Query,
-  SQLSourceDef,
   StructDef,
   isSourceDef,
 } from '../../../model/malloy_types';
@@ -500,7 +499,6 @@ export class Document extends MalloyElement implements NameSpace {
   globalNameSpace: NameSpace = new GlobalNameSpace();
   documentModel: Record<string, ModelEntry> = {};
   queryList: Query[] = [];
-  sqlBlocks: SQLSourceDef[] = [];
   statements: DocStatementList;
   didInitModel = false;
   annotation: Annotation = {};
@@ -518,7 +516,6 @@ export class Document extends MalloyElement implements NameSpace {
     }
     this.documentModel = {};
     this.queryList = [];
-    this.sqlBlocks = [];
     if (extendingModelDef) {
       if (extendingModelDef.annotation) {
         this.annotation.inherits = extendingModelDef.annotation;
@@ -547,21 +544,17 @@ export class Document extends MalloyElement implements NameSpace {
           q.modelAnnotation = modelDef.annotation;
         }
       }
-      for (const q of this.sqlBlocks) {
-        if (q.modelAnnotation === undefined && modelDef.annotation) {
-          q.modelAnnotation = modelDef.annotation;
-        }
-      }
     }
     if (modelDef.annotation) {
       for (const sd of this.modelAnnotationTodoList) {
         sd.modelAnnotation ||= modelDef.annotation;
       }
     }
     const ret: DocumentCompileResult = {
-      modelDef,
-      queryList: this.queryList,
-      sqlBlocks: this.sqlBlocks,
+      modelDef: {
+        ...modelDef,
+        queryList: this.queryList,
+      },
       needs,
     };
     return ret;
@@ -588,7 +581,13 @@ export class Document extends MalloyElement implements NameSpace {
   }
 
   modelDef(): ModelDef {
-    const def: ModelDef = {name: '', exports: [], contents: {}};
+    const def: ModelDef = {
+      name: '',
+      exports: [],
+      contents: {},
+      queryList: [],
+      dependencies: {},
+    };
     if (this.hasAnnotation()) {
       def.annotation = this.currentModelAnnotation();
     }
@@ -608,22 +607,6 @@ export class Document extends MalloyElement implements NameSpace {
     return def;
   }
 
-  defineSQL(sql: SQLSourceDef, name?: string): boolean {
-    const ret = {
-      ...sql,
-      as: `$${this.sqlBlocks.length}`,
-    };
-    if (name) {
-      if (this.getEntry(name)) {
-        return false;
-      }
-      ret.as = name;
-      this.setEntry(name, {entry: ret});
-    }
-    this.sqlBlocks.push(ret);
-    return true;
-  }
-
   getEntry(str: string): ModelEntry {
     return this.globalNameSpace.getEntry(str) ?? this.documentModel[str];
   }

packages/malloy/src/lang/index.ts

@@ -21,7 +21,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-export {MalloyTranslator} from './parse-malloy';
+export {MalloyTranslator, MalloyTranslation} from './parse-malloy';
 export type {
   UpdateData,
   SchemaData,