wip: extract dependency columns

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts

@@ -59,10 +59,16 @@ async function populatePk(model: Model, insertObj: any) {
   }
 }
 
-function checkColumnRequired(column: Column<any>, fields: string[]) {
+function checkColumnRequired(
+  column: Column<any>,
+  fields: string[],
+  extractPkAndPv?: boolean
+) {
   // if primary key or foreign key included in fields, it's required
   if (column.pk || column.uidt === UITypes.ForeignKey) return true;
 
+  if (extractPkAndPv && column.pv) return true;
+
   // check fields defined and if not, then select all
   // if defined check if it is in the fields
   return !fields || fields.includes(column.title);
@@ -189,7 +195,7 @@ class BaseModelSqlv2 {
     const { where, fields, ...rest } = this._getListArgs(args as any);
 
     const qb = this.dbDriver(this.tnPath);
-    await this.selectObject({ qb, fields });
+    await this.selectObject({ qb, fields, viewId: this.viewId });
     if (+rest?.shuffle) {
       await this.shuffle({ qb });
     }
@@ -264,7 +270,7 @@ class BaseModelSqlv2 {
     const proto = await this.getProto();
     const data = await this.extractRawQueryAndExec(qb);
 
-    console.log(qb.toQuery());
+    // console.log(qb.toQuery());
 
     return data?.map((d) => {
       d.__proto__ = proto;
@@ -404,7 +410,7 @@ class BaseModelSqlv2 {
       await parentTable.getColumns();
 
       const qb = this.dbDriver(childTable.table_name);
-      await childModel.selectObject({ qb });
+      await childModel.selectObject({ qb, extractPkAndPv: true });
 
       const childQb = this.dbDriver.queryBuilder().from(
         this.dbDriver
@@ -431,6 +437,9 @@ class BaseModelSqlv2 {
           .as('list')
       );
 
+
+      // console.log(childQb.toQuery())
+
       const children = await this.extractRawQueryAndExec(childQb);
       const proto = await (
         await Model.getBaseModelSQL({
@@ -439,6 +448,7 @@ class BaseModelSqlv2 {
         })
       ).getProto();
 
+
       return _.groupBy(
         children.map((c) => {
           c.__proto__ = proto;
@@ -1303,15 +1313,22 @@ class BaseModelSqlv2 {
     }
   }
 
+  // todo:
+  //  pass view id as argument
+  //  add option to get only pk and pv
   public async selectObject({
     qb,
     fields: _fields,
+    extractPkAndPv,
+    viewId,
   }: {
     qb: QueryBuilder;
     fields?: string[] | string;
+    extractPkAndPv?: boolean;
+    viewId?: string;
   }): Promise<void> {
-    const view = await View.get(this.viewId);
-    const viewColumns = this.viewId && (await View.getColumns(this.viewId));
+    const view = await View.get(viewId);
+    const viewColumns = viewId && (await View.getColumns(viewId));
     const fields = Array.isArray(_fields) ? _fields : _fields?.split(',');
     const res = {};
     const viewOrTableColumns = viewColumns || (await this.model.getColumns());
@@ -1325,14 +1342,17 @@ class BaseModelSqlv2 {
       // hide if column marked as hidden in view
       // of if column is system field and system field is hidden
       if (
+        !extractPkAndPv &&
         !(viewOrTableColumn instanceof Column) &&
         (!(viewOrTableColumn as GridViewColumn)?.show ||
           (!view?.show_system_fields &&
             column.uidt !== UITypes.ForeignKey &&
+            !column.pk &&
             isSystemColumn(column)))
       )
         continue;
-      if (!checkColumnRequired(column, fields)) continue;
+
+      if (!checkColumnRequired(column, fields, extractPkAndPv)) continue;
 
       switch (column.uidt) {
         case 'LinkToAnotherRecord':

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts

@@ -1,7 +1,9 @@
-import View from '../../../../../models/View';
-import { isSystemColumn, UITypes } from 'nocodb-sdk';
-import Model from '../../../../../models/Model';
+import { isSystemColumn, RelationTypes, UITypes } from 'nocodb-sdk';
+import Column from '../../../../../models/Column';
 import LinkToAnotherRecordColumn from '../../../../../models/LinkToAnotherRecordColumn';
+import LookupColumn from '../../../../../models/LookupColumn';
+import Model from '../../../../../models/Model';
+import View from '../../../../../models/View';
 
 const getAst = async ({
   query,
@@ -9,23 +11,34 @@ const getAst = async ({
   includePkByDefault = true,
   model,
   view,
+  dependencyFields = {
+    nested: {},
+    fields: new Set(),
+  },
 }: {
   query?: RequestQuery;
   extractOnlyPrimaries?: boolean;
   includePkByDefault?: boolean;
   model: Model;
   view?: View;
+  dependencyFields?: DependantFields;
 }) => {
   if (!model.columns?.length) await model.getColumns();
 
   // extract only pk and pv
   if (extractOnlyPrimaries) {
-    return {
+    const ast = {
       ...(model.primaryKeys
         ? model.primaryKeys.reduce((o, pk) => ({ ...o, [pk.title]: 1 }), {})
         : {}),
       ...(model.primaryValue ? { [model.primaryValue.title]: 1 } : {}),
     };
+    await Promise.all(
+      model.primaryKeys.map((c) => extractDependencies(c, dependencyFields))
+    );
+    await extractDependencies(model.primaryValue, dependencyFields);
+
+    return { ast, dependencyFields };
   }
 
   let fields = query?.fields || query?.f;
@@ -45,7 +58,7 @@ const getAst = async ({
       {}
     );
 
-  return model.columns.reduce(async (obj, col) => {
+  const ast = await model.columns.reduce(async (obj, col) => {
     let value: number | boolean | { [key: string]: any } = 1;
     const nestedFields =
       query?.nested?.[col.title]?.fields || query?.nested?.[col.title]?.f;
@@ -55,10 +68,19 @@ const getAst = async ({
           .getColOptions<LinkToAnotherRecordColumn>()
           .then((colOpt) => colOpt.getRelatedTable());
 
-        value = await getAst({
+        const { ast } = await getAst({
           model,
           query: query?.nested?.[col.title],
+          dependencyFields: (dependencyFields.nested[col.title] =
+            dependencyFields.nested[col.title] || {
+              nested: {},
+              fields: new Set(),
+            }),
         });
+
+        value = ast;
+
+        // todo: include field relative to the relation => pk / fk
       } else {
         value = (Array.isArray(fields) ? fields : fields.split(',')).reduce(
           (o, f) => ({ ...o, [f]: 1 }),
@@ -74,22 +96,98 @@ const getAst = async ({
         model,
         query: query?.nested?.[col.title],
         extractOnlyPrimaries: nestedFields !== '*',
+        dependencyFields:(dependencyFields.nested[col.title] =
+        dependencyFields.nested[col.title] || {
+          nested: {},
+          fields: new Set(),
+        })
       });
     }
 
+    const isRequested =
+      allowedCols && (!includePkByDefault || !col.pk)
+        ? allowedCols[col.id] &&
+          (!isSystemColumn(col) || view.show_system_fields) &&
+          (!fields?.length || fields.includes(col.title)) &&
+          value
+        : fields?.length
+        ? fields.includes(col.title) && value
+        : value;
+    if (isRequested) await extractDependencies(col, dependencyFields);
+
     return {
       ...(await obj),
-      [col.title]:
-        allowedCols && (!includePkByDefault || !col.pk)
-          ? allowedCols[col.id] &&
-            (!isSystemColumn(col) || view.show_system_fields) &&
-            (!fields?.length || fields.includes(col.title)) &&
-            value
-          : fields?.length
-          ? fields.includes(col.title) && value
-          : value,
+      [col.title]: isRequested,
     };
   }, Promise.resolve({}));
+
+  return { ast, dependencyFields };
+};
+
+const extractDependencies = async (
+  column: Column,
+  dependencyFields: DependantFields = {
+    nested: {},
+    fields: new Set(),
+  }
+) => {
+  switch (column.uidt) {
+    case UITypes.Lookup:
+      await extractLookupDependencies(column, dependencyFields);
+      break;
+    case UITypes.LinkToAnotherRecord:
+      await extractRelationDependencies(column, dependencyFields);
+      break;
+    default:
+      dependencyFields.fields.add(column.title);
+      break;
+  }
+};
+
+const extractLookupDependencies = async (
+  lookUpColumn: Column<LookupColumn>,
+  dependencyFields: DependantFields = {
+    nested: {},
+    fields: new Set(),
+  }
+) => {
+  const lookupColumnOpts = await lookUpColumn.getColOptions();
+  const relationColumn = await lookupColumnOpts.getRelationColumn();
+  await extractRelationDependencies(relationColumn, dependencyFields);
+  await extractDependencies(
+    await lookupColumnOpts.getLookupColumn(),
+    (dependencyFields.nested[relationColumn.title] = dependencyFields.nested[
+      relationColumn.title
+    ] || {
+      nested: {},
+      fields: new Set(),
+    })
+  );
+};
+
+const extractRelationDependencies = async (
+  relationColumn: Column<LinkToAnotherRecordColumn>,
+  dependencyFields: DependantFields = {
+    nested: {},
+    fields: new Set(),
+  }
+) => {
+  const relationColumnOpts = await relationColumn.getColOptions();
+
+  switch (relationColumnOpts.type) {
+    case RelationTypes.HAS_MANY:
+      dependencyFields.fields.add(
+        await relationColumnOpts.getParentColumn().then((col) => col.title)
+      );
+      break;
+    case RelationTypes.BELONGS_TO:
+    case RelationTypes.MANY_TO_MANY:
+      dependencyFields.fields.add(
+        await relationColumnOpts.getChildColumn().then((col) => col.title)
+      );
+
+      break;
+  }
 };
 
 type RequestQuery = {
@@ -100,4 +198,9 @@ type RequestQuery = {
   };
 };
 
+interface DependantFields {
+  fields?: Set<string>;
+  nested?: DependantFields;
+}
+
 export default getAst;

packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts

@@ -97,7 +97,11 @@ async function getDataList(model, view: View, req) {
     dbDriver: NcConnectionMgrv2.get(base),
   });
 
-  const requestObj = await getAst({ model, query: req.query, view });
+  const { ast, dependencyFields } = await getAst({
+    model,
+    query: req.query,
+    view,
+  });
 
   const listArgs: any = { ...req.query };
   try {
@@ -107,8 +111,21 @@ async function getDataList(model, view: View, req) {
     listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
   } catch (e) {}
 
+  console.log(
+    JSON.stringify(
+      dependencyFields,
+      (_v, o) => {
+        if (o instanceof Set) {
+          return [...o];
+        }
+        return o;
+      },
+      2
+    )
+  );
+  console.log(JSON.stringify(ast, null, 2));
   const data = await nocoExecute(
-    requestObj,
+    ast,
     await baseModel.list(listArgs),
     {},
     listArgs

packages/nocodb/src/lib/models/Column.ts

@@ -308,7 +308,7 @@ export default class Column<T = any> implements ColumnType {
     }
   }
 
-  public async getColOptions<T>(ncMeta = Noco.ncMeta): Promise<T> {
+  public async getColOptions<U = T>(ncMeta = Noco.ncMeta): Promise<U> {
     let res: any;
 
     switch (this.uidt) {