feat: export-import using SDK (WIP)

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

packages/nocodb/tests/export-import/ReadMe.md

@@ -0,0 +1,23 @@
+## config.json
+{
+    "srcProject": "sample",
+    "dstProject": "sample-copy",
+    "baseURL": "http://localhost:8080",
+    "xc-auth": "Copy Auth Token"
+}
+- baseURL & xc-auth are common configurations for both import & export
+
+## Export
+- `srcProject`: specify source project name to be exported.
+- Export JSON file will be created as `srcProject.json`
+- execute 
+ `cd packages/nocodb/tests/export-import`
+ `node exportSchema.js`
+
+## Import
+- `srcProject`: specify JSON file name to be imported (sans .JSON suffix)
+- `dstProject`: new project name to be imported as
+- Data will also be imported if `srcProject` exists in NocoDB. Note that, data import isn't via exported JSON
+- execute
+  `cd packages/nocodb/tests/export-import`
+  `node importSchema.js`

packages/nocodb/tests/export-import/config.json

@@ -0,0 +1,6 @@
+{
+  "srcProject": "sample",
+  "dstProject": "sample-copy",
+  "baseURL": "http://localhost:8080",
+  "xc-auth": "Copy Auth Token"
+}

packages/nocodb/tests/export-import/exportSchema.js

@@ -0,0 +1,297 @@
+const Api = require('nocodb-sdk').Api;
+const { UITypes } = require('nocodb-sdk');
+const jsonfile = require('jsonfile');
+
+const GRID = 3, GALLERY = 2, FORM = 1;
+
+let ncMap = { /* id: name <string> */ };
+let tblSchema = [];
+let api = {};
+let viewStore = { columns: {}, sort: {}, filter: {} };
+
+let inputConfig = jsonfile.readFileSync(`config.json`)
+let ncConfig = {
+  projectName: inputConfig.srcProject,
+  baseURL: inputConfig.baseURL,
+  headers: {
+    'xc-auth': `${inputConfig["xc-auth"]}`
+  }
+};
+
+
+// helper routines
+// remove objects containing 0/ false/ null
+// fixme: how to handle when cdf (default value) is configured as 0/ null/ false
+function removeEmpty(obj) {
+  return Object.fromEntries(
+    Object.entries(obj)
+      .filter(([_, v]) => v != null && v != 0 && v != false)
+      .map(([k, v]) => [k, v === Object(v) ? removeEmpty(v) : v])
+  );
+}
+
+function addColumnSpecificData(c) {
+  // pick required fields to proceed further
+  let col = removeEmpty(
+    (({ id, title, column_name, uidt, dt, pk, pv, rqd, dtxp, system }) => ({
+      id,
+      title,
+      column_name,
+      uidt,
+      dt,
+      pk,
+      pv,
+      rqd,
+      dtxp,
+      system
+    }))(c)
+  );
+
+  switch (c.uidt) {
+    case UITypes.Formula:
+      col.formula = c.colOptions.formula;
+      col.formula_raw = c.colOptions.formula_raw;
+      break;
+    case UITypes.LinkToAnotherRecord:
+      col[`colOptions`] = {
+        fk_model_id: c.fk_model_id,
+        fk_related_model_id: c.colOptions.fk_related_model_id,
+        fk_child_column_id: c.colOptions.fk_child_column_id,
+        fk_parent_column_id: c.colOptions.fk_parent_column_id,
+        type: c.colOptions.type
+      };
+      break;
+    case UITypes.Lookup:
+      col[`colOptions`] = {
+        fk_model_id: c.fk_model_id,
+        fk_relation_column_id: c.colOptions.fk_relation_column_id,
+        fk_lookup_column_id: c.colOptions.fk_lookup_column_id
+      };
+      break;
+    case UITypes.Rollup:
+      col[`colOptions`] = {
+        fk_model_id: c.fk_model_id,
+        fk_relation_column_id: c.colOptions.fk_relation_column_id,
+        fk_rollup_column_id: c.colOptions.fk_rollup_column_id,
+        rollup_function: c.colOptions.rollup_function
+      };
+      break;
+  }
+
+  return col;
+}
+
+function addViewDetails(v) {
+  // pick required fields to proceed further
+  let view = (({ id, title, type, show_system_fields, lock_type, order }) => ({
+    id,
+    title,
+    type,
+    show_system_fields,
+    lock_type,
+    order
+  }))(v);
+
+  // form view
+  if (v.type === FORM) {
+    view.property = (({
+      heading,
+      subheading,
+      success_msg,
+      redirect_after_secs,
+      email,
+      submit_another_form,
+      show_blank_form
+    }) => ({
+      heading,
+      subheading,
+      success_msg,
+      redirect_after_secs,
+      email,
+      submit_another_form,
+      show_blank_form
+    }))(v.view);
+  }
+
+  // gallery view
+  else if (v.type === GALLERY) {
+    view.property = {
+      fk_cover_image_col_id: ncMap[v.view.fk_cover_image_col_id]
+    };
+  }
+
+  // gallery view doesn't share column information in api yet
+  if (v.type !== GALLERY) {
+    if (v.type === GRID)
+      view.columns = viewStore.columns[v.id].map(a =>
+        (({ id, width, order, show }) => ({ id, width, order, show }))(a)
+      );
+    if (v.type === FORM)
+      view.columns = viewStore.columns[v.id].map(a =>
+        (({ id, order, show, label, help, description, required }) => ({
+          id,
+          order,
+          show,
+          label,
+          help,
+          description,
+          required
+        }))(a)
+      );
+
+    for (let i = 0; i < view.columns?.length; i++)
+      view.columns[i].title = ncMap[viewStore.columns[v.id][i].id];
+
+    // skip hm & mm columns
+    view.columns = view.columns
+      ?.filter(a => a.title?.includes('_nc_m2m_') === false)
+      .filter(a => a.title?.includes('nc_') === false);
+  }
+
+  // filter & sort configurations
+  if (v.type !== FORM) {
+    view.sort = viewStore.sort[v.id].map(a =>
+      (({ fk_column_id, direction, order }) => ({
+        fk_column_id,
+        direction,
+        order
+      }))(a)
+    );
+    view.filter = viewStore.filter[v.id].map(a =>
+      (({ fk_column_id, logical_op, comparison_op, value, order }) => ({
+        fk_column_id,
+        logical_op,
+        comparison_op,
+        value,
+        order
+      }))(a)
+    );
+  }
+  return view;
+}
+
+// view data stored as is for quick access
+async function storeViewDetails(tableId) {
+  // read view data for each table
+  let viewList = await api.dbView.list(tableId);
+  for (let j = 0; j < viewList.list.length; j++) {
+    let v = viewList.list[j];
+    let viewDetails = [];
+
+    // invoke view specific read to populate columns information
+    if (v.type === FORM) viewDetails = (await api.dbView.formRead(v.id)).columns;
+    else if (v.type === GALLERY) viewDetails = await api.dbView.galleryRead(v.id);
+    else if (v.type === GRID) viewDetails = await api.dbView.gridColumnsList(v.id);
+    viewStore.columns[v.id] = viewDetails;
+
+    // populate sort information
+    let vSort = await api.dbTableSort.list(v.id);
+    viewStore.sort[v.id] = vSort.sorts.list;
+
+    let vFilter = await api.dbTableFilter.read(v.id);
+    viewStore.filter[v.id] = vFilter;
+  }
+}
+
+// mapping table for quick information access
+// store maps for tableId, columnId, viewColumnId & viewId to their names
+async function generateMapTbl(pId) {
+  const tblList = await api.dbTable.list(pId);
+
+  for (let i = 0; i < tblList.list.length; i++) {
+    let tblId = tblList.list[i].id;
+    let tbl = await api.dbTable.read(tblId);
+
+    // table ID <> name
+    ncMap[tblId] = tbl.title;
+
+    // column ID <> name
+    tbl.columns.map(x => (ncMap[x.id] = x.title));
+
+    // view ID <> name
+    tbl.views.map(x => (ncMap[x.id] = x.tn));
+
+    for (let i = 0; i < tbl.views.length; i++) {
+      let x = tbl.views[i];
+      let viewColumns = [];
+      if (x.type === FORM) viewColumns = (await api.dbView.formRead(x.id)).columns;
+      else if (x.type === GALLERY)
+        viewColumns = (await api.dbView.galleryRead(x.id)).columns;
+      else if (x.type === GRID) viewColumns = await api.dbView.gridColumnsList(x.id);
+
+      // view column ID <> name
+      viewColumns?.map(a => (ncMap[a.id] = ncMap[a.fk_column_id]));
+    }
+  }
+}
+
+// main
+//
+async function exportSchema() {
+  api = new Api(ncConfig);
+
+  // fetch project details (id et.al)
+  const x = await api.project.list();
+  const p = x.list.find(a => a.title === ncConfig.projectName);
+
+  await generateMapTbl(p.id);
+
+  // read project
+  const tblList = await api.dbTable.list(p.id);
+
+  // for each table
+  for (let i = 0; i < tblList.list.length; i++) {
+    let tblId = tblList.list[i].id;
+    await storeViewDetails(tblId);
+
+    let tbl = await api.dbTable.read(tblId);
+
+    // prepare schema
+    let tSchema = {
+      id: tbl.id,
+      title: tbl.title,
+      table_name: tbl?.table_name,
+      columns: [...tbl.columns.map(c => addColumnSpecificData(c))]
+        .filter(a => a.title.includes('_nc_m2m_') === false)        // mm
+        .filter(a => a.title.includes(p.prefix) === false)          // hm
+        .filter(
+          a => !(a?.system === 1 && a.uidt === UITypes.LinkToAnotherRecord)
+        ),
+      views: [...tbl.views.map(v => addViewDetails(v))]
+    };
+    tblSchema.push(tSchema);
+  }
+}
+
+(async () => {
+  await exportSchema();
+  jsonfile.writeFileSync(
+    `${ncConfig.projectName.replace(/ /g, '_')}.json`,
+    tblSchema,
+    { spaces: 2 }
+  );
+})().catch(e => {
+  console.log(e);
+});
+
+/**
+ * @copyright Copyright (c) 2021, Xgene Cloud Ltd
+ *
+ * @author Raju Udava <sivadstala@gmail.com>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */

packages/nocodb/tests/export-import/importSchema.js

@@ -0,0 +1,537 @@
+// tbd
+// - formula dependency list
+// - nested lookup/ rollup
+
+const Api = require('nocodb-sdk').Api;
+const { UITypes } = require('nocodb-sdk');
+const jsonfile = require('jsonfile');
+
+let inputConfig = jsonfile.readFileSync(`config.json`)
+let ncConfig = {
+  srcProject: inputConfig.srcProject,
+  projectName: inputConfig.dstProject,
+  baseURL: inputConfig.baseURL,
+  headers: {
+    'xc-auth': `${inputConfig["xc-auth"]}`
+  }
+};
+let ncIn = jsonfile.readFileSync(`${ncConfig.srcProject}.json`);
+
+let api = {};
+let ncProject = {};
+let link = [];
+let lookup = [];
+let rollup = [];
+let formula = [];
+
+let rootLinks = [];
+
+// maps v1 table ID, v2 table ID & table title to table schema
+let ncTables = {};
+
+
+async function createBaseTables() {
+  console.log(`createBaseTables`);
+  for (let i = 0; i < ncIn.length; i++) {
+    let tblSchema = ncIn[i];
+    let reducedColumnSet = tblSchema.columns.filter(
+      a =>
+        a.uidt !== UITypes.LinkToAnotherRecord &&
+        a.uidt !== UITypes.Lookup &&
+        a.uidt !== UITypes.Rollup &&
+        a.uidt !== UITypes.Formula
+    );
+    link.push(
+      ...tblSchema.columns.filter(a => a.uidt === UITypes.LinkToAnotherRecord)
+    );
+    lookup.push(...tblSchema.columns.filter(a => a.uidt === UITypes.Lookup));
+    rollup.push(...tblSchema.columns.filter(a => a.uidt === UITypes.Rollup));
+    formula.push(...tblSchema.columns.filter(a => a.uidt === UITypes.Formula));
+    formula.map(a => (a['table_id'] = tblSchema.id));
+
+    let tbl = await api.dbTable.create(ncProject.id, {
+      title: tblSchema.title,
+      table_name: tblSchema.title,
+      columns: reducedColumnSet.map(({ id, ...rest }) => ({ ...rest }))
+    });
+    ncTables[tbl.title] = tbl;
+    ncTables[tbl.id] = tbl;
+    ncTables[tblSchema.id] = tbl;
+  }
+}
+
+let linksCreated = [];
+function isLinkCreated(pId, cId) {
+  let idx = linksCreated.findIndex(a => a.cId === pId && a.pId === cId);
+  if (idx === -1) {
+    linksCreated.push({ pId: pId, cId: cId });
+    return false;
+  }
+  return true;
+}
+
+// retrieve nc-view column ID from corresponding nc-column ID
+async function nc_getViewColumnId(viewId, viewType, ncColumnId) {
+  // retrieve view Info
+  let viewDetails;
+
+  if (viewType === 'form')
+    viewDetails = (await api.dbView.formRead(viewId)).columns;
+  else if (viewType === 'gallery')
+    viewDetails = (await api.dbView.galleryRead(viewId)).columns;
+  else viewDetails = await api.dbView.gridColumnsList(viewId);
+
+  return viewDetails.find(x => x.fk_column_id === ncColumnId)?.id;
+}
+
+async function createFormula() {
+  for (let i = 0; i < formula.length; i++) {
+    let tbl = await api.dbTableColumn.create(ncTables[formula[i].table_id].id, {
+      uidt: UITypes.Formula,
+      title: formula[i].title,
+      formula_raw: formula[i].formula_raw
+    });
+  }
+}
+
+async function createLinks() {
+  console.log(`createLinks`);
+
+  for (let i = 0; i < link.length; i++) {
+    if (
+      (link[i].colOptions.type === 'mm' &&
+        false ===
+          isLinkCreated(
+            link[i].colOptions.fk_parent_column_id,
+            link[i].colOptions.fk_child_column_id
+          )) ||
+      link[i].colOptions.type === 'hm'
+    ) {
+      let srcTbl = ncTables[link[i].colOptions.fk_model_id];
+      let dstTbl = ncTables[link[i].colOptions.fk_related_model_id];
+
+      // create link
+      let tbl = await api.dbTableColumn.create(srcTbl.id, {
+        uidt: UITypes.LinkToAnotherRecord,
+        title: link[i].title,
+        parentId: srcTbl.id,
+        childId: dstTbl.id,
+        type: link[i].colOptions.type
+      });
+      ncTables[tbl.title] = tbl;
+      ncTables[tbl.id] = tbl;
+      ncTables[link[i].colOptions.fk_model_id] = tbl;
+
+      // for data-link procedure later
+      rootLinks.push({ linkColumn: link[i], linkSrcTbl: srcTbl });
+
+      // symmetry field update
+      //
+      let v2ColSchema = tbl.columns.find(x => x.title === link[i].title);
+      // read related table again after link is created
+      dstTbl = await api.dbTable.read(dstTbl.id);
+      let v2SymmetricColumn =
+        link[i].colOptions.type === 'mm'
+          ? dstTbl.columns.find(
+              x =>
+                x.uidt === UITypes.LinkToAnotherRecord &&
+                x?.colOptions.fk_parent_column_id ===
+                  v2ColSchema.colOptions.fk_child_column_id &&
+                x?.colOptions.fk_child_column_id ===
+                  v2ColSchema.colOptions.fk_parent_column_id
+            )
+          : dstTbl.columns.find(
+              x =>
+                x.uidt === UITypes.LinkToAnotherRecord &&
+                x?.colOptions.fk_parent_column_id ===
+                  v2ColSchema.colOptions.fk_parent_column_id &&
+                x?.colOptions.fk_child_column_id ===
+                  v2ColSchema.colOptions.fk_child_column_id
+            );
+      let v1SymmetricColumn =
+        link[i].colOptions.type === 'mm'
+          ? link.find(
+              x =>
+                x.colOptions.fk_parent_column_id ===
+                  link[i].colOptions.fk_child_column_id &&
+                x.colOptions.fk_child_column_id ===
+                  link[i].colOptions.fk_parent_column_id &&
+                x.colOptions.type === 'mm'
+            )
+          : link.find(
+              x =>
+                x.colOptions.fk_parent_column_id ===
+                  link[i].colOptions.fk_parent_column_id &&
+                x.colOptions.fk_child_column_id ===
+                  link[i].colOptions.fk_child_column_id &&
+                x.colOptions.type === 'bt'
+            );
+
+      tbl = await api.dbTableColumn.update(v2SymmetricColumn.id, {
+        ...v2SymmetricColumn,
+        title: v1SymmetricColumn.title,
+        column_name: null
+      });
+      ncTables[tbl.title] = tbl;
+      ncTables[tbl.id] = tbl;
+      ncTables[v1SymmetricColumn.colOptions.fk_model_id] = tbl;
+    }
+  }
+}
+
+function get_v2Id(v1ColId) {
+  for (let i = 0; i < ncIn.length; i++) {
+    let tblSchema = ncIn[i];
+    let colSchema = {};
+    if (
+      undefined !== (colSchema = tblSchema.columns.find(x => x.id === v1ColId))
+    ) {
+      let colName = colSchema.title;
+      let v2Tbl = ncTables[tblSchema.id];
+      return v2Tbl.columns.find(y => y.title === colName)?.id;
+    }
+  }
+}
+
+async function createLookup() {
+  console.log(`createLookup`);
+
+  for (let i = 0; i < lookup.length; i++) {
+    let srcTbl = ncTables[lookup[i].colOptions.fk_model_id];
+    let v2_fk_relation_column_id = get_v2Id(
+      lookup[i].colOptions.fk_relation_column_id
+    );
+    let v2_lookup_column_id = get_v2Id(
+      lookup[i].colOptions.fk_lookup_column_id
+    );
+
+    if (v2_lookup_column_id) {
+      let tbl = await api.dbTableColumn.create(srcTbl.id, {
+        uidt: UITypes.Lookup,
+        title: lookup[i].title,
+        fk_relation_column_id: v2_fk_relation_column_id,
+        fk_lookup_column_id: v2_lookup_column_id
+      });
+      ncTables[tbl.title] = tbl;
+      ncTables[tbl.id] = tbl;
+      ncTables[lookup[i].colOptions.fk_model_id] = tbl;
+    }
+  }
+}
+
+async function createRollup() {
+  console.log(`createRollup`);
+
+  for (let i = 0; i < rollup.length; i++) {
+    let srcTbl = ncTables[rollup[i].colOptions.fk_model_id];
+    let v2_fk_relation_column_id = get_v2Id(
+      rollup[i].colOptions.fk_relation_column_id
+    );
+    let v2_rollup_column_id = get_v2Id(
+      rollup[i].colOptions.fk_rollup_column_id
+    );
+
+    if (v2_rollup_column_id) {
+      let tbl = await api.dbTableColumn.create(srcTbl.id, {
+        uidt: UITypes.Rollup,
+        title: rollup[i].title,
+        column_name: rollup[i].title,
+        fk_relation_column_id: v2_fk_relation_column_id,
+        fk_rollup_column_id: v2_rollup_column_id,
+        rollup_function: rollup[i].colOptions.rollup_function
+      });
+      ncTables[tbl.title] = tbl;
+      ncTables[tbl.id] = tbl;
+      ncTables[rollup[i].colOptions.fk_model_id] = tbl;
+    }
+  }
+}
+
+async function configureGrid() {
+  console.log(`configureGrid`);
+
+  for (let i = 0; i < ncIn.length; i++) {
+    let tblSchema = ncIn[i];
+    let tblId = ncTables[tblSchema.id].id;
+    let gridList = tblSchema.views.filter(a => a.type === 3);
+    let srcTbl = await api.dbTable.read(tblId);
+
+    const view = await api.dbView.list(tblId);
+
+    // create / rename view
+    for (let gridCnt = 0; gridCnt < gridList.length; gridCnt++) {
+      let viewCreated = {};
+      // rename first view; default view already created
+      if (gridCnt === 0) {
+        viewCreated = await api.dbView.update(view.list[0].id, {
+          title: gridList[gridCnt].title
+        });
+      }
+      // create new views
+      else {
+        viewCreated = await api.dbView.gridCreate(tblId, {
+          title: gridList[gridCnt].title
+        });
+      }
+
+      // retrieve view Info
+      let viewId = viewCreated.id;
+      let viewDetails = await api.dbView.gridColumnsList(viewId);
+
+      // column visibility
+      for (
+        let colCnt = 0;
+        colCnt < gridList[gridCnt].columns.length;
+        colCnt++
+      ) {
+        let ncColumnId = srcTbl.columns.find(
+          a => a.title === gridList[gridCnt].columns[colCnt].title
+        )?.id;
+        // let ncViewColumnId = await nc_getViewColumnId( viewCreated.id, "grid", ncColumnId )
+        let ncViewColumnId = viewDetails.find(
+          x => x.fk_column_id === ncColumnId
+        )?.id;
+        // column order & visibility
+        await api.dbViewColumn.update(viewCreated.id, ncViewColumnId, {
+          show: gridList[gridCnt].columns[colCnt].show,
+          order: gridList[gridCnt].columns[colCnt].order
+        });
+        await api.dbView.gridColumnUpdate(ncViewColumnId, {
+          width: gridList[gridCnt].columns[colCnt].width
+        });
+      }
+
+      // sort
+      for (let sCnt = 0; sCnt < gridList[gridCnt].sort.length; sCnt++) {
+        let sColName = tblSchema.columns.find(
+          a => gridList[gridCnt].sort[sCnt].fk_column_id === a.id
+        ).title;
+        await api.dbTableSort.create(viewId, {
+          fk_column_id: srcTbl.columns.find(a => a.title === sColName)?.id,
+          direction: gridList[gridCnt].sort[sCnt].direction
+        });
+      }
+
+      // filter
+      for (let fCnt = 0; fCnt < gridList[gridCnt].filter.length; fCnt++) {
+        let fColName = tblSchema.columns.find(
+          a => gridList[gridCnt].sort[fCnt].fk_column_id === a.id
+        ).title;
+        await api.dbTableFilter.create(viewId, {
+          ...gridList[gridCnt].filter[fCnt],
+          fk_column_id: srcTbl.columns.find(a => a.title === fColName)?.id
+        });
+      }
+    }
+  }
+}
+
+async function configureGallery() {
+  console.log(`configureGallery`);
+
+  for (let i = 0; i < ncIn.length; i++) {
+    let tblSchema = ncIn[i];
+    let tblId = ncTables[tblSchema.id].id;
+    let galleryList = tblSchema.views.filter(a => a.type === 2);
+    for (let cnt = 0; cnt < galleryList.length; cnt++) {
+      const viewCreated = await api.dbView.galleryCreate(tblId, {
+        title: galleryList[cnt].title
+      });
+    }
+  }
+}
+
+async function configureForm() {
+  console.log(`configureForm`);
+
+  for (let i = 0; i < ncIn.length; i++) {
+    let tblSchema = ncIn[i];
+    let tblId = ncTables[tblSchema.id].id;
+    let formList = tblSchema.views.filter(a => a.type === 1);
+    let srcTbl = await api.dbTable.read(tblId);
+
+    for (let formCnt = 0; formCnt < formList.length; formCnt++) {
+      const formData = {
+        title: formList[formCnt].title,
+        ...formList[formCnt].property
+      };
+      const viewCreated = await api.dbView.formCreate(tblId, formData);
+
+      // column visibility
+      for (
+        let colCnt = 0;
+        colCnt < formList[formCnt].columns.length;
+        colCnt++
+      ) {
+        let ncColumnId = srcTbl.columns.find(
+          a => a.title === formList[formCnt].columns[colCnt].title
+        )?.id;
+        let ncViewColumnId = await nc_getViewColumnId(
+          viewCreated.id,
+          'form',
+          ncColumnId
+        );
+        // column order & visibility
+        await api.dbView.formColumnUpdate(ncViewColumnId, {
+          show: formList[formCnt].columns[colCnt].show,
+          order: formList[formCnt].columns[colCnt].order,
+          label: formList[formCnt].columns[colCnt].label,
+          description: formList[formCnt].columns[colCnt].description,
+          required: formList[formCnt].columns[colCnt].required
+        });
+      }
+    }
+  }
+}
+
+async function restoreBaseData() {
+  console.log(`restoreBaseData`);
+
+  for (let i = 0; i < ncIn.length; i++) {
+    let tblSchema = ncIn[i];
+    let tblId = ncTables[tblSchema.id].id;
+    let pk = tblSchema.columns.find(a => a.pk).title;
+
+    let moreRecords = true;
+    let offset = 0,
+      limit = 25;
+
+    while (moreRecords) {
+      let recList = await api.dbTableRow.list(
+        'nc',
+        ncConfig.srcProject,
+        tblSchema.title,
+        {},
+        {
+          query: { limit: limit, offset: offset }
+        }
+      );
+      moreRecords = !recList.pageInfo.isLastPage;
+      offset += limit;
+
+      for (let recCnt = 0; recCnt < recList.list.length; recCnt++) {
+        let record = await api.dbTableRow.read(
+          'nc',
+          ncConfig.srcProject,
+          tblSchema.title,
+          recList.list[recCnt][pk]
+        );
+
+        // post-processing on the record
+        for (const [key, value] of Object.entries(record)) {
+          let table = ncTables[tblId];
+          // retrieve datatype
+          const dt = table.columns.find(x => x.title === key)?.uidt;
+          if (dt === UITypes.LinkToAnotherRecord) delete record[key];
+          if (dt === UITypes.Lookup) delete record[key];
+          if (dt === UITypes.Rollup) delete record[key];
+        }
+        await api.dbTableRow.create(
+          'nc',
+          ncConfig.projectName,
+          tblSchema.title,
+          record
+        );
+      }
+    }
+  }
+}
+
+async function restoreLinks() {
+  console.log(`restoreLinks`);
+
+  for (let i = 0; i < rootLinks.length; i++) {
+    let pk = rootLinks[i].linkSrcTbl.columns.find(a => a.pk).title;
+    let moreRecords = true;
+    let offset = 0,
+      limit = 25;
+
+    while (moreRecords) {
+      let recList = await api.dbTableRow.list(
+        'nc',
+        ncConfig.srcProject,
+        rootLinks[i].linkSrcTbl.title,
+        {},
+        {
+          query: { limit: limit, offset: offset }
+        }
+      );
+      moreRecords = !recList.pageInfo.isLastPage;
+      offset += limit;
+
+      for (let recCnt = 0; recCnt < recList.list.length; recCnt++) {
+        let record = await api.dbTableRow.read(
+          'nc',
+          ncConfig.srcProject,
+          rootLinks[i].linkSrcTbl.title,
+          recList.list[recCnt][pk]
+        );
+        let linkField = record[rootLinks[i].linkColumn.title];
+        if (linkField.length) {
+          await api.dbTableRow.nestedAdd(
+            'nc',
+            ncConfig.projectName,
+            rootLinks[i].linkSrcTbl.title,
+            record[pk],
+            rootLinks[i].linkColumn.colOptions.type,
+            encodeURIComponent(rootLinks[i].linkColumn.title),
+            linkField[0][pk]
+          );
+        }
+      }
+    }
+  }
+}
+
+async function importSchema() {
+  api = new Api(ncConfig);
+
+  const x = await api.project.list();
+  const p = x.list.find(a => a.title === ncConfig.projectName);
+  if (p) await api.project.delete(p.id);
+  ncProject = await api.project.create({ title: ncConfig.projectName });
+
+  await createBaseTables();
+  await createLinks();
+  await createLookup();
+  await createRollup();
+  await createFormula();
+
+  // configure views
+  await configureGrid();
+  await configureGallery();
+  await configureForm();
+
+  // restore data only if source project exists
+  const p2 = x.list.find(a => a.title === ncConfig.srcProject);
+  if (p2 !== undefined) {
+    await restoreBaseData();
+    await restoreLinks();
+  }
+}
+(async () => {
+  await importSchema();
+  console.log('completed');
+})().catch(e => console.log(e));
+
+/**
+ * @copyright Copyright (c) 2021, Xgene Cloud Ltd
+ *
+ * @author Raju Udava <sivadstala@gmail.com>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */