sync: attachments (wip)

TODO: use api when changed to ts
Signed-off-by: mertmit <mertmit99@gmail.com>

packages/nocodb/tests/sync/attachment.json

@@ -0,0 +1,337 @@
+{
+  "appBlanket": {
+    "userInfoById": {
+      "usrcTFn14vKTIgbW3": {
+        "id": "usrcTFn14vKTIgbW3",
+        "firstName": "Steyer",
+        "lastName": "Rom",
+        "email": "steyerrom@gmail.com",
+        "profilePicUrl": "https://static.airtable.com/images/userIcons/user_icon_9.png",
+        "permissionLevel": "owner",
+        "appBlanketUserState": "active"
+      }
+    },
+    "externalAccountInfoById": {},
+    "userGroupInfoById": {},
+    "workspaceSyncSources": [],
+    "activeUserIdByAcceptedInviteId": {},
+    "isWorkspaceOptedOutOfUserContentCdnAuth": false,
+    "isEnterpriseAccountOptedOutOfUserContentCdnAuth": false,
+    "enterpriseAttachmentRestrictions": {
+      "restrictionType": "unrestricted",
+      "attachmentTypeAllowlist": []
+    },
+    "isWorkspaceLinkedToEnterpriseAccount": false
+  },
+  "description": null,
+  "sortTiebreakerKey": "appEHTLsc4lSaia9A",
+  "defaultViewMutability": null,
+  "maintenanceModeSettings": null,
+  "sharesById": {
+    "shrqM5QS9sSZ94mQx": {
+      "id": "shrqM5QS9sSZ94mQx",
+      "modelId": "appEHTLsc4lSaia9A",
+      "createdByUserId": "usrcTFn14vKTIgbW3",
+      "canBeCloned": false,
+      "canBeExported": false,
+      "includeHiddenColumns": false,
+      "includeBlocks": true,
+      "emailDomain": null,
+      "hasPassword": false,
+      "generationNumber": 0,
+      "metadata": null
+    }
+  },
+  "workflowSectionsById": {},
+  "applicationTransactionNumber": 21,
+  "tableSchemas": [
+    {
+      "id": "tblXYuhMZ3hWZkBCa",
+      "name": "Table 1",
+      "primaryColumnId": "fldrhmH0EYnOXfnUA",
+      "columns": [
+        {
+          "id": "fldrhmH0EYnOXfnUA",
+          "name": "Name",
+          "type": "text"
+        },
+        {
+          "id": "fldVFoK7t6aC92xzj",
+          "name": "Notes",
+          "type": "multilineText"
+        },
+        {
+          "id": "fld66bK6Pq8AG4m3h",
+          "name": "Attachments",
+          "type": "multipleAttachment",
+          "typeOptions": {
+            "unreversed": true
+          }
+        },
+        {
+          "id": "fldRC8zKoyGzM6agG",
+          "name": "Status",
+          "type": "select",
+          "typeOptions": {
+            "choices": {
+              "selQSYarqhTyVrwZw": {
+                "id": "selQSYarqhTyVrwZw",
+                "name": "Todo",
+                "color": "red"
+              },
+              "selOFyoifOyV50QI7": {
+                "id": "selOFyoifOyV50QI7",
+                "name": "In progress",
+                "color": "yellow"
+              },
+              "selsxseijq4XvcTB8": {
+                "id": "selsxseijq4XvcTB8",
+                "name": "Done",
+                "color": "green"
+              }
+            },
+            "choiceOrder": [
+              "selQSYarqhTyVrwZw",
+              "selOFyoifOyV50QI7",
+              "selsxseijq4XvcTB8"
+            ]
+          }
+        }
+      ],
+      "meaningfulColumnOrder": [
+        {
+          "columnId": "fldrhmH0EYnOXfnUA",
+          "visibility": true
+        },
+        {
+          "columnId": "fldVFoK7t6aC92xzj",
+          "visibility": true
+        },
+        {
+          "columnId": "fld66bK6Pq8AG4m3h",
+          "visibility": true
+        },
+        {
+          "columnId": "fldRC8zKoyGzM6agG",
+          "visibility": true
+        }
+      ],
+      "views": [
+        {
+          "id": "viwbgKWGvUoZCosF1",
+          "name": "Grid view",
+          "type": "grid",
+          "createdByUserId": "usrcTFn14vKTIgbW3"
+        }
+      ],
+      "viewOrder": [
+        "viwbgKWGvUoZCosF1"
+      ],
+      "viewsById": {
+        "viwbgKWGvUoZCosF1": {
+          "id": "viwbgKWGvUoZCosF1",
+          "name": "Grid view",
+          "type": "grid",
+          "createdByUserId": "usrcTFn14vKTIgbW3"
+        }
+      },
+      "viewSectionsById": {},
+      "schemaChecksum": "412180368d81674e723b957501f16a57c9264fc69d19668b70d1547888c29413"
+    }
+  ],
+  "tableDatas": [
+    {
+      "id": "tblXYuhMZ3hWZkBCa",
+      "rows": [
+        {
+          "id": "recv9Z8uFFNt50rqX",
+          "createdTime": "2022-04-27T18:48:37.000Z",
+          "cellValuesByColumnId": {
+            "fld66bK6Pq8AG4m3h": [
+              {
+                "id": "attdRIU80oCDC8u6X",
+                "url": "https://dl.airtable.com/.attachments/247a5881e7742c2d55cb8f814fe7263a/964d1018/512x512.png",
+                "filename": "512x512.png",
+                "type": "image/png",
+                "size": 77822,
+                "width": 2763,
+                "height": 2763,
+                "smallThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/f5f0717d00f1f61c402fec203d16efd7/61bea79e",
+                "smallThumbWidth": 36,
+                "smallThumbHeight": 36,
+                "largeThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/8f49ea3dcdaf3aa9788e84f1a3e3f3e2/81284101",
+                "largeThumbWidth": 512,
+                "largeThumbHeight": 512,
+                "fullThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/c5300e8cda92de966de82c760cd44533/89d51367",
+                "fullThumbWidth": 3000,
+                "fullThumbHeight": 3000
+              }
+            ],
+            "fldrhmH0EYnOXfnUA": "nc"
+          }
+        },
+        {
+          "id": "recmAktd3OQe3Wg8C",
+          "createdTime": "2022-04-27T18:48:37.000Z",
+          "cellValuesByColumnId": {
+            "fld66bK6Pq8AG4m3h": [
+              {
+                "id": "attLBB2eqE9grLlUU",
+                "url": "https://dl.airtable.com/.attachments/a67aaa1efa29d40c3633ca03f0b366e6/9b56c6e2/Abstract-Nord.png",
+                "filename": "Abstract-Nord.png",
+                "type": "image/png",
+                "size": 140219,
+                "width": 1920,
+                "height": 1080,
+                "smallThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/d5daa7de864a8302ad9d2c43a257a52d/2b709aa8",
+                "smallThumbWidth": 64,
+                "smallThumbHeight": 36,
+                "largeThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/7b388a8484a1bc8f9f91bbf304c9b42e/4663133b",
+                "largeThumbWidth": 910,
+                "largeThumbHeight": 512,
+                "fullThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/4d5dfa21efa34198c740443551531eea/ea51b594",
+                "fullThumbWidth": 3000,
+                "fullThumbHeight": 3000
+              },
+              {
+                "id": "attIzZpRBBWwRI5Wz",
+                "url": "https://dl.airtable.com/.attachments/107f2f1d886b1f1ae0d528ba7f76df03/931109ad/archlinux.png",
+                "filename": "archlinux.png",
+                "type": "image/png",
+                "size": 184887,
+                "width": 3840,
+                "height": 2160,
+                "smallThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/cb9894ae30f7a264a87639d5ce4980ec/e000fb24",
+                "smallThumbWidth": 64,
+                "smallThumbHeight": 36,
+                "largeThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/1f4146a90346325a8ab51d5d7f4f40be/9a73928f",
+                "largeThumbWidth": 910,
+                "largeThumbHeight": 512,
+                "fullThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/9954669a1ba7bc2d4e4d50f9c0130435/7d46d64a",
+                "fullThumbWidth": 3000,
+                "fullThumbHeight": 3000
+              },
+              {
+                "id": "atttj7OPAZ1iDtHBt",
+                "url": "https://dl.airtable.com/.attachments/4eec95e15a951829c2b12e4375bece7f/8ff26ede/arctic-landscape.png",
+                "filename": "arctic-landscape.png",
+                "type": "image/png",
+                "size": 155548,
+                "width": 1920,
+                "height": 1080,
+                "smallThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/dd63a0d18ed1c4bf3e7096145fc5fa0f/ebfc95ac",
+                "smallThumbWidth": 64,
+                "smallThumbHeight": 36,
+                "largeThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/c9b441a88304b53cdc44fa0fcfdb57e6/8c6b629d",
+                "largeThumbWidth": 910,
+                "largeThumbHeight": 512,
+                "fullThumbUrl": "https://dl.airtable.com/.attachmentThumbnails/f5b74f24a31f22e8e859954499abfa67/fd11b916",
+                "fullThumbWidth": 3000,
+                "fullThumbHeight": 3000
+              }
+            ],
+            "fldrhmH0EYnOXfnUA": "wp"
+          }
+        },
+        {
+          "id": "recAFj2eQVsynoDJa",
+          "createdTime": "2022-04-27T18:48:37.000Z",
+          "cellValuesByColumnId": {
+            "fldrhmH0EYnOXfnUA": "test"
+          }
+        }
+      ],
+      "viewDatas": [
+        {
+          "id": "viwbgKWGvUoZCosF1",
+          "frozenColumnCount": 1,
+          "columnOrder": [
+            {
+              "columnId": "fldrhmH0EYnOXfnUA",
+              "visibility": true
+            },
+            {
+              "columnId": "fldVFoK7t6aC92xzj",
+              "visibility": true
+            },
+            {
+              "columnId": "fld66bK6Pq8AG4m3h",
+              "visibility": true
+            },
+            {
+              "columnId": "fldRC8zKoyGzM6agG",
+              "visibility": true
+            }
+          ],
+          "sharesById": {},
+          "createdByUserId": "usrcTFn14vKTIgbW3",
+          "applicationTransactionNumber": 21,
+          "rowOrder": [
+            {
+              "rowId": "recv9Z8uFFNt50rqX",
+              "visibility": true
+            },
+            {
+              "rowId": "recmAktd3OQe3Wg8C",
+              "visibility": true
+            },
+            {
+              "rowId": "recAFj2eQVsynoDJa",
+              "visibility": true
+            }
+          ]
+        }
+      ],
+      "hasOnlyIncludedRowAndCellDataForIncludedViews": false
+    }
+  ],
+  "hasBlockInstallations": false,
+  "applicationAdminFlags": {
+    "UPDATE_PRIMITIVE_CELL_THROTTLE_MS": null,
+    "MAX_WORKFLOWS_PER_APPLICATION": null,
+    "MAX_SYNC_SOURCES_PER_APPLICATION": null,
+    "MAX_SYNC_SOURCES_PER_TABLE": null,
+    "MAX_SYNCED_TABLES_PER_APPLICATION": null,
+    "CUSTOM_MAX_NUM_ROWS_PER_TABLE": null
+  },
+  "pageBundles": [],
+  "uploadedUserContentCdnSetting": {
+    "applicationScopedAuthMode": "public"
+  },
+  "applicationV2TargetedFeatureFlagClientConfiguration": {
+    "nonCollaboratorsInCollaboratorField": {
+      "trafficLevel": 0
+    },
+    "applicationInsights": {
+      "trafficLevel": 0
+    },
+    "autoOpenInsightsPaneOnUnseenSuggestion": {
+      "trafficLevel": 0
+    },
+    "disabledWorkflowOnSchemaChangeSuggestion": {
+      "trafficLevel": 0
+    },
+    "syncFailureSuggestion": {
+      "trafficLevel": 0
+    },
+    "unusedViewsSuggestion": {
+      "trafficLevel": 0
+    },
+    "filterUnusedViewsUsingDependencyGraph": {
+      "trafficLevel": 100
+    },
+    "unusedSelectChoicesSuggestion": {
+      "trafficLevel": 0
+    },
+    "unifiedEventLog": {
+      "trafficLevel": 0
+    },
+    "constantPoolingForCrudResponses": {
+      "trafficLevel": 0
+    }
+  },
+  "applicationV2EnabledFeatureNames": [
+    "filterUnusedViewsUsingDependencyGraph"
+  ],
+  "isConstantPooledData": false
+}

packages/nocodb/tests/sync/sync.js

@@ -1,6 +1,8 @@
 const Api = require('nocodb-sdk').Api;
 const jsonfile = require('jsonfile');
 const { UITypes } = require('nocodb-sdk');
+const axios = require('axios').default;
+const FormData = require('form-data');
 
 function syncLog(log) {
   console.log(log)
@@ -560,8 +562,50 @@ function nocoBaseDataProcessing(table, record) {
       if (dt === 'Lookup') delete rec[key];
       if (dt === 'Rollup') delete rec[key];
 
-      // attachment types not handled currently
-      if (dt === 'Attachment') delete rec[key];
+      if (dt === 'Attachment') {
+        let tempArr = [];
+        for (const v of value) {
+          const binaryImage = await axios
+            .get(v.url, {
+              responseType: 'stream',
+              headers: {
+                'Content-Type': v.type
+              }
+            })
+            .then(response => {
+              return response.data;
+            })
+            .catch(error => {
+              console.log(error);
+              return false;
+            });
+
+          var imageFile = new FormData();
+          imageFile.append('files', binaryImage, {
+            filename: v.filename
+          });
+
+          const rs = await axios
+            .post(syncDB.baseURL + '/api/v1/db/storage/upload', imageFile, {
+              params: {
+                path: `noco/${syncDB.projectName}/${table.title}/${key}`
+              },
+              headers: {
+                'Content-Type': `multipart/form-data; boundary=${imageFile._boundary}`,
+                'xc-auth': syncDB.authToken
+              }
+            })
+            .then(response => {
+              return response.data;
+            })
+            .catch(e => {
+              console.log(e);
+            });
+
+          tempArr.push(...rs);
+        }
+        rec[key] = JSON.stringify(tempArr);
+      }
     }
 
     // insert airtable record ID explicitly into each records