wix · Idokah · Jan 4, 2024 · Jan 4, 2024 · Jan 4, 2024 · Jan 4, 2024
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -6,18 +6,21 @@ on:
 
   workflow_dispatch:
 
+  schedule:
+    - cron: '0 7 * * 0' # Runs every Sunday at 7:00 AM UTC (which is 9:00 AM TLV)
+
 jobs:
   test_main:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Use Node.js 16
+      - name: Use Node.js 20
         uses: actions/setup-node@v2
         with:
-          node-version: lts/*
+          node-version: 20
 
       - name: Restore Dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: |
             node_modules
@@ -39,13 +42,12 @@ jobs:
       matrix:
         database: [
                    "postgres", "postgres13", "postgres12", "postgres11", "postgres10", "postgres9",
-                   "spanner",
                    "mysql", "mysql5",
-                   "mssql", "mssql17",
+                   "mssql",
+                   spanner,
                    "mongo", "mongo4",
-                   "firestore",
                    "dynamodb",
-                   "google-sheets"
+                   "firestore"
                    ]
 
     env:
@@ -54,19 +56,19 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Use Node.js 16
+      - name: Use Node.js 20
         uses: actions/setup-node@v2
         with:
-          node-version: lts/*
+          node-version: 20
 
       - name: Restore Dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: |
             node_modules
             */*/node_modules
           key: ${{ runner.os }}-${{ hashFiles('**/package.json') }}
-          
+
       - name: Downloading Image
         if: ${{ ( matrix.database != 'google-sheets') }}
         run: |

diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
@@ -6,35 +6,49 @@ on:
       gcp:
         type: boolean
         description: GCP container registry
-        default: true
+        default: false
 
       aws:
         type: boolean
         description: AWS ECR container registry
-        default: true
+        default: false
 
       dockerHub:
         type: boolean
         description: Docker container registry
-        default: true
+        default: false
+
+      image-tag:
+        type: string
+        description: Image tag
+        default: latest
+
+      container-registry-folder:
+        type: choice
+        description: Container registry folder
+        options:
+          - velo-manged-external-db
+          - velo-manged-external-db-test
+          - velo-external-db
+          - velo-external-db-testing
 
 jobs:
   Build-and-push:
     runs-on: ubuntu-latest
-  
+
     env:
       ECR_REPOSITORY: ${{ secrets.AWS_REPO_NAME }}
       PUBLIC_ECR_URL: ${{ secrets.AWS_REPO_URL }}
       IMAGE_NAME: ${{ secrets.IMAGE_NAME }}
       DOCKERHUB_REPOSITORY: ${{ secrets.DOCKERHUB_REPOSITORY }}
       GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
       GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-      CONTAINER_REGISTRY_FOLDER: ${{ secrets.GCP_CONTAINER_REGISTRY_FOLDER }}
-      IMAGE_TAG: latest
+      IMAGE_TAG: ${{ github.event.inputs.image-tag }}
+      CONTAINER_REGISTRY_FOLDER: ${{ github.event.inputs.container-registry-folder }}
 
     steps:
       - name: Git checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       # Build docker image
       - name: Build Docker image
@@ -56,46 +70,84 @@ jobs:
         if: github.event.inputs.aws == 'true'
         id: aws-push-image
         run: |
-          docker tag $IMAGE_NAME:$IMAGE_TAG $PUBLIC_ECR_URL/$IMAGE_NAME:$IMAGE_TAG
+          # Setting image tag to image tag input
+          docker tag $IMAGE_NAME $PUBLIC_ECR_URL/$IMAGE_NAME:$IMAGE_TAG
+          # Setting image to with github branch name 
+          docker tag $IMAGE_NAME $PUBLIC_ECR_URL/$IMAGE_NAME:branch-${GITHUB_REF##*/}
+          # Setting image to with github sha
+          docker tag $IMAGE_NAME $PUBLIC_ECR_URL/$IMAGE_NAME:sha-${GITHUB_SHA}
+
           echo "Pushing image to ECR..."
-          docker push $PUBLIC_ECR_URL/$IMAGE_NAME:$IMAGE_TAG
+          docker push $PUBLIC_ECR_URL/$IMAGE_NAME  --all-tags
+
+          echo "Pushing image to ECR with following tags:"
           echo "::set-output name=image::$PUBLIC_ECR_URL/$IMAGE_NAME:$IMAGE_TAG"
+          echo "::set-output name=image::$PUBLIC_ECR_URL/$IMAGE_NAME:branch-${GITHUB_REF##*/}"
+          echo "::set-output name=image::$PUBLIC_ECR_URL/$IMAGE_NAME:sha-${GITHUB_SHA}"
 
       - name: Login to Docker Hub
         if: github.event.inputs.dockerHub == 'true'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-          
+
       - name: Tag and push the image to Docker Hub
         if: github.event.inputs.dockerHub == 'true'
         id: dockerhub-push-image
         run: |
+          # Setting image tag to image tag input
           docker tag $IMAGE_NAME $DOCKERHUB_REPOSITORY/$IMAGE_NAME:$IMAGE_TAG
+          # Setting image to with github branch name
+          docker tag $IMAGE_NAME $DOCKERHUB_REPOSITORY/$IMAGE_NAME:branch-${GITHUB_REF##*/}
+          # Setting image to with github sha
+          docker tag $IMAGE_NAME $DOCKERHUB_REPOSITORY/$IMAGE_NAME:sha-${GITHUB_SHA}
+
           echo "Pushing image to Docker Hub..."
-          docker push $DOCKERHUB_REPOSITORY/$IMAGE_NAME:$IMAGE_TAG
+          docker push $DOCKERHUB_REPOSITORY/$IMAGE_NAME  --all-tags
+
+          echo "Pushing image to Docker Hub with following tags:"
           echo "::set-output name=image::$DOCKERHUB_REPOSITORY/$IMAGE_NAME:$IMAGE_TAG"
+          echo "::set-output name=image::$DOCKERHUB_REPOSITORY/$IMAGE_NAME:branch-${GITHUB_REF##*/}"
+          echo "::set-output name=image::$DOCKERHUB_REPOSITORY/$IMAGE_NAME:sha-${GITHUB_SHA}"
 
       #  Authenticate to Google Cloud
       - id: auth
         name: Authenticate to Google Cloud
         if: github.event.inputs.gcp == 'true'
-        uses: 'google-github-actions/auth@v0'
+        uses: 'google-github-actions/auth@v2'
         with:
           credentials_json: '${{ secrets.GCP_SA_KEY }}'
 
       # Set up Cloud SDK
       - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v0
+        uses: google-github-actions/setup-gcloud@v2
         if: github.event.inputs.gcp == 'true'
         with:
           project_id: ${{ secrets.GCP_PROJECT_ID }}
 
+      - name: Docker Auth
+        if: github.event.inputs.gcp == 'true'
+        run: |-
+          gcloud auth configure-docker us-docker.pkg.dev --quiet
+
       # Push image to Google Container Registry
       - name: Tag and push image to GCP container registry
         if: github.event.inputs.gcp == 'true'
         run: |
           gcloud auth configure-docker -q
-          docker tag $IMAGE_NAME:$IMAGE_TAG gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER
-          docker push gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER:$IMAGE_TAG
+
+          # Setting image tag to image tag input
+          docker tag $IMAGE_NAME gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER:$IMAGE_TAG
+          # Setting image to with github branch name
+          docker tag $IMAGE_NAME gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER:branch-${GITHUB_REF##*/}
+          # Setting image to with github sha
+          docker tag $IMAGE_NAME gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER:sha-${GITHUB_SHA}
+
+          echo "Pushing image to GCP container registry..."
+          docker push gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER --all-tags
+
+          echo "Pushing image to GCP container registry with following tags:"
+          echo "::set-output name=image::gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER:$IMAGE_TAG"
+          echo "::set-output name=image::gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER:branch-${GITHUB_REF##*/}"
+          echo "::set-output name=image::gcr.io/$GCP_PROJECT_ID/$CONTAINER_REGISTRY_FOLDER:sha-${GITHUB_SHA}"
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # Compile stage
-FROM node:lts-slim 
+FROM node:20-slim 
 WORKDIR /usr/lib/app
 
 # Copy the project dir
@@ -13,7 +13,7 @@ RUN npm install
 RUN nx run-many --target=build --all
 
 
-FROM node:lts-alpine
+FROM node:20-alpine
 
 WORKDIR /usr/lib/app
 

diff --git a/apps/velo-external-db/test/resources/docker-compose.yaml b/apps/velo-external-db/test/resources/docker-compose.yaml
@@ -126,7 +126,7 @@ services:
       POSTGRES_DB: test-db
 
   mysql:
-    image: mysql:8
+    image: mysql:8.0
     command: --default-authentication-plugin=mysql_native_password
     restart: always
     environment:

diff --git a/libs/external-db-mysql/tests/e2e-testkit/docker-compose.yaml b/libs/external-db-mysql/tests/e2e-testkit/docker-compose.yaml
@@ -1,7 +1,7 @@
 version: "3.9"
 services:
   mysql:
-    image: mysql:8
+    image: mysql:8.0
     command: --default-authentication-plugin=mysql_native_password
     restart: always
     environment:

diff --git a/libs/velo-external-db-core/src/converters/data_utils.ts b/libs/velo-external-db-core/src/converters/data_utils.ts
@@ -6,12 +6,19 @@ export const asWixData = (item: Item) => {
     return generateIdsIfNeeded(packDates(item))
 }
 
+const replaceNonAlphanumeric = (str: string) => {
+    // Replace non-alphanumeric characters with dashes
+    return str.replace(/[^a-zA-Z0-9]/g, '-')
+}
+
 export const generateIdsIfNeeded = (item: Item): ItemWithId => {
     if ('_id' in item)
         return item as ItemWithId
     const sha = crypto.createHash('sha1')
     const fieldsConcat = Object.values(item).join('')
-    return { ...item, _id: sha.update(fieldsConcat).digest('base64') }
+    const base64Digest = sha.update(fieldsConcat).digest('base64')
+    const validId = replaceNonAlphanumeric(base64Digest)
+    return { ...item, _id: validId }
 }
 
 const packDates = (item: Item) => Object.entries(item)

diff --git a/libs/velo-external-db-core/src/router.ts b/libs/velo-external-db-core/src/router.ts
@@ -79,11 +79,23 @@ const executeHook = async(hooks: DataHooks | SchemaHooks, _actionName: string, p
 
 export const createRouter = () => {
     const router = express.Router()
-    router.use(express.json())
+    router.use(express.json({ limit: '4mb' }))
+    router.use(express.urlencoded({ limit: '4mb', extended: true }))
     router.use(compression())
     router.use('/assets', express.static(path.join(__dirname, 'assets')))
     router.use(unless(['/', '/provision', '/favicon.ico'], secretKeyAuthMiddleware({ secretKey: cfg.secretKey })))
 
+    //set timeout of 1.5 minutes per request
+    router.use((req, res, next) => {
+        const NinetySecondsInMs = 90 * 1000
+        res.setTimeout(NinetySecondsInMs, () => {
+            console.warn(`Request has timed out - ${req.method} ${req.url}`)
+            console.dir({ body: req.body }, { depth: 3 })
+            res.status(408).send('Request Timeout')
+        })
+        next()
+    })
+
     config.forEach(({ pathPrefix, roles }) => router.use(includes([pathPrefix], authRoleMiddleware({ roles }))))
 
     // *************** INFO **********************
@@ -201,7 +213,7 @@ export const createRouter = () => {
             const { collectionName } = req.body
             const customContext = {}
             const { items } = await executeDataHooksFor(DataActions.BeforeBulkUpdate, dataPayloadFor(BULK_UPDATE, req.body), requestContextFor(BULK_UPDATE, req.body), customContext)
-            await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body))
+            roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body))
             const data = await schemaAwareDataService.bulkUpdate(collectionName, items)
 
             const dataAfterAction = await executeDataHooksFor(DataActions.AfterBulkUpdate, data, requestContextFor(BULK_UPDATE, req.body), customContext)

diff --git a/package.json b/package.json
@@ -51,7 +51,7 @@
     "google-spreadsheet": "^3.3.0",
     "moment": "^2.29.3",
     "mongodb": "^4.6.0",
-    "mssql": "^8.1.0",
+    "mssql": "^11.0.1",
     "mysql": "^2.18.1",
     "nested-property": "^4.0.0",
     "node-cache": "^5.1.2",
@@ -78,13 +78,13 @@
     "@types/chance": "^1.1.3",
     "@types/compression": "^1.7.2",
     "@types/ejs": "^3.1.1",
-    "@types/express": "4.17.13",
+    "@types/express": "^5.0.1",
     "@types/google-spreadsheet": "^3.3.0",
     "@types/jest": "^27.4.1",
     "@types/jest-when": "^3.5.0",
     "@types/mssql": "^8.0.2",
     "@types/mysql": "^2.15.21",
-    "@types/node": "^16.11.7",
+    "@types/node": "^22.14.1",
     "@types/pg": "8.6.5",
     "@types/rewire": "^2.5.28",
     "@types/sqlstring": "^2.3.0",
@@ -107,8 +107,8 @@
     "prettier": "^2.5.1",
     "rewire": "^6.0.0",
     "ts-jest": "27.1.4",
-    "ts-node": "9.1.1",
-    "typescript": "~4.6.2",
+    "ts-node": "^10.9.2",
+    "typescript": "^4.9.5",
     "uuid-validate": "^0.0.3"
   },
   "lint-staged": {