Merge pull request #295 from aleksandrychev/ENT-14726

ENT-14726: Excluded staging from the search indexes

Author: olehermanse

.gitlab-ci.yml

@@ -6,7 +6,7 @@ variables:
   # build.js authenticates to the GitHub API with this token to avoid rate limits.
   # The multiplatform buildx job reads EXTRA_DOCKER_ARGS (not the legacy inline
   # --build-arg), so pass the token through there.
-  EXTRA_DOCKER_ARGS: "--build-arg GITHUB_USERNAME_TOKEN"
+  EXTRA_DOCKER_ARGS: "--build-arg GITHUB_USERNAME_TOKEN --build-arg SEARCH_ENGINES_DISABLED=false" # do not disable search engines by default
   DOCKER_HOST:
     value: 'tcp://docker:2376'
     description: "Required to run dind inside k8s runners with TLS"
@@ -39,6 +39,9 @@ build:docker:
 
 build:docker-multiplatform:
   rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      variables:
+        EXTRA_DOCKER_ARGS: "--build-arg GITHUB_USERNAME_TOKEN --build-arg SEARCH_ENGINES_DISABLED=true" # disable if the branch is master (staging)
     - if: '$CI_COMMIT_BRANCH =~ /^(master|production)$/'
 
 publish:image:

Dockerfile

@@ -1,5 +1,9 @@
 FROM node:26-alpine AS build
 ARG GITHUB_USERNAME_TOKEN
+
+ARG SEARCH_ENGINES_DISABLED
+ENV HUGO_PARAMS_SEARCHENGINES_DISABLED=$SEARCH_ENGINES_DISABLED
+
 WORKDIR /build-website
 ADD https://github.com/gohugoio/hugo/releases/download/v0.167.0/hugo_0.167.0_Linux-64bit.tar.gz hugo.tar.gz
 RUN echo "4d84519b9f619e6d4c3fb45a50157abeabeb724f859c60605f44c23def6e1169  hugo.tar.gz" | sha256sum -c

config.toml

@@ -6,6 +6,12 @@ theme = "cfbs-theme"
 enableRobotsTXT = true
 syntaxCSS = true
 
+# this parameter is set by Hugo from HUGO_PARAMS_SEARCHENGINES_DISABLED environment variable
+# that we define on the build stage
+# it is false by defaut to avoid a case when it was accedntly not defined
+[params.searchengines]
+disabled = false 
+
 # Number of posts per page
 [pagination]
 pagerSize = 10

themes/cfbs-theme/layouts/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+{{ if .Site.Params.searchengines.disabled }}Disallow: /{{ end }}