diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 43b8a5966..4c51043de 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -128,12 +128,13 @@ jobs: - detect-changes if: ${{ needs.detect-changes.outputs.ckan == 'true' }} container: - image: ckan/ckan-dev:2.10.3 + image: ckan/ckan-dev:2.11.4 + options: --user root services: solr: - image: ckan/ckan-solr:2.10-solr9 + image: ckan/ckan-solr:2.11-solr9 postgres: - image: postgres:12 + image: ckan/ckan-postgres-dev:2.11 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -153,16 +154,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: true - - name: Create Database - run: | - psql --host=postgres --username=postgres --command="CREATE USER ckan_default WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" - createdb --encoding=utf-8 --host=postgres --username=postgres --owner=ckan_default ckan_test - psql --host=postgres --username=postgres --command="CREATE USER datastore_write WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" - psql --host=postgres --username=postgres --command="CREATE USER datastore_read WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" - createdb --encoding=utf-8 --host=postgres --username=postgres --owner=datastore_write datastore_test - name: Install requirements run: | - apk add proj proj-dev proj-util geos + apt-get update + apt-get install -y proj-bin geos-bin cd ckan/ckanext/ckanext-restricteddata pip install -r dev-requirements.txt pip install -r requirements.txt @@ -183,18 +178,26 @@ jobs: cd ../ckanext-markdown_editor pip install -r requirements.txt pip install -e . + + - name: Initialize database + run: | + ckan -c ckan/ckanext/ckanext-restricteddata/test.ini db init + ckan -c ckan/ckanext/ckanext-restricteddata/test.ini db upgrade + ckan -c ckan/ckanext/ckanext-restricteddata/test.ini db upgrade -p restricteddata + ckan -c ckan/ckanext/ckanext-restricteddata/test.ini db upgrade -p pages + ckan -c ckan/ckanext/ckanext-restricteddata/test.ini db upgrade -p activity - name: Run tests run: pytest --ckan-ini=ckan/ckanext/ckanext-restricteddata/test.ini --cov=ckanext.restricteddata --disable-warnings ckan/ckanext/ckanext-restricteddata/ckanext/restricteddata/tests - name: install codecov requirements run: | - apk add gpg gpg-agent + apt-get install -y gpg gpg-agent curl - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5 with: flags: ckan - os: alpine + os: debian token: ${{ secrets.CODECOV_TOKEN }} test-e2e: diff --git a/cdk/lib/ckan-stack.ts b/cdk/lib/ckan-stack.ts index a52290b84..32b435706 100644 --- a/cdk/lib/ckan-stack.ts +++ b/cdk/lib/ckan-stack.ts @@ -181,7 +181,7 @@ export class CkanStack extends Stack { const ckanContainerSecrets: { [key: string]: aws_ecs.Secret; } = { // .env.ckan - CKAN_BEAKER_SESSION_SECRET: aws_ecs.Secret.fromSecretsManager(beakerSecret), + CKAN_SESSION_SECRET: aws_ecs.Secret.fromSecretsManager(beakerSecret), CKAN_APP_INSTANCE_UUID: aws_ecs.Secret.fromSecretsManager(appUUIDSecret), CKAN_BEAKER_SESSION_VALIDATE_KEY: aws_ecs.Secret.fromSecretsManager(beakerValidateKeySecret), CKAN_PAHA_JWT_KEY: aws_ecs.Secret.fromSecretsManager(pahaJwtKeySecret), diff --git a/ckan/Dockerfile b/ckan/Dockerfile index f24d90233..138c28c3a 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -83,11 +83,10 @@ RUN \ mkdir -p ${DATA_DIR} && \ # Install CKAN and requirements pip install --find-links=/srv/app/wheels -r ${SRC_DIR}/ckan-requirements.txt && \ - pip install -e git+https://github.com/ckan/ckan.git@ckan-2.10.9#egg=ckan && \ + pip install -e git+https://github.com/ckan/ckan.git@ckan-2.11.4#egg=ckan && \ # Configure environment addgroup -g 92 ckan && \ adduser -u 92 -h ${APP_DIR} -H -D -G ckan ckan && \ - cp ${CKAN_DIR}/who.ini ${APP_DIR} && \ # Set timezone echo "UTC" > /etc/timezone && \ # Change ownership to app user @@ -189,6 +188,8 @@ RUN \ addgroup sudo && \ adduser ckan sudo && \ echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \ + # Add development packages required by pip-compile + apk add --no-cache postgresql-dev && \ # fix permissions chown -R ckan:ckan ${APP_DIR} diff --git a/ckan/ckan-requirements.in b/ckan/ckan-requirements.in index 13491e063..cebde70b7 100644 --- a/ckan/ckan-requirements.in +++ b/ckan/ckan-requirements.in @@ -1,7 +1,7 @@ # This file defines the used CKAN version and any overrides # Compile with pip-compile to produce the ckan-requirements.txt used in build -ckan[requirements]==2.10.9 +ckan[requirements]==2.11.4 # Runtime requirements uWSGI==2.0.30 diff --git a/ckan/ckan-requirements.txt b/ckan/ckan-requirements.txt index e47bc3246..bdb2573b3 100644 --- a/ckan/ckan-requirements.txt +++ b/ckan/ckan-requirements.txt @@ -6,58 +6,62 @@ # --extra-index-url https://alpine-wheels.github.io/index -alembic==1.8.1 +alembic==1.13.2 # via ckan -async-timeout==5.0.1 - # via redis -babel==2.10.3 +async-timeout==4.0.3 + # via + # ckan + # redis +babel==2.15.0 # via # ckan # flask-babel -beaker==1.11.0 - # via ckan -bleach==5.0.1 - # via ckan -blinker==1.5 +bleach==6.1.0 # via ckan +blinker==1.8.2 + # via + # ckan + # flask +cachelib==0.13.0 + # via + # ckan + # flask-session certifi==2024.7.4 # via # ckan # requests -charset-normalizer==2.0.12 +charset-normalizer==3.3.2 # via # ckan # requests # Removed manually for installing editable separately -# ckan[requirements]==2.10.9 +# ckan[requirements]==2.11.4 # via -r ckan-requirements.in -click==8.1.3 +click==8.1.7 # via # ckan # flask # rq -deprecated==1.2.13 +dominate==2.9.1 # via ckan -dominate==2.7.0 +feedgen==1.0.0 # via ckan -feedgen==0.9.0 - # via ckan -flask==2.0.3 +flask==3.0.3 # via # ckan # flask-babel # flask-login - # flask-multistatic + # flask-session # flask-wtf -flask-babel==1.0.0 +flask-babel==4.0.0 # via ckan -flask-login==0.6.1 +flask-login==0.6.3 # via ckan -flask-multistatic==1.0 +flask-session==0.8.0 # via ckan -flask-wtf==1.0.1 +flask-wtf==1.2.1 # via ckan -greenlet==2.0.2 +greenlet==3.0.3 # via # ckan # sqlalchemy @@ -65,11 +69,9 @@ idna==3.7 # via # ckan # requests -importlib-metadata==4.11.4 - # via ckan -importlib-resources==6.4.0 +importlib-metadata==8.0.0 # via ckan -itsdangerous==2.1.1 +itsdangerous==2.2.0 # via # ckan # flask @@ -79,68 +81,62 @@ jinja2==3.1.6 # ckan # flask # flask-babel -lxml==4.9.1 +lxml==5.2.2 # via # ckan # feedgen -mako==1.2.2 +mako==1.3.5 # via # alembic # ckan -markdown==3.4.1 +markdown==3.6 # via ckan -markupsafe==2.1.1 +markupsafe==2.1.5 # via # ckan # jinja2 # mako + # werkzeug # wtforms -mypy==0.971 +msgspec==0.18.6 # via # ckan - # sqlalchemy -mypy-extensions==0.4.3 + # flask-session +mypy==1.10.1 # via # ckan - # mypy -nose==1.3.7 + # sqlalchemy +mypy-extensions==1.0.0 # via # ckan - # pyutilib + # mypy packaging==24.1 # via ckan passlib==1.7.4 # via ckan -polib==1.1.1 +polib==1.2.0 # via ckan -psycopg2==2.9.3 +psycopg2==2.9.9 # via ckan -pyjwt==2.4.0 +pyjwt==2.8.0 # via ckan pyparsing==3.1.2 # via ckan pysolr==3.9.0 # via ckan -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # ckan # feedgen python-magic==0.4.27 # via ckan -pytz==2021.3 +pytz==2024.1 # via - # babel # ckan # flask-babel -pytz-deprecation-shim==0.1.0.post0 - # via - # ckan - # tzlocal -pyutilib==6.0.0 - # via ckan pyyaml==6.0.1 # via ckan -redis==4.6.0 +redis==5.0.7 # via # ckan # rq @@ -148,21 +144,20 @@ requests==2.32.4 # via # ckan # pysolr -rq==1.11.0 +rq==1.16.2 # via ckan -simplejson==3.17.6 +simplejson==3.19.2 # via ckan six==1.16.0 # via # bleach # ckan # python-dateutil - # pyutilib -sqlalchemy[mypy]==1.4.41 +sqlalchemy[mypy]==1.4.52 # via # alembic # ckan -sqlalchemy2-stubs==0.0.2a27 +sqlalchemy2-stubs==0.0.2a38 # via # ckan # sqlalchemy @@ -172,24 +167,21 @@ tomli==2.0.1 # via # ckan # mypy -typing-extensions==4.3.0 +typing-extensions==4.12.2 # via + # alembic # ckan # mypy # sqlalchemy2-stubs -tzdata==2022.1 - # via - # ckan - # pytz-deprecation-shim -tzlocal==4.2 +tzlocal==5.2 # via ckan -urllib3==1.26.19 +urllib3==2.2.2 # via # ckan # requests uwsgi==2.0.30 # via -r ckan-requirements.in -watchdog==2.1.6 +watchdog==4.0.1 # via # ckan # werkzeug @@ -199,24 +191,20 @@ webencodings==0.5.1 # via # bleach # ckan -werkzeug[watchdog]==2.0.3 +werkzeug[watchdog]==3.0.6 # via # ckan # flask # flask-login -wrapt==1.14.0 - # via - # ckan - # deprecated -wtforms==3.0.1 +wtforms==3.1.2 # via # ckan # flask-wtf -zipp==3.19.1 +zipp==3.19.2 # via # ckan # importlib-metadata -zope-interface==5.4.0 +zope-interface==6.4.post2 # via ckan # The following packages are considered to be unsafe in a requirements file: diff --git a/ckan/ckanext/ckanext-harvest b/ckan/ckanext/ckanext-harvest index d83d9d71a..501bc49ed 160000 --- a/ckan/ckanext/ckanext-harvest +++ b/ckan/ckanext/ckanext-harvest @@ -1 +1 @@ -Subproject commit d83d9d71a079d9526474c6263fc486e81ad3d70c +Subproject commit 501bc49ed9f81d300507e7aeb64ed405d240d23f diff --git a/ckan/ckanext/ckanext-pages b/ckan/ckanext/ckanext-pages index ca68f37fa..4b5f0f439 160000 --- a/ckan/ckanext/ckanext-pages +++ b/ckan/ckanext/ckanext-pages @@ -1 +1 @@ -Subproject commit ca68f37faf2c43eff251bd16fb5af8f5b4cd8b1b +Subproject commit 4b5f0f439554235972c669574ed8cab3c2279add diff --git a/ckan/ckanext/ckanext-restricteddata/ckanext/restricteddata/templates/organization/member_new.html b/ckan/ckanext/ckanext-restricteddata/ckanext/restricteddata/templates/organization/member_new.html index e0fc920b0..8ff67efcb 100644 --- a/ckan/ckanext/ckanext-restricteddata/ckanext/restricteddata/templates/organization/member_new.html +++ b/ckan/ckanext/ckanext-restricteddata/ckanext/restricteddata/templates/organization/member_new.html @@ -1,7 +1,7 @@ {% ckan_extends %} {% block secondary_content %} - {% snippet 'snippets/organization.html', organization=organization %} + {{ super () }}

diff --git a/ckan/ckanext/ckanext-restricteddata/ckanext/restricteddata/tests/conftest.py b/ckan/ckanext/ckanext-restricteddata/ckanext/restricteddata/tests/conftest.py index 334972e0b..81e5b4050 100644 --- a/ckan/ckanext/ckanext-restricteddata/ckanext/restricteddata/tests/conftest.py +++ b/ckan/ckanext/ckanext-restricteddata/ckanext/restricteddata/tests/conftest.py @@ -5,3 +5,5 @@ def clean_db(reset_db, migrate_db_for): reset_db() migrate_db_for("restricteddata") + migrate_db_for("activity") # https://github.com/ckan/ckan/issues/8540 + migrate_db_for("pages") # probably the same issue? diff --git a/ckan/ckanext/ckanext-restricteddata/ckanext/restricteddata/tests/test_plugin.py b/ckan/ckanext/ckanext-restricteddata/ckanext/restricteddata/tests/test_plugin.py index 78cc4e133..3953331fa 100644 --- a/ckan/ckanext/ckanext-restricteddata/ckanext/restricteddata/tests/test_plugin.py +++ b/ckan/ckanext/ckanext-restricteddata/ckanext/restricteddata/tests/test_plugin.py @@ -65,6 +65,7 @@ def test_some_action(): import pytest import datetime import jwt +import uuid from pathlib import Path from ckan.plugins import plugin_loaded, toolkit @@ -516,11 +517,11 @@ def test_groups_are_removed(app): def test_paha_authentication_with_missing_fields(app): valid_token = { "iss": "PAHA", - "id": "user-id", + "id": str(uuid.uuid4()), "email": "foo.bar@example.com", "firstName": "Foo", "lastName": "Bar", - "activeOrganizationId": "organization-id", + "activeOrganizationId": str(uuid.uuid4()), "activeOrganizationNameFi": "organization name fi", "activeOrganizationNameSv": "organization name sv", "activeOrganizationNameEn": "organization name en", @@ -546,7 +547,7 @@ def test_paha_authentication_with_missing_fields(app): @pytest.mark.usefixtures("with_plugins", "clean_db", "with_request_context") def test_paha_authentication_creates_organization(app): some_user = User() - organization_id = "paha-organization-id" + organization_id = str(uuid.uuid4()) organization_name_fi = "paha organization fi" organization_name_sv = "paha organization sv" organization_name_en = "paha organization en" @@ -573,7 +574,7 @@ def test_paha_authentication_creates_organization(app): } # Test using fallback organization title - organization_id = "paha-organization-id-2" + organization_id = str(uuid.uuid4()) paha_token = create_paha_token({ "id": some_user['id'], "activeOrganizationId": organization_id, @@ -602,11 +603,11 @@ def test_paha_authentication_creates_new_user(app): email = "foo@example.com" # Get access token with a PAHA token - paha_token = create_paha_token({"id":"test-id", - "email":email, - "firstName":"Foo", - "lastName":"Bar-Baz von Bärzügə", - "activeOrganizationId":organization["id"]}) + paha_token = create_paha_token({"id": str(uuid.uuid4()), + "email" :email, + "firstName": "Foo", + "lastName": "Bar-Baz von Bärzügə", + "activeOrganizationId": organization["id"]}) _auth_token = get_auth_token_for_paha_token(app, paha_token).json['token'] # Verify that the user has been created @@ -618,7 +619,7 @@ def test_paha_authentication_creates_new_user(app): @pytest.mark.usefixtures("with_plugins", "clean_db", "with_request_context") def test_paha_authentication_logs_in_user(app): organization = RestrictedDataOrganization() - test_id = "test-id" + test_id = str(uuid.uuid4()) some_user = User(id=test_id) # Get access token with a PAHA token @@ -723,7 +724,7 @@ def test_paha_auth_fails_with_reused_token(app): client = app.test_client() response = client.get(toolkit.url_for("paha.authorized", token=auth_token)) assert response.status_code == 200 - + # Try to log in again with a different client using the same token client = app.test_client() response = client.get(toolkit.url_for("paha.authorized", token=auth_token)) @@ -904,7 +905,7 @@ def test_normal_user_cannot_edit_user_profile(app): context={"user": user["name"], "ignore_auth": False}, id=user["name"], fullname="Test full name") - + @pytest.mark.usefixtures("with_plugins", "clean_db") def test_normal_user_cannot_request_reset(app): client = app.test_client(use_cookies=True) @@ -914,7 +915,7 @@ def test_normal_user_cannot_request_reset(app): # Anonymous user get result = client.get(request_reset) assert result.status_code == 403 - + # Anonymous user post result = client.post(request_reset, data={'id': user['id']}) assert result.status_code == 403 diff --git a/ckan/scripts/init_ckan.sh b/ckan/scripts/init_ckan.sh index 28ed0afb0..4e8152125 100644 --- a/ckan/scripts/init_ckan.sh +++ b/ckan/scripts/init_ckan.sh @@ -11,25 +11,36 @@ jinja2 ${TEMPLATE_DIR}/ckan.ini.j2 -o ${APP_DIR}/ckan.ini jinja2 ${TEMPLATE_DIR}/who.ini.j2 -o ${APP_DIR}/who.ini # run prerun script that checks connections and inits db -python prerun.py || { echo '[CKAN prerun] FAILED. Exiting...' ; exit 1; } +python connection_check.py || { echo '[CKAN connection check] FAILED. Exiting...' ; exit 1; } + +echo "Init CKAN database ..." +ckan -c ${APP_DIR}/ckan.ini db init echo "Upgrade CKAN database ..." ckan -c ${APP_DIR}/ckan.ini db upgrade -if [[ "${DEV_MODE}" == "true" ]]; then - echo "Initializing test database" - echo ${DB_CKAN_PASS} | psql -h ${DB_CKAN_HOST} -U ${DB_CKAN_USER} -c "CREATE DATABASE ckan_test OWNER ${DB_CKAN_USER} ENCODING 'utf-8'" -fi +#if [[ "${DEV_MODE}" == "true" ]]; then + #echo "Initializing test database" + #echo ${DB_CKAN_PASS} | psql -h ${DB_CKAN_HOST} -U ${DB_CKAN_USER} -c "CREATE DATABASE ckan_test OWNER ${DB_CKAN_USER} ENCODING 'utf-8'" +#fi + # init ckan extensions #echo "init ckan extensions ..." # init ckan extension databases echo "init ckan extension databases ..." ckan -c ${APP_DIR}/ckan.ini db upgrade -p restricteddata -ckan -c ${APP_DIR}/ckan.ini harvester initdb +ckan -c ${APP_DIR}/ckan.ini db upgrade -p activity +ckan -c ${APP_DIR}/ckan.ini db upgrade -p harvest +ckan -c ${APP_DIR}/ckan.ini db upgrade -p pages + if [[ "${MATOMO_ENABLED}" == "true" ]]; then ckan -c ${APP_DIR}/ckan.ini matomo init_db && ckan -c ${APP_DIR}/ckan.ini db upgrade -p matomo fi +# Update client translations +echo "Update client translations ..." +ckan -c ${APP_DIR}/ckan.ini translation js + # set init flag to done echo "$CKAN_IMAGE_TAG" > ${DATA_DIR}/.init-done diff --git a/ckan/scripts/reinit_ckan.sh b/ckan/scripts/reinit_ckan.sh index 7ce07758f..496c68226 100644 --- a/ckan/scripts/reinit_ckan.sh +++ b/ckan/scripts/reinit_ckan.sh @@ -8,7 +8,18 @@ jinja2 ${TEMPLATE_DIR}/ckan.ini.j2 -o ${APP_DIR}/ckan.ini jinja2 ${TEMPLATE_DIR}/who.ini.j2 -o ${APP_DIR}/who.ini # run prerun script that checks connections and inits db -python prerun.py || { echo '[CKAN prerun] FAILED. Exiting...' ; exit 1; } +python connection_check.py || { echo '[CKAN connection check] FAILED. Exiting...' ; exit 1; } + +echo "Init CKAN database ..." +ckan -c ${APP_DIR}/ckan.ini db init echo "Upgrade CKAN database ..." ckan -c ${APP_DIR}/ckan.ini db upgrade +ckan -c ${APP_DIR}/ckan.ini db upgrade -p restricteddata +ckan -c ${APP_DIR}/ckan.ini db upgrade -p activity +ckan -c ${APP_DIR}/ckan.ini db upgrade -p pages +ckan -c ${APP_DIR}/ckan.ini db upgrade -p harvest + +# Update client translations +echo "Update client translations ..." +ckan -c ${APP_DIR}/ckan.ini translation js diff --git a/ckan/setup/app/connection_check.py b/ckan/setup/app/connection_check.py new file mode 100644 index 000000000..df4d44c5e --- /dev/null +++ b/ckan/setup/app/connection_check.py @@ -0,0 +1,108 @@ +""" +Copyright (c) 2016 Keitaro AB +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import base64 +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request +from typing import Optional + +import psycopg2 +from sqlalchemy.engine.url import make_url + +ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini') + +RETRY = 5 + + +def check_db_connection(retry: Optional[int] = None) -> None: + + print('[connection_check] Start check_db_connection...') + + if retry is None: + retry = RETRY + elif retry == 0: + print('[connection_check] Giving up after 5 tries...') + sys.exit(1) + + conn_str = os.environ.get('CKAN_SQLALCHEMY_URL', '') + try: + db_user = make_url(conn_str).username + db_passwd = make_url(conn_str).password + db_host = make_url(conn_str).host + db_name = make_url(conn_str).database + connection = psycopg2.connect(user=db_user, + host=db_host, + password=db_passwd, + database=db_name) + + except psycopg2.Error as e: + print(str(e)) + print('[connection_check] Unable to connect to the database...try again in a while.') + import time + time.sleep(10) + check_db_connection(retry=retry - 1) + else: + connection.close() + + +def check_solr_connection(retry: Optional[int] = None) -> None: + + print('[connection_check] Start check_solr_connection...') + + if retry is None: + retry = RETRY + elif retry == 0: + print('[connection_check] Giving up after 5 tries...') + sys.exit(1) + + url = os.environ.get('CKAN_SOLR_URL', '') + username = os.environ.get('CKAN_SOLR_USER', '') + password = os.environ.get('CKAN_SOLR_PASSWORD', '') + search_url = f'{url}/schema/name?wt=json' + + try: + if not username: + connection = urllib.request.urlopen(search_url) + else: + request = urllib.request.Request(search_url) + base64string = base64.b64encode(bytes(f'{username}:{password}', 'ascii')) + request.add_header("Authorization", f"Basic {base64string.decode('utf-8')}") + connection = urllib.request.urlopen(request) + except urllib.error.URLError: + print('[connection_check] Unable to connect to solr...try again in a while.') + import time + time.sleep(10) + check_solr_connection(retry=retry - 1) + else: + conn_info = connection.read() + schema_name = json.loads(conn_info) + if 'ckan' in schema_name['name']: + print('[connection_check] Succesfully connected to solr and CKAN schema loaded') + else: + print('[connection_check] Succesfully connected to solr, but CKAN schema not found') + sys.exit(1) + + +if __name__ == '__main__': + + maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true' + + if maintenance: + print('[connection_check] Maintenance mode, skipping setup...') + else: + check_db_connection() + check_solr_connection() diff --git a/ckan/setup/app/prerun.py b/ckan/setup/app/prerun.py deleted file mode 100644 index 39f43c4b5..000000000 --- a/ckan/setup/app/prerun.py +++ /dev/null @@ -1,229 +0,0 @@ -""" -Copyright (c) 2016 Keitaro AB -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - https://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import os -import sys -import subprocess -import psycopg2 -from sqlalchemy.engine.url import make_url -import urllib.request -import urllib.error -import urllib.parse -import base64 -import re -import json - -import time - -ckan_ini = os.environ.get('CKAN_INI', '/srv/app/ckan.ini') - -RETRY = 5 - -def check_db_connection(retry=None): - - print('[prerun] Start check_db_connection...') - - if retry is None: - retry = RETRY - elif retry == 0: - print('[prerun] Giving up after 5 tries...') - sys.exit(1) - - conn_str = os.environ.get('CKAN_SQLALCHEMY_URL', '') - try: - db_user = make_url(conn_str).username - db_passwd = make_url(conn_str).password - db_host = make_url(conn_str).host - db_name = make_url(conn_str).database - connection = psycopg2.connect(user=db_user, - host=db_host, - password=db_passwd, - database=db_name) - - except psycopg2.Error as e: - print((str(e))) - print('[prerun] Unable to connect to the database...try again in a while.') - import time - time.sleep(10) - check_db_connection(retry = retry - 1) - else: - connection.close() - -def check_solr_connection(retry=None): - - print('[prerun] Start check_solr_connection...') - - if retry is None: - retry = RETRY - elif retry == 0: - print('[prerun] Giving up after 5 tries...') - sys.exit(1) - - url = os.environ.get('CKAN_SOLR_URL', '') - username = os.environ.get('CKAN_SOLR_USER', '') - password = os.environ.get('CKAN_SOLR_PASSWORD', '') - search_url = '{url}/schema/name?wt=json'.format(url=url) - - - - try: - if not username: - connection = urllib.request.urlopen(search_url, timeout=20) - else: - request = urllib.request.Request(search_url) - base64string = base64.b64encode(bytes('%s:%s' % (username, password),'ascii')) - request.add_header("Authorization", "Basic %s" % base64string.decode('utf-8')) - connection = urllib.request.urlopen(request, timeout=20) - except urllib.error.URLError: - print('[prerun] Unable to connect to solr...try again in a while.') - import time - time.sleep(10) - check_solr_connection(retry = retry - 1) - else: - conn_info = connection.read() - schema_name = json.loads(conn_info) - if 'ckan' in schema_name['name']: - print('[prerun] Succesfully connected to solr and CKAN schema loaded') - else: - print('[prerun] Succesfully connected to solr, but CKAN schema not found') - sys.exit(1) - -def init_db(): - - print('[prerun] Start init_db...') - - db_command = ['ckan', '-c', ckan_ini, 'db', 'init'] - - print('[prerun] Initializing or upgrading db - start using ckan db init') - try: - # run init scripts - subprocess.check_output(db_command, stderr=subprocess.STDOUT) - - print('[prerun] Initializing or upgrading db - end') - except subprocess.CalledProcessError as e: - if 'OperationalError' in str(e.output): - print(e.output.decode('utf-8')) - print('[prerun] Database not ready, waiting a bit before exit...') - import time - time.sleep(5) - sys.exit(1) - else: - print(e.output.decode('utf-8')) - raise e - print('[prerun] Initializing or upgrading db - finish') - - -def init_datastore(): - - conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') - if not conn_str: - print('[prerun] Skipping datastore initialization') - return - - datastore_perms_command = ['ckan', '-c', ckan_ini, 'datastore', - 'set-permissions'] - - db_user = make_url(conn_str).username - db_passwd = make_url(conn_str).password - db_host = make_url(conn_str).host - db_name = make_url(conn_str).database - connection = psycopg2.connect(user=db_user, - host=db_host, - password=db_passwd, - database=db_name) - cursor = connection.cursor() - - print('[prerun] Initializing datastore db - start') - try: - datastore_perms = subprocess.Popen( - datastore_perms_command, - stdout=subprocess.PIPE) - - perms_sql = datastore_perms.stdout.read() - perms_sql = perms_sql.decode('utf-8') - perms_sql = perms_sql.replace("@"+db_host, "") - # Remove internal pg command as psycopg2 does not like it - perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql) - cursor.execute(perms_sql) - for notice in connection.notices: - print(notice) - - connection.commit() - - print('[prerun] Initializing datastore db - end') - print((datastore_perms.stdout.read())) - except psycopg2.Error as e: - print('[prerun] Could not initialize datastore') - print(e.decode('utf-8')) - - except subprocess.CalledProcessError as e: - if 'OperationalError' in str(e.output): - print(e.output.decode('utf-8')) - print('[prerun] Database not ready, waiting a bit before exit...') - time.sleep(5) - sys.exit(1) - else: - print(e.output.decode('utf-8')) - raise e - finally: - cursor.close() - connection.close() - - -def create_sysadmin(): - - print('[prerun] Start create_sysadmin...') - - name = os.environ.get('CKAN_SYSADMIN_NAME') - password = os.environ.get('CKAN_SYSADMIN_PASSWORD') - email = os.environ.get('CKAN_SYSADMIN_EMAIL') - - if name and password and email: - - # Check if user exists - command = ['ckan', '-c', ckan_ini, 'user', 'show', name] - - out = subprocess.check_output(command) - if 'User:None' not in re.sub(r'\s', '', out.decode('utf-8')): - print('[prerun] Sysadmin user exists, skipping creation') - return - - # Create user - command = ['ckan', '-c', ckan_ini, 'user', 'add', - name, - 'password=' + password, - 'email=' + email] - - subprocess.call(command) - print(('[prerun] Created user {0}'.format(name))) - - # Make it sysadmin - command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add', - name] - - subprocess.call(command) - print(('[prerun] Made user {0} a sysadmin'.format(name))) - -if __name__ == '__main__': - - maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true' - - if maintenance: - print('[prerun] Maintenance mode, skipping setup...') - else: - check_db_connection() - check_solr_connection() - init_db() - if os.environ.get('CKAN_DATASTORE_WRITE_URL'): - init_datastore() - create_sysadmin() diff --git a/ckan/templates/ckan.ini.j2 b/ckan/templates/ckan.ini.j2 index df4cd0569..99b6cbb5f 100644 --- a/ckan/templates/ckan.ini.j2 +++ b/ckan/templates/ckan.ini.j2 @@ -20,14 +20,12 @@ use = egg:ckan full_stack = true cache_dir = /tmp/%(ckan.site_id)s/ -beaker.session.key = ckan -beaker.session.secret = {{ environ('CKAN_BEAKER_SESSION_SECRET') }} -beaker.session.cookie_expires = {% if environ('DEV_MODE') == 'true' %}False{% else %}True{% endif %} -# Secure session does not currently work in our environments as ssl is terminated on Load balancerreq -#beaker.session.secure = True -beaker.session.httponly = True -beaker.session.type = cookie -beaker.session.validate_key = {{ environ('CKAN_BEAKER_SESSION_VALIDATE_KEY') }} +SECRET_KEY = {{ environ('CKAN_SESSION_SECRET') }} +SESSION_COOKIE_NAME = ckan +SESSION_COOKIE_SECURE = True +SESSION_COOKIE_HTTPONLY = True +SESSION_COOKIE_SAMESITE = Strict +SESSION_TYPE = cookie app_instance_uuid = {{ environ('CKAN_APP_INSTANCE_UUID') }} diff --git a/docker/.env.ckan.local b/docker/.env.ckan.local index f6bc3df25..04638a1bd 100644 --- a/docker/.env.ckan.local +++ b/docker/.env.ckan.local @@ -2,8 +2,7 @@ # ckan.ini CKAN_DRUPAL_SITE_URL="http://nginx" -CKAN_BEAKER_SESSION_SECRET="9PBdkxokLGWW4M4jeTI25h+4t" -CKAN_BEAKER_SESSION_VALIDATE_KEY="iPBD4P87SdAXPaDERKXhgtiX4P7xhh" +CKAN_SESSION_SECRET="9PBdkxokLGWW4M4jeTI25h+4t" CKAN_APP_INSTANCE_UUID="{dc6259b8-f112-4d23-8816-aadcede1895c}" CKAN_SITE_ID=default CKAN_PLUGINS_DEFAULT="fluent scheming_datasets scheming_groups scheming_organizations" diff --git a/robot/restricteddata.robot b/robot/restricteddata.robot index cc8412be9..177289005 100644 --- a/robot/restricteddata.robot +++ b/robot/restricteddata.robot @@ -136,7 +136,7 @@ Create Test Organisation Page Should Contain Testiorganisaatio Add Test User To Test Organisation - Open URL Path /organization/members/testiorganisaatio + Open URL Path /organization/manage_members/testiorganisaatio Click Link link:Lisää jäsen Input Text Into Select2 username ${TEST_USER_USERNAME} Submit Primary Form diff --git a/robot/tests/site_information.robot b/robot/tests/site_information.robot index 12941243b..1082cba20 100644 --- a/robot/tests/site_information.robot +++ b/robot/tests/site_information.robot @@ -5,7 +5,7 @@ Resource ../restricteddata.robot *** Test cases *** CKAN version - CKAN Version Should Be 2.10.9 + CKAN Version Should Be 2.11.4 Site title Open Chromium