Merge branch 'develop' into release/4.1.1

Author: stchris

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Aleph Community Discourse
+    url: https://aleph.discourse.group
+    about: Get support for running and developing Aleph and related projects here.

.github/workflows/build-ui.yml

@@ -38,6 +38,9 @@ jobs:
       - name: Run tests
         run: make test-ui
 
+      - name: Build
+        run: make build-ui
+
       - name: Build production image
         if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
         run: docker build -t ghcr.io/alephdata/aleph-ui-production:${GITHUB_SHA} -f ui/Dockerfile.production ui

Dockerfile

@@ -1,21 +1,19 @@
-FROM ubuntu:20.04
+FROM python:3.9
 ENV DEBIAN_FRONTEND noninteractive
 
 # build-essential
 RUN apt-get -qq -y update \
-    && apt-get -qq --no-install-recommends -y install locales \
-    ca-certificates postgresql-client libpq-dev curl jq \
-    python3-pip python3-icu python3-psycopg2 \
-    python3-lxml python3-crypto git \
-    && apt-get -qq -y autoremove \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
-    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
+  && apt-get -qq --no-install-recommends -y install locales \
+  postgresql-client jq python3-dev \
+  && apt-get -qq -y autoremove \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+  && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
 
 ENV LANG='en_US.UTF-8'
 
 RUN groupadd -g 1000 -r app \
-    && useradd -m -u 1000 -s /bin/false -g app app
+  && useradd -m -u 1000 -s /bin/false -g app app
 
 # Install Python dependencies
 RUN pip3 install --no-cache-dir -q -U pip setuptools six
@@ -32,19 +30,19 @@ RUN pip install --no-cache-dir -q -e /aleph
 ENV ALEPH_WORD_FREQUENCY_URI=https://public.data.occrp.org/develop/models/word-frequencies/word_frequencies-v0.4.1.zip
 ENV ALEPH_FTM_COMPARE_MODEL_URI=https://public.data.occrp.org/develop/models/xref/glm_bernoulli_2e_wf-v0.4.1.pkl
 RUN mkdir -p /opt/ftm-compare/word-frequencies/ && \
-    curl -L -o "/opt/ftm-compare/word-frequencies/word-frequencies.zip" "$ALEPH_WORD_FREQUENCY_URI" && \
-    python3 -m zipfile --extract /opt/ftm-compare/word-frequencies/word-frequencies.zip /opt/ftm-compare/word-frequencies/ && \
-    curl -L -o "/opt/ftm-compare/model.pkl" "$ALEPH_FTM_COMPARE_MODEL_URI"
+  curl -L -o "/opt/ftm-compare/word-frequencies/word-frequencies.zip" "$ALEPH_WORD_FREQUENCY_URI" && \
+  python3 -m zipfile --extract /opt/ftm-compare/word-frequencies/word-frequencies.zip /opt/ftm-compare/word-frequencies/ && \
+  curl -L -o "/opt/ftm-compare/model.pkl" "$ALEPH_FTM_COMPARE_MODEL_URI"
 
 # Configure some docker defaults:
 ENV ALEPH_ELASTICSEARCH_URI=http://elasticsearch:9200/ \
-    ALEPH_DATABASE_URI=postgresql://aleph:aleph@postgres/aleph \
-    FTM_STORE_URI=postgresql://aleph:aleph@postgres/aleph \
-    REDIS_URL=redis://redis:6379/0 \
-    ARCHIVE_TYPE=file \
-    ARCHIVE_PATH=/data \
-    FTM_COMPARE_FREQUENCIES_DIR=/opt/ftm-compare/word-frequencies/ \
-    FTM_COMPARE_MODEL=/opt/ftm-compare/model.pkl
+  ALEPH_DATABASE_URI=postgresql://aleph:aleph@postgres/aleph \
+  FTM_STORE_URI=postgresql://aleph:aleph@postgres/aleph \
+  REDIS_URL=redis://redis:6379/0 \
+  ARCHIVE_TYPE=file \
+  ARCHIVE_PATH=/data \
+  FTM_COMPARE_FREQUENCIES_DIR=/opt/ftm-compare/word-frequencies/ \
+  FTM_COMPARE_MODEL=/opt/ftm-compare/model.pkl
 
 RUN mkdir /run/prometheus
 

SUPPORT.md

@@ -8,7 +8,7 @@ The objective of the Aleph project is to provide powerful software to those who
 
 ## Eligible use cases
 
-The maintainers of this project limit support and responses both on GitHub and in the Slack channel to authorised groups and individuals. In order to receive authorisation, we require that you disclose the manner in which you use our technology. **We will decide if that use falls within the intended uses of Aleph.**
+The maintainers of this project limit support and responses both on GitHub and on Discourse to authorised groups and individuals. In order to receive authorisation, we require that you disclose the manner in which you use our technology. **We will decide if that use falls within the intended uses of Aleph.**
 
 Examples of intended uses could include:
 
@@ -20,7 +20,7 @@ Support includes installation support, requests for new features or issues speci
 
 Please submit a description of your use case along with your name, affiliation and email address using one of the following channels:
 
-- The Slack #intros channel
+- As part of the [Discourse](https://aleph.discourse.group/) signup process
 - As part of the first GitHub issue that you file
 - Via our [contact form](https://requests.occrp.org/datadesk)
 
@@ -30,4 +30,4 @@ Make sure you describe your goals, rather than the set of techniques that define
 
 The Aleph team supports feature versions for 12 months after the first major iteration of that version was released. For example, we support Aleph 3.12.x for 12 months after Aleph 3.12.0 was released. The Aleph team supports upgrades, but only from supported feature versions of the product. Support means helping to ensure that you can get your Aleph instance up and running. As we're a small team we don't have the capacity to backport bugs to supported versions. In the case of critical secruity vulnrabilities we'll endeavour to ensure that all currently supported versions, but we recommend administrators upgrade to the latest version as soon as possible.
 
-For versions that are supported, if you are having problems, you can reach out to us in Slack or by raising an issue in Github.
+For versions that are supported, if you are having problems, you can reach out to us on Discourse or by raising an issue in Github.

aleph.env.tmpl

@@ -91,6 +91,10 @@ ALEPH_OAUTH_SECRET=
 # Define a different ftm entity store:
 # FTM_STORE_URI=postgresql://<username>:<password>@<host>/<database>
 
+# ingest-file uses a document cache to speed up operations
+# (it defaults to either one of the two above postgres connection strings)
+# TAGS_DATABASE_URI=postgresql://<username>:<password>@<host>/<database>
+
 # Queue mechanism
 # REDIS_URL=redis://redis:6379/0
 # RABBITMQ_URL=rabbitmq

aleph/__init__.py

@@ -1,9 +1,9 @@
 import logging
 import warnings
 from sqlalchemy.exc import SAWarning
-from pkg_resources import get_distribution
+from importlib.metadata import version
 
-__version__ = get_distribution("aleph").version
+__version__ = version("aleph")
 
 # shut up useless SA warning:
 warnings.filterwarnings(

aleph/manage.py

@@ -1,4 +1,6 @@
 # coding: utf-8
+import importlib.metadata
+import platform
 import sys
 import json
 import click
@@ -8,6 +10,7 @@
 from itertools import count
 from normality import slugify
 from tabulate import tabulate
+from typing import Any
 from flask.cli import FlaskGroup
 from followthemoney.cli.util import write_object
 from servicelayer.taskqueue import flush_queues
@@ -25,7 +28,6 @@
     hash_plaintext_api_keys as _hash_plaintext_api_keys,
 )
 from aleph.index.entities import iter_proxies
-from aleph.index.util import AlephOperationalException
 from aleph.logic.collections import create_collection, update_collection
 from aleph.logic.collections import delete_collection, reindex_collection
 from aleph.logic.collections import upgrade_collections, reingest_collection
@@ -85,7 +87,28 @@ def ensure_collection(foreign_id, label):
     return collection
 
 
-@click.group(cls=FlaskGroup, create_app=create_app)
+def print_aleph_version(ctx: click.Context, _: click.Parameter, value: Any) -> None:
+    if not value or ctx.resilient_parsing:
+        return
+
+    click.echo(
+        f"Aleph {importlib.metadata.version('aleph')}\n"
+        f"Python {platform.python_version()}\n"
+        f"Flask {importlib.metadata.version('flask')}\n"
+        f"Werkzeug {importlib.metadata.version('werkzeug')}",
+    )
+    ctx.exit()
+
+
+@click.group(cls=FlaskGroup, create_app=create_app, add_version_option=False)
+@click.option(
+    "-v",
+    "--version",
+    help="Show the Aleph version.",
+    callback=print_aleph_version,
+    expose_value=False,
+    is_flag=True,
+)
 def cli():
     """Server-side command line for aleph."""
 
@@ -503,11 +526,11 @@ def publish(foreign_id):
 
 @cli.command()
 def upgrade():
-    """Create or upgrade the search index and database."""
-    try:
-        upgrade_system()
-    except AlephOperationalException:
-        log.exception("Failed to upgrade.")
+    """Create or upgrade the search index and database.
+
+    :raises AlephOperationalException:
+    """
+    upgrade_system()
 
 
 @cli.command()

aleph/settings.py

@@ -148,7 +148,9 @@ def __init__(self) -> None:
 
         # API rate limiting (req/min for anonymous users)
         self.API_RATE_LIMIT = env.to_int("ALEPH_API_RATE_LIMIT", 30)
-        self.API_RATE_WINDOW = 15  # minutes
+
+        # API rate window (in minutes)
+        self.API_RATE_WINDOW = env.to_int("ALEPH_API_RATE_WINDOW", 15)
 
         # Export file size limit
         self.EXPORT_MAX_SIZE = env.to_int(

aleph/tests/test_entities_api.py

@@ -716,4 +716,21 @@ def test_expand(self):
             prop = res["property"]
             assert prop == "holder", prop
             assert res["count"] == 1, pformat(res)
-            assert len(res["entities"]) == 1, pformat(res)
+        assert len(res["entities"]) == 1, pformat(res)
+
+    def test_view_transliterate(self):
+        _, headers = self.login(is_admin=True)
+
+        data = {
+            "id": "1",
+            "schema": "Person",
+            "properties": {
+                "name": ["İlham Əliyev"],
+            },
+        }
+        entity = self.create_entity(data, self.col)
+        index_entity(entity)
+
+        res = self.client.get(f"/api/2/entities/{entity.id}", headers=headers)
+        assert res.json["properties"]["name"][0] == "İlham Əliyev"
+        assert res.json["latinized"]["İlham Əliyev"] == "Ilham Aliyev"

aleph/tests/test_manage.py

@@ -308,3 +308,9 @@ def test_userdel_nonexistent_group(self):
         )
         assert result.exit_code != 0
         assert isinstance(result.exception, SystemExit)
+
+    def test_version_option_shows_aleph_version(self):
+        for option in ["-v", "--version"]:
+            result = self.runner.invoke(manage.cli, option)
+            assert result.exit_code == 0
+            assert "Aleph" in result.output