Enhance anon import script to download test user CSV and update accounts

data_import_settings/dev_extras.json

@@ -4,7 +4,7 @@
 	"confirm" : false,
 	"max_content_length" : 40000000,
 	"types" : {
-		"account" : {"import" : true, "limit" : -1},
+		"account" : {"import" : true, "limit" : -1, "download_test_users": true},
 		"application" : {"import" : true, "limit" : -1},
 		"background_job" : {"import" : true, "limit" : 10000},
 		"cache" : {"import" : true, "limit" : -1},
@@ -19,7 +19,6 @@
 		"preserve" : {"import" : true, "limit" : -1},
 		"provenance" : {"import" : true, "limit" : 10000},
 		"upload" : {"import" : true, "limit" : 10000},
-		"datalog_journal_added": {"import" : true, "limit" : -1},
 		"article" : {"import" : true, "limit" : 100000}
 	}
 }

data_import_settings/test_server.json

@@ -4,7 +4,7 @@
 	"confirm" : false,
 	"max_content_length" : 50000000,
 	"types" : {
-		"account" : {"import" : true, "limit" : -1},
+		"account" : {"import" : true, "limit" : -1, "download_test_users": true},
 		"application" : {"import" : true, "limit" : -1},
 		"background_job" : {"import" : true, "limit" : -1},
 		"cache" : {"import" : true, "limit" : -1},

doajtest/helpers.py

@@ -20,15 +20,7 @@
 from portality.lib.dates import FMT_DATE_STD
 from portality.lib.thread_utils import wait_until
 from portality.tasks.redis_huey import events_queue, scheduled_short_queue, scheduled_long_queue
-from portality.util import url_for
-
-
-def patch_config(inst, properties):
-    originals = {}
-    for k, v in properties.items():
-        originals[k] = inst.config.get(k)
-        inst.config[k] = v
-    return originals
+from portality.util import url_for, patch_config
 
 
 def with_es(_func=None, *, indices=None, warm_mappings=None):

doajtest/selenium_helpers.py

@@ -11,7 +11,8 @@
 from selenium.webdriver.common.by import By
 
 from doajtest.fixtures.url_path import URL_LOGOUT
-from doajtest.helpers import DoajTestCase, patch_config
+from doajtest.helpers import DoajTestCase
+from portality.util import patch_config
 from portality import app, models, core
 from portality.dao import ESMappingMissingError
 

doajtest/unit/test_background_helper.py

@@ -2,6 +2,7 @@
 
 import huey.api
 
+import portality.util
 from doajtest import helpers
 from portality import constants
 from portality.background import BackgroundTask
@@ -45,7 +46,7 @@ class TestRedisHueyTaskHelper(TestCase):
     @classmethod
     def setUpClass(cls) -> None:
         super().setUpClass()
-        cls.org_config = helpers.patch_config(app, {
+        cls.org_config = portality.util.patch_config(app, {
             'HUEY_SCHEDULE': {
                 cls.task_name_a: constants.CRON_NEVER,
             },
@@ -57,7 +58,7 @@ def setUpClass(cls) -> None:
     @classmethod
     def tearDownClass(cls) -> None:
         super().tearDownClass()
-        helpers.patch_config(app, cls.org_config)
+        portality.util.patch_config(app, cls.org_config)
 
     def test_01_register_schedule(self):
         helper = background_helper.RedisHueyTaskHelper(scheduled_short_queue, fixture_bgtask_class(self.task_name_a))

doajtest/unit/test_background_task_status_parameterised.py

@@ -13,7 +13,8 @@ def load_cases():
 import json
 
 from doajtest.fixtures.background import save_mock_bgjob
-from doajtest.helpers import DoajTestCase, apply_test_case_config, patch_config
+from doajtest.helpers import DoajTestCase, apply_test_case_config
+from portality.util import patch_config
 from portality import constants
 from portality.bll import DOAJ
 from portality.tasks.anon_export import AnonExportBackgroundTask

doajtest/unit/test_bll_journal_csv.py

@@ -7,7 +7,8 @@
 
 from doajtest import helpers
 from doajtest.fixtures import ArticleFixtureFactory, JournalFixtureFactory
-from doajtest.helpers import DoajTestCase, patch_config
+from doajtest.helpers import DoajTestCase
+from portality.util import patch_config
 from doajtest.mocks.models_Cache import ModelCacheMockFactory
 from doajtest.mocks.store import StoreMockFactory
 from portality import models

doajtest/unit/test_bll_site_sitemap.py

@@ -7,7 +7,7 @@
 
 from doajtest import helpers
 from doajtest.fixtures import JournalFixtureFactory, ArticleFixtureFactory
-from doajtest.helpers import DoajTestCase, patch_config
+from doajtest.helpers import DoajTestCase
 from doajtest.mocks.models_Cache import ModelCacheMockFactory
 from doajtest.mocks.store import StoreMockFactory
 from portality import models
@@ -17,7 +17,7 @@
 from portality.core import app
 from portality.lib import nav
 from portality.lib.paths import rel2abs
-from portality.util import get_full_url_safe
+from portality.util import get_full_url_safe, patch_config
 
 
 def load_cases():

doajtest/unit/test_concurrent_saves.py

@@ -1,4 +1,5 @@
-from doajtest.helpers import DoajTestCase, patch_config
+from doajtest.helpers import DoajTestCase
+from portality.util import patch_config
 from doajtest.fixtures import JournalFixtureFactory
 from portality.models import Journal
 from portality.bll.exceptions import ConcurrentUpdateRequestException

doajtest/unit/test_query_filters.py

@@ -2,7 +2,8 @@
 from flask_login import login_user
 
 from doajtest.fixtures import AccountFixtureFactory, EditorGroupFixtureFactory, ApplicationFixtureFactory
-from doajtest.helpers import DoajTestCase, patch_config
+from doajtest.helpers import DoajTestCase
+from portality.util import patch_config
 from portality import models
 from portality.lib import query_filters
 from portality.bll.services.query import Query

doajtest/unit/test_task_discontinued_soon.py

@@ -1,7 +1,8 @@
 import unittest
 import datetime
 
-from doajtest.helpers import DoajTestCase, patch_config
+from doajtest.helpers import DoajTestCase
+from portality.util import patch_config
 
 from portality import models
 from portality.tasks import find_discontinued_soon

doajtest/unit/test_tasks_old_data_cleanup.py

@@ -2,6 +2,7 @@
 import time
 from typing import Iterable
 
+import portality.util
 from doajtest import helpers
 from doajtest.helpers import DoajTestCase
 from portality.dao import DomainObject
@@ -40,7 +41,7 @@ def test_clean_all_old_data__normal(self):
         self.assertEqual(BackgroundJob.hit_count(es_queries.query_all()),
                          len(days_for_test))
 
-        org_config = helpers.patch_config(self.app_test, {
+        org_config = portality.util.patch_config(self.app_test, {
             "TASK_DATA_RETENTION_DAYS": {
                 "notification": 4,
                 "background_job": 7,
@@ -55,7 +56,7 @@ def test_clean_all_old_data__normal(self):
         self.assertEqual(Notification.hit_count(es_queries.query_all()), 4)
         self.assertEqual(BackgroundJob.hit_count(es_queries.query_all()), 6)
 
-        helpers.patch_config(self.app_test, org_config)
+        portality.util.patch_config(self.app_test, org_config)
 
     def test_clean_all_old_data__no_obj_deleted(self):
         # prepare data
@@ -67,7 +68,7 @@ def test_clean_all_old_data__no_obj_deleted(self):
         self.assertEqual(org_size, len(days_for_test))
         self.assertEqual(BackgroundJob.hit_count(es_queries.query_all()), 0)
 
-        org_config = helpers.patch_config(self.app_test, {
+        org_config = portality.util.patch_config(self.app_test, {
             "TASK_DATA_RETENTION_DAYS": {
                 "notification": 1000,
                 "background_job": 7,
@@ -82,4 +83,4 @@ def test_clean_all_old_data__no_obj_deleted(self):
         self.assertEqual(Notification.hit_count(es_queries.query_all()), org_size)
         self.assertEqual(BackgroundJob.hit_count(es_queries.query_all()), 0)
 
-        helpers.patch_config(self.app_test, org_config)
+        portality.util.patch_config(self.app_test, org_config)

portality/core.py

@@ -171,7 +171,6 @@ def load_crossref_schema(app):
                 message="There was an error attempting to load schema from " + path, inner=e)
 
 
-
 ############################################
 # Elasticsearch initialisation
 
@@ -185,23 +184,45 @@ def create_es_connection(app):
     return conn
 
 
+def prepare_type(es_type):
+    """ Ensure a type has an index correctly prepared - e.g. LCC on app startup """
+    expected_alias = app.config['ELASTIC_SEARCH_DB_PREFIX'] + es_type
+
+    if not es_connection.indices.exists(expected_alias):
+        initialise_index(app, es_connection, only_mappings=es_type)
+
+
 def put_mappings(conn, mappings):
 
     for key, mapping in iter(mappings.items()):
         altered_key = app.config['ELASTIC_SEARCH_DB_PREFIX'] + key
-        if not conn.indices.exists(altered_key):
-            r = conn.indices.create(index=altered_key, body=mapping,
-                                    request_timeout=app.config.get("ES_SOCKET_TIMEOUT", None))
-            print("Creating ES Type + Mapping in index {0} for {1}; status: {2}".format(altered_key, key, r))
+
+        # If the alias exists, we don't create any new indices (app already initialised)
+        if conn.indices.exists(altered_key):
+            print("Alias {0} already exists for type {1}".format(altered_key, key))
         else:
-            print("ES Type + Mapping already exists in index {0} for {1}".format(altered_key, key))
+            print("Preparing new index / alias for " + key)
+            # Set up a new index and corresponding alias
+            idx_name = altered_key + '-{}'.format(dates.today(dates.FMT_DATE_SHORT))
+
+            try:
+                resp = es_connection.indices.create(index=idx_name,
+                                                    body=mapping,
+                                                    request_timeout=app.config.get("ES_SOCKET_TIMEOUT", None))
+                print("Initialised index: {}".format(resp['index']))
+            except elasticsearch.exceptions.RequestError as e:
+                print('Could not create index: ' + str(e))
+
+            resp2 = es_connection.indices.put_alias(index=idx_name, name=altered_key)
+            print("Created alias:     {:<25} -> {},  status {}".format(idx_name, altered_key, resp2))
 
 
 def initialise_index(app, conn, only_mappings=None):
     """
     ~~InitialiseIndex:Framework->Elasticsearch:Technology~~
     :param app:
     :param conn:
+    :param only_mappings: Init a subset of the index types
     :return:
     """
     if not app.config['INITIALISE_INDEX']:
@@ -216,7 +237,7 @@ def initialise_index(app, conn, only_mappings=None):
     mappings = es_data_mapping.get_mappings(app)
 
     if only_mappings is not None:
-        mappings = {key:value for (key, value) in mappings.items() if key in only_mappings}
+        mappings = {key: value for (key, value) in mappings.items() if key in only_mappings}
 
     # Send the mappings to ES
     put_mappings(conn, mappings)

portality/lcc.py

@@ -3,6 +3,7 @@
 """
 from portality.lib import paths
 from portality.models import LCC
+from portality.core import prepare_type
 
 
 def loadLCC(source=None):
@@ -57,6 +58,9 @@ def loadLCC(source=None):
         pn["children"].append(cn)
 
     lcc = LCC(**tree)
+
+    # Initialise the LCC index correctly if it doesn't exist - otherwise it'll be created badly on save
+    prepare_type(lcc.__type__)
     lcc.save()
 
 

portality/scripts/anon_import.py

@@ -20,17 +20,19 @@
 import json
 import re
 import shutil
+import tempfile
+import requests
 from dataclasses import dataclass
 from time import sleep
 
 import portality.dao
-from doajtest.helpers import patch_config
 from portality import models
+from portality.scripts.createuser import create_users
 from portality.core import app, es_connection
 from portality.dao import DomainObject
 from portality.lib import dates, es_data_mapping
 from portality.store import StoreFactory
-from portality.util import ipt_prefix
+from portality.util import ipt_prefix, patch_config
 
 
 @dataclass
@@ -46,6 +48,24 @@ def find_toberemoved_indexes(prefix):
             yield index
 
 
+def dl_test_users():
+    """ Download the test user CSV from google sheets and save to tmp file for reading """
+
+    csv_url = app.config.get('TEST_USERS_CSV_DL_PATH')
+    if not csv_url:
+        print("No test user CSV supplied. Skipping.")
+    else:
+
+        with tempfile.NamedTemporaryFile(delete=False) as tf:
+            with requests.get(csv_url, stream=True) as r:
+                for line in r.iter_lines():
+                    tf.write(line + '\n'.encode())
+                tf.close()
+
+            # Supply the temp file to the create_users function
+            create_users(tf.name)
+
+
 def do_import(config):
     # filter for the types we are going to work with
     import_types = {}
@@ -60,9 +80,9 @@ def do_import(config):
     print("\n")
 
     toberemoved_index_prefixes = [ipt_prefix(import_type) for import_type in import_types.keys()]
-    toberemoved_indexes = itertools.chain.from_iterable(
+    toberemoved_indexes = list(itertools.chain.from_iterable(
         find_toberemoved_indexes(p) for p in toberemoved_index_prefixes
-    )
+    ))
     toberemoved_index_aliases = list(portality.dao.find_index_aliases(toberemoved_index_prefixes))
 
     if toberemoved_indexes:
@@ -160,8 +180,11 @@ def do_import(config):
 
                 n += 1
 
-            else:
-                print(("dao class not available for the import {x}. Skipping import {x}".format(x=import_type)))
+        else:
+            print(("dao class not available for the import {x}. Skipping import {x}".format(x=import_type)))
+
+        if cfg.get("download_test_users"):
+            dl_test_users()
 
     # once we've finished importing, clean up by deleting the entire temporary container
     tempStore.delete_container(container)

portality/settings.py

@@ -1601,6 +1601,10 @@
 # value should be key file path of json, empty string means disabled
 GOOGLE_KEY_PATH = ''
 
+# The /export path to test users CSV file on google sheets (file is public)
+TEST_USERS_CSV_DL_PATH = ""
+
+
 #############################################
 # Datalog
 # ~~->Datalog:Feature~~

portality/util.py

@@ -178,4 +178,12 @@ def get_full_url_safe(endpoint):
 
 def no_op(*args, **kwargs):
     """ noop (no operation) function """
-    pass
+    pass
+
+
+def patch_config(inst, properties):
+    originals = {}
+    for k, v in properties.items():
+        originals[k] = inst.config.get(k)
+        inst.config[k] = v
+    return originals