diff --git a/.github/workflows/MCUsReleaseTest.yaml b/.github/workflows/MCUsReleaseTest.yaml index 3fc212e98..8e8ea7eae 100644 --- a/.github/workflows/MCUsReleaseTest.yaml +++ b/.github/workflows/MCUsReleaseTest.yaml @@ -102,9 +102,12 @@ jobs: ES_PASSWORD: ${{ secrets.ES_PASSWORD }} ES_INDEX_TEST: ${{ secrets.ES_INDEX_TEST }} ES_INDEX_LIVE: ${{ secrets.ES_INDEX_LIVE }} + ES_INDEX_EXPERIMENTAL: ${{ secrets.ES_INDEX_EXPERIMENTAL }} run: | echo "Indexing to Test." python -u scripts/index.py ${{ github.repository }} ${{ steps.app-token.outputs.token }} ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} "False" ${{ github.event.inputs.release_version }} "False" "False" + echo "Indexing to Experimental." + python -u scripts/index.py ${{ github.repository }} ${{ steps.app-token.outputs.token }} ${{ secrets.ES_INDEX_EXPERIMENTAL }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} "False" ${{ github.event.inputs.release_version }} "False" "False" - name: Notify Mattermost - Test ready env: diff --git a/.github/workflows/checkIndexes.yaml b/.github/workflows/checkIndexes.yaml index 71cb11f7a..e21ddc2e9 100644 --- a/.github/workflows/checkIndexes.yaml +++ b/.github/workflows/checkIndexes.yaml @@ -64,6 +64,15 @@ jobs: python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY" "--index_legacy_packages" "${{ github.event.inputs.index_legacy }}" continue-on-error: true # Ensure the workflow continues + - name: Check Indexed Links - Experimental + if: ${{ github.event.inputs.select_index == 'Test' || github.event.inputs.select_index == 'Both' }} + run: | + FIX_ACTION=${{ github.event.inputs.fix }} # Capture the fix input + LOG_ONLY=$([[ "$FIX_ACTION" == "false" ]] && echo true || echo false) # Negate the fix input + echo "LOG_ONLY is set to $LOG_ONLY" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_EXPERIMENTAL }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY" "--index_legacy_packages" "${{ github.event.inputs.index_legacy }}" + continue-on-error: true # Ensure the workflow continues + - name: Check Indexed Links - Test if: ${{ github.event.inputs.select_index == 'Test' || github.event.inputs.select_index == 'Both' }} run: | @@ -95,6 +104,11 @@ jobs: python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX" continue-on-error: true # Ensure the workflow continues + - name: Check Indexed Links - Experimental + run: | + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_EXPERIMENTAL }} "--es_regex" "$GLOBAL_REGEX" + continue-on-error: true # Ensure the workflow continues + - name: Check Indexed Links - Test run: | python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX" @@ -122,6 +136,11 @@ jobs: python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX" continue-on-error: true # Ensure the workflow continues + - name: Check Indexed Links - Experimental + run: | + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_EXPERIMENTAL }} "--es_regex" "$GLOBAL_REGEX" + continue-on-error: true # Ensure the workflow continues + - name: Check Indexed Links - Test run: | python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX" diff --git a/.github/workflows/index.yaml b/.github/workflows/index.yaml index 43aec22fe..9c619b70b 100644 --- a/.github/workflows/index.yaml +++ b/.github/workflows/index.yaml @@ -70,6 +70,8 @@ jobs: echo "Promote to latest requested, but ignored. Only available for LIVE updates." fi python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} ${{ github.event.inputs.force_index }} ${{ github.event.inputs.release_version }} False False "--keep_previous_dates" ${{ github.event.inputs.keep_dates }} + echo "Indexing to Experimental." + python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_EXPERIMENTAL }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} ${{ github.event.inputs.force_index }} ${{ github.event.inputs.release_version }} False False "--keep_previous_dates" ${{ github.event.inputs.keep_dates }} fi - name: Send notification to Mattermost diff --git a/.github/workflows/releaseCodegripPackages.yaml b/.github/workflows/releaseCodegripPackages.yaml index a092ec43f..b1ed55598 100644 --- a/.github/workflows/releaseCodegripPackages.yaml +++ b/.github/workflows/releaseCodegripPackages.yaml @@ -116,6 +116,8 @@ jobs: if [[ ${{ github.event.inputs.select_index }} == "Test" ]]; then echo "Indexing to Test." python -u scripts/index_codegrip_packages.py ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ github.event.inputs.start_date }} ${{ github.event.inputs.end_date }} > message.txt + echo "Indexing to Experimental." + python -u scripts/index_codegrip_packages.py ${{ secrets.ES_INDEX_EXPERIMENTAL }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ github.event.inputs.start_date }} ${{ github.event.inputs.end_date }} > message.txt else echo "Indexing to Live." python -u scripts/index_codegrip_packages.py ${{ secrets.ES_INDEX_LIVE }} ${{ secrets.PROG_DEBUG_CODEGRIP_LIVE }} ${{ github.event.inputs.start_date }} ${{ github.event.inputs.end_date }} > message.txt diff --git a/.github/workflows/updateDb.yaml b/.github/workflows/updateDb.yaml index d33066505..4b5f37b49 100644 --- a/.github/workflows/updateDb.yaml +++ b/.github/workflows/updateDb.yaml @@ -80,4 +80,6 @@ jobs: else echo "Indexing database to TEST." python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} False ${{ github.event.inputs.release_version }} True False + echo "Indexing database to EXPERIMENTAL." + python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_EXPERIMENTAL }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} False ${{ github.event.inputs.release_version }} True False fi diff --git a/.github/workflows/updateDbDevices.yaml b/.github/workflows/updateDbDevices.yaml index dba90546e..65ae977b8 100644 --- a/.github/workflows/updateDbDevices.yaml +++ b/.github/workflows/updateDbDevices.yaml @@ -228,6 +228,8 @@ jobs: if [[ ${{ steps.mikrosdk_payload.outputs.index }} == "Test" ]]; then echo "Indexing database to TEST." python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} "True" "latest" "True" "False" + echo "Indexing database to EXPERIMENTAL." + python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_EXPERIMENTAL }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} "True" "latest" "True" "False" fi if [[ ${{ steps.mikrosdk_payload.outputs.index }} == "Live" ]]; then echo "Indexing database to LIVE." diff --git a/.github/workflows/updateDbFromSdk.yaml b/.github/workflows/updateDbFromSdk.yaml index b8af55013..3f2c62f77 100644 --- a/.github/workflows/updateDbFromSdk.yaml +++ b/.github/workflows/updateDbFromSdk.yaml @@ -82,6 +82,7 @@ jobs: ES_PASSWORD: ${{ secrets.ES_PASSWORD }} ES_INDEX_TEST: ${{ secrets.ES_INDEX_TEST }} ES_INDEX_LIVE: ${{ secrets.ES_INDEX_LIVE }} + ES_INDEX_EXPERIMENTAL: ${{ secrets.ES_INDEX_EXPERIMENTAL }} run: | if [[ ${{ steps.mikrosdk_payload.outputs.index }} == "Live" ]]; then echo "Uploading and indexing to Live." @@ -118,6 +119,8 @@ jobs: if [[ ${{ steps.mikrosdk_payload.outputs.index }} == "Test" ]]; then echo "Indexing database to TEST." python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_TEST }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} "True" "latest" "True" "False" + echo "Indexing database to EXPERIMENTAL." + python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_INDEX_EXPERIMENTAL }} ${{ secrets.PROG_DEBUG_CODEGRIP_TEST }} "True" "latest" "True" "False" fi if [[ ${{ steps.mikrosdk_payload.outputs.index }} == "Live" ]]; then echo "Indexing database to LIVE." diff --git a/.github/workflows/updateSchemasAndClocks.yaml b/.github/workflows/updateSchemasAndClocks.yaml index f74e7a788..4d3081797 100644 --- a/.github/workflows/updateSchemasAndClocks.yaml +++ b/.github/workflows/updateSchemasAndClocks.yaml @@ -77,6 +77,7 @@ jobs: ES_PASSWORD: ${{ secrets.ES_PASSWORD }} ES_INDEX_TEST: ${{ secrets.ES_INDEX_TEST }} ES_INDEX_LIVE: ${{ secrets.ES_INDEX_LIVE }} + ES_INDEX_EXPERIMENTAL: ${{ secrets.ES_INDEX_EXPERIMENTAL }} run: | INDEX_TEST="True" if [[ ${{ steps.determine-trigger.outputs.run_test }} == "False" ]]; then @@ -87,7 +88,7 @@ jobs: echo "Uploading and indexing to Live." python -u scripts/update_schemas.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ steps.determine-trigger.outputs.run_version }} ${{ secrets.ES_INDEX_LIVE }} "$INDEX_TEST" else - echo "Uploading and indexing to Test and Live." + echo "Uploading and indexing to Test, Experimental and Live." python -u scripts/update_schemas.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ steps.determine-trigger.outputs.run_version }} ${{ secrets.ES_INDEX_TEST }} "$INDEX_TEST" fi @@ -98,6 +99,7 @@ jobs: ES_PASSWORD: ${{ secrets.ES_PASSWORD }} ES_INDEX_TEST: ${{ secrets.ES_INDEX_TEST }} ES_INDEX_LIVE: ${{ secrets.ES_INDEX_LIVE }} + ES_INDEX_EXPERIMENTAL: ${{ secrets.ES_INDEX_EXPERIMENTAL }} run: | - echo "Uploading and indexing to Test and Live." + echo "Uploading and indexing to Test, Experimental and Live." python -u scripts/update_clocks.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} "latest" diff --git a/scripts/check_indexes.py b/scripts/check_indexes.py index 9e81fa2b3..224a60408 100644 --- a/scripts/check_indexes.py +++ b/scripts/check_indexes.py @@ -101,7 +101,7 @@ def str2bool(v): package_name = f'{indexed_item['source']['name']}.7z' else: package_name = f'{indexed_item['source']['name']}.json' - if indexed_item['source']['name'] == 'database' and 'test' in args.es_index: + if indexed_item['source']['name'] == 'database' and ('test' in args.es_index or 'experimental' in args.es_index): package_name = 'database_dev.7z' # Set gh_package_name only for github assets if 'gh_package_name' not in indexed_item['source'] and 'Device Pack' not in indexed_item['source']['category'] and indexed_item['source']['author'] not in thirdparty_authors: diff --git a/scripts/index.py b/scripts/index.py index a0cf14312..84b9d64df 100644 --- a/scripts/index.py +++ b/scripts/index.py @@ -397,7 +397,7 @@ def index_release_to_elasticsearch(es : Elasticsearch, index_name, release_detai package_name = name_without_extension if 'database' in name_without_extension: package_name = 'database' - if ('dev' in name_without_extension) and ('test' in index_name): + if ('dev' in name_without_extension) and ('test' in index_name or 'experimental' in index_name): print("Database test version.") elif ('dev' not in name_without_extension) and ('live' in index_name): print("Database live version.") @@ -481,11 +481,14 @@ def index_release_to_elasticsearch(es : Elasticsearch, index_name, release_detai print(f"{resp["result"]} {resp['_id']}") # Database is indexed as separate ID for both indexes, so skip it in this step if (name_without_extension in always_index) and ('database' not in name_without_extension): - if ('ES_INDEX_TEST' in os.environ) and ('ES_INDEX_LIVE' in os.environ): + if ('ES_INDEX_TEST' in os.environ) and ('ES_INDEX_LIVE' in os.environ) and ('ES_INDEX_EXPERIMENTAL' in os.environ): if index_name == os.environ['ES_INDEX_TEST']: resp = es.index(index=os.environ['ES_INDEX_LIVE'], doc_type=None, id=name_without_extension, body=doc) print(f"Indexed to LIVE as well.") print(f"{resp["result"]} {resp['_id']}") + resp = es.index(index=os.environ['ES_INDEX_EXPERIMENTAL'], doc_type=None, id=name_without_extension, body=doc) + print(f"Indexed to EXPERIMENTAL as well.") + print(f"{resp["result"]} {resp['_id']}") # Print new version after indexing if previous_version != doc['version'] and True == doc['package_changed']: diff --git a/scripts/update_clocks.py b/scripts/update_clocks.py index ff4e683be..acbf8b0e9 100644 --- a/scripts/update_clocks.py +++ b/scripts/update_clocks.py @@ -57,6 +57,10 @@ def index_clocks(es: Elasticsearch, release_details, version, current_hash): resp = es.index(index=os.environ['ES_INDEX_TEST'], doc_type=None, id=f'clocks', body=doc) print(f"{resp['result']} {resp['_id']}") + doc['_type'] = '_doc' + resp = es.index(index=os.environ['ES_INDEX_EXPERIMENTAL'], doc_type=None, id=f'clocks', body=doc) + print(f"{resp['result']} {resp['_id']}") + resp = es.index(index=os.environ['ES_INDEX_LIVE'], doc_type=None, id=f'clocks', body=doc) print(f"{resp['result']} {resp['_id']}") diff --git a/scripts/update_schemas.py b/scripts/update_schemas.py index 85bda5d0c..a4702166c 100644 --- a/scripts/update_schemas.py +++ b/scripts/update_schemas.py @@ -183,10 +183,12 @@ def index_schemas(es: Elasticsearch, release_details, version, index_name, curre print(f"{resp['result']} {resp['_id']}") # Special case - update live index Elasticsearch base as well - if 'ES_INDEX_TEST' in os.environ and 'ES_INDEX_LIVE' in os.environ: + if 'ES_INDEX_TEST' in os.environ and 'ES_INDEX_LIVE' in os.environ and 'ES_INDEX_EXPERIMENTAL' in os.environ: if index_name == os.environ['ES_INDEX_TEST']: resp = es.index(index=os.environ['ES_INDEX_LIVE'], doc_type=None, id=f'schemas{test_version}', body=doc) print(f"{resp['result']} {resp['_id']}") + resp = es.index(index=os.environ['ES_INDEX_EXPERIMENTAL'], doc_type=None, id=f'schemas{test_version}', body=doc) + print(f"{resp['result']} {resp['_id']}") async def upload_asset(session, token, repo, tag_name, asset_path): """ Upload a release asset to GitHub asynchronously """