diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 24d504ada34..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,344 +0,0 @@ -version: 2 -jobs: - build_test_deploy: - machine: - image: ubuntu-2004:202111-02 - docker_layer_caching: true - resource_class: large - environment: - USER_NAME: circleci - USER_UID: 1000 - USER_GID: 1000 - TZ: Europe/Berlin - steps: - - checkout - - run: - name: "Custom environment variables" - command: | - if [ -n "$CIRCLE_BRANCH" ]; then - NORMALIZED_BRANCH=${CIRCLE_BRANCH//[\/-]/_} - echo "export NORMALIZED_BRANCH=${NORMALIZED_BRANCH}" >> $BASH_ENV - echo "export DOCKER_TAG=${NORMALIZED_BRANCH}__${CIRCLE_BUILD_NUM}" >> $BASH_ENV - fi - if [ -n "$CIRCLE_TAG" ]; then - echo "export NORMALIZED_BRANCH=master" >> $BASH_ENV - echo "export DOCKER_TAG=${CIRCLE_TAG}" >> $BASH_ENV - fi - - run: - name: Build webknossos-dev docker image - command: | - docker pull lincbrain/webknossos-dev:$NORMALIZED_BRANCH || true - DEV_CACHE=$NORMALIZED_BRANCH docker-compose build base - - run: - name: Prepare dependency folders - command: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier ~/.cache/yarn - - restore_cache: - name: Restore target cache - keys: - - target-cache-{{ checksum ".circleci/cache_version" }}-{{ .Branch }} - - target-cache-{{ checksum ".circleci/cache_version" }}-master - - restore_cache: - name: Restore sbt cache - keys: - - sbt-cache-{{ checksum ".circleci/cache_version" }}-{{ checksum "project/Dependencies.scala" }} - - sbt-cache-{{ checksum ".circleci/cache_version" }}- - - restore_cache: - name: Restore yarn cache - keys: - - yarn-cache-{{ checksum ".circleci/cache_version" }}-{{ checksum "yarn.lock" }} - - yarn-cache-{{ checksum ".circleci/cache_version" }}- - - run: - name: Install frontend dependencies - command: docker-compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --frozen-lockfile - - run: - name: Assert unique evolution numbers - command: docker-compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers - - restore_cache: - name: Restore webpack cache - keys: - - webpack-cache-{{ checksum ".circleci/cache_version" }}-{{ .Branch }} - - webpack-cache-{{ checksum ".circleci/cache_version" }}-master -# - run: -# name: Assert schema.sql and evolutions are equal -# command: | -# docker-compose up -d postgres -# sleep 3 -# docker-compose run compile tools/postgres/dbtool.js check-evolutions-schema -# - run: -# name: Build frontend documentation -# command: | -# WK_VERSION=${CIRCLE_TAG:-${CIRCLE_BUILD_NUM:-dev}} -# docker-compose run base yarn run docs --project-version $WK_VERSION - - run: - name: Build webknossos (webpack) - command: | - docker-compose run base yarn build - - run: - name: Build webknossos (sbt) - command: | - if [ "${CIRCLE_BRANCH}" == "master" ]; then - docker-compose run compile sbt -no-colors clean compile stage - else - docker-compose run compile sbt -no-colors -DfailOnWarning compile stage - fi - - run: - name: Build webknossos-datastore (sbt) - command: docker-compose run base sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage - - - run: - name: Build webknossos-tracingstore (sbt) - command: docker-compose run base sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage - - - save_cache: - name: Save target cache - key: target-cache-{{ checksum ".circleci/cache_version" }}-{{ .Branch }}-{{ .Revision }} - paths: - - "project/target" - - "target" - - - save_cache: - name: Save sbt cache - key: sbt-cache-{{ checksum ".circleci/cache_version" }}-{{ checksum "project/Dependencies.scala" }} - paths: - - "~/.ivy2" - - "~/.cache/coursier" - - - save_cache: - name: Save yarn cache - key: yarn-cache-{{ checksum ".circleci/cache_version" }}-{{ checksum "yarn.lock" }} - paths: - - "~/.cache/yarn" - - - save_cache: - name: Save webpack cache - key: webpack-cache-{{ checksum ".circleci/cache_version" }}-{{ .Branch }}-{{ .Revision }} - paths: - - "node_modules/.cache/webpack" - - - run: - name: Build webknossos docker image - command: docker-compose build --pull webknossos - - - run: - name: Build webknossos-datastore docker image - command: docker-compose build --pull webknossos-datastore - - - run: - name: Build webknossos-tracingstore docker image - command: docker-compose build --pull webknossos-tracingstore - -# - run: -# name: Lint frontend code and check formatting -# command: | -# .circleci/not-on-master.sh docker-compose run base bash -c "yarn run check-frontend" -# - run: -# name: Check for cyclic dependencies in front-end -# command: | -# .circleci/not-on-master.sh docker-compose run base yarn check-cyclic-dependencies -# - run: -# name: Run frontend tests -# command: | -# .circleci/not-on-master.sh docker-compose run base yarn test-verbose -# - run: -# name: Lint backend code and check formatting -# command: | -# .circleci/not-on-master.sh docker-compose run backend-lint-format -# - run: -# name: Run backend tests -# command: | -# .circleci/not-on-master.sh docker-compose run backend-tests -# - run: -# name: Run end-to-end tests -# command: | -# for i in {1..3}; do # retry -# .circleci/not-on-master.sh docker-compose run e2e-tests && s=0 && break || s=$? -# done -# (exit $s) -# - run: -# name: Validate frontend types -# command: | -# .circleci/not-on-master.sh docker-compose run base yarn typecheck -# - run: -# name: Start webknossos -# background: true -# command: docker-compose up webknossos -# - run: -# name: Run webknossos smoke test -# command: | -# for i in {1..10}; do # retry -# sleep 10 -# curl --fail -v http://localhost:9000/api/health && s=0 && break || s=$? -# done -# (exit $s) -# - run: -# name: Stop webknossos -# command: docker-compose down --volumes --remove-orphans -# -# - run: -# name: Start webknossos-datastore -# background: true -# command: docker-compose up webknossos-datastore -# - run: -# name: Run webknossos-datastore smoke test -# command: | -# for i in {1..10}; do # retry -# sleep 10 -# curl --fail -v http://localhost:9090/data/health && s=0 && break || s=$? -# done -# (exit $s) -# - run: -# name: Stop webknossos-datastore -# command: docker-compose down --volumes --remove-orphans -# -# - run: -# name: Start webknossos-tracingstore -# background: true -# command: docker-compose up webknossos-tracingstore -# - run: -# name: Run webknossos-tracingstore smoke test -# command: | -# for i in {1..10}; do # retry -# sleep 10 -# curl --fail -v http://localhost:9050/tracings/health && s=0 && break || s=$? -# done -# (exit $s) -# - run: -# name: Stop webknossos-tracingstore -# command: docker-compose down --volumes --remove-orphans - - - run: - name: Push docker images - command: | - function retry() { - for i in {1..5}; do - "$@" && s=0 && break || s=$? - sleep 10 - done - return $s - } - retry sh -c 'echo $DOCKER_TOKEN | docker login -u $DOCKER_USER --password-stdin' - retry docker-compose push webknossos - retry docker-compose push webknossos-datastore - retry docker-compose push webknossos-tracingstore - if [ -n "$CIRCLE_BRANCH" ]; then - docker tag \ - lincbrain/webknossos:${DOCKER_TAG} \ - lincbrain/webknossos:${NORMALIZED_BRANCH} - retry docker push lincbrain/webknossos:${NORMALIZED_BRANCH} - docker tag \ - lincbrain/webknossos-datastore:${DOCKER_TAG} \ - lincbrain/webknossos-datastore:${NORMALIZED_BRANCH} - retry docker push lincbrain/webknossos-datastore:${NORMALIZED_BRANCH} - docker tag \ - lincbrain/webknossos-tracingstore:${DOCKER_TAG} \ - lincbrain/webknossos-tracingstore:${NORMALIZED_BRANCH} - retry docker push lincbrain/webknossos-tracingstore:${NORMALIZED_BRANCH} - fi - docker logout - - run: - name: Report coverage - command: .circleci/not-on-master.sh docker-compose run base yarn coverage || true - - run: - name: Send Slack notification (master only) - command: .circleci/slack-notification.sh - - nightly: - docker: - - image: scalableminds/puppeteer:master - resource_class: large - steps: - - checkout - - run: - name: Remove dev-deployment - command: > - curl - -X POST - -H "X-Auth-Token: $RELEASE_API_TOKEN" - https://kubernetix.scm.io/hooks/remove/webknossos/dev/master?user=CI+%28nightly%29 - - run: - name: Wait 3min - command: sleep 180 - - run: - name: Install dev-deployment - command: > - curl - -X POST - -H "X-Auth-Token: $RELEASE_API_TOKEN" - https://kubernetix.scm.io/hooks/install/webknossos/dev/master?user=CI+%28nightly%29 - - run: - name: Install dependencies and sleep at least 3min - command: | - yarn install --frozen-lockfile & - sleep 180 & - wait - - run: - name: Refresh datasets - command: curl -X POST --fail https://master.webknossos.xyz/data/triggers/checkInboxBlocking?token=$WK_AUTH_TOKEN - - run: - name: Run screenshot-tests - command: | - # CircleCI cancels the job after 60 minutes. To ensure that screenshots are still - # uploaded as artifacts, we define a timeout of 50 minutes for the screenshot tests. - URL=https://master.webknossos.xyz/ \ - timeout 3000 \ - yarn test-screenshot - - - store_artifacts: - path: frontend/javascripts/test/screenshots - - - store_artifacts: - path: frontend/javascripts/test/snapshots/type-check - - wkorg_nightly: - docker: - - image: scalableminds/puppeteer:master - resource_class: large - steps: - - checkout - - run: - name: Install dependencies - command: | - yarn install --frozen-lockfile - - run: - name: Run screenshot-tests - command: | - # CircleCI cancels the job after 60 minutes. To ensure that screenshots are still - # uploaded as artifacts, we define a timeout of 50 minutes for the screenshot tests. - timeout 3000 \ - yarn test-wkorg-screenshot - - - store_artifacts: - path: frontend/javascripts/test/screenshots-wkorg - -workflows: - version: 2 - circleci_build: - jobs: - - build_test_deploy: - context: - - DockerHub - filters: - tags: - only: /.*/ - circleci_nightly: - jobs: - - nightly - triggers: - - schedule: - # 02:15 AM UTC - cron: "15 2 * * *" - filters: - branches: - only: - - master - circleci_wkorg_nightly: - jobs: - - wkorg_nightly - triggers: - - schedule: - # 03:15 AM UTC - cron: "15 3 * * *" - filters: - branches: - only: - - master diff --git a/.circleci/not-on-master.sh b/.circleci/not-on-master.sh deleted file mode 100755 index 581393ebead..00000000000 --- a/.circleci/not-on-master.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -if [ "${CIRCLE_BRANCH}" == "master" ]; then - echo "Skipping this step on master..." -else - exec "$@" -fi diff --git a/.circleci/slack-notification.sh b/.circleci/slack-notification.sh deleted file mode 100755 index c0be28a1cac..00000000000 --- a/.circleci/slack-notification.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -if [ "${CIRCLE_BRANCH}" == "master" ] ; then - author=${CIRCLE_USERNAME} - author=${author/fm3/<@florian>} - author=${author/daniel-wer/<@daniel>} - author=${author/hotzenklotz/<@tom>} - author=${author/MichaelBuessemeyer/<@michael>} - author=${author/normanrz/<@norman>} - author=${author/philippotto/<@philipp>} - author=${author/valentin-pinkau/<@valentin>} - author=${author/frcroth/<@felix.roth>} - author=${author/dieknolle3333/<@charlie.meister>} - channel="webknossos-bots" - commitmsg="$(git log --format=%s -n 1)" - pullregex="(.*)#([0-9]+)(.*)" - while [[ "$commitmsg" =~ $pullregex ]] - do - commitmsg="${BASH_REMATCH[1]}#${BASH_REMATCH[3]}" - done - buildlink="" - mesg="${author} WEBKNOSSOS docker image \`master__${CIRCLE_BUILD_NUM}\` for “${commitmsg}” is ${buildlink}." - user="ci-notify" - token="${SLACK_NOTIFY_TOKEN:-}" - res=$(curl -s \ - -X POST \ - -d "token=${token}" \ - -d "channel=${channel}" \ - -d "text=${mesg}" \ - -d "username=${user}" \ - -d "icon_url=https://a.slack-edge.com/41b0a/img/plugins/circleci/service_48.png" \ - https://slack.com/api/chat.postMessage - ) - if [[ "$(echo ${res} | jq '.ok')" == "false" ]]; then - echo "[WARN] Error sending Slack notification: $(echo ${res} | jq -r '.error')." - fi - echo "[INFO] Sent Slack notification to ${channel}." -fi diff --git a/.codespellrc b/.codespellrc index 1a7bf949281..07607d91669 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,6 +1,6 @@ [codespell] # Ref: https://github.com/codespell-project/codespell#using-a-config-file -skip = *.svg,*.sublime-workspace,*.lock,.codespellrc,./util/target/,./binaryData,./node_modules,./pg,./project/target,./target,./webknossos-datastore/target,./webknossos-jni/target,./webknossos-tracingstore/target,./util/target,./coverage,./public-test,./tools/proxy/node_modules,./docs/publications.md,./public/bundle +skip = *.svg,*.sublime-workspace,*.lock,.codespellrc,./util/target/,./binaryData,./node_modules,./pg,./project/target,./target,./webknossos-datastore/target,./webknossos-jni/target,./webknossos-tracingstore/target,./util/target,./coverage,./public-test,./tools/proxy/node_modules,./docs/publications.md,./public/bundle,./tools/migration-unified-annotation-versioning/venv,./tools/migration-unified-annotation-versioning/*.json # some names and camelCased variables etc ignore-regex = \b([a-z]+[A-Z][a-zA-Z]*|H Mattern|Manuel|Nat Commun)\b -ignore-words-list = lod,nd,ue +ignore-words-list = lod,nd,ue,DoubleClick diff --git a/.dependency-cruiser.js b/.dependency-cruiser.js new file mode 100644 index 00000000000..44748a4a7dc --- /dev/null +++ b/.dependency-cruiser.js @@ -0,0 +1,414 @@ +/** @type {import('dependency-cruiser').IConfiguration} */ +module.exports = { + forbidden: [ + // { + // name: 'no-circular', + // severity: 'warn', + // comment: + // 'This dependency is part of a circular relationship. You might want to revise ' + + // 'your solution (i.e. use dependency inversion, make sure the modules have a single responsibility) ', + // from: {}, + // to: { + // circular: true + // } + // }, + // { + // name: 'no-orphans', + // comment: + // "This is an orphan module - it's likely not used (anymore?). Either use it or " + + // "remove it. If it's logical this module is an orphan (i.e. it's a config file), " + + // "add an exception for it in your dependency-cruiser configuration. By default " + + // "this rule does not scrutinize dot-files (e.g. .eslintrc.js), TypeScript declaration " + + // "files (.d.ts), tsconfig.json and some of the babel and webpack configs.", + // severity: 'warn', + // from: { + // orphan: true, + // pathNot: [ + // '(^|/)[.][^/]+[.](?:js|cjs|mjs|ts|cts|mts|json)$', // dot files + // '[.]d[.]ts$', // TypeScript declaration files + // '(^|/)tsconfig[.]json$', // TypeScript config + // '(^|/)(?:babel|webpack)[.]config[.](?:js|cjs|mjs|ts|cts|mts|json)$' // other configs + // ] + // }, + // to: {}, + // }, + // { + // name: 'no-deprecated-core', + // comment: + // 'A module depends on a node core module that has been deprecated. Find an alternative - these are ' + + // "bound to exist - node doesn't deprecate lightly.", + // severity: 'warn', + // from: {}, + // to: { + // dependencyTypes: [ + // 'core' + // ], + // path: [ + // '^v8/tools/codemap$', + // '^v8/tools/consarray$', + // '^v8/tools/csvparser$', + // '^v8/tools/logreader$', + // '^v8/tools/profile_view$', + // '^v8/tools/profile$', + // '^v8/tools/SourceMap$', + // '^v8/tools/splaytree$', + // '^v8/tools/tickprocessor-driver$', + // '^v8/tools/tickprocessor$', + // '^node-inspect/lib/_inspect$', + // '^node-inspect/lib/internal/inspect_client$', + // '^node-inspect/lib/internal/inspect_repl$', + // '^async_hooks$', + // '^punycode$', + // '^domain$', + // '^constants$', + // '^sys$', + // '^_linklist$', + // '^_stream_wrap$' + // ], + // } + // }, + // { + // name: 'not-to-deprecated', + // comment: + // 'This module uses a (version of an) npm module that has been deprecated. Either upgrade to a later ' + + // 'version of that module, or find an alternative. Deprecated modules are a security risk.', + // severity: 'warn', + // from: {}, + // to: { + // dependencyTypes: [ + // 'deprecated' + // ] + // } + // }, + // { + // name: 'no-non-package-json', + // severity: 'error', + // comment: + // "This module depends on an npm package that isn't in the 'dependencies' section of your package.json. " + + // "That's problematic as the package either (1) won't be available on live (2 - worse) will be " + + // "available on live with an non-guaranteed version. Fix it by adding the package to the dependencies " + + // "in your package.json.", + // from: {}, + // to: { + // dependencyTypes: [ + // 'npm-no-pkg', + // 'npm-unknown' + // ] + // } + // }, + { + name: "not-to-unresolvable", + comment: + "This module depends on a module that cannot be found ('resolved to disk'). If it's an npm " + + "module: add it to your package.json. In all other cases you likely already know what to do.", + severity: "error", + from: {}, + to: { + couldNotResolve: true, + }, + }, + { + name: "no-duplicate-dep-types", + comment: + "Likely this module depends on an external ('npm') package that occurs more than once " + + "in your package.json i.e. bot as a devDependencies and in dependencies. This will cause " + + "maintenance problems later on.", + severity: "warn", + from: {}, + to: { + moreThanOneDependencyType: true, + // as it's pretty common to have a type import be a type only import + // _and_ (e.g.) a devDependency - don't consider type-only dependency + // types for this rule + dependencyTypesNot: ["type-only"], + }, + }, + + /* rules you might want to tweak for your specific situation: */ + { + name: "not-to-test", + comment: + "This module depends on code within a folder that should only contain tests. As tests don't " + + "implement functionality this is odd. Either you're writing a test outside the test folder " + + "or there's something in the test folder that isn't a test.", + severity: "error", + from: { + pathNot: "^(frontend/javascripts/test)", + }, + to: { + path: "^(frontend/javascripts/test)", + }, + }, + { + name: "not-to-spec", + comment: + "This module depends on a spec (test) file. The sole responsibility of a spec file is to test code. " + + "If there's something in a spec that's of use to other modules, it doesn't have that single " + + "responsibility anymore. Factor it out into (e.g.) a separate utility/ helper or a mock.", + severity: "error", + from: {}, + to: { + path: "[.](?:spec|test)[.](?:js|mjs|cjs|jsx|ts|mts|cts|tsx)$", + }, + }, + { + name: "not-to-dev-dep", + severity: "error", + comment: + "This module depends on an npm package from the 'devDependencies' section of your " + + "package.json. It looks like something that ships to production, though. To prevent problems " + + "with npm packages that aren't there on production declare it (only!) in the 'dependencies'" + + "section of your package.json. If this module is development only - add it to the " + + "from.pathNot re of the not-to-dev-dep rule in the dependency-cruiser configuration", + from: { + path: "^(frontend/javascripts)", + pathNot: "frontend/javascripts/test", + }, + to: { + dependencyTypes: ["npm-dev"], + // type only dependencies are not a problem as they don't end up in the + // production code or are ignored by the runtime. + dependencyTypesNot: ["type-only"], + pathNot: ["node_modules/@types/"], + }, + }, + { + name: "optional-deps-used", + severity: "info", + comment: + "This module depends on an npm package that is declared as an optional dependency " + + "in your package.json. As this makes sense in limited situations only, it's flagged here. " + + "If you're using an optional dependency here by design - add an exception to your" + + "dependency-cruiser configuration.", + from: {}, + to: { + dependencyTypes: ["npm-optional"], + }, + }, + { + name: "peer-deps-used", + comment: + "This module depends on an npm package that is declared as a peer dependency " + + "in your package.json. This makes sense if your package is e.g. a plugin, but in " + + "other cases - maybe not so much. If the use of a peer dependency is intentional " + + "add an exception to your dependency-cruiser configuration.", + severity: "warn", + from: {}, + to: { + dependencyTypes: ["npm-peer"], + }, + }, + { + name: "accessors-should-not-import-sagas", + comment: "", + severity: "warn", + from: { + path: "^(frontend/javascripts/oxalis/model/accessors/).*[.]ts$", + }, + to: { + path: "^(frontend/javascripts/oxalis/model/sagas/).*[.]ts$", + reachable: true, + }, + }, + ], + options: { + /* Which modules not to follow further when encountered */ + doNotFollow: { + /* path: an array of regular expressions in strings to match against */ + path: ["node_modules"], + }, + + /* Which modules to exclude */ + // exclude : { + // /* path: an array of regular expressions in strings to match against */ + // path: '', + // }, + + /* Which modules to exclusively include (array of regular expressions in strings) + dependency-cruiser will skip everything not matching this pattern + */ + // includeOnly : [''], + + /* List of module systems to cruise. + When left out dependency-cruiser will fall back to the list of _all_ + module systems it knows of. It's the default because it's the safe option + It might come at a performance penalty, though. + moduleSystems: ['amd', 'cjs', 'es6', 'tsd'] + + As in practice only commonjs ('cjs') and ecmascript modules ('es6') + are widely used, you can limit the moduleSystems to those. + */ + + // moduleSystems: ['cjs', 'es6'], + + /* + false: don't look at JSDoc imports (the default) + true: dependency-cruiser will detect dependencies in JSDoc-style + import statements. Implies "parser": "tsc", so the dependency-cruiser + will use the typescript parser for JavaScript files. + + For this to work the typescript compiler will need to be installed in the + same spot as you're running dependency-cruiser from. + */ + // detectJSDocImports: true, + + /* prefix for links in html and svg output (e.g. 'https://github.com/you/yourrepo/blob/main/' + to open it on your online repo or `vscode://file/${process.cwd()}/` to + open it in visual studio code), + */ + // prefix: `vscode://file/${process.cwd()}/`, + + /* false (the default): ignore dependencies that only exist before typescript-to-javascript compilation + true: also detect dependencies that only exist before typescript-to-javascript compilation + "specify": for each dependency identify whether it only exists before compilation or also after + */ + // tsPreCompilationDeps: false, + + /* list of extensions to scan that aren't javascript or compile-to-javascript. + Empty by default. Only put extensions in here that you want to take into + account that are _not_ parsable. + */ + // extraExtensionsToScan: [".json", ".jpg", ".png", ".svg", ".webp"], + + /* if true combines the package.jsons found from the module up to the base + folder the cruise is initiated from. Useful for how (some) mono-repos + manage dependencies & dependency definitions. + */ + // combinedDependencies: false, + + /* if true leave symlinks untouched, otherwise use the realpath */ + // preserveSymlinks: false, + + /* TypeScript project file ('tsconfig.json') to use for + (1) compilation and + (2) resolution (e.g. with the paths property) + + The (optional) fileName attribute specifies which file to take (relative to + dependency-cruiser's current working directory). When not provided + defaults to './tsconfig.json'. + */ + tsConfig: { + fileName: "tsconfig.json", + }, + + /* Webpack configuration to use to get resolve options from. + + The (optional) fileName attribute specifies which file to take (relative + to dependency-cruiser's current working directory. When not provided defaults + to './webpack.conf.js'. + + The (optional) `env` and `arguments` attributes contain the parameters + to be passed if your webpack config is a function and takes them (see + webpack documentation for details) + */ + // webpackConfig: { + // fileName: 'webpack.config.js', + // env: {}, + // arguments: {} + // }, + + /* Babel config ('.babelrc', '.babelrc.json', '.babelrc.json5', ...) to use + for compilation + */ + // babelConfig: { + // fileName: '.babelrc', + // }, + + /* List of strings you have in use in addition to cjs/ es6 requires + & imports to declare module dependencies. Use this e.g. if you've + re-declared require, use a require-wrapper or use window.require as + a hack. + */ + // exoticRequireStrings: [], + + /* options to pass on to enhanced-resolve, the package dependency-cruiser + uses to resolve module references to disk. The values below should be + suitable for most situations + + If you use webpack: you can also set these in webpack.conf.js. The set + there will override the ones specified here. + */ + enhancedResolveOptions: { + /* What to consider as an 'exports' field in package.jsons */ + exportsFields: ["exports"], + /* List of conditions to check for in the exports field. + Only works when the 'exportsFields' array is non-empty. + */ + conditionNames: ["import", "require", "node", "default", "types"], + /* The extensions, by default are the same as the ones dependency-cruiser + can access (run `npx depcruise --info` to see which ones that are in + _your_ environment). If that list is larger than you need you can pass + the extensions you actually use (e.g. [".js", ".jsx"]). This can speed + up module resolution, which is the most expensive step. + */ + // extensions: [".js", ".jsx", ".ts", ".tsx", ".d.ts"], + /* What to consider a 'main' field in package.json */ + + // if you migrate to ESM (or are in an ESM environment already) you will want to + // have "module" in the list of mainFields, like so: + // mainFields: ["module", "main", "types", "typings"], + mainFields: ["main", "types", "typings"], + /* A list of alias fields in package.jsons + + See [this specification](https://github.com/defunctzombie/package-browser-field-spec) and + the webpack [resolve.alias](https://webpack.js.org/configuration/resolve/#resolvealiasfields) + documentation. + + Defaults to an empty array (= don't use alias fields). + */ + // aliasFields: ["browser"], + }, + + /* skipAnalysisNotInRules will make dependency-cruiser execute + analysis strictly necessary for checking the rule set only. + + See https://github.com/sverweij/dependency-cruiser/blob/main/doc/options-reference.md#skipanalysisnotinrules + for details + */ + skipAnalysisNotInRules: true, + + reporterOptions: { + dot: { + /* pattern of modules that can be consolidated in the detailed + graphical dependency graph. The default pattern in this configuration + collapses everything in node_modules to one folder deep so you see + the external modules, but their innards. + */ + collapsePattern: "node_modules/(?:@[^/]+/[^/]+|[^/]+)", + + /* Options to tweak the appearance of your graph.See + https://github.com/sverweij/dependency-cruiser/blob/main/doc/options-reference.md#reporteroptions + for details and some examples. If you don't specify a theme + dependency-cruiser falls back to a built-in one. + */ + // theme: { + // graph: { + // /* splines: "ortho" gives straight lines, but is slow on big graphs + // splines: "true" gives bezier curves (fast, not as nice as ortho) + // */ + // splines: "true" + // }, + // } + }, + archi: { + /* pattern of modules that can be consolidated in the high level + graphical dependency graph. If you use the high level graphical + dependency graph reporter (`archi`) you probably want to tweak + this collapsePattern to your situation. + */ + collapsePattern: + "^(?:packages|src|lib(s?)|app(s?)|bin|test(s?)|spec(s?))/[^/]+|node_modules/(?:@[^/]+/[^/]+|[^/]+)", + + /* Options to tweak the appearance of your graph. If you don't specify a + theme for 'archi' dependency-cruiser will use the one specified in the + dot section above and otherwise use the default one. + */ + // theme: { }, + }, + text: { + highlightFocused: true, + }, + }, + }, +}; +// generated: dependency-cruiser@16.10.0 on 2025-03-14T08:50:28.513Z diff --git a/.editorconfig b/.editorconfig index 426b13cd4a2..e179e77a44d 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,6 +9,12 @@ charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true +[*.py] +indent_size = 4 + +[*.cpp] +indent_size = 4 + [*.md] trim_trailing_whitespace = false diff --git a/webknossos-datastore/Dockerfile b/.github/Dockerfile.datastore similarity index 95% rename from webknossos-datastore/Dockerfile rename to .github/Dockerfile.datastore index 49e19ebe6db..26385ae71ed 100644 --- a/webknossos-datastore/Dockerfile +++ b/.github/Dockerfile.datastore @@ -14,7 +14,7 @@ VOLUME /webknossos-datastore/binaryData /tmp COPY target/universal/stage . RUN chown -R webknossos . \ - && chmod go+x bin/webknossos-datastore \ + && chmod +x bin/webknossos-datastore \ && chmod go+w . RUN echo '#!/bin/bash\numask 002\nbin/webknossos-datastore "$@"\n' > /docker-entrypoint.sh \ diff --git a/webknossos-tracingstore/Dockerfile b/.github/Dockerfile.tracingstore similarity index 87% rename from webknossos-tracingstore/Dockerfile rename to .github/Dockerfile.tracingstore index 5df708e82cd..c9354f5029a 100644 --- a/webknossos-tracingstore/Dockerfile +++ b/.github/Dockerfile.tracingstore @@ -1,7 +1,7 @@ FROM eclipse-temurin:21-jammy RUN apt-get update \ - && apt-get -y install libdraco4 \ + && apt-get -y install libblosc1 libbrotli1 libdraco4 \ && rm -rf /var/lib/apt/lists/* RUN mkdir -p /webknossos-tracingstore \ @@ -16,7 +16,7 @@ VOLUME /webknossos-tracingstore/tracingData /tmp COPY target/universal/stage . RUN chown -R webknossos . \ - && chmod go+x bin/webknossos-tracingstore \ + && chmod +x bin/webknossos-tracingstore \ && chmod go+w . RUN echo '#!/bin/bash\numask 002\nbin/webknossos-tracingstore "$@"\n' > /docker-entrypoint.sh \ @@ -31,4 +31,4 @@ HEALTHCHECK \ EXPOSE 9050 ENTRYPOINT ["/docker-entrypoint.sh"] -CMD ["-J-Xmx20G", "-J-Xms1G", "-Dconfig.file=conf/standalone-tracingstore.conf", "-Dlogger.file=conf/logback-docker.xml", "-Dlogback.configurationFile=conf/logback-docker.xml", "-Dhttp.port=9090", "-Dhttp.address=0.0.0.0"] +CMD ["-J-Xmx20G", "-J-Xms1G", "-Dconfig.file=conf/standalone-tracingstore.conf", "-Dlogger.file=conf/logback-docker.xml", "-Dlogback.configurationFile=conf/logback-docker.xml", "-Dhttp.port=9050", "-Dhttp.address=0.0.0.0"] diff --git a/Dockerfile b/.github/Dockerfile.webknossos similarity index 62% rename from Dockerfile rename to .github/Dockerfile.webknossos index 3b9642e09a4..f39a76546e5 100644 --- a/Dockerfile +++ b/.github/Dockerfile.webknossos @@ -1,5 +1,5 @@ FROM eclipse-temurin:21-jammy -ARG VERSION_NODE="18.x" +ARG VERSION_NODE="22.x" # Install dependencies, including Node.js RUN curl -sL "https://deb.nodesource.com/setup_${VERSION_NODE}" | bash - \ @@ -12,15 +12,17 @@ RUN curl -sL "https://deb.nodesource.com/setup_${VERSION_NODE}" | bash - \ RUN mkdir -p /webknossos WORKDIR /webknossos -# Copy the application's binaries to the container +# Copy compiled Scala output from a previous build step, e.g. output of the Docker-dev image COPY target/universal/stage . # Setup user and group RUN addgroup --system --gid 999 webknossos \ && adduser --system --uid 999 --ingroup webknossos webknossos \ && mkdir disk \ - && chown -R webknossos:webknossos . \ - && chmod -R go+rwx . + && chown -R webknossos . \ + && chmod +x bin/webknossos \ + && chmod +x tools/postgres/dbtool.js \ + && chmod go+w . # Create a custom entrypoint RUN echo '#!/bin/bash\numask 000\nbin/webknossos "$@"\n' > /docker-entrypoint.sh \ @@ -36,5 +38,6 @@ USER webknossos # Expose the port the app runs on EXPOSE 9000 -# Set the container's entry point +ENV CERTIFICATE "eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzY2FsYWJsZW1pbmRzIiwiaWF0IjoxNzM5ODExMTc3LCJleHAiOjE4OTM0NTYwMDAsImZlYXR1cmVzIjp7Im9wZW5JZENvbm5lY3RFbmFibGVkIjpmYWxzZSwic2VnbWVudEFueXRoaW5nRW5hYmxlZCI6ZmFsc2UsImVkaXRhYmxlTWFwcGluZ3NFbmFibGVkIjpmYWxzZX19.c_zBzj7bKPCXp5eTjFrCC4YmOxSH9uQx3BRMYbVx9dPsiCwlyvi7jepqOP7bS8QIcdoSHnrhHpZ-tQKkTi3rkw" + ENTRYPOINT [ "/docker-entrypoint.sh" ] diff --git a/.github/actions/retry/action.yml b/.github/actions/retry/action.yml new file mode 100644 index 00000000000..9508c7499e3 --- /dev/null +++ b/.github/actions/retry/action.yml @@ -0,0 +1,24 @@ +name: "Shell Retry" +description: "Runs a small shell script with retries" +inputs: + run: + description: "Command to execute." + required: true + retries: + description: "Maximum number of retries until failed" + required: true + retry_delay_seconds: + description: "Amount of time to sleep between each retry" + default: 0 +runs: + using: 'composite' + steps: + - shell: bash + run: | + for i in {1..${{ inputs.retries }}}; do + bash -c "$INPUT_COMMAND" && exit 0 + sleep "${{ inputs.retry_delay_seconds }}" + done + exit 1 + env: + INPUT_COMMAND: ${{ inputs.run }} diff --git a/.github/common_edge_cases.md b/.github/common_edge_cases.md index c9fa1dda17f..63bdcbd288c 100644 --- a/.github/common_edge_cases.md +++ b/.github/common_edge_cases.md @@ -14,3 +14,5 @@ Consider SQL pitfalls: - `x IN ()` statements must never called with empty list - `ARRAY_AGG(x)` may have nullable values (use `ARRAY_REMOVE(ARRAY_AGG(x), null)` instead) - Complex SQL queries may have a fanout effect due to multiple left joins, leading to duplicates + +When changing the API version, also adapt `ApiVersioning.CURRENT_API_VERSION` diff --git a/.github/docker-compose.yml b/.github/docker-compose.yml new file mode 100644 index 00000000000..2f58d8c7221 --- /dev/null +++ b/.github/docker-compose.yml @@ -0,0 +1,117 @@ +# For CI only +name: webknossos-ci + +services: + webknossos: + image: akanzer/webknossos:${DOCKER_TAG} + pull_policy: never + ports: + - "9000:9000" + command: + - -Dconfig.file=conf/application.conf + - -Djava.net.preferIPv4Stack=true + - -Dtracingstore.fossildb.address=fossildb + - -Dtracingstore.redis.address=redis + - -Ddatastore.redis.address=redis + - -DwebKnossos.sampleOrganization.enabled=false + volumes: + - ../binaryData:/webknossos/binaryData + environment: + - CERTIFICATE + - POSTGRES_URL=jdbc:postgresql://postgres/webknossos + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + depends_on: + postgres: + condition: service_healthy + fossildb: + condition: service_healthy + redis: + condition: service_healthy + + webknossos-datastore: + image: akanzer/webknossos-datastore:${DOCKER_TAG} + pull_policy: never + ports: + - "9090:9090" + volumes: + - ../binaryData:/webknossos-datastore/binaryData + command: + - -J-Xmx20G + - -J-Xms1G + - -Dconfig.file=conf/standalone-datastore.conf + - -Dhttp.port=9090 + - -Dhttp.uri=http://webknossos-datastore:9090 + - -Ddatastore.webKnossos.uri=webknossos:9000 + - -Ddatastore.redis.address=redis + depends_on: + redis: + condition: service_healthy + + webknossos-tracingstore: + image: akanzer/webknossos-tracingstore:${DOCKER_TAG} + pull_policy: never + ports: + - "9050:9050" + command: + - -J-Xmx20G + - -J-Xms1G + - -Dconfig.file=conf/standalone-tracingstore.conf + - -Dhttp.port=9050 + - -Dhttp.uri=http://webknossos-tracingstore:9050 + - -Dtracingstore.fossildb.address=fossildb + - -Dtracingstore.redis.address=redis + - -Ddatastore.webknossos.uri=webknossos:9000 + depends_on: + redis: + condition: service_healthy + fossildb: + condition: service_healthy + + fossildb: + image: scalableminds/fossildb:master__510 + command: + - fossildb + - -c + - skeletons,skeletonTreeBodies,volumes,volumeData,volumeSegmentIndex,editableMappingsInfo,editableMappingsAgglomerateToGraph,editableMappingsSegmentToAgglomerate,annotations,annotationUpdates + ports: + - "7155:7155" + volumes: + - "../fossildb/data:/fossildb/data" + - "../fossildb/backup:/fossildb/backup" + + postgres: + image: postgres:17-bookworm + environment: + - POSTGRES_DB=webknossos + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + ports: + - "5434:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -d webknossos -U ${POSTGRES_USER} -h 127.0.0.1 -p 5432"] + interval: 2s + timeout: 5s + retries: 30 + + redis: + image: redis:7.4 + command: + - redis-server + ports: + - 6379:6379 + healthcheck: + test: + - CMD + - bash + - -c + - "exec 3<> /dev/tcp/127.0.0.1/6379 && echo PING >&3 && head -1 <&3 | grep PONG" + timeout: 1s + interval: 5s + retries: 10 + +volumes: + postgres_data: + external: false diff --git a/.github/slack-notification.sh b/.github/slack-notification.sh new file mode 100755 index 00000000000..2e3a29230a9 --- /dev/null +++ b/.github/slack-notification.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +author=${GITHUB_ACTOR} +author=${author/fm3/<@florian>} +author=${author/daniel-wer/<@daniel>} +author=${author/hotzenklotz/<@tom>} +author=${author/MichaelBuessemeyer/<@michael>} +author=${author/normanrz/<@norman>} +author=${author/philippotto/<@philipp>} +author=${author/valentin-pinkau/<@valentin>} +author=${author/frcroth/<@felix.roth>} +author=${author/knollengewaechs/<@charlie.meister>} +channel="webknossos-bots" +commitmsg=$(git log --format=%s -n 1) +pullregex="(.*)#([0-9]+)(.*)" + +while [[ "$commitmsg" =~ $pullregex ]]; do + commitmsg="${BASH_REMATCH[1]}#${BASH_REMATCH[3]}" +done + +buildlink="<${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}|ready>" +mesg="${author} WEBKNOSSOS docker image \`${DOCKER_TAG}\` for “${commitmsg}” is ${buildlink}." +user="ci-notify" +token="${SLACK_NOTIFY_TOKEN:-}" +res=$(curl -s \ + -X POST \ + -d "token=${token}" \ + -d "channel=${channel}" \ + -d "text=${mesg}" \ + -d "username=${user}" \ + -d "icon_url=https://a.slack-edge.com/41b0a/img/plugins/circleci/service_48.png" \ + https://slack.com/api/chat.postMessage +) +if [[ "$(echo "${res}" | jq '.ok')" == "false" ]]; then + echo "[WARN] Error sending Slack notification: $(echo "${res}" | jq -r '.error')." +fi +echo "[INFO] Sent Slack notification to ${channel}." diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml new file mode 100644 index 00000000000..010b24a13b6 --- /dev/null +++ b/.github/workflows/build_test_deploy.yml @@ -0,0 +1,308 @@ +name: CI Pipeline + +on: + push: + branches: + - master + tags: + - '*' + pull_request: + branches: + - '*' + +env: + USER_UID: 1001 + USER_GID: 1001 + POSTGRES_URL: "jdbc:postgresql://localhost:5434/webknossos" + POSTGRES_USER: "webknossos_user" + POSTGRES_PASSWORD: "secret_password" + CERTIFICATE: ${{ secrets.CERTIFICATE_FOR_CI_ONLY }} + CERTIFICATE_PUBLIC_KEY: ${{ secrets.CERTIFICATE_PUBLIC_KEY }} + +jobs: + frontend-tests: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 5 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: corepack enable + - name: Install frontend dependencies + run: yarn install --immutable + + # - name: Lint frontend code and check formatting + # run: yarn run check-frontend + # - name: Typecheck frontend code + # run: yarn typecheck + - name: Check for cyclic dependencies in frontend + run: yarn check-cyclic-dependencies + - name: Run frontend tests + run: yarn run vitest run --coverage.enabled true frontend/javascripts/test/**/*.spec.ts + # - name: Download Coverage Artifacts + # # Can not use actions/download-artifact@v4 because it does not support downloading artifacts from other GA runs + # run: | + # gh_last_success_run_id=$(gh run list --workflow "CI Pipeline" --json conclusion,headBranch,databaseId --branch master --jq 'first(.[] | select(.conclusion | contains("success"))) | .databaseId') + # [ -z "$gh_last_success_run_id" ] && echo "No successful run found" && exit 1 || true + # gh run download $gh_last_success_run_id -n $ARTIFACT_NAME -D $OUTPUT_DIR + # env: + # GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # ARTIFACT_NAME: vitest-coverage-master + # OUTPUT_DIR: coverage-master + # - name: 'Report Coverage' + # uses: davelosert/vitest-coverage-report-action@v2 + # with: + # comment-on: none # alternative: pr + # json-summary-compare-path: coverage-master/coverage-summary.json + # - name: "Upload Coverage" + # if: github.ref == 'refs/heads/master' + # uses: actions/upload-artifact@v4 + # with: + # name: vitest-coverage-master + # path: coverage + + backend-tests: + runs-on: ubuntu-22.04 + if: github.ref != 'refs/heads/master' + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 5 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: corepack enable + + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + - uses: sbt/setup-sbt@v1 + + - name: Start Docker Services Postgres, Redis + run: docker compose --file .github/docker-compose.yml up -d postgres redis + + - name: Install frontend dependencies + run: yarn install --immutable + + - name: "Install System Dependencies" + run: sudo apt-get update && sudo apt-get install -y libdraco-dev libblosc1 + + - name: Assert unique evolution numbers + run: tools/postgres/dbtool.js assert-unique-evolution-numbers + - name: Assert schema.sql and evolutions are equal + run: tools/postgres/dbtool.js check-evolutions-schema + - name: Assert that all migrations are mentioned in one migration guide and that they have a reversion counterpart. + run: tools/assert-complete-migrations.sh + + - name: Lint backend code and check formatting + run: sbt ";scapegoat; scalafmtCheck; util/scalafmtCheck; webknossosTracingstore/scalafmtCheck; webknossosDatastore/scalafmtCheck" + + - name: Run backend unit tests + run: sbt -v "testOnly backend.*" + + - name: Start Fossildb for e2e tests + run: docker compose --file .github/docker-compose.yml up -d fossildb + - name: Prepare binaryData folder for e2e tests + run: mkdir -p binaryData/Organization_X && chmod 777 binaryData/Organization_X + - name: Run end-to-end tests + uses: ./.github/actions/retry + with: + run: | + sbt "testOnly e2e.* -- -Dtracingstore.fossildb.address=localhost -Dtracingstore.redis.address=localhost -Ddatastore.redis.address=localhost -Ddatastore.watchFileSystem.enabled=false" + retries: 3 + + + build-smoketest-push: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 5 + + - uses: docker/setup-buildx-action@v3 + with: + driver: docker + + - name: Start Docker Services Postgres, Redis + run: docker compose --file .github/docker-compose.yml up -d postgres redis + + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: corepack enable + + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + - uses: sbt/setup-sbt@v1 + + - name: "Install System Dependencies" + run: sudo apt-get update && sudo apt-get install -y libdraco-dev libblosc1 + + - name: "Custom environment variables (Pull Request)" + if: ${{ github.event_name == 'pull_request' }} + run: | + NORMALIZED_BRANCH="${GITHUB_HEAD_REF//[\/-]/_}" + DOCKER_TAG="${NORMALIZED_BRANCH}__${GITHUB_RUN_ID}" + echo "DOCKER_TAG=$DOCKER_TAG" | tee -a "$GITHUB_ENV" + echo "NORMALIZED_BRANCH=$NORMALIZED_BRANCH" | tee -a "$GITHUB_ENV" + echo "CI_BUILD_NUM=$GITHUB_RUN_ID" | tee -a "$GITHUB_ENV" + + - name: "Custom environment variables (Not Pull Request)" + if: ${{ github.event_name != 'pull_request' }} + run: | + CI_BRANCH=$(git rev-parse --abbrev-ref HEAD) + NORMALIZED_BRANCH=${CI_BRANCH//[\/-]/_} + DOCKER_TAG="${NORMALIZED_BRANCH}__${GITHUB_RUN_ID}" + echo "DOCKER_TAG=$DOCKER_TAG" | tee -a "$GITHUB_ENV" + if [ "$GITHUB_REF_TYPE" == "branch" ]; then + echo "NORMALIZED_BRANCH=$NORMALIZED_BRANCH" | tee -a "$GITHUB_ENV" + else + CI_TAG=$GITHUB_REF_NAME + echo "CI_TAG=$CI_TAG" | tee -a "$GITHUB_ENV" + echo "NORMALIZED_BRANCH=$CI_TAG" | tee -a "$GITHUB_ENV" + fi + echo "CI_BUILD_NUM=$GITHUB_RUN_ID" | tee -a "$GITHUB_ENV" + + - name: Prepare Dependency Folders + run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier + + - name: Install frontend dependencies + run: yarn install --immutable + env: + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: "true" + + - name: Build frontend documentation + run: yarn run docs --project-version "$WK_VERSION" + env: + WK_VERSION: ${{ github.run_id }} + + - name: Build webknossos (webpack) + run: yarn build + + - name: Build webknossos (sbt) + run: sbt -DfailOnWarning compile stage + - name: Build webknossos-datastore (sbt) + run: sbt -DfailOnWarning "project webknossosDatastore" copyMessages compile stage + - name: Build webknossos-tracingstore (sbt) + run: sbt -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage + - name: Checksum App Dirs + run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt + + - name: Build webknossos docker image + uses: docker/build-push-action@v6 + with: + file: .github/Dockerfile.webknossos + context: . + push: false + tags: | + akanzer/webknossos:${{ env.NORMALIZED_BRANCH }} + akanzer/webknossos:${{ env.DOCKER_TAG }} + outputs: | + type=image + - name: Build webknossos-datastore docker image + uses: docker/build-push-action@v6 + with: + file: .github/Dockerfile.datastore + context: webknossos-datastore + push: false + tags: | + akanzer/webknossos-datastore:${{ env.NORMALIZED_BRANCH }} + akanzer/webknossos-datastore:${{ env.DOCKER_TAG }} + outputs: | + type=image + - name: Build webknossos-tracingstore docker image + uses: docker/build-push-action@v6 + with: + file: .github/Dockerfile.tracingstore + context: webknossos-tracingstore + push: false + tags: | + akanzer/webknossos-tracingstore:${{ env.NORMALIZED_BRANCH }} + akanzer/webknossos-tracingstore:${{ env.DOCKER_TAG }} + outputs: | + type=image + + - name: Start webknossos, datastore and tracingstore for smoke test + run: | + set -e + docker compose --file .github/docker-compose.yml up -d \ + webknossos \ + webknossos-datastore \ + webknossos-tracingstore + - name: Run webknossos smoke test + uses: ./.github/actions/retry + with: + run: curl -v --fail-with-body "http://localhost:9000/api/health" + retries: 20 + retry_delay_seconds: 5 + - name: Run webknossos-datastore smoke test + uses: ./.github/actions/retry + with: + run: curl -v --fail-with-body "http://localhost:9090/data/health" + retries: 20 + retry_delay_seconds: 5 + - name: Run webknossos-tracingstore smoke test + uses: ./.github/actions/retry + with: + run: curl -v --fail-with-body "http://localhost:9050/tracings/health" + retries: 20 + retry_delay_seconds: 5 + - name: Stop webknossos, datastore and tracingstore + run: docker compose --file .github/docker-compose.yml down --volumes --remove-orphans + + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASS }} + + - name: Push webknossos image + if: ${{ ! startsWith(github.ref, 'refs/tags/') }} + uses: ./.github/actions/retry + with: + run: docker push "akanzer/webknossos:${{ env.DOCKER_TAG }}" + retries: 5 + retry_delay_seconds: 10 + - name: Push normalized webknossos image + uses: ./.github/actions/retry + with: + run: docker push "akanzer/webknossos:${{ env.NORMALIZED_BRANCH }}" + retries: 5 + retry_delay_seconds: 10 + - name: Push webknossos-datastore image + if: ${{ ! startsWith(github.ref, 'refs/tags/') }} + uses: ./.github/actions/retry + with: + run: docker push "akanzer/webknossos-datastore:${{ env.DOCKER_TAG }}" + retries: 5 + retry_delay_seconds: 10 + - name: Push normalized webknossos-datastore image + uses: ./.github/actions/retry + with: + run: docker push "akanzer/webknossos-datastore:${{ env.NORMALIZED_BRANCH }}" + retries: 5 + retry_delay_seconds: 10 + - name: Push webknossos-tracingstore image + if: ${{ ! startsWith(github.ref, 'refs/tags/') }} + uses: ./.github/actions/retry + with: + run: docker push "akanzer/webknossos-tracingstore:${{ env.DOCKER_TAG }}" + retries: 5 + retry_delay_seconds: 10 + - name: Push normalized webknossos-tracingstore image + uses: ./.github/actions/retry + with: + run: docker push "akanzer/webknossos-tracingstore:${{ env.NORMALIZED_BRANCH }}" + retries: 5 + retry_delay_seconds: 10 + - name: Slack Notification + if: github.ref == 'refs/heads/master' + run: .github/slack-notification.sh + env: + SLACK_NOTIFY_TOKEN: ${{ secrets.SLACK_NOTIFY_TOKEN }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000000..bbb71f814da --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,74 @@ +name: Nightly Test Pipeline + +on: + workflow_dispatch: {} + schedule: + # Runs every day at 12:00 AM UTC + - cron: '0 0 * * *' + +jobs: + nightly-screenshot-tests: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Process and Log Branch Name + run: | + echo "BRANCH_NAME=${{ github.ref_name }}" >> $GITHUB_ENV + SUBDOMAIN=$(echo "${{ github.ref_name }}" | tr -dc '[:alpha:]' | tr '[:upper:]' '[:lower:]') + TAG_NAME=$(echo "${{ github.ref_name }}" | sed 's/[\/-]/_/g') + + echo "SUBDOMAIN=$SUBDOMAIN" >> $GITHUB_ENV + echo "TAG_NAME=$TAG_NAME" >> $GITHUB_ENV + + echo "Sanitized branch name: $SUBDOMAIN" + echo "Tag name: $TAG_NAME" + + - name: Remove dev-deployment + run: | + curl -X POST \ + -H "X-Auth-Token: ${{ secrets.RELEASE_API_TOKEN }}" \ + https://kubernetix.scm.io/hooks/remove/webknossos/dev/${{ env.TAG_NAME }}?user=CI+%28nightly%29 + + - name: Wait 3 minutes + run: sleep 180 + + - name: Install dev-deployment + run: | + curl -X POST \ + -H "X-Auth-Token: ${{ secrets.RELEASE_API_TOKEN }}" \ + https://kubernetix.scm.io/hooks/install/webknossos/dev/${{ env.TAG_NAME }}?user=CI+%28nightly%29 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install dependencies and wait + run: | + corepack enable && yarn install --immutable & + sleep 180 & + wait + + - name: Refresh datasets + run: | + curl -X POST --fail https://${{ env.SUBDOMAIN }}.webknossos.xyz/data/triggers/checkInboxBlocking?token=${{ secrets.WK_AUTH_TOKEN }} + + - name: Run screenshot tests + run: | + URL=https://${{ env.SUBDOMAIN }}.webknossos.xyz/ \ + timeout 3000 \ + yarn test-screenshot + env: + URL: https://${{ env.SUBDOMAIN }}.webknossos.xyz/ + WK_AUTH_TOKEN: ${{ secrets.WK_AUTH_TOKEN }} + BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }} + BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} + + - name: Upload screenshots as artifact + uses: actions/upload-artifact@v4 + if: always() + with: + name: screenshots + path: frontend/javascripts/test/screenshots diff --git a/.github/workflows/wkorg-nightly.yaml b/.github/workflows/wkorg-nightly.yaml new file mode 100644 index 00000000000..2fad1e797b4 --- /dev/null +++ b/.github/workflows/wkorg-nightly.yaml @@ -0,0 +1,60 @@ +name: Nightly WK.org Test Pipeline + +on: + workflow_dispatch: {} + schedule: + # Runs every day at 12:00 AM UTC + - cron: '0 0 * * *' + +jobs: + nightly-screenshot-tests: + runs-on: ubuntu-latest + timeout-minutes: 50 + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install dependencies + run: | + corepack enable && yarn install --immutable + + - name: Run screenshot tests + run: | + yarn test-wkorg-screenshot + env: + URL: https://webknossos.org/ + WK_AUTH_TOKEN: ${{ secrets.WK_AUTH_TOKEN }} + BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }} + BROWSERSTACK_ACCESS_KEY : ${{ secrets.BROWSERSTACK_ACCESS_KEY }} + + - name: Upload screenshots as artifact + uses: actions/upload-artifact@v4 + if: always() + with: + name: screenshots + path: frontend/javascripts/test/screenshots-wkorg + + + nightly-gzip-test: + runs-on: ubuntu-latest + + steps: + - name: Assert GZIP is enabled + run: | + # Test gzipped assets + curl -s -I -H "Accept-Encoding: gzip" https://webknossos.org/assets/bundle/main.js | grep -q "content-encoding: gzip" + curl -s -I -H "Accept-Encoding: gzip" https://webknossos.org/assets/bundle/main.css | grep -q "content-encoding: gzip" + # Test gzipped buckets + curl -s -i \ + -H 'accept: application/octet-stream' \ + -H 'Accept-Encoding: gzip' \ + -H 'content-type: application/json' \ + --data-raw '[{"position":[2752,4320,1728],"additionalCoordinates":[],"mag":[1,1,1],"cubeSize":32,"fourBit":false}]' \ + 'https://data-humerus.webknossos.org/data/datasets/scalable_minds/l4dense_motta_et_al_demo/layers/segmentation/data?token=' \ + | grep -q "content-encoding: gzip" + echo Success. \ No newline at end of file diff --git a/.gitignore b/.gitignore index dbedca4f61d..ad1e0583cbe 100755 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ RUNNING_PID .bloop .metals metals.sbt +__pycache__/ # Webknossos @@ -78,10 +79,9 @@ lifttemplate.iws conf/github.conf tmp/ errorShots/ -flow-coverage/ -# Sublime +# Sublime *.sublime-project *.sublime-workspace @@ -89,10 +89,19 @@ flow-coverage/ .nyc_output coverage coverage-ts +*.vscode +.bsp +temp-webknossos-schema** +conf/application.conf-e +.env +.bloop +.metals +metals.sbt + **/screenshots/*.diff.png **/screenshots/*.new.png -# Yarn +# Yarn .yarn/* !.yarn/cache !.yarn/patches @@ -100,4 +109,4 @@ coverage-ts !.yarn/releases !.yarn/sdks !.yarn/versions -tools/proxy/.yarn/* +tools/**/.yarn/* diff --git a/CHANGELOG.released.md b/CHANGELOG.released.md index 196473c0ba0..6cf27d42400 100644 --- a/CHANGELOG.released.md +++ b/CHANGELOG.released.md @@ -7,6 +7,291 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Calendar Versioning](http://calver.org/) `0Y.0M.MICRO`. For upgrade instructions, please check the [migration guide](MIGRATIONS.released.md). +## [25.05.0](https://github.com/scalableminds/webknossos/releases/tag/25.05.0) - 2025-04-29 +[Commits](https://github.com/scalableminds/webknossos/compare/25.04.0...25.05.0) + +### Highlights +- Added the concept of "toolkits". By default, all tools are available to the user (as before), but one can select a specific toolkit to only see certain tools. Some toolkits also change the behavior of the tools. For example, the "Split Segments" toolkit (see below). [#8434](https://github.com/scalableminds/webknossos/pull/8434) +- Added a workflow for splitting segments. Select the "Split Segments" toolkit and create a bounding box in which you want to execute the split. Then, use the skeleton tool to place nodes on the boundary between two (merged) segments. The nodes will be used to construct a 3D surface. Finally, use the floodfill tool (enable 3D and bounding-box restriction) to relabel a part of the segment. the floodfill won't cross the 3D surface. [#8434](https://github.com/scalableminds/webknossos/pull/8434) +- Added a new "draw" mode for the skeleton tool. When enabled, one can rapidly create notes by keeping the left mouse button pressed and moving the cursor. [#8434](https://github.com/scalableminds/webknossos/pull/8434) + +### Added +- Added more layer specific settings to the configurations included in sharing links. [#8539](https://github.com/scalableminds/webknossos/pull/8539) +- When uploading multiple NMLs at once, the description is now kept, if all NMLs with non-empty descriptions have the same description. [#8533](https://github.com/scalableminds/webknossos/pull/8533) + +### Changed +- Updated E2E tests to use `vitest` framework instead of `ava`. [#8543](https://github.com/scalableminds/webknossos/pull/8543) +- Adjusted the names of custom model inference jobs and train model jobs to match the worker's naming. [#8524](https://github.com/scalableminds/webknossos/pull/8524) +- Updated screenshot tests to use `vitest` framework instead of `ava`. [#8553](https://github.com/scalableminds/webknossos/pull/8553) +- The mapping dropdown for segmentation is wider now so that mapping names are fully readable. [#8570](https://github.com/scalableminds/webknossos/pull/8570) + +### Fixed +- Fixed that the Version Restore sidebar did load forever when very large volume annotations were created. [#8574](https://github.com/scalableminds/webknossos/pull/8574) +- Fixed rendering bug that could occur for transformed datasets when toggling the transform. [#8568](https://github.com/scalableminds/webknossos/pull/8568) +- Fixed a bug in the trees tab where the color change of a tree would affect the tree on which the context menu was previously opened. [#8562](https://github.com/scalableminds/webknossos/pull/8562) + +### Removed +- The Annotation update route can no longer update the description of the annotation. Please set the description before uploading the annotation instead. You can still edit the description in the UI. [#8533](https://github.com/scalableminds/webknossos/pull/8533) + +## [25.04.0](https://github.com/scalableminds/webknossos/releases/tag/25.04.0) - 2025-04-22 +[Commits](https://github.com/scalableminds/webknossos/compare/25.03.1...25.04.0) + +### Highlights +- The opacity of meshes can be adjusted using the 'Change Segment Color' context menu entry in the segments tab. [#8443](https://github.com/scalableminds/webknossos/pull/8443) +- Allow to mirror datasets along an axis in the dataset settings as part of the rotation feature. [#8485](https://github.com/scalableminds/webknossos/pull/8485) + +### Added +- Performance improvements for volume annotation save requests. [#8460](https://github.com/scalableminds/webknossos/pull/8460) +- Performance improvements for segment statistics (volume + bounding box in context menu). [#8469](https://github.com/scalableminds/webknossos/pull/8469) +- Added that the dataset name is automatically corrected in view mode URLs upon loading. [#8514](https://github.com/scalableminds/webknossos/pull/8514) +- Upgraded backend dependencies for improved performance and stability. [#8507](https://github.com/scalableminds/webknossos/pull/8507) +- New config option `datastore.dataVaults.credentials` allows admins to set up global credentials for remote dataset loading. [#8509](https://github.com/scalableminds/webknossos/pull/8509) + +### Changed +- When deleting a dataset / layer, layers that are referenced in other datasets are moved there instead of being deleted. [#8437](https://github.com/scalableminds/webknossos/pull/8437/) +- Added a parameter to the reserve manual upload route allowing to make the request fail if the name is already taken. Moreover, the new dataset's id and directory name are returned in the response. [#8476](https://github.com/scalableminds/webknossos/pull/8476) +- The skeleton tool can no longer be activated if the skeleton layer is invisible. [#8501](https://github.com/scalableminds/webknossos/pull/8501) +- Improved speed of mesh rendering and mouse interaction in 3D viewport. [#8106](https://github.com/scalableminds/webknossos/pull/8106) +- Numbered docker image now use different and larger numbers. [#8147](https://github.com/scalableminds/webknossos/pull/8147) +- Replace frontend unit testing framework `ava` with `vitest`. Minimum required nodejs version is now `22+`. [#8479](https://github.com/scalableminds/webknossos/pull/8479) + +### Fixed +- Fixed a Bug where the "Save view configuration as default" modal's text included undefined. [#8514](https://github.com/scalableminds/webknossos/pull/8514) +- Fixed the alignment of the button that allows restricting floodfill operations to a bounding box. [#8388](https://github.com/scalableminds/webknossos/pull/8388) +- Fixed that it was possible to trigger the find largest segment id job on layers which are not stored as segmentation layers on the server. [#8503](https://github.com/scalableminds/webknossos/pull/8503) +- Fixed a rare and subtle bug related to volume annotation and undo/redo. [#7506](https://github.com/scalableminds/webknossos/pull/7506) +- Fixed a bug where segment statistics would sometimes be wrong in case of an on-disk segmentation fallback layer with segment index file. [#8460](https://github.com/scalableminds/webknossos/pull/8460) +- Fixed a bug where sometimes outdated segment statistics would be displayed. [#8460](https://github.com/scalableminds/webknossos/pull/8460) +- Fixed a bug where outbound zarr streaming would contain a typo in the zarr header dimension_separator field. [#8510](https://github.com/scalableminds/webknossos/pull/8510) +- Fixed a bug where sometimes large skeletons were not saved correctly, making them inaccessible on the next load. [#8513](https://github.com/scalableminds/webknossos/pull/8513) +- Fixed that meshes weren't loaded correctly if the precomputed mesh file contained multiple levels-of-detail. [#8519](https://github.com/scalableminds/webknossos/pull/8519) +- Fixed that authentication-related token renewal did not work properly in certain scenarios. [#8532](https://github.com/scalableminds/webknossos/pull/8532) +- Fixed resolution of legacy links having dataset names with `-`. [#8548](https://github.com/scalableminds/webknossos/pull/8548) +- Fixed that adding a layer using the dataset settings' advanced tab would crash WEBKNOSSOS. Bug was introduced by [#8503](https://github.com/scalableminds/webknossos/pull/8503). [#8550](https://github.com/scalableminds/webknossos/pull/8550) + +### Breaking Changes +- Removed `docker-compose.yml` in favor of `tools/hosting/docker-compose.yml` [#8147](https://github.com/scalableminds/webknossos/pull/8147) + + +## [25.03.1](https://github.com/scalableminds/webknossos/releases/tag/25.03.1) - 2025-04-10 +[Commits](https://github.com/scalableminds/webknossos/compare/25.03.0...25.03.1) + +### Fixed +- Fixed a bug where the annotation list would sometimes load very long if you have many annotations. [#8498](https://github.com/scalableminds/webknossos/pull/8498) +- Fixed a bug where sometimes large skeletons were not saved correctly, making them inaccessible on the next load. [#8513](https://github.com/scalableminds/webknossos/pull/8513) + +## [25.03.0](https://github.com/scalableminds/webknossos/releases/tag/25.03.0) - 2025-04-01 +[Commits](https://github.com/scalableminds/webknossos/compare/25.02.1...25.03.0) + +### Highlights +- Added a command palette that allows navigating between pages, switching tools and accessing some user settings via Ctrl+P. [#8447](https://github.com/scalableminds/webknossos/pull/8447/) +- Added support for datasets with the following data types: int8, int16, int32, uint32 (support for color was added, support for segmentation already existed before) and int64 (segmentation only). [#8325](https://github.com/scalableminds/webknossos/pull/8325) + +### Added +- Super users can now share the trained AI models with other organizations. [#8418](https://github.com/scalableminds/webknossos/pull/8418) +- Failed jobs may be retried by super-users. [#8377](https://github.com/scalableminds/webknossos/pull/8377) +- Optimized server-side storage of skeleton annotation layers. [#8423](https://github.com/scalableminds/webknossos/pull/8423) +- Added support for adding N5 datasets with compact-style multiscale metadata. [#8456](https://github.com/scalableminds/webknossos/pull/8456) +- Added a credit system making payment for long running jobs possible. For now it is in testing phase. [#8352](https://github.com/scalableminds/webknossos/pull/8352) +- The maximum available storage of an organization is now enforced during upload. [#8385](https://github.com/scalableminds/webknossos/pull/8385) + +### Changed +- When using a zarr link to a wk-served data layer as another layer’s source, the user’s token is used to access the data. [#8322](https://github.com/scalableminds/webknossos/pull/8322/) +- Compound annotations (created when viewing all annotations of a task) no longer permanently store data in the FossilDB. [#8422](https://github.com/scalableminds/webknossos/pull/8422) +- When creating multiple tasks at once (bulk task creation), they now all need to have the same task type. [#8405](https://github.com/scalableminds/webknossos/pull/8405) +- Improved performance when changing the layout/viewports. [#8448](https://github.com/scalableminds/webknossos/pull/8448) +- Annotation upload will now always add a skeleton annotation layer, even if the downloaded annotation was volume-only. [#8466](https://github.com/scalableminds/webknossos/pull/8466) + +### Fixed +- Fixed a bug that would lock a non-existing mapping to an empty segmentation layer under certain conditions. [#8401](https://github.com/scalableminds/webknossos/pull/8401) +- Fixed the alignment of the button that allows restricting floodfill operations to a bounding box. [#8388](https://github.com/scalableminds/webknossos/pull/8388) +- Fixed a bug for rotated dataset where volume tools were disabled although the dataset was rendered untransformed. [#8432](https://github.com/scalableminds/webknossos/pull/8432) +- Fixed rare bug where saving got stuck. [#8409](https://github.com/scalableminds/webknossos/pull/8409) +- Fixed some rendering bugs for float datasets that used a large dynamic range. [#8325](https://github.com/scalableminds/webknossos/pull/8325) +- Fixed a bug where reverting annotations could get stuck if some of its layers had been deleted in the meantime. [#8405](https://github.com/scalableminds/webknossos/pull/8405) +- When removing a segment from the segment list, a corresponding precomputed mesh was not removed automatically. [#8428](https://github.com/scalableminds/webknossos/pull/8428) +- Fixed a bug where newly added remote datasets would always appear in root folder, regardless of actual selected folder. [#8425](https://github.com/scalableminds/webknossos/pull/8425) +- Fixed a bug where the python libs functionality `wk.RemoteDataset.explore_and_add_remote` would error. [#8425](https://github.com/scalableminds/webknossos/pull/8425) +- Fixed a bug where various UI dialogs would be dark mode even the user preferred a light theme. [#8445](https://github.com/scalableminds/webknossos/pull/8445) +- Fixed an issue with icon spacing on the task dashboard page. [#8452](https://github.com/scalableminds/webknossos/pull/8452) +- Fixed a spacing issue in the statusbar. [#8455](https://github.com/scalableminds/webknossos/pull/8455) +- Fixed a bug where the "Create Animation" modal did not open when selecting the corresponding feature from the navbar menu. [#8444](https://github.com/scalableminds/webknossos/pull/8444) +- Fixed that the brightness/contrast settings (in the histogram) could not be changed if the histogram data could not be loaded for some reason. [#8459](https://github.com/scalableminds/webknossos/pull/8459) +- Fixed that downloading + reupload of an annotation, in which the skeleton tools were never used, actually made them unreachable. [#8466](https://github.com/scalableminds/webknossos/pull/8466) +- Fixed a bug where task creation with volume zip as input would fail. [#8468](https://github.com/scalableminds/webknossos/pull/8468) +- Fixed visual alignment of actions in ai model list. [#8474](https://github.com/scalableminds/webknossos/pull/8474) +- Improve formatting of credits amount in organization management page [#8487](https://github.com/scalableminds/webknossos/pull/8487) +- Re-enabled jobs planned to be paid with credits for organizations without a paid plan. [#8478](https://github.com/scalableminds/webknossos/pull/8478) +- Fixed that the dataset extent tooltip in the right details bar in the dashboard did not properly update when switching datasets. [#8477](https://github.com/scalableminds/webknossos/pull/8477) +- Fixed a bug where task creation with volume zip as input would fail. [#8468](https://github.com/scalableminds/webknossos/pull/8468) +- Fixed that a warning message about a newer version of an annotation was shown multiple times. [#8486](https://github.com/scalableminds/webknossos/pull/8486) + +## [25.02.1](https://github.com/scalableminds/webknossos/releases/tag/25.02.1) - 2025-02-26 +[Commits](https://github.com/scalableminds/webknossos/compare/25.02.0...25.02.1) + +### Fixed +- Fixed rare bug where saving got stuck. [#8409](https://github.com/scalableminds/webknossos/pull/8409) +- Fixed the correct highlighting of navbar menu items depending on the current open page. [#8473](https://github.com/scalableminds/webknossos/pull/8473) + +## [25.02.0](https://github.com/scalableminds/webknossos/releases/tag/25.02.0) - 2025-02-17 +[Commits](https://github.com/scalableminds/webknossos/compare/25.01.0...25.02.0) + +### Highlights +- Starting now, proofreading (editable mapping) annotations will have a revertible history. Also the annotation version history is no longer split in individual tabs per layer. [#7917](https://github.com/scalableminds/webknossos/pull/7917) +- Measurement tools are now accessible when viewing datasets outside of an annotation. [#8334](https://github.com/scalableminds/webknossos/pull/8334) + +### Added +- It is now possible to start a split-merger evaluation when starting a neuron inference. [#8221](https://github.com/scalableminds/webknossos/pull/8221) +- Added the possibility to configure a rotation for a dataset, which can be toggled off and on when viewing and annotating data. [#8159](https://github.com/scalableminds/webknossos/pull/8159) +- When using the “Restore older Version” feature, there are no longer separate tabs for the different annotation layers. Only one linear annotation history is now used, and if you revert to an older version, all layers are reverted. If layers were added/deleted since then, that is also reverted. This also means that proofreading annotations can now be reverted to older versions as well. The description text of annotations is now versioned as well. [#7917](https://github.com/scalableminds/webknossos/pull/7917) +- Added the possibility to use the "merger mode" even when the user has annotated volume data in the current layer (as long as no other mapping is active). [#8335](https://github.com/scalableminds/webknossos/pull/8335) +- Added a note explaining that imported data is private by default to the dataset upload view. [#8354](https://github.com/scalableminds/webknossos/pull/8354) +- Measurement tools are now accessible when viewing datasets outside of an annotation. [#8334](https://github.com/scalableminds/webknossos/pull/8334) +- Support reading Blosc-compressed datasets with the “autoshuffle” parameter. [#8387](https://github.com/scalableminds/webknossos/pull/8387) + +### Changed +- Improved the scrolling behaviour of sliders: Sliders must be focused to scroll them. Additionally, parent element scrolling is prevented when using the slider. [#8321](https://github.com/scalableminds/webknossos/pull/8321) [#8321](https://github.com/scalableminds/webknossos/pull/8321) +- Increase the flood fill maximum bounding box size limits for segmentation layers restricted to coarser mags. [#8382](https://github.com/scalableminds/webknossos/pull/8382) + +### Fixed +- Fixed a silent bug in the dashboard when refreshing newest dataset list. [#8386](https://github.com/scalableminds/webknossos/pull/8386) +- Fixed a bug that lead to trees being dropped when merging to trees together. [#8359](https://github.com/scalableminds/webknossos/pull/8359) +- Fixed that the onboarding screen incorrectly appeared when a certain request failed. [#8356](https://github.com/scalableminds/webknossos/pull/8356) +- Fixed the segment registering in coarser mags for non-mag-aligned bounding boxes. [#8364](https://github.com/scalableminds/webknossos/pull/8364) +- Fixed using the flood fill tool in 2D mode for mags other than the finest one. [#8382](https://github.com/scalableminds/webknossos/pull/8382) + +### Removed + - Removed the feature to downsample existing volume annotations. All new volume annotations had a whole mag stack since [#4755](https://github.com/scalableminds/webknossos/pull/4755) (four years ago). [#7917](https://github.com/scalableminds/webknossos/pull/7917) + +## [25.01.0](https://github.com/scalableminds/webknossos/releases/tag/25.01.0) - 2025-01-22 +[Commits](https://github.com/scalableminds/webknossos/compare/24.12.0...25.01.0) + +### Highlights +- The fill tool can now be adapted so that it only acts within a specified bounding box. Use the new "Restrict Floodfill" mode for that in the toolbar. [#8267](https://github.com/scalableminds/webknossos/pull/8267) +- Added the option for "Selective Segment Visibility" for segmentation layers. Select this option in the left sidebar to only show segments that are currently active or hovered. [#8281](https://github.com/scalableminds/webknossos/pull/8281) +- Segment and tree names can be edited by double-clicking them. [#8316](https://github.com/scalableminds/webknossos/pull/8316) + +### Added +- Added the total volume of a dataset to a tooltip in the dataset info tab. [#8229](https://github.com/scalableminds/webknossos/pull/8229) +- Optimized performance of data loading with “fill value“ chunks. [#8271](https://github.com/scalableminds/webknossos/pull/8271) +- Added the option to export a segmentation that was corrected with the proofreading tool to a new segmentation. [#8286](https://github.com/scalableminds/webknossos/pull/8286) +- A segment can be activated with double-click now. [#8281](https://github.com/scalableminds/webknossos/pull/8281) +- It is now possible to select the magnification of the layers on which an AI model will be trained. [#8266](https://github.com/scalableminds/webknossos/pull/8266) +- Added support for translations in OME NGFF zarr datasets (translation within coordinateTransformations on datasets). [#8311](https://github.com/scalableminds/webknossos/pull/8311) +- When the eraser tool is active, one can switch temporarily to the fill-segment tool by pressing shift and ctrl. Only pressing shift, switches to the pick-segment tool. [#8314](https://github.com/scalableminds/webknossos/pull/8314) +- Enabled auto sorting of Typescript imports in Biome linter. [#8313](https://github.com/scalableminds/webknossos/pull/8313) +- Clicking on a segment or tree group will show some details in the details table. [#8316](https://github.com/scalableminds/webknossos/pull/8316) + +### Changed +- Renamed "resolution" to "magnification" in more places within the codebase, including local variables. [#8168](https://github.com/scalableminds/webknossos/pull/8168) +- Layer names are now allowed to contain `$` as special characters. [#8241](https://github.com/scalableminds/webknossos/pull/8241) +- Datasets can now be renamed and can have duplicate names. [#8075](https://github.com/scalableminds/webknossos/pull/8075) +- Starting an AI training job using multiple annotations now supports inputting task-IDs and considers their task bounding boxes. [#8310](https://github.com/scalableminds/webknossos/pull/8310) +- Improved the default colors for skeleton trees. [#8228](https://github.com/scalableminds/webknossos/pull/8228) +- Allowed to train an AI model using differently sized bounding boxes. We recommend all bounding boxes to have equal dimensions or to have dimensions which are multiples of the smallest bounding box. [#8222](https://github.com/scalableminds/webknossos/pull/8222) +- Within the bounding box tool, the cursor updates immediately after pressing `ctrl`, indicating that a bounding box can be moved instead of resized. [#8253](https://github.com/scalableminds/webknossos/pull/8253) +- Improved the styling of active tools and modes in the toolbar. [#8295](https://github.com/scalableminds/webknossos/pull/8295) + +### Fixed +- Fixed that listing datasets with the `api/datasets` route without compression failed due to missing permissions regarding public datasets. [#8249](https://github.com/scalableminds/webknossos/pull/8249) +- A "Locked by anonymous user" banner is no longer shown when opening public editable annotations of other organizations. [#8273](https://github.com/scalableminds/webknossos/pull/8273) +- Fixed a bug that uploading a zarr dataset with an already existing `datasource-properties.json` file failed. [#8268](https://github.com/scalableminds/webknossos/pull/8268) +- Fixed the organization switching feature for datasets opened via old links. [#8257](https://github.com/scalableminds/webknossos/pull/8257) +- Fixed that uploading an NML file without an organization id failed. Now the user's organization is used as fallback. [#8277](https://github.com/scalableminds/webknossos/pull/8277) +- Fixed that the frontend did not ensure a minimum length for annotation layer names. Moreover, names starting with a `.` are also disallowed now. [#8244](https://github.com/scalableminds/webknossos/pull/8244) +- Fixed a bug where in the add remote dataset view the dataset name setting was not in sync with the datasource setting of the advanced tab making the form not submittable. [#8245](https://github.com/scalableminds/webknossos/pull/8245) +- Fixed read and update dataset route for versions 8 and lower. [#8263](https://github.com/scalableminds/webknossos/pull/8263) +- Fixed that task bounding boxes are again converted to user bounding boxes when uploading annotations via nmls. [#8280](https://github.com/scalableminds/webknossos/pull/8280) +- Added missing legacy support for `isValidNewName` route. [#8252](https://github.com/scalableminds/webknossos/pull/8252) +- Fixed some layout issues in the upload view. [#8231](https://github.com/scalableminds/webknossos/pull/8231) +- Fixed `FATAL: role "postgres" does not exist` error message in Docker compose. [#8240](https://github.com/scalableminds/webknossos/pull/8240) +- Fixed the Zarr 3 implementation not accepting BytesCodec without "configuration" key. [#8282](https://github.com/scalableminds/webknossos/pull/8282) +- Fixed that reloading the data of a volume annotation layer did not work properly. [#8298](https://github.com/scalableminds/webknossos/pull/8298) +- Removed the magnification slider for the TIFF export within the download modal if only one magnification is available for the selected layer. [#8297](https://github.com/scalableminds/webknossos/pull/8297) +- Fixed regression in styling of segment and skeleton tree tab. [#8307](https://github.com/scalableminds/webknossos/pull/8307) +- Fixed the template for neuron inferral using a custom workflow. [#8312](https://github.com/scalableminds/webknossos/pull/8312) +- Fixed that the list of processing jobs crashed for deleted job types. [#8300](https://github.com/scalableminds/webknossos/pull/8300) +- Fixed an issue where you could not maximize or reposition the 3D/XZ viewport in Safari. [#8337](https://github.com/scalableminds/webknossos/pull/8337) +- Fixed upload of NGFF datasets with only one directory named "color". [#8341](https://github.com/scalableminds/webknossos/pull/8341/) +- Fixed an issue that could occur if the NGFF multiscale name was set to "/" when exploring. [#8341](https://github.com/scalableminds/webknossos/pull/8341/) + +### Removed +- Removed support for HTTP API versions 3 and 4. [#8075](https://github.com/scalableminds/webknossos/pull/8075) +- Removed that a warning is shown when a dataset is served from a datastore that was marked with isScratch=true. [#8296](https://github.com/scalableminds/webknossos/pull/8296) + +## [24.12.0](https://github.com/scalableminds/webknossos/releases/tag/24.12.0) - 2024-12-05 +[Commits](https://github.com/scalableminds/webknossos/compare/24.11.1...24.12.0) + +### Added +- When exploring remote URIs pasted from Neuroglancer, the format prefixes like `precomputed://` are now ignored, so users don’t have to remove them. [#8195](https://github.com/scalableminds/webknossos/pull/8195) + +### Changed +- Reading image files on datastore filesystem is now done asynchronously. [#8126](https://github.com/scalableminds/webknossos/pull/8126) +- Improved error messages for starting jobs on datasets from other organizations. [#8181](https://github.com/scalableminds/webknossos/pull/8181) +- Terms of Service for Webknossos are now accepted at registration, not afterward. [#8193](https://github.com/scalableminds/webknossos/pull/8193) +- Removed bounding box size restriction for inferral jobs for super users. [#8200](https://github.com/scalableminds/webknossos/pull/8200) +- Improved logging for errors when loading datasets and problems arise during a conversion step. [#8202](https://github.com/scalableminds/webknossos/pull/8202) + +### Fixed +- Fixed performance bottleneck when deleting a lot of trees at once. [#8176](https://github.com/scalableminds/webknossos/pull/8176) +- Fixed a bug where changing the color of a segment via the menu in the segments tab would update the segment color of the previous segment, on which the context menu was opened. [#8225](https://github.com/scalableminds/webknossos/pull/8225) +- Fixed a bug when importing an NML with groups when only groups but no trees exist in an annotation. [#8176](https://github.com/scalableminds/webknossos/pull/8176) +- Fixed a bug where trying to delete a non-existing node (via the API, for example) would delete the whole active tree. [#8176](https://github.com/scalableminds/webknossos/pull/8176) +- Fixed a bug where dataset uploads would fail if the organization directory on disk is missing. [#8230](https://github.com/scalableminds/webknossos/pull/8230) + +### Removed +- Removed Google Analytics integration. [#8201](https://github.com/scalableminds/webknossos/pull/8201) + +## [24.11.1](https://github.com/scalableminds/webknossos/releases/tag/24.11.1) - 2024-11-13 +[Commits](https://github.com/scalableminds/webknossos/compare/24.10.0...24.11.1) + +### Highlights +- It is now possible to add metadata in annotations to Trees and Segments. [#7875](https://github.com/scalableminds/webknossos/pull/7875) +- Added a button to the search popover in the skeleton and segment tab to select all matching non-group results. [#8123](https://github.com/scalableminds/webknossos/pull/8123) + +### Added +- It is now possible to add metadata in annotations to Trees and Segments. [#7875](https://github.com/scalableminds/webknossos/pull/7875) +- Added a summary row to the time tracking overview, where times and annotations/tasks are summed. [#8092](https://github.com/scalableminds/webknossos/pull/8092) +- Most sliders have been improved: Wheeling above a slider now changes its value and double-clicking its knob resets it to its default value. [#8095](https://github.com/scalableminds/webknossos/pull/8095) +- It is now possible to search for unnamed segments with the full default name instead of only their id. [#8133](https://github.com/scalableminds/webknossos/pull/8133) +- Increased loading speed for precomputed meshes. [#8110](https://github.com/scalableminds/webknossos/pull/8110) +- Added a button to the search popover in the skeleton and segment tab to select all matching non-group results. [#8123](https://github.com/scalableminds/webknossos/pull/8123) +- Unified wording in UI and code: “Magnification”/“mag” is now used in place of “Resolution“ most of the time, compare [terminology document](https://docs.webknossos.org/webknossos/terminology.html). [#8111](https://github.com/scalableminds/webknossos/pull/8111) +- Added support for adding remote OME-Zarr NGFF version 0.5 datasets. [#8122](https://github.com/scalableminds/webknossos/pull/8122) +- Workflow reports may be deleted by superusers. [#8156](https://github.com/scalableminds/webknossos/pull/8156) + +### Changed +- Some mesh-related actions were disabled in proofreading-mode when using meshfiles that were created for a mapping rather than an oversegmentation. [#8091](https://github.com/scalableminds/webknossos/pull/8091) +- Admins can now see and cancel all jobs. The owner of the job is shown in the job list. [#8112](https://github.com/scalableminds/webknossos/pull/8112) +- Migrated nightly screenshot tests from CircleCI to GitHub actions. [#8134](https://github.com/scalableminds/webknossos/pull/8134) +- Migrated nightly screenshot tests for wk.org from CircleCI to GitHub actions. [#8135](https://github.com/scalableminds/webknossos/pull/8135) +- Thumbnails for datasets now use the selected mapping from the view configuration if available. [#8157](https://github.com/scalableminds/webknossos/pull/8157) + +### Fixed +- Fixed a bug during dataset upload in case the configured `datastore.baseFolder` is an absolute path. [#8098](https://github.com/scalableminds/webknossos/pull/8098) [#8103](https://github.com/scalableminds/webknossos/pull/8103) +- Fixed bbox export menu item [#8152](https://github.com/scalableminds/webknossos/pull/8152) +- When trying to save an annotation opened via a link including a sharing token, the token is automatically discarded in case it is insufficient for update actions but the users token is. [#8139](https://github.com/scalableminds/webknossos/pull/8139) +- Fix that scrolling in the trees and segments tab did not work while dragging. [#8162](https://github.com/scalableminds/webknossos/pull/8162) +- Fixed that uploading a dataset which needs a conversion failed when the angstrom unit was configured for the conversion. [#8173](https://github.com/scalableminds/webknossos/pull/8173) +- Fixed that the skeleton search did not automatically expand groups that contained the selected tree [#8129](https://github.com/scalableminds/webknossos/pull/8129) +- Fixed interactions in the trees and segments tab like the search due to a bug introduced by [#8162](https://github.com/scalableminds/webknossos/pull/8162). [#8186](https://github.com/scalableminds/webknossos/pull/8186) +- Fixed a bug that zarr streaming version 3 returned the shape of mag (1, 1, 1) / the finest mag for all mags. [#8116](https://github.com/scalableminds/webknossos/pull/8116) +- Fixed sorting of mags in outbound zarr streaming. [#8125](https://github.com/scalableminds/webknossos/pull/8125) +- Fixed a bug where you could not create annotations for public datasets of other organizations. [#8107](https://github.com/scalableminds/webknossos/pull/8107) +- Users without edit permissions to a dataset can no longer delete sharing tokens via the API. [#8083](https://github.com/scalableminds/webknossos/issues/8083) +- Fixed downloading task annotations of teams you are not in, when accessing directly via URI. [#8155](https://github.com/scalableminds/webknossos/pull/8155) +- Removed unnecessary scrollbars in skeleton tab that occurred especially after resizing. [#8148](https://github.com/scalableminds/webknossos/pull/8148) +- Deleting a bounding box is now possible independently of a visible segmentation layer. [#8164](https://github.com/scalableminds/webknossos/pull/8164) +- S3-compliant object storages can now be accessed via HTTPS. [#8167](https://github.com/scalableminds/webknossos/pull/8167) +- Fixed that skeleton tree nodes were created with the wrong mag. [#8185](https://github.com/scalableminds/webknossos/pull/8185) +- Fixed the expected type of a tree node received from the server. Fixes nml export to include the `inMag` field correctly. [#8187](https://github.com/scalableminds/webknossos/pull/8187) +- Fixed a layout persistence bug leading to empty viewports, triggered when switching between orthogonal, flight, or oblique mode. [#8177](https://github.com/scalableminds/webknossos/pull/8177) + +### Removed + +### Breaking Changes + + ## [24.10.0](https://github.com/scalableminds/webknossos/releases/tag/24.10.0) - 2024-09-24 [Commits](https://github.com/scalableminds/webknossos/compare/24.08.1...24.10.0) @@ -113,7 +398,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Added the option to set a default mapping for a dataset in the dataset view configuration. The default mapping is loaded when the dataset is opened and the user / url does not configure something else. [#7858](https://github.com/scalableminds/webknossos/pull/7858) - Uploading an annotation into a dataset that it was not created for now also works if the dataset is in a different organization. [#7816](https://github.com/scalableminds/webknossos/pull/7816) - When downloading + reuploading an annotation that is based on a segmentation layer with active mapping, that mapping is now still selected after the reupload. [#7822](https://github.com/scalableminds/webknossos/pull/7822) -- In the Voxelytics workflow list, the name of the WEBKNOSSOS user who started the job is displayed. [#7794](https://github.com/scalableminds/webknossos/pull/7795) +- In the Voxelytics workflow list, the name of the WEBKNOSSOS user who started the job is displayed. [#7794](https://github.com/scalableminds/webknossos/pull/7794) - Start an alignment job (aligns the section in a dataset) via the "AI Analysis" button (not available to all users yet). [#7820](https://github.com/scalableminds/webknossos/pull/7820) - Added additional validation for the animation job modal. Bounding boxes must be larger then zero. [#7883](https://github.com/scalableminds/webknossos/pull/7883) @@ -401,7 +686,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released ### Fixed - Fixed that some segment (group) actions were not properly disabled for non-editable segmentation layers. [#7207](https://github.com/scalableminds/webknossos/issues/7207) - Fixed a bug where data from zarr2 datasets that have a channel axis was broken. [#7374](https://github.com/scalableminds/webknossos/pull/7374) -- Fixed a bug which changed the cursor position while editing the name of a tree or the comment of a node. [#7390](#https://github.com/scalableminds/webknossos/pull/7390) +- Fixed a bug which changed the cursor position while editing the name of a tree or the comment of a node. [#7390](https://github.com/scalableminds/webknossos/pull/7390) - Streaming sharded zarr3 datasets from servers which do not respond with Accept-Ranges header is now possible. [#7392](https://github.com/scalableminds/webknossos/pull/7392) ## [23.10.2](https://github.com/scalableminds/webknossos/releases/tag/23.10.2) - 2023-09-26 @@ -527,7 +812,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released [Commits](https://github.com/scalableminds/webknossos/compare/23.06.0...23.07.0) ### Highlights -- Added new shortcuts for fast tool switching. Look at the updated [Keyboard Shortcuts documentation](https://docs.webknossos.org/webknossos/keyboard_shortcuts.html#tool-switching-shortcuts) to see the new shortcuts. [#7112](https://github.com/scalableminds/webknossos/pull/7112) +- Added new shortcuts for fast tool switching. Look at the updated [Keyboard Shortcuts documentation](https://docs.webknossos.org/webknossos/ui/keyboard_shortcuts.html#tool-switching-shortcuts) to see the new shortcuts. [#7112](https://github.com/scalableminds/webknossos/pull/7112) - Creating bounding boxes can now be done by dragging the left mouse button (when the bounding box tool is selected). To move around in the dataset while this tool is active, keep ALT pressed. [#7118](https://github.com/scalableminds/webknossos/pull/7118) ### Added @@ -627,7 +912,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Editing the meta data of segments (e.g., the name) is now undoable. [#6944](https://github.com/scalableminds/webknossos/pull/6944) - Added more icons and small redesigns for various pages in line with the new branding. [#6938](https://github.com/scalableminds/webknossos/pull/6938) - Added more graphics and branding to job notification emails. [#6994](https://github.com/scalableminds/webknossos/pull/6994) -- Added action button in Team Admin page that links to the User Admin page to edit team members there. [#6958](https://github.com/scalableminds/webknossos/pull/6999) +- Added action button in Team Admin page that links to the User Admin page to edit team members there. [#6999](https://github.com/scalableminds/webknossos/pull/6999) ### Changed - Moved the view mode selection in the toolbar next to the position field. [#6949](https://github.com/scalableminds/webknossos/pull/6949) @@ -714,7 +999,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Added functions to get and set segment colors to the frontend API (`api.data.{getSegmentColor,setSegmentColor}`). [#6853](https://github.com/scalableminds/webknossos/pull/6853) ### Changed -- Limit paid team sharing features to respective organization plans. [#6767](https://github.com/scalableminds/webknossos/pull/6776) +- Limit paid team sharing features to respective organization plans. [#6767](https://github.com/scalableminds/webknossos/pull/6767) - Rewrite the database tools in `tools/postgres` to JavaScript and adding support for non-default Postgres username-password combinations. [#6803](https://github.com/scalableminds/webknossos/pull/6803) - Added owner name to organization page. [#6811](https://github.com/scalableminds/webknossos/pull/6811) - Remove multiline