Commit e300befc authored by Daniel Lee's avatar Daniel Lee 🐐
Browse files

Merge tag '2.11.0' into main

parents e4c9b39a 5619cecd
......@@ -48,6 +48,8 @@
variables: &variables
EPCT_TEST_DATA_DIR: /data/data-tailor
NETCDFGEN_RUNTIME_DIR: /netcdfgen-runtime
DOCKER_DRIVER: overlay2
GIT_DEPTH: 1
BUILD_OPTS: "--output-folder conda-channel"
LC_ALL: "en_US.utf8"
LANG: "en_US.utf8"
......@@ -66,19 +68,17 @@ stages:
- build
- installer
- test
- test_installer
- test_proc
- test_installer
- quality
- deploy
# cache using branch name
# https://gitlab.com/help/ci/caching/index.md
cache:
key: ${CI_COMMIT_REF_SLUG}
paths:
- .npm
- cache/Cypress
- node_modules
workflow:
rules:
# no pipeline if commit on a branch with open merge requests (MR detached pipeline already running)
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
when: never
- when: always
.common_linux: &common_linux
image: miniconda3-build-centos:latest
......@@ -122,6 +122,7 @@ pdf docs:
paths:
- conda-channel
expire_in: 1 days
interruptible: true
build linux core:
<<: *build_common_linux
......@@ -130,7 +131,14 @@ build linux core:
build linux webui:
<<: *build_common_linux
cache:
key: ${CI_COMMIT_REF_SLUG}
paths:
- .npm
- cache/Cypress
- node_modules
script:
- export DT_VERSION="0.0.0"
- conda build $CHANNEL_OPTS $BUILD_OPTS ./epct-webui
build linux plugin-gis:
......@@ -141,7 +149,11 @@ build linux plugin-gis:
build linux plugins:
<<: *build_common_linux
script:
- conda install git
- conda install git && conda clean -afy \
&& find /opt/conda/ -follow -type f -name '*.a' -delete \
&& find /opt/conda/ -follow -type f -name '*.pyc' -delete \
&& find /opt/conda/ -follow -type f -name '*.js.map' -delete \
&& find /opt/conda/lib/python*/site-packages/bokeh/server/static -follow -type f -name '*.js' ! -name '*.min.js' -delete
- git clone -b development https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eumetsat.int/data-tailor/umarf-plugins/pfd-plugins-master.git epct_plugin_umarf/pfd-plugins-master
- git clone -b development https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eumetsat.int/data-tailor/umarf-plugins/netcdf.git epct_plugin_umarf/netcdf
- git clone -b development https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eumetsat.int/data-tailor/umarf-plugins/msgclmk_grib.git epct_plugin_umarf/msgclmk_grib
......@@ -169,6 +181,7 @@ build linux gdal:
paths:
- conda-channel
expire_in: 1 days
interruptible: true
build win core:
<<: *build_common_win
......@@ -179,6 +192,7 @@ build win core:
build win webui:
<<: *build_common_win
script:
- set DT_VERSION=0.0.0
- conda build %CHANNEL_OPTS_WIN% %BUILD_OPTS% ./epct-webui
- IF %errorlevel% NEQ 0 exit /b %errorlevel%
......@@ -198,6 +212,8 @@ build win gdal:
installer linux:
extends: .build_common_linux
stage: installer
rules:
- if: $CI_COMMIT_TAG
needs:
- build linux core
- build linux webui
......@@ -209,7 +225,9 @@ installer linux:
- conda index $CI_PROJECT_DIR/conda-channel
- conda create --name epct-installer python=3.6
- conda init bash && source ~/.bashrc && conda activate epct-installer
- export DT_VERSION="0.0.0"
script:
- conda install jinja2
- conda install constructor
- ln -s $CI_PROJECT_DIR/conda-channel /tmp/conda-channel
- cp conda/post_install.sh /tmp/post_install.sh
......@@ -235,14 +253,22 @@ installer linux:
- conda init bash && source ~/.bashrc && conda activate epct-tests
needs:
- build linux core
- build linux webui
- build linux plugin-gis
- build linux gdal
interruptible: true
tests linux functional:
<<: *test_common_linux
dependencies:
- build linux core
- build linux plugin-gis
- build linux gdal
script:
- conda install -y $CHANNEL_OPTS epct epct_plugin_gis msg-gdal-driver epct_restapi
- conda install -y $CHANNEL_OPTS epct epct_plugin_gis msg-gdal-driver epct_restapi && conda clean -afy \
&& find /opt/conda/ -follow -type f -name '*.a' -delete \
&& find /opt/conda/ -follow -type f -name '*.pyc' -delete \
&& find /opt/conda/ -follow -type f -name '*.js.map' -delete \
&& find /opt/conda/lib/python*/site-packages/bokeh/server/static -follow -type f -name '*.js' ! -name '*.min.js' -delete
- pip install --no-deps --ignore-installed falcon_multipart
- epct info
- pytest -vv --durations=0 --junitxml=$CI_PROJECT_DIR/linux-epct-functional-tests.xml -k test_ functional_tests/
......@@ -259,8 +285,16 @@ tests linux dtws-validation:
<<: *test_common_linux
rules:
- if: $CI_COMMIT_TAG
dependencies:
- build linux core
- build linux plugin-gis
- build linux gdal
script:
- conda install -y $CHANNEL_OPTS epct epct_plugin_gis msg-gdal-driver epct_restapi
- conda install -y $CHANNEL_OPTS epct epct_plugin_gis msg-gdal-driver epct_restapi && conda clean -afy \
&& find /opt/conda/ -follow -type f -name '*.a' -delete \
&& find /opt/conda/ -follow -type f -name '*.pyc' -delete \
&& find /opt/conda/ -follow -type f -name '*.js.map' -delete \
&& find /opt/conda/lib/python*/site-packages/bokeh/server/static -follow -type f -name '*.js' ! -name '*.min.js' -delete
- pip install --no-deps --ignore-installed falcon_multipart
- epct info
- pytest --durations=0 --junitxml=$CI_PROJECT_DIR/linux-epct-validation-tests-DTWS.xml -k test_DTWS validation_tests/
......@@ -275,8 +309,14 @@ tests linux dtws-validation:
tests linux epct:
<<: *test_common_linux
dependencies:
- build linux core
script:
- conda install -y $CHANNEL_OPTS epct
- conda install -y $CHANNEL_OPTS epct && conda clean -afy \
&& find /opt/conda/ -follow -type f -name '*.a' -delete \
&& find /opt/conda/ -follow -type f -name '*.pyc' -delete \
&& find /opt/conda/ -follow -type f -name '*.js.map' -delete \
&& find /opt/conda/lib/python*/site-packages/bokeh/server/static -follow -type f -name '*.js' ! -name '*.min.js' -delete
- epct info
- cd epct/tests
- pytest -vv --durations=0 --cov=epct --junitxml=$CI_PROJECT_DIR/linux-epct-functional-tests.xml test_*
......@@ -291,8 +331,16 @@ tests linux epct:
tests linux epct-plugin-gis:
<<: *test_common_linux
dependencies:
- build linux core
- build linux plugin-gis
- build linux gdal
script:
- conda install -y $CHANNEL_OPTS epct epct_plugin_gis msg-gdal-driver
- conda install -y $CHANNEL_OPTS epct epct_plugin_gis msg-gdal-driver && conda clean -afy \
&& find /opt/conda/ -follow -type f -name '*.a' -delete \
&& find /opt/conda/ -follow -type f -name '*.pyc' -delete \
&& find /opt/conda/ -follow -type f -name '*.js.map' -delete \
&& find /opt/conda/lib/python*/site-packages/bokeh/server/static -follow -type f -name '*.js' ! -name '*.min.js' -delete
- epct info
- cd epct_plugin_gis/tests
- pytest -vv --durations=0 --cov=epct_plugin_gis --junitxml=$CI_PROJECT_DIR/linux-epct-plugin-gis-functional-tests.xml test_*
......@@ -307,8 +355,14 @@ tests linux epct-plugin-gis:
tests linux epct-restapi:
<<: *test_common_linux
dependencies:
- build linux core
script:
- conda install -y $CHANNEL_OPTS epct epct_restapi
- conda install -y $CHANNEL_OPTS epct epct_restapi && conda clean -afy \
&& find /opt/conda/ -follow -type f -name '*.a' -delete \
&& find /opt/conda/ -follow -type f -name '*.pyc' -delete \
&& find /opt/conda/ -follow -type f -name '*.js.map' -delete \
&& find /opt/conda/lib/python*/site-packages/bokeh/server/static -follow -type f -name '*.js' ! -name '*.min.js' -delete
- pip install --no-deps --ignore-installed falcon_multipart
- epct info
- cd epct-restapi/epct_restapi
......@@ -324,13 +378,29 @@ tests linux epct-restapi:
tests linux epct-webui:
<<: *test_common_linux
dependencies:
- build linux core
- build linux webui
needs:
- build linux core
- build linux webui
cache:
key: ${CI_COMMIT_REF_SLUG}
paths:
- .npm
- cache/Cypress
- node_modules
rules:
- if: $BUILD_LINUX
- if: $CI_MERGE_REQUEST_ID
- if: $CI_COMMIT_TAG
script:
- yum install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib
- conda install -y $CHANNEL_OPTS epct_webui openjdk=8 git
- conda install -y $CHANNEL_OPTS epct_webui openjdk=8 git && conda clean -afy \
&& find /opt/conda/ -follow -type f -name '*.a' -delete \
&& find /opt/conda/ -follow -type f -name '*.pyc' -delete \
&& find /opt/conda/ -follow -type f -name '*.js.map' -delete \
&& find /opt/conda/lib/python*/site-packages/bokeh/server/static -follow -type f -name '*.js' ! -name '*.min.js' -delete
- git clone https://github.com/outofcoffee/imposter.git && cd imposter
- ./gradlew clean shadowJar
- java -jar distro/all/build/libs/imposter-all.jar --configDir $CI_PROJECT_DIR/epct-restapi/ &
......@@ -346,6 +416,12 @@ tests linux epct-webui:
tests linux validation:
<<: *test_common_linux
dependencies:
- build linux core
- build linux webui
- build linux plugin-gis
- build linux gdal
- build linux plugins
needs:
- build linux core
- build linux webui
......@@ -358,7 +434,11 @@ tests linux validation:
- if: $CI_COMMIT_TAG
script:
- conda install -c eumetsat perl libiconv libjpeg-turbo-cos6-x86_64 libpng eugene && ln -s $CONDA_PREFIX/x86_64-conda_cos6-linux-gnu/sysroot/usr/lib64/libjpeg.so.62 $CONDA_PREFIX/lib/libjpeg.so.62
- conda install -y $CHANNEL_OPTS epct epct_plugin_gis msg-gdal-driver epct_restapi epct_plugin_umarf
- conda install -y $CHANNEL_OPTS epct epct_plugin_gis msg-gdal-driver epct_restapi epct_plugin_umarf && conda clean -afy \
&& find /opt/conda/ -follow -type f -name '*.a' -delete \
&& find /opt/conda/ -follow -type f -name '*.pyc' -delete \
&& find /opt/conda/ -follow -type f -name '*.js.map' -delete \
&& find /opt/conda/lib/python*/site-packages/bokeh/server/static -follow -type f -name '*.js' ! -name '*.min.js' -delete
- pip install --no-deps --ignore-installed falcon_multipart
- epct info
- pytest --durations=0 --junitxml=$CI_PROJECT_DIR/linux-epct-validation-tests-API.xml -k test_API -m "not longrunning" validation_tests/
......@@ -388,11 +468,19 @@ tests linux validation:
tests linux long-validation:
<<: *test_common_linux
dependencies:
- build linux core
- build linux plugin-gis
- build linux gdal
rules:
- if: $CI_COMMIT_TAG
script:
- conda install -c eumetsat perl libiconv libjpeg-turbo-cos6-x86_64 libpng eugene && ln -s $CONDA_PREFIX/x86_64-conda_cos6-linux-gnu/sysroot/usr/lib64/libjpeg.so.62 $CONDA_PREFIX/lib/libjpeg.so.62
- conda install -y $CHANNEL_OPTS epct epct_plugin_gis msg-gdal-driver epct_restapi
- conda install -y $CHANNEL_OPTS epct epct_plugin_gis msg-gdal-driver epct_restapi && conda clean -afy \
&& find /opt/conda/ -follow -type f -name '*.a' -delete \
&& find /opt/conda/ -follow -type f -name '*.pyc' -delete \
&& find /opt/conda/ -follow -type f -name '*.js.map' -delete \
&& find /opt/conda/lib/python*/site-packages/bokeh/server/static -follow -type f -name '*.js' ! -name '*.min.js' -delete
- pip install --no-deps --ignore-installed falcon_multipart
- pytest --durations=0 --junitxml=$CI_PROJECT_DIR/linux-epct-longtests.xml -m longrunning validation_tests/
artifacts:
......@@ -406,12 +494,32 @@ tests linux long-validation:
tests linux webui validation:
<<: *test_common_linux
image: webui-image:latest
dependencies:
- build linux core
- build linux plugin-gis
- build linux gdal
- build linux webui
needs:
- build linux core
- build linux plugin-gis
- build linux gdal
- build linux webui
cache:
key: ${CI_COMMIT_REF_SLUG}
paths:
- .npm
- cache/Cypress
- node_modules
stage: test
tags:
- linux
script:
- yum install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib
- conda install -y $CHANNEL_OPTS epct epct_restapi epct_webui epct_plugin_gis msg-gdal-driver
- conda install -y $CHANNEL_OPTS epct epct_restapi epct_webui epct_plugin_gis msg-gdal-driver && conda clean -afy \
&& find /opt/conda/ -follow -type f -name '*.a' -delete \
&& find /opt/conda/ -follow -type f -name '*.pyc' -delete \
&& find /opt/conda/ -follow -type f -name '*.js.map' -delete \
&& find /opt/conda/lib/python*/site-packages/bokeh/server/static -follow -type f -name '*.js' ! -name '*.min.js' -delete
- pip install --no-deps --ignore-installed falcon_multipart
- epct info
- cd epct-webui
......@@ -436,6 +544,44 @@ tests linux webui validation:
- epct-webui/cypress/screenshots/**/*.png
expire_in: 1 day
tests linux proc validation:
<<: *common_linux
stage: test_proc
dependencies:
- build linux core
- build linux plugin-gis
- build linux gdal
before_script:
- conda index $CI_PROJECT_DIR/conda-channel
- conda create --name epct-tests python=3.6 pytest pytest-cov
- conda init bash && source ~/.bashrc && conda activate epct-tests
needs:
- build linux core
- build linux plugin-gis
- build linux gdal
- tests linux validation
rules:
- if: $BUILD_LINUX
- if: $CI_MERGE_REQUEST_ID
- if: $CI_COMMIT_TAG
script:
- conda install -c eumetsat perl libiconv libjpeg-turbo-cos6-x86_64 libpng eugene && ln -s $CONDA_PREFIX/x86_64-conda_cos6-linux-gnu/sysroot/usr/lib64/libjpeg.so.62 $CONDA_PREFIX/lib/libjpeg.so.62
- conda install -y $CHANNEL_OPTS epct epct_plugin_gis msg-gdal-driver epct_restapi && conda clean -afy \
&& find /opt/conda/ -follow -type f -name '*.a' -delete \
&& find /opt/conda/ -follow -type f -name '*.pyc' -delete \
&& find /opt/conda/ -follow -type f -name '*.js.map' -delete \
&& find /opt/conda/lib/python*/site-packages/bokeh/server/static -follow -type f -name '*.js' ! -name '*.min.js' -delete
- pip install --no-deps --ignore-installed falcon_multipart
- pytest --durations=0 --junitxml=$CI_PROJECT_DIR/linux-epct-validation-tests-PROC.xml -k test_PROC -m "not longrunning" validation_tests/
artifacts:
reports:
junit:
- $CI_PROJECT_DIR/linux-epct-validation-tests-PROC.xml
paths:
- $CI_PROJECT_DIR/linux-epct-validation-tests-PROC.xml
expire_in: 4 days
when: always
tests linux installer:
image: centos:centos7
stage: test_installer
......@@ -443,10 +589,8 @@ tests linux installer:
- linux
needs:
- installer linux
- tests linux validation
- tests linux proc validation
rules:
- if: $BUILD_LINUX
- if: $CI_MERGE_REQUEST_ID
- if: $CI_COMMIT_TAG
before_script:
- mkdir /tmp/conda-channel
......@@ -494,37 +638,6 @@ tests linux installer:
- epct-webui/cypress/videos/**/*.mp4
- epct-webui/cypress/screenshots/**/*.png
tests linux proc validation:
<<: *common_linux
stage: test_proc
before_script:
- conda index $CI_PROJECT_DIR/conda-channel
- conda create --name epct-tests python=3.6 pytest pytest-cov
- conda init bash && source ~/.bashrc && conda activate epct-tests
needs:
- build linux core
- build linux webui
- build linux plugin-gis
- build linux gdal
- tests linux validation
rules:
- if: $BUILD_LINUX
- if: $CI_MERGE_REQUEST_ID
- if: $CI_COMMIT_TAG
script:
- conda install -c eumetsat perl libiconv libjpeg-turbo-cos6-x86_64 libpng eugene && ln -s $CONDA_PREFIX/x86_64-conda_cos6-linux-gnu/sysroot/usr/lib64/libjpeg.so.62 $CONDA_PREFIX/lib/libjpeg.so.62
- conda install -y $CHANNEL_OPTS epct epct_plugin_gis msg-gdal-driver epct_restapi
- pip install --no-deps --ignore-installed falcon_multipart
- pytest --durations=0 --junitxml=$CI_PROJECT_DIR/linux-epct-validation-tests-PROC.xml -k test_PROC -m "not longrunning" validation_tests/
artifacts:
reports:
junit:
- $CI_PROJECT_DIR/linux-epct-validation-tests-PROC.xml
paths:
- $CI_PROJECT_DIR/linux-epct-validation-tests-PROC.xml
expire_in: 4 days
when: always
# WINDOWS test section
.tests_common_win: &test_common_win
......@@ -551,9 +664,14 @@ tests linux proc validation:
- if: $CI_COMMIT_TAG
- if: $CI_MERGE_REQUEST_ID
- if: $START_WIN
interruptible: true
tests win validation:
<<: *test_common_win
dependencies:
- build win core
- build win plugin-gis
- build win gdal
script:
- conda install -y %CHANNEL_OPTS_WIN% epct epct_restapi epct_plugin_gis msg-gdal-driver && epct info && pip install --no-deps --ignore-installed falcon_multipart
- pytest --durations=0 --junitxml=%CI_PROJECT_DIR%\win-epct-validation-tests-API.xml -m "not longrunning" -k test_API validation_tests\
......@@ -598,6 +716,10 @@ tests win validation:
tests win long-validation:
<<: *test_common_win
dependencies:
- build win core
- build win plugin-gis
- build win gdal
script:
- conda install -y %CHANNEL_OPTS_WIN% epct epct_restapi epct_plugin_gis msg-gdal-driver && epct info && pip install --no-deps --ignore-installed falcon_multipart
- pytest --durations=0 --junitxml=%CI_PROJECT_DIR%\win-epct-validation-longtests.xml -m longrunning validation_tests\
......@@ -615,6 +737,10 @@ tests win long-validation:
tests win functional:
<<: *test_common_win
dependencies:
- build win core
- build win plugin-gis
- build win gdal
script:
- conda install -y %CHANNEL_OPTS_WIN% epct epct_restapi epct_plugin_gis msg-gdal-driver && epct info && pip install --no-deps --ignore-installed falcon_multipart && pytest --durations=0 --junitxml=%CI_PROJECT_DIR%\win-epct-functional-tests.xml -k test_ functional_tests\
- IF %errorlevel% NEQ 0 exit /b %errorlevel%
......@@ -629,6 +755,8 @@ tests win functional:
tests win epct:
<<: *test_common_win
dependencies:
- build win core
script:
- conda install -y %CHANNEL_OPTS_WIN% epct && epct info && cd epct\tests && dir && pytest -vv --durations=0 --cov=epct --junitxml=%CI_PROJECT_DIR%\win-epct-functional-tests.xml -k "test_"
- IF %errorlevel% NEQ 0 exit /b %errorlevel%
......@@ -643,6 +771,10 @@ tests win epct:
tests win epct-plugin-gis:
<<: *test_common_win
dependencies:
- build win core
- build win plugin-gis
- build win gdal
script:
- conda install -y %CHANNEL_OPTS_WIN% epct epct_plugin_gis msg-gdal-driver && epct info && cd epct_plugin_gis\tests && pytest -vv --durations=0 --cov=epct_plugin_gis --junitxml=%CI_PROJECT_DIR%\win-epct-plugin-gis-functional-tests.xml -k "test_"
- IF %errorlevel% NEQ 0 exit /b %errorlevel%
......@@ -657,6 +789,8 @@ tests win epct-plugin-gis:
tests win epct-restapi:
<<: *test_common_win
dependencies:
- build win core
script:
- conda install -y %CHANNEL_OPTS_WIN% epct epct_restapi && pip install --no-deps --ignore-installed falcon_multipart && epct info && cd epct-restapi\tests && pytest -vv --durations=0 --cov=epct_restapi --junitxml=%CI_PROJECT_DIR%\win-epct-restapi-functional-tests.xml -k "test_"
- IF %errorlevel% NEQ 0 exit /b %errorlevel%
......@@ -673,6 +807,10 @@ tests win proc validation:
variables:
<<: *variables
EPCT_TEST_DATA_DIR: C:\data\data-tailor
dependencies:
- build win core
- build win plugin-gis
- build win gdal
stage: test_proc
tags:
- windows
......@@ -686,7 +824,6 @@ tests win proc validation:
when: runner_system_failure
needs:
- build win core
- build win webui
- build win plugin-gis
- build win gdal
- tests win validation
......@@ -716,6 +853,8 @@ code_quality:
stage: quality
needs: []
dependencies: []
rules:
- if: $CI_COMMIT_TAG
tags:
- linux
script:
......
......@@ -4,6 +4,27 @@ All notable changes to this project are documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [2.11.0]
### Added
- Implemented xRIT channel segment selection (#897)
- Allow for GEO products the ROI extraction if the input data specifies or known to be in a known projection (#885)
- Re-projection by chunks for ASCATL1SZO and ASCATL1SZR full orbit products (#825)
- Implementation of new KPI, documenting step-by-step operational procedures (#738, #889)
### Changed
- Optimised pipeline jobs duration and structure (#900)
- Improve error information shown to users (#883)
- Improvements in the way static resources are served (#882)
- Improved DTWS nginx logging with response time (#881)
- Improved REST API logging: more details, traceback and customisation submission tracking (#880)
- Updated product discovery after Data Store API change (#847)
- ROI customisations as internal metric for downstream analytics (#838)
- Data Tailor version always discoverable (#836)
### Fixed
- Webapp not displaying a proper error when a download operation fails (#873)
## [2.10.0]
### Added
......
......@@ -46,7 +46,7 @@ The DTWS provides monitoring capabilities at several levels. They are described
DTWS performance (ELK stack)
----------------------------
DTWS performance can be monitored by opening `$SERVICE_NODE_INTERNAL_IP:5607` with a browser,
DTWS performance can be monitored by opening `$SERVICE_NODE_INTERNAL_IP:5601` with a browser,
then accessing the Dashboard.
Scheduler and worker health
......@@ -337,6 +337,39 @@ Nginx configuration
`Nginx` configuration for the DTWS is found on the service node at `/etc/nginx/sites.d/epcs.conf`.
Configure Dask logging
----------------------
The Dask distributed scheduler provides feedback via an interactive dashboard containing live information.
The scheduler, workers, and client all emit logs using `Python standard logging module <https://docs.python.org/3/library/logging.html>`_.
Note that the default settings are: `INFO` for scheduler and client, and `WARNING` for workers.
Dask logging verbosity can be controlled by modifying a configuration file called `distributed.yaml`.
Logging for specific components like `distributed.client`, `distributed.scheduler`, `distributed.nanny`,
`distributed.worker`, etc. can each be independently configured.
To enable this functionality, we need to:
- Create a `dask` folder under the host path `$DTWS_CONFIG/`
- In the `dask` folder add the file `distributed.yaml`
- Open the file and add the following code:
.. code-block::
distributed:
version: 2
logging:
distributed: info
distributed.client: warning
distributed.scheduler: info
distributed.worker: warning
- Restart both the scheduler and the worker
In case of troubles, the verbosity level can be substituted with any of the available ones from the Python module
(for details see `here <https://docs.python.org/3/library/logging.html#logging-levels>`_).
ELK Stack configuration
-----------------------
......@@ -501,12 +534,54 @@ The file can use *markdown metadata* too, for example:
To disable the message, just remove the file.
Deployment Monitoring via the ELK Stack
---------------------------------------
The KPI of the DTWS are monitored via the ELK stack,
installed on the service node as explained in `assets/dtws/ELK.README.rst
<assets/dtws/ELK.README.rst>`_.
In order to connect to the Kibana GUI, from an external network,
connect to that node in ssh with port forwarding on port 5601, as follows:
`ssh $SERVICE_NODE_IP -L 5601:localhost:5601`
Then open `localhost:5601/status` in a browser on the local pc.
A page will then appear, displaying various features available.
Click on `Dashboard` in the upper left corner of the screen.
Two distinct entries will appear:
`Data Tailor` and `Data Tailor per Users`.
First access the `Data Tailor` dashboard.
Up on the right, select a suitable time range encompassing the dates from which
logged data will be gathered by Logstash, elaborated by Elastic Search,
and visualised in this page by Kibana for analysis. Click on `Refresh`.
A series of general analytics and plots summing up all activity by
users on the Data Tailor deployment should now be visible.
Section details can be further inspected by clicking on the small wheel
in the upper right corner of the respective subpanel.
Return to the previous page, and access the `Data Tailor per Users` dashboard.
When compared to the former dashboard, in this case an additional `Users` subpanel
is visible at the top of the page. Choose some users as suited,
select again the time range above, and apply the selections.
The bottom panels will now populate with decomposed data and plots,
each corresponding to statistics of user-individual activity.
Troubleshoooting
================
DTWS accepting customisation requests but not executing them
--------------------------------------------------------------
It has been noted on the pilot deployment that, starting from a give time, the users can submit
It has been noted on the pilot deployment that, starting from a given time, users can submit
customisation requests but such requests are not processed (they remain running, then become "inactive").
This is usually due to communication problems between the scheduler and the worker, because:
......
......@@ -23,7 +23,8 @@ EUMETSAT Cloud services and infrastructure
The DTWS is integrated with the other services in the EUMETSAT cloud,
that must be active and configured in advance.
:note: This guide assumes that such services can access the DTWS nodes on internal IPs.
.. note::
This guide assumes that such services can access the DTWS nodes on internal IPs.
If this is not true, the proxy on the service node needs to be configured
to serve the DTWS services (in particular the REST API) as well.
This is discouraged for security reasons.
......@@ -139,7 +140,8 @@ All the commands below are given as an administrator.
Service node
------------
:note: The service node in principle needs to be setup only the first time a DTWS is installed on the ICSI.
.. note::
The service node in principle needs to be setup only the first time a DTWS is installed on the ICSI.
DTWS updates do not require a new installation of the node, just a configuration
......@@ -202,9 +204,13 @@ Start `nginx` and ensure it will restart after a reboot:
.. note::
The stock `nginx` configuration on CentOS7 may behave strangely. In case, replace `nginx.conf`
The stock `nginx` configuration on CentOS7 may behave strangely. In case, replace ``nginx.conf``
with the sample one provided in `assets/dtws/nginx/nginx-sample.conf
<assets/dtws/nginx/nginx-sample.conf>`_
<assets/dtws/nginx/nginx-sample.conf>`_.
To configure the **access log file format** properly: put the following configuration in the ``nginx.conf``::
log_format combined_with_time '$remote_addr - $upstr