Upgrade to REDIS8

celery-cmd

@@ -12,7 +12,6 @@ CELERY__OPTS=${CELERY__OPTS:-"--without-gossip --without-mingle -Ofair -E"}
 CELERY__LOG_LEVEL=${CELERY__LOG_LEVEL:-"ERROR"}
 CELERY__LOG_FILE=${CELERY__LOG_FILE:-"/var/log/celery.log"}
 CELERY__WORKER_NAME=${CELERY__WORKER_NAME:-"worker1@%h"}
-CELERY__WORKER_CONCURRENCY=${CELERY__WORKER_CONCURRENCY:-"4"}
 
 # Celery beat settings
 CELERY__BEAT_SCHEDULE=${CELERY__BEAT_SCHEDULE:-"celery.beat:PersistentScheduler"}
@@ -52,7 +51,7 @@ $CELERY_BIN -A $CELERY_APP worker --autoscale=$CELERY__AUTOSCALE_VALUES \
     --max-memory-per-child=$CELERY__MAX_MEMORY_PER_CHILD $CELERY__OPTS \
     --statedb=$CELERY__STATE_DB \
     --loglevel=$CELERY__LOG_LEVEL -n $CELERY__WORKER_NAME -f $CELERY__LOG_FILE \
-    --concurrency=$CELERY__WORKER_CONCURRENCY --max-tasks-per-child=$CELERY__MAX_TASKS_PER_CHILD \
+    --prefetch-multiplier=1 --max-tasks-per-child=$CELERY__MAX_TASKS_PER_CHILD \
     -X harvesting &
 
 echo "Starting Harvester Celery Worker..."

docker-compose.yml

@@ -103,8 +103,8 @@ services:
       retries: 3
     env_file:
       - ${COMPOSE_ENV_FILES:-.env}
-    ports:
-      - "8080:8080"
+    #ports:
+    #  - "8080:8080"
     volumes:
       - statics:/mnt/volumes/statics
       - geoserver-data-dir:/geoserver_data/data
@@ -147,8 +147,9 @@ services:
 
   # Vanilla Redis service. This is needed by celery
   redis:
-    image: redis:7-alpine
+    image: redis:8-alpine
     container_name: redis4${COMPOSE_PROJECT_NAME}
+    command: redis-server --maxmemory 512mb --maxmemory-policy noeviction --appendonly yes
     volumes:
       - redisdata:/data
     restart: unless-stopped

geonode/settings.py

@@ -1458,10 +1458,10 @@ def get_geonode_catalogue_service():
 
 # Disabling the heartbeat because workers seems often disabled in flower,
 # thanks to http://stackoverflow.com/a/14831904/654755
-BROKER_HEARTBEAT = 0
+CELERY_BROKER_HEARTBEAT = 0
 
 # Avoid long running and retried tasks to be run over-and-over again.
-BROKER_TRANSPORT_OPTIONS = {
+CELERY_BROKER_TRANSPORT_OPTIONS = {
     "fanout_prefix": True,
     "fanout_patterns": True,
     "socket_timeout": 60,
@@ -1496,7 +1496,7 @@ def get_geonode_catalogue_service():
 CELERY_ACKS_LATE = ast.literal_eval(os.environ.get("CELERY_ACKS_LATE", "True"))
 
 # Add a ten-minutes timeout to all Celery tasks.
-CELERYD_SOFT_TIME_LIMIT = 600
+CELERY_TASK_SOFT_TIME_LIMIT = 600
 
 # Set this to False in order to run async
 _EAGER_FLAG = "False" if ASYNC_SIGNALS else "True"
@@ -1654,7 +1654,7 @@ def get_geonode_catalogue_service():
 CELERY_MAX_CACHED_RESULTS = os.environ.get("CELERY_MAX_CACHED_RESULTS", 32768)
 
 # NOTE: I don't know if this is compatible with upstart.
-CELERYD_POOL_RESTARTS = ast.literal_eval(os.environ.get("CELERYD_POOL_RESTARTS", "True"))
+CELERY_WORKER_POOL_RESTARTS = ast.literal_eval(os.environ.get("CELERYD_POOL_RESTARTS", "True"))
 CELERY_TRACK_STARTED = ast.literal_eval(os.environ.get("CELERY_TRACK_STARTED", "True"))
 CELERY_SEND_TASK_SENT_EVENT = ast.literal_eval(os.environ.get("CELERY_SEND_TASK_SENT_EVENT", "True"))
 

pyproject.toml

@@ -64,7 +64,7 @@ dependencies = [
     "django-sequences==3.0",
     "oauthlib==3.3.1",
     "pyjwt==2.13.0",
-    "redis==6.2.0",
+    "redis==8.1.0",
 
     # geopython dependencies
     "pycsw @ https://github.com/geopython/pycsw/archive/refs/tags/3.0.0-beta1.zip",