From 0562286d8c2321b3d72bdd5fa3512d4cc8e06d3c Mon Sep 17 00:00:00 2001 From: Amrit Sarkar Date: Tue, 17 Nov 2026 01:52:49 -0800 Subject: [PATCH 1/9] SOLR-17704: Add curator-recipes dependency --- gradle/libs.versions.toml | 1 + solr/api/gradle.lockfile | 1 + solr/benchmark/gradle.lockfile | 1 + solr/core/build.gradle | 3 +++ solr/core/gradle.lockfile | 1 + solr/cross-dc-manager/gradle.lockfile | 1 + solr/licenses/curator-recipes-5.9.0.jar.sha1 | 1 + solr/modules/analysis-extras/gradle.lockfile | 1 + solr/modules/clustering/gradle.lockfile | 1 + solr/modules/cross-dc/gradle.lockfile | 1 + solr/modules/cuvs/gradle.lockfile | 1 + solr/modules/extraction/gradle.lockfile | 1 + solr/modules/gcs-repository/gradle.lockfile | 1 + solr/modules/jwt-auth/gradle.lockfile | 1 + solr/modules/langid/gradle.lockfile | 1 + solr/modules/language-models/gradle.lockfile | 1 + solr/modules/ltr/gradle.lockfile | 1 + solr/modules/opentelemetry/gradle.lockfile | 1 + solr/modules/s3-repository/gradle.lockfile | 1 + solr/modules/scripting/gradle.lockfile | 1 + solr/modules/sql/gradle.lockfile | 1 + solr/server/gradle.lockfile | 1 + solr/solr-ref-guide/gradle.lockfile | 1 + solr/solrj-jetty/gradle.lockfile | 1 + solr/solrj-streaming/gradle.lockfile | 1 + solr/solrj-zookeeper/gradle.lockfile | 1 + solr/solrj/gradle.lockfile | 1 + solr/test-framework/gradle.lockfile | 1 + solr/webapp/gradle.lockfile | 1 + 29 files changed, 31 insertions(+) create mode 100644 solr/licenses/curator-recipes-5.9.0.jar.sha1 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5a160315f627..4627462a6bdc 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -224,6 +224,7 @@ apache-commons-math3 = { module = "org.apache.commons:commons-math3", version.re apache-commons-text = { module = "org.apache.commons:commons-text", version.ref = "apache-commons-text" } apache-curator-client = { module = "org.apache.curator:curator-client", version.ref = "apache-curator" } apache-curator-framework = { module = "org.apache.curator:curator-framework", version.ref = "apache-curator" } +apache-curator-recipes = { module = "org.apache.curator:curator-recipes", version.ref = "apache-curator" } apache-curator-test = { module = "org.apache.curator:curator-test", version.ref = "apache-curator" } apache-httpcomponents-httpclient = { module = "org.apache.httpcomponents:httpclient", version.ref = "apache-httpcomponents-httpclient" } apache-httpcomponents-httpcore = { module = "org.apache.httpcomponents:httpcore", version.ref = "apache-httpcomponents-httpcore" } diff --git a/solr/api/gradle.lockfile b/solr/api/gradle.lockfile index c6b03d24b736..e4a614b61a2c 100644 --- a/solr/api/gradle.lockfile +++ b/solr/api/gradle.lockfile @@ -100,6 +100,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,swaggerBuild,testRuntimeCl org.apache.commons:commons-math3:3.6.1=jarValidation,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=jarValidation,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=jarValidation,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-core:2.26.1=jarValidation,testRuntimeClasspath diff --git a/solr/benchmark/gradle.lockfile b/solr/benchmark/gradle.lockfile index e9e9d01611e2..2fc9b5ed87ab 100644 --- a/solr/benchmark/gradle.lockfile +++ b/solr/benchmark/gradle.lockfile @@ -90,6 +90,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,testRunti org.apache.commons:commons-math3:3.6.1=annotationProcessor,compileClasspath,jarValidation,runtimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,runtimeClasspath,testRuntimeClasspath org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,runtimeClasspath,testRuntimeClasspath org.apache.logging.log4j:log4j-core:2.26.1=jarValidation,runtimeClasspath,testRuntimeClasspath diff --git a/solr/core/build.gradle b/solr/core/build.gradle index 9d2e45e51f82..efba06f72088 100644 --- a/solr/core/build.gradle +++ b/solr/core/build.gradle @@ -115,6 +115,9 @@ dependencies { implementation(libs.apache.curator.framework, { exclude group: 'org.apache.zookeeper', module: 'zookeeper' }) + implementation(libs.apache.curator.recipes, { + exclude group: 'org.apache.zookeeper', module: 'zookeeper' + }) implementation(libs.apache.curator.client, { exclude group: 'org.apache.zookeeper', module: 'zookeeper' }) diff --git a/solr/core/gradle.lockfile b/solr/core/gradle.lockfile index a3a44b5bc858..55399890b67a 100644 --- a/solr/core/gradle.lockfile +++ b/solr/core/gradle.lockfile @@ -97,6 +97,7 @@ org.apache.commons:commons-lang3:3.20.0=cliDocsRuntime,compileClasspath,jarValid org.apache.commons:commons-math3:3.6.1=cliDocsRuntime,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=cliDocsRuntime,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=cliDocsRuntime,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=cliDocsRuntime,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testCompileClasspath,testRuntimeClasspath org.apache.logging.log4j:log4j-api:2.26.1=cliDocsRuntime,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath org.apache.logging.log4j:log4j-core:2.26.1=cliDocsRuntime,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath diff --git a/solr/cross-dc-manager/gradle.lockfile b/solr/cross-dc-manager/gradle.lockfile index f67dfa496fb6..0565f1db2c58 100644 --- a/solr/cross-dc-manager/gradle.lockfile +++ b/solr/cross-dc-manager/gradle.lockfile @@ -103,6 +103,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,runtimeLi org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.kafka:kafka-clients:4.3.1=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.logging.log4j:log4j-1.2-api:2.26.1=solrPlatformLibs diff --git a/solr/licenses/curator-recipes-5.9.0.jar.sha1 b/solr/licenses/curator-recipes-5.9.0.jar.sha1 new file mode 100644 index 000000000000..f288b6ef0eb2 --- /dev/null +++ b/solr/licenses/curator-recipes-5.9.0.jar.sha1 @@ -0,0 +1 @@ +4ddd58155916ce7a26073087ce4c1a58c8397eff diff --git a/solr/modules/analysis-extras/gradle.lockfile b/solr/modules/analysis-extras/gradle.lockfile index 24b256e04f13..a7b4676429aa 100644 --- a/solr/modules/analysis-extras/gradle.lockfile +++ b/solr/modules/analysis-extras/gradle.lockfile @@ -92,6 +92,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,runtimeLi org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-1.2-api:2.26.1=solrPlatformLibs org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath diff --git a/solr/modules/clustering/gradle.lockfile b/solr/modules/clustering/gradle.lockfile index 13d0336f4f4b..0c49d70c7e33 100644 --- a/solr/modules/clustering/gradle.lockfile +++ b/solr/modules/clustering/gradle.lockfile @@ -89,6 +89,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,runtimeLi org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-1.2-api:2.26.1=solrPlatformLibs org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath diff --git a/solr/modules/cross-dc/gradle.lockfile b/solr/modules/cross-dc/gradle.lockfile index 524e52ff7f5a..c89b399ebeb3 100644 --- a/solr/modules/cross-dc/gradle.lockfile +++ b/solr/modules/cross-dc/gradle.lockfile @@ -93,6 +93,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,runtimeLi org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.kafka:kafka-clients:4.3.1=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath org.apache.logging.log4j:log4j-1.2-api:2.26.1=solrPlatformLibs diff --git a/solr/modules/cuvs/gradle.lockfile b/solr/modules/cuvs/gradle.lockfile index 4e488d77b617..8b06ef9e76ab 100644 --- a/solr/modules/cuvs/gradle.lockfile +++ b/solr/modules/cuvs/gradle.lockfile @@ -91,6 +91,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,runtimeLi org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-1.2-api:2.26.1=solrPlatformLibs org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath diff --git a/solr/modules/extraction/gradle.lockfile b/solr/modules/extraction/gradle.lockfile index e2c3b8477fb9..7eaf9bde23e4 100644 --- a/solr/modules/extraction/gradle.lockfile +++ b/solr/modules/extraction/gradle.lockfile @@ -89,6 +89,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,runtimeLi org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-1.2-api:2.26.1=solrPlatformLibs org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath diff --git a/solr/modules/gcs-repository/gradle.lockfile b/solr/modules/gcs-repository/gradle.lockfile index d26beb18fecb..66ee39984733 100644 --- a/solr/modules/gcs-repository/gradle.lockfile +++ b/solr/modules/gcs-repository/gradle.lockfile @@ -144,6 +144,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,runtimeLi org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.httpcomponents:httpclient:4.5.14=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath org.apache.httpcomponents:httpcore:4.4.16=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath diff --git a/solr/modules/jwt-auth/gradle.lockfile b/solr/modules/jwt-auth/gradle.lockfile index 002380a64ddf..c172a267598a 100644 --- a/solr/modules/jwt-auth/gradle.lockfile +++ b/solr/modules/jwt-auth/gradle.lockfile @@ -108,6 +108,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,runtimeLi org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-1.2-api:2.26.1=solrPlatformLibs org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath diff --git a/solr/modules/langid/gradle.lockfile b/solr/modules/langid/gradle.lockfile index 4ae137ade57e..b8eb259cfa02 100644 --- a/solr/modules/langid/gradle.lockfile +++ b/solr/modules/langid/gradle.lockfile @@ -90,6 +90,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,runtimeLi org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-1.2-api:2.26.1=solrPlatformLibs org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath diff --git a/solr/modules/language-models/gradle.lockfile b/solr/modules/language-models/gradle.lockfile index f60aa615c81d..fb75d906556e 100644 --- a/solr/modules/language-models/gradle.lockfile +++ b/solr/modules/language-models/gradle.lockfile @@ -102,6 +102,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,runtimeLi org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-1.2-api:2.26.1=solrPlatformLibs org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath diff --git a/solr/modules/ltr/gradle.lockfile b/solr/modules/ltr/gradle.lockfile index 125fe1f25f71..98e133248425 100644 --- a/solr/modules/ltr/gradle.lockfile +++ b/solr/modules/ltr/gradle.lockfile @@ -90,6 +90,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,runtimeLi org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-1.2-api:2.26.1=solrPlatformLibs org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath diff --git a/solr/modules/opentelemetry/gradle.lockfile b/solr/modules/opentelemetry/gradle.lockfile index 84bf53a6fe77..9d43709f8f91 100644 --- a/solr/modules/opentelemetry/gradle.lockfile +++ b/solr/modules/opentelemetry/gradle.lockfile @@ -98,6 +98,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,runtimeLi org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-1.2-api:2.26.1=solrPlatformLibs org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath diff --git a/solr/modules/s3-repository/gradle.lockfile b/solr/modules/s3-repository/gradle.lockfile index cdc167ade64c..86493ed28b41 100644 --- a/solr/modules/s3-repository/gradle.lockfile +++ b/solr/modules/s3-repository/gradle.lockfile @@ -91,6 +91,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,runtimeLi org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.httpcomponents.client5:httpclient5:5.6.4=jarValidation,runtimeClasspath,runtimeLibs,testRuntimeClasspath org.apache.httpcomponents.core5:httpcore5-h2:5.4.3=jarValidation,runtimeClasspath,runtimeLibs,testRuntimeClasspath diff --git a/solr/modules/scripting/gradle.lockfile b/solr/modules/scripting/gradle.lockfile index b7140034ed13..00b4a3f311a9 100644 --- a/solr/modules/scripting/gradle.lockfile +++ b/solr/modules/scripting/gradle.lockfile @@ -89,6 +89,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,runtimeLi org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-1.2-api:2.26.1=solrPlatformLibs org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath diff --git a/solr/modules/sql/gradle.lockfile b/solr/modules/sql/gradle.lockfile index ea463d4727ab..434077398546 100644 --- a/solr/modules/sql/gradle.lockfile +++ b/solr/modules/sql/gradle.lockfile @@ -97,6 +97,7 @@ org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,runtimeLib org.apache.commons:commons-text:1.15.0=jarValidation,runtimeClasspath,runtimeLibs,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.httpcomponents.client5:httpclient5:5.5=jarValidation,runtimeClasspath,runtimeLibs,testRuntimeClasspath org.apache.httpcomponents.core5:httpcore5-h2:5.3.4=jarValidation,runtimeClasspath,runtimeLibs,testRuntimeClasspath diff --git a/solr/server/gradle.lockfile b/solr/server/gradle.lockfile index 715d32d966ab..df01d81a7f4a 100644 --- a/solr/server/gradle.lockfile +++ b/solr/server/gradle.lockfile @@ -83,6 +83,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,runtimeClasspath,solrCore org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,solrCore org.apache.curator:curator-client:5.9.0=jarValidation,runtimeClasspath,solrCore org.apache.curator:curator-framework:5.9.0=jarValidation,runtimeClasspath,solrCore +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,solrCore org.apache.logging.log4j:log4j-1.2-api:2.26.1=jarValidation,libExt,runtimeClasspath org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,libExt,runtimeClasspath,solrCore org.apache.logging.log4j:log4j-core:2.26.1=jarValidation,libExt,runtimeClasspath,solrCore diff --git a/solr/solr-ref-guide/gradle.lockfile b/solr/solr-ref-guide/gradle.lockfile index 21fb78467d67..03138cd2d7b2 100644 --- a/solr/solr-ref-guide/gradle.lockfile +++ b/solr/solr-ref-guide/gradle.lockfile @@ -95,6 +95,7 @@ org.apache.commons:commons-math3:3.6.1=cliClasspath,testRuntimeClasspath org.apache.commons:commons-text:1.15.0=testRuntimeClasspath org.apache.curator:curator-client:5.9.0=cliClasspath,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=cliClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=cliClasspath,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=testRuntimeClasspath org.apache.httpcomponents.client5:httpclient5:5.5=testRuntimeClasspath org.apache.httpcomponents.core5:httpcore5-h2:5.3.4=testRuntimeClasspath diff --git a/solr/solrj-jetty/gradle.lockfile b/solr/solrj-jetty/gradle.lockfile index 986cc17910b8..099c73b6cf60 100644 --- a/solr/solrj-jetty/gradle.lockfile +++ b/solr/solrj-jetty/gradle.lockfile @@ -89,6 +89,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,testRuntimeClasspath org.apache.commons:commons-math3:3.6.1=jarValidation,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=jarValidation,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=jarValidation,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-core:2.26.1=jarValidation,testRuntimeClasspath diff --git a/solr/solrj-streaming/gradle.lockfile b/solr/solrj-streaming/gradle.lockfile index 38fef6fcf209..86c46cfe1cbe 100644 --- a/solr/solrj-streaming/gradle.lockfile +++ b/solr/solrj-streaming/gradle.lockfile @@ -94,6 +94,7 @@ org.apache.commons:commons-math3:3.6.1=compileClasspath,jarValidation,runtimeCla org.apache.commons:commons-text:1.15.0=jarValidation,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=jarValidation,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=jarValidation,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.httpcomponents.client5:httpclient5:5.5=jarValidation,testRuntimeClasspath org.apache.httpcomponents.core5:httpcore5-h2:5.3.4=jarValidation,testRuntimeClasspath diff --git a/solr/solrj-zookeeper/gradle.lockfile b/solr/solrj-zookeeper/gradle.lockfile index aeeeab9a2e25..1d0f76ff26ab 100644 --- a/solr/solrj-zookeeper/gradle.lockfile +++ b/solr/solrj-zookeeper/gradle.lockfile @@ -89,6 +89,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,testRuntimeClasspath org.apache.commons:commons-math3:3.6.1=jarValidation,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-core:2.26.1=jarValidation,testRuntimeClasspath diff --git a/solr/solrj/gradle.lockfile b/solr/solrj/gradle.lockfile index 747aa01123aa..db14c5722fb7 100644 --- a/solr/solrj/gradle.lockfile +++ b/solr/solrj/gradle.lockfile @@ -90,6 +90,7 @@ org.apache.commons:commons-lang3:3.20.0=jarValidation,testRuntimeClasspath org.apache.commons:commons-math3:3.6.1=jarValidation,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=jarValidation,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=jarValidation,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-api:2.26.1=jarValidation,testRuntimeClasspath org.apache.logging.log4j:log4j-core:2.26.1=jarValidation,testRuntimeClasspath diff --git a/solr/test-framework/gradle.lockfile b/solr/test-framework/gradle.lockfile index 370025bbd4a7..38a0cfc4e4cf 100644 --- a/solr/test-framework/gradle.lockfile +++ b/solr/test-framework/gradle.lockfile @@ -90,6 +90,7 @@ org.apache.commons:commons-lang3:3.20.0=compileClasspath,jarValidation,runtimeCl org.apache.commons:commons-math3:3.6.1=jarValidation,runtimeClasspath,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,runtimeClasspath,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=compileClasspath,jarValidation,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.apache.logging.log4j:log4j-api:2.26.1=compileClasspath,jarValidation,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.apache.logging.log4j:log4j-core:2.26.1=compileClasspath,jarValidation,runtimeClasspath,testCompileClasspath,testRuntimeClasspath diff --git a/solr/webapp/gradle.lockfile b/solr/webapp/gradle.lockfile index a64f6c31ce7e..bc9740c71b11 100644 --- a/solr/webapp/gradle.lockfile +++ b/solr/webapp/gradle.lockfile @@ -96,6 +96,7 @@ org.apache.commons:commons-math3:3.6.1=jarValidation,solrCore,testRuntimeClasspa org.apache.commons:commons-text:1.15.0=jarValidation,testRuntimeClasspath org.apache.curator:curator-client:5.9.0=jarValidation,solrCore,testCompileClasspath,testRuntimeClasspath org.apache.curator:curator-framework:5.9.0=jarValidation,solrCore,testCompileClasspath,testRuntimeClasspath +org.apache.curator:curator-recipes:5.9.0=jarValidation,solrCore,testRuntimeClasspath org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath org.apache.httpcomponents.client5:httpclient5:5.5=jarValidation,testRuntimeClasspath org.apache.httpcomponents.core5:httpcore5-h2:5.3.4=jarValidation,testRuntimeClasspath From b6b2015bfc6618d500499b64454d3a109770ad41 Mon Sep 17 00:00:00 2001 From: Amrit Sarkar Date: Tue, 17 Nov 2026 01:52:49 -0800 Subject: [PATCH 2/9] SOLR-17704: Run lock tests without isAcquired --- .../solr/cloud/ConfigSetApiLockingTest.java | 139 +++++---- .../apache/solr/cloud/ThreadBoundLock.java | 79 +++++ .../solr/cloud/ZkDistributedLockTest.java | 282 ++++++++--------- .../collections/CollectionApiLockingTest.java | 289 ++++++++++-------- .../DeleteCollectionLockCleanupTest.java | 4 +- 5 files changed, 450 insertions(+), 343 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/cloud/ThreadBoundLock.java diff --git a/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java index 30edcbd8a937..ec345ef8be52 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java @@ -17,8 +17,11 @@ package org.apache.solr.cloud; -import java.util.concurrent.CountDownLatch; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.cloud.SolrZkClient; import org.junit.Test; @@ -32,6 +35,9 @@ public class ConfigSetApiLockingTest extends SolrTestCaseJ4 { static final int TIMEOUT = 10000; + private final Map boundLocks = new IdentityHashMap<>(); + private SolrZkClient zkClient; + /** * Tests the Config Set API locking. All tests run from single test method to save on setup time. */ @@ -40,111 +46,118 @@ public void monothreadedApiLockTests() throws Exception { ZkTestServer server = new ZkTestServer(createTempDir("zkData")); try { server.run(); - try (SolrZkClient zkClient = + try (SolrZkClient client = new SolrZkClient.Builder() .withUrl(server.getZkAddress()) .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) .build()) { + zkClient = client; ConfigSetApiLockFactory apiLockFactory = new ConfigSetApiLockFactory( new ZkDistributedConfigSetLockFactory(zkClient, "/apiLockTestRoot")); - monothreadedTests(apiLockFactory); - multithreadedTests(apiLockFactory); + try { + monothreadedTests(apiLockFactory); + multithreadedTests(apiLockFactory); + } finally { + boundLocks.values().forEach(ThreadBoundLock::close); + } } } finally { server.shutdown(); } } - private void monothreadedTests(ConfigSetApiLockFactory apiLockingHelper) { + private ThreadBoundLock bound(DistributedMultiLock lock) { + return boundLocks.computeIfAbsent( + lock, l -> new ThreadBoundLock(l::waitUntilAcquired, l::release)); + } + + private void assertAcquired(String message, DistributedMultiLock lock) throws Exception { + awaitAcquired(message, bound(lock).acquire()); + } + + private void awaitAcquired(String message, Future acquisition) throws Exception { + try { + acquisition.get(TIMEOUT, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + throw new AssertionError(message, e); + } + } + + private Future assertBlocked( + String message, DistributedMultiLock lock, DistributedMultiLock holder, int level, int count) + throws Exception { + Future acquisition = bound(lock).acquire(); + ThreadBoundLock.waitForLockNodes( + zkClient, DistributedMultiLock.splitLockIds(holder.getLockId()).get(level), count); + assertFalse(message, acquisition.isDone()); + return acquisition; + } + + private void release(DistributedMultiLock lock) throws Exception { + bound(lock).release(); + } + + private void monothreadedTests(ConfigSetApiLockFactory apiLockingHelper) throws Exception { // Config set lock without a base config set DistributedMultiLock cs1Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_1, null); - assertTrue("cs1Lock should have been acquired", cs1Lock.isAcquired()); + assertAcquired("cs1Lock should have been acquired", cs1Lock); // This lock does have a base config set DistributedMultiLock cs2Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_2, BASE_CONFIG_SET_NAME); - assertTrue("cs2Lock should have been acquired", cs2Lock.isAcquired()); + assertAcquired("cs2Lock should have been acquired", cs2Lock); // This lock has the same base config set, but that shouldn't prevent acquiring it DistributedMultiLock cs3Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_3, BASE_CONFIG_SET_NAME); - assertTrue("cs3Lock should have been acquired", cs3Lock.isAcquired()); + assertAcquired("cs3Lock should have been acquired", cs3Lock); // But we shouldn't be able to lock at this stage the base config set DistributedMultiLock csBaseLock = apiLockingHelper.createConfigSetApiLock(BASE_CONFIG_SET_NAME, null); - assertFalse("csBaseLock should not have been acquired", csBaseLock.isAcquired()); + Future csBaseAcquisition = + assertBlocked("csBaseLock should not have been acquired", csBaseLock, cs2Lock, 1, 3); - cs2Lock.release(); + release(cs2Lock); assertFalse( - "csBaseLock should not have been acquired, cs3Lock still there", csBaseLock.isAcquired()); - cs3Lock.release(); - assertTrue("csBaseLock should have been acquired", csBaseLock.isAcquired()); + "csBaseLock should not have been acquired, cs3Lock still there", + csBaseAcquisition.isDone()); + release(cs3Lock); + awaitAcquired("csBaseLock should have been acquired", csBaseAcquisition); // Acquiring a lock with a locked base config set should not be possible cs2Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_2, BASE_CONFIG_SET_NAME); - assertFalse( - "cs2Lock should not have been acquired, csBaseLock still held", cs2Lock.isAcquired()); - - csBaseLock.release(); - assertTrue("cs2Lock should now be acquired, csBaseLock was freed", cs2Lock.isAcquired()); - - cs2Lock.release(); + Future cs2Acquisition = + assertBlocked( + "cs2Lock should not have been acquired, csBaseLock still held", + cs2Lock, + csBaseLock, + 0, + 2); + + release(csBaseLock); + awaitAcquired("cs2Lock should now be acquired, csBaseLock was freed", cs2Acquisition); + + release(cs2Lock); } private void multithreadedTests(ConfigSetApiLockFactory apiLockingHelper) throws Exception { // This lock does have a base config set DistributedMultiLock cs2Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_2, BASE_CONFIG_SET_NAME); - assertTrue("cs2Lock should have been acquired", cs2Lock.isAcquired()); + assertAcquired("cs2Lock should have been acquired", cs2Lock); // But we shouldn't be able to lock at this stage the base config set DistributedMultiLock csBaseLock = apiLockingHelper.createConfigSetApiLock(BASE_CONFIG_SET_NAME, null); - assertFalse("csBaseLock should not have been acquired", csBaseLock.isAcquired()); - - // Wait for acquisition of the base config set lock on another thread (and be notified via a - // latch) - final CountDownLatch latch = new CountDownLatch(1); - new Thread( - () -> { - csBaseLock.waitUntilAcquired(); - // countDown() will not be called if waitUntilAcquired() threw an exception - latch.countDown(); - }) - .start(); - - // Wait for the thread to start and to get blocked in waitUntilAcquired() - // (thread start could have been checked more reliably using another latch, and verifying the - // thread is in waitUntilAcquired done through that thread stacktrace, but that would be - // overkill compared to the very slight race condition of waiting 30ms, but a race that would - // not cause the test to fail since we're testing... that nothing happened yet). - Thread.sleep(30); - - assertEquals( - "we should not have been notified that base config set lock was acquired", - 1, - latch.getCount()); - assertFalse("base config set lock should not have been acquired", csBaseLock.isAcquired()); - - cs2Lock.release(); - assertTrue( + Future csBaseAcquisition = + assertBlocked("csBaseLock should not have been acquired", csBaseLock, cs2Lock, 1, 2); + + release(cs2Lock); + awaitAcquired( "base config set lock should have been acquired now that other lock was released", - csBaseLock.isAcquired()); - - // Wait for the Zookeeper watch to fire + the thread to be unblocked and countdown the latch - // We'll wait up to 10 seconds here, so should be safe even if GC is extraordinarily high with a - // pause - int i = 0; - while (i < 1000 && latch.getCount() != 0) { - Thread.sleep(10); - i++; - } - assertEquals( - "we should have been notified that the base config set lock was acquired", - 0, - latch.getCount()); + csBaseAcquisition); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/ThreadBoundLock.java b/solr/core/src/test/org/apache/solr/cloud/ThreadBoundLock.java new file mode 100644 index 000000000000..7342f1049a64 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/ThreadBoundLock.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + +import java.io.Closeable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.common.util.TimeSource; +import org.apache.solr.util.TimeOut; + +/** + * Runs a lock's acquire and release on one dedicated thread, since Curator locks belong to the + * thread that acquired them. + */ +public class ThreadBoundLock implements Closeable { + + static final int RELEASE_TIMEOUT_MS = ZkDistributedLockTest.TIMEOUT; + static final int LOCK_NODES_TIMEOUT_MS = ZkDistributedLockTest.TIMEOUT; + + private final ExecutorService executor = + ExecutorUtil.newMDCAwareSingleThreadExecutor(new SolrNamedThreadFactory("lockTest")); + private final Runnable waitUntilAcquired; + private final Runnable release; + + public ThreadBoundLock(Runnable waitUntilAcquired, Runnable release) { + this.waitUntilAcquired = waitUntilAcquired; + this.release = release; + } + + public Future acquire() { + return executor.submit(waitUntilAcquired); + } + + /** Only valid once {@link #acquire()} completed, since a blocked acquire occupies the thread. */ + public void release() throws Exception { + executor.submit(release).get(RELEASE_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } + + /** Interrupts a blocked acquire. */ + @Override + public void close() { + ExecutorUtil.shutdownNowAndAwaitTermination(executor); + } + + /** Waits until the directory of {@code heldLockId} has {@code count} children. */ + public static void waitForLockNodes(SolrZkClient zkClient, String heldLockId, int count) + throws Exception { + String dir = heldLockId.substring(0, heldLockId.lastIndexOf('/')); + new TimeOut(LOCK_NODES_TIMEOUT_MS, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME) + .waitFor( + "expected " + count + " lock nodes in " + dir, + () -> { + try { + return zkClient.getChildren(dir, null).size() == count; + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java index 8a538ccc11ae..5f2a4fecc1d3 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java @@ -17,8 +17,9 @@ package org.apache.solr.cloud; +import java.util.ArrayList; import java.util.List; -import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -29,6 +30,7 @@ import org.apache.solr.common.params.CollectionParams; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; +import org.junit.After; import org.junit.Test; public class ZkDistributedLockTest extends SolrTestCaseJ4 { @@ -41,6 +43,19 @@ public class ZkDistributedLockTest extends SolrTestCaseJ4 { static final int TIMEOUT = 10000; + private final List boundLocks = new ArrayList<>(); + + @After + public void closeBoundLocks() { + boundLocks.forEach(ThreadBoundLock::close); + } + + private ThreadBoundLock bind(DistributedLock lock) { + ThreadBoundLock bound = new ThreadBoundLock(lock::waitUntilAcquired, lock::release); + boundLocks.add(bound); + return bound; + } + /** * Tests the obtaining of a single read or write lock at a specific hierarchical level, for * distributed Collection API locking as well as distributed Config Set API locking.
@@ -64,14 +79,14 @@ public void testSingleLocks() throws Exception { DistributedCollectionLockFactory collLockFactory = new ZkDistributedCollectionLockFactory(zkClient, "/lockTestCollectionRoot"); - monothreadedCollectionTests(collLockFactory); - multithreadedCollectionTests(collLockFactory); + monothreadedCollectionTests(zkClient, collLockFactory); + multithreadedCollectionTests(zkClient, collLockFactory); DistributedConfigSetLockFactory configSetLockFactory = new ZkDistributedConfigSetLockFactory(zkClient, "/lockTestConfigSetRoot"); - monothreadedConfigSetTests(configSetLockFactory); - multithreadedConfigSetTests(configSetLockFactory); + monothreadedConfigSetTests(zkClient, configSetLockFactory); + multithreadedConfigSetTests(zkClient, configSetLockFactory); } } finally { server.shutdown(); @@ -103,32 +118,40 @@ public void testLockDirsAreContainers() throws Exception { SHARD_NAME, REPLICA_NAME, null); + replicaLock.waitUntilAcquired(); + String replicaLockId = replicaLock.getLockId(); replicaLock.release(); DistributedLock shardLock = collLockFactory.createLock( true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); + shardLock.waitUntilAcquired(); + String shardLockId = shardLock.getLockId(); shardLock.release(); DistributedLock collLock = collLockFactory.createLock( true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); + collLock.waitUntilAcquired(); + String collLockId = collLock.getLockId(); collLock.release(); - assertLockDirsAreContainers(server, replicaLock, collRoot); - assertLockDirsAreContainers(server, shardLock, collRoot); - assertLockDirsAreContainers(server, collLock, collRoot); + assertLockDirsAreContainers(server, replicaLockId, collRoot); + assertLockDirsAreContainers(server, shardLockId, collRoot); + assertLockDirsAreContainers(server, collLockId, collRoot); String configSetRoot = "/lockTestConfigSetRoot"; DistributedConfigSetLockFactory configSetLockFactory = new ZkDistributedConfigSetLockFactory(zkClient, configSetRoot); DistributedLock configSetLock = configSetLockFactory.createLock(true, CONFIG_SET_NAME); + configSetLock.waitUntilAcquired(); + String configSetLockId = configSetLock.getLockId(); configSetLock.release(); - assertLockDirsAreContainers(server, configSetLock, configSetRoot); + assertLockDirsAreContainers(server, configSetLockId, configSetRoot); // Simulate ZooKeeper removing the empty containers - zkClient.clean(parent(parent(collLock.getLockId()))); + zkClient.clean(parent(parent(collLockId))); DistributedLock lock = collLockFactory.createLock( true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); - assertTrue(lock.isAcquired()); + lock.waitUntilAcquired(); lock.release(); } } finally { @@ -190,12 +213,12 @@ public void testDeleteEmptyLockDirs() throws Exception { DistributedLock shardLock = factory.createLock( true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); + shardLock.waitUntilAcquired(); factory.deleteEmptyLockDirs(COLLECTION_NAME); assertFalse(zkClient.exists(collLockDir)); assertFalse(zkClient.exists(replicaLockDir)); assertTrue(zkClient.exists(shardLock.getLockId())); - assertTrue(shardLock.isAcquired()); shardLock.release(); factory.deleteEmptyLockDirs(COLLECTION_NAME); @@ -247,7 +270,7 @@ public String create(String path, byte[] data, CreateMode createMode) factory.createLock( true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); assertTrue(removed.get()); - assertTrue(lock.isAcquired()); + lock.waitUntilAcquired(); lock.release(); } } finally { @@ -299,10 +322,9 @@ public String create(String path, byte[] data, CreateMode createMode) } } - /** Asserts every node between the lock and {@code root} is a container. */ - private static void assertLockDirsAreContainers( - ZkTestServer server, DistributedLock lock, String root) { - for (String dir = parent(lock.getLockId()); !dir.equals(root); dir = parent(dir)) { + /** Asserts every node between the lock node and {@code root} is a container. */ + private static void assertLockDirsAreContainers(ZkTestServer server, String lockId, String root) { + for (String dir = parent(lockId); !dir.equals(root); dir = parent(dir)) { assertTrue( dir + " should be a container node", server.getZKDatabase().getDataTree().getContainers().contains(dir)); @@ -311,47 +333,54 @@ private static void assertLockDirsAreContainers( /** Releases the lock and returns the dir it was created in. */ private static String lockDir(DistributedLock lock) { + lock.waitUntilAcquired(); + String lockId = lock.getLockId(); lock.release(); - return parent(lock.getLockId()); + return parent(lockId); } private static String parent(String path) { return path.substring(0, path.lastIndexOf('/')); } - private void monothreadedCollectionTests(DistributedCollectionLockFactory factory) { + private void monothreadedCollectionTests( + SolrZkClient zkClient, DistributedCollectionLockFactory factory) throws Exception { // Collection level locks DistributedLock collRL1 = factory.createLock( false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); - assertTrue("collRL1 should have been acquired", collRL1.isAcquired()); + ThreadBoundLock collRL1Thread = bind(collRL1); + collRL1Thread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); DistributedLock collRL2 = factory.createLock( false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); - assertTrue("collRL1 should have been acquired", collRL2.isAcquired()); + ThreadBoundLock collRL2Thread = bind(collRL2); + collRL2Thread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); DistributedLock collWL3 = factory.createLock( true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); + ThreadBoundLock collWL3Thread = bind(collWL3); + Future collWL3Acquired = collWL3Thread.acquire(); + ThreadBoundLock.waitForLockNodes(zkClient, collRL1.getLockId(), 3); assertFalse( - "collWL3 should not have been acquired, due to collRL1 and collRL2", collWL3.isAcquired()); - - assertTrue( - "collRL2 should have been acquired, that should not have changed", collRL2.isAcquired()); + "collWL3 should not have been acquired, due to collRL1 and collRL2", + collWL3Acquired.isDone()); - collRL1.release(); - collRL2.release(); - assertTrue( - "collWL3 should have been acquired, collRL1 and collRL2 were released", - collWL3.isAcquired()); + collRL1Thread.release(); + collRL2Thread.release(); + collWL3Acquired.get(TIMEOUT, TimeUnit.MILLISECONDS); DistributedLock collRL4 = factory.createLock( false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); + ThreadBoundLock collRL4Thread = bind(collRL4); + Future collRL4Acquired = collRL4Thread.acquire(); + ThreadBoundLock.waitForLockNodes(zkClient, collWL3.getLockId(), 2); assertFalse( "collRL4 should not have been acquired, due to collWL3 locking the collection", - collRL4.isAcquired()); + collRL4Acquired.isDone()); // Collection is write locked by collWL3 and collRL4 read lock waiting behind. Now moving to // request shard level locks. These are totally independent of the Collection level locks so @@ -359,16 +388,18 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor DistributedLock shardWL5 = factory.createLock( true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); - assertTrue( - "shardWL5 should have been acquired, there is no lock on that shard", - shardWL5.isAcquired()); + ThreadBoundLock shardWL5Thread = bind(shardWL5); + shardWL5Thread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); DistributedLock shardWL6 = factory.createLock( true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); + ThreadBoundLock shardWL6Thread = bind(shardWL6); + Future shardWL6Acquired = shardWL6Thread.acquire(); + ThreadBoundLock.waitForLockNodes(zkClient, shardWL5.getLockId(), 2); assertFalse( "shardWL6 should not have been acquired, shardWL5 is locking that shard", - shardWL6.isAcquired()); + shardWL6Acquired.isDone()); // Get a lock on a Replica. Again this is independent of collection or shard level DistributedLock replicaRL7 = @@ -379,7 +410,8 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor SHARD_NAME, REPLICA_NAME, null); - assertTrue("replicaRL7 should have been acquired", replicaRL7.isAcquired()); + ThreadBoundLock replicaRL7Thread = bind(replicaRL7); + replicaRL7Thread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); DistributedLock replicaWL8 = factory.createLock( @@ -389,170 +421,104 @@ private void monothreadedCollectionTests(DistributedCollectionLockFactory factor SHARD_NAME, REPLICA_NAME, null); + ThreadBoundLock replicaWL8Thread = bind(replicaWL8); + Future replicaWL8Acquired = replicaWL8Thread.acquire(); + ThreadBoundLock.waitForLockNodes(zkClient, replicaRL7.getLockId(), 2); assertFalse( "replicaWL8 should not have been acquired, replicaRL7 is read locking that replica", - replicaWL8.isAcquired()); + replicaWL8Acquired.isDone()); - replicaRL7.release(); - assertTrue( - "replicaWL8 should have been acquired, as replicaRL7 got released", - replicaWL8.isAcquired()); + replicaRL7Thread.release(); + replicaWL8Acquired.get(TIMEOUT, TimeUnit.MILLISECONDS); - collWL3.release(); - assertTrue("collRL4 should have been acquired given collWL3 released", collRL4.isAcquired()); - shardWL5.release(); - assertTrue( - "shardWL6 should have been acquired, now that shardWL5 was released", - shardWL6.isAcquired()); + collWL3Thread.release(); + collRL4Acquired.get(TIMEOUT, TimeUnit.MILLISECONDS); + shardWL5Thread.release(); + shardWL6Acquired.get(TIMEOUT, TimeUnit.MILLISECONDS); - replicaWL8.release(); - try { - replicaWL8.isAcquired(); - fail("isAcquired() called after release() on a lock should have thrown exception"); - } catch (IllegalStateException ise) { - // expected - } + replicaWL8Thread.release(); // Releasing the collection lock used in the multithreaded phase - collRL4.release(); + collRL4Thread.release(); } - private void multithreadedCollectionTests(DistributedCollectionLockFactory factory) - throws Exception { + private void multithreadedCollectionTests( + SolrZkClient zkClient, DistributedCollectionLockFactory factory) throws Exception { // Acquiring right away a read lock DistributedLock readLock = factory.createLock( false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); - assertTrue("readLock should have been acquired", readLock.isAcquired()); + ThreadBoundLock readLockThread = bind(readLock); + readLockThread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); // And now creating a write lock, that can't be acquired just yet, because of the read lock DistributedLock writeLock = factory.createLock( true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); - assertFalse("writeLock should not have been acquired", writeLock.isAcquired()); - - // Wait for acquisition of the write lock on another thread (and be notified via a latch) - final CountDownLatch latch = new CountDownLatch(1); - new Thread( - () -> { - writeLock.waitUntilAcquired(); - // countDown() will not be called if waitUntilAcquired() threw any kind of exception - latch.countDown(); - }) - .start(); - - // Wait for the thread to start and to get blocked in waitUntilAcquired() (thread start could - // have been checked more reliably using another latch, and verifying the thread is in - // waitUntilAcquired done through that thread stacktrace, but that would be overkill compared to - // the very slight race condition of waiting 30ms, but a race that would not cause the test to - // fail since we're testing... that nothing happened yet). - Thread.sleep(30); - - assertEquals( - "we should not have been notified that writeLock was acquired", 1, latch.getCount()); - assertFalse("writeLock should not have been acquired", writeLock.isAcquired()); - - readLock.release(); - assertTrue( - "writeLock should have been acquired now that readlock was released", - writeLock.isAcquired()); - - // Wait for the Zookeeper watch to fire + the thread to be unblocked and countdown the latch - // We'll wait up to 10 seconds here, so should be safe even if GC is extraordinarily high with a - // pause - int i = 0; - while (i < 1000 && latch.getCount() != 0) { - Thread.sleep(10); - i++; - } - assertEquals("we should have been notified that writeLock was acquired", 0, latch.getCount()); + ThreadBoundLock writeLockThread = bind(writeLock); + Future writeLockAcquired = writeLockThread.acquire(); + ThreadBoundLock.waitForLockNodes(zkClient, readLock.getLockId(), 2); + assertFalse("writeLock should not have been acquired", writeLockAcquired.isDone()); + + readLockThread.release(); + writeLockAcquired.get(TIMEOUT, TimeUnit.MILLISECONDS); + writeLockThread.release(); } - private void monothreadedConfigSetTests(DistributedConfigSetLockFactory factory) { + private void monothreadedConfigSetTests( + SolrZkClient zkClient, DistributedConfigSetLockFactory factory) throws Exception { DistributedLock configSetRL1 = factory.createLock(false, CONFIG_SET_NAME); - assertTrue("configSetRL1 should have been acquired", configSetRL1.isAcquired()); + ThreadBoundLock configSetRL1Thread = bind(configSetRL1); + configSetRL1Thread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); DistributedLock configSetRL2 = factory.createLock(false, CONFIG_SET_NAME); - assertTrue("configSetRL2 should have been acquired", configSetRL2.isAcquired()); + ThreadBoundLock configSetRL2Thread = bind(configSetRL2); + configSetRL2Thread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); DistributedLock configSetWL1 = factory.createLock(true, CONFIG_SET_NAME); + ThreadBoundLock configSetWL1Thread = bind(configSetWL1); + Future configSetWL1Acquired = configSetWL1Thread.acquire(); + ThreadBoundLock.waitForLockNodes(zkClient, configSetRL1.getLockId(), 3); assertFalse( "configSetWL1 should not have been acquired due to configSetRL1 and configSetRL2", - configSetWL1.isAcquired()); + configSetWL1Acquired.isDone()); - configSetRL1.release(); - configSetRL2.release(); - assertTrue( - "configSetWL1 should have been acquired, configSetRL1 and configSetRL2 were released", - configSetWL1.isAcquired()); + configSetRL1Thread.release(); + configSetRL2Thread.release(); + configSetWL1Acquired.get(TIMEOUT, TimeUnit.MILLISECONDS); DistributedLock configSetRL3 = factory.createLock(false, CONFIG_SET_NAME); + ThreadBoundLock configSetRL3Thread = bind(configSetRL3); + Future configSetRL3Acquired = configSetRL3Thread.acquire(); + ThreadBoundLock.waitForLockNodes(zkClient, configSetWL1.getLockId(), 2); assertFalse( "configSetRL3 should not have been acquired due to configSetWL1", - configSetRL3.isAcquired()); + configSetRL3Acquired.isDone()); - configSetWL1.release(); - assertTrue( - "configSetRL3 should have been acquired, configSetWL1 was released", - configSetRL3.isAcquired()); + configSetWL1Thread.release(); + configSetRL3Acquired.get(TIMEOUT, TimeUnit.MILLISECONDS); - configSetRL3.release(); - - try { - configSetRL3.isAcquired(); - fail("isAcquired() called after release() on a lock should have thrown exception"); - } catch (IllegalStateException ise) { - // expected - } + configSetRL3Thread.release(); } - private void multithreadedConfigSetTests(DistributedConfigSetLockFactory factory) - throws Exception { + private void multithreadedConfigSetTests( + SolrZkClient zkClient, DistributedConfigSetLockFactory factory) throws Exception { // Acquiring right away a read lock DistributedLock configSetRL1 = factory.createLock(false, CONFIG_SET_NAME); - assertTrue("configSetRL1 should have been acquired", configSetRL1.isAcquired()); + ThreadBoundLock configSetRL1Thread = bind(configSetRL1); + configSetRL1Thread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); // And now creating a write lock, that can't be acquired just yet, because of the read lock DistributedLock configSetWL1 = factory.createLock(true, CONFIG_SET_NAME); + ThreadBoundLock configSetWL1Thread = bind(configSetWL1); + Future configSetWL1Acquired = configSetWL1Thread.acquire(); + ThreadBoundLock.waitForLockNodes(zkClient, configSetRL1.getLockId(), 2); assertFalse( "configSetWL1 should not have been acquired due to configSetRL1", - configSetWL1.isAcquired()); - - // Wait for acquisition of the write lock on another thread (and be notified via a latch) - final CountDownLatch latch = new CountDownLatch(1); - new Thread( - () -> { - configSetWL1.waitUntilAcquired(); - // countDown() will not be called if waitUntilAcquired() threw any kind of exception - latch.countDown(); - }) - .start(); - - // Wait for the thread to start and to get blocked in waitUntilAcquired() (thread start could - // have been checked more reliably using another latch, and verifying the thread is in - // waitUntilAcquired done through that thread stacktrace, but that would be overkill compared to - // the very slight race condition of waiting 30ms, but a race that would not cause the test to - // fail since we're testing... that nothing happened yet). - Thread.sleep(30); - - assertEquals( - "we should not have been notified that configSetWL1 was acquired", 1, latch.getCount()); - assertFalse("configSetWL1 should not have been acquired", configSetWL1.isAcquired()); - - configSetRL1.release(); - assertTrue( - "configSetWL1 should have been acquired now that configSetRL1 was released", - configSetWL1.isAcquired()); - - // Wait for the Zookeeper watch to fire + the thread to be unblocked and countdown the latch. - // We'll wait up to 10 seconds here, so should be safe even if GC is extraordinarily high with a - // pause - int i = 0; - while (i < 1000 && latch.getCount() != 0) { - Thread.sleep(10); - i++; - } - assertEquals( - "we should have been notified that configSetWL1 was acquired", 0, latch.getCount()); + configSetWL1Acquired.isDone()); + + configSetRL1Thread.release(); + configSetWL1Acquired.get(TIMEOUT, TimeUnit.MILLISECONDS); + configSetWL1Thread.release(); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java index ee0da8853b82..ed51af1dbe6d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java @@ -17,11 +17,15 @@ package org.apache.solr.cloud.api.collections; +import java.util.IdentityHashMap; import java.util.List; -import java.util.concurrent.CountDownLatch; +import java.util.Map; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.cloud.DistributedMultiLock; +import org.apache.solr.cloud.ThreadBoundLock; import org.apache.solr.cloud.ZkDistributedCollectionLockFactory; import org.apache.solr.cloud.ZkTestServer; import org.apache.solr.common.SolrException; @@ -37,6 +41,9 @@ public class CollectionApiLockingTest extends SolrTestCaseJ4 { static final int TIMEOUT = 10000; + private final Map boundLocks = new IdentityHashMap<>(); + private SolrZkClient zkClient; + /** * Tests the Collection API locking. All tests run from single test method to save on setup time. */ @@ -45,25 +52,66 @@ public void monothreadedApiLockTests() throws Exception { ZkTestServer server = new ZkTestServer(createTempDir("zkData")); try { server.run(); - try (SolrZkClient zkClient = + try (SolrZkClient client = new SolrZkClient.Builder() .withUrl(server.getZkAddress()) .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) .build()) { + zkClient = client; CollectionApiLockFactory apiLockFactory = new CollectionApiLockFactory( new ZkDistributedCollectionLockFactory(zkClient, "/apiLockTestRoot")); - monothreadedTests(apiLockFactory); - multithreadedTests(apiLockFactory); - testCallingLockIdSubLocks(apiLockFactory); + try { + monothreadedTests(apiLockFactory); + multithreadedTests(apiLockFactory); + testCallingLockIdSubLocks(apiLockFactory); + } finally { + boundLocks.values().forEach(ThreadBoundLock::close); + } } } finally { server.shutdown(); } } - private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { + private ThreadBoundLock bound(DistributedMultiLock lock) { + return boundLocks.computeIfAbsent( + lock, l -> new ThreadBoundLock(l::waitUntilAcquired, l::release)); + } + + private void assertAcquired(String message, DistributedMultiLock lock) throws Exception { + awaitAcquired(message, bound(lock).acquire()); + } + + private void awaitAcquired(String message, Future acquisition) throws Exception { + try { + acquisition.get(TIMEOUT, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + throw new AssertionError(message, e); + } + } + + private Future assertBlocked( + String message, DistributedMultiLock lock, DistributedMultiLock holder, int level, int count) + throws Exception { + Future acquisition = bound(lock).acquire(); + ThreadBoundLock.waitForLockNodes( + zkClient, DistributedMultiLock.splitLockIds(holder.getLockId()).get(level), count); + assertFalse(message, acquisition.isDone()); + return acquisition; + } + + private void release(DistributedMultiLock lock) throws Exception { + bound(lock).release(); + } + + private void abandon(DistributedMultiLock lock) { + bound(lock).close(); + lock.release(); + } + + private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) throws Exception { // Lock at collection level (which prevents locking + acquiring on any other level of the // hierarchy) DistributedMultiLock collLock = @@ -72,7 +120,7 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { COLLECTION_NAME, null, null); - assertTrue("Collection should have been acquired", collLock.isAcquired()); + assertAcquired("Collection should have been acquired", collLock); assertEquals( "Lock at collection level expected to need one distributed lock", 1, @@ -86,7 +134,8 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { COLLECTION_NAME, SHARD1_NAME, null); - assertFalse("Shard1 should not have been acquired", shard1Lock.isAcquired()); + Future shard1Acquisition = + assertBlocked("Shard1 should not have been acquired", shard1Lock, collLock, 0, 2); assertEquals( "Lock at shard level expected to need two distributed locks", 2, @@ -100,18 +149,17 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { COLLECTION_NAME, SHARD2_NAME, null); - assertFalse("Shard2 should not have been acquired", shard2Lock.isAcquired()); + Future shard2Acquisition = + assertBlocked("Shard2 should not have been acquired", shard2Lock, collLock, 0, 3); - assertTrue("Collection should still be acquired", collLock.isAcquired()); + assertAcquired("Collection should still be acquired", collLock); - collLock.release(); + release(collLock); - assertTrue( - "Shard1 should have been acquired now that collection lock released", - shard1Lock.isAcquired()); - assertTrue( - "Shard2 should have been acquired now that collection lock released", - shard2Lock.isAcquired()); + awaitAcquired( + "Shard1 should have been acquired now that collection lock released", shard1Acquisition); + awaitAcquired( + "Shard2 should have been acquired now that collection lock released", shard2Acquisition); // Request a lock on replica of shard1 DistributedMultiLock replicaShard1Lock = @@ -120,9 +168,13 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME); - assertFalse( - "replicaShard1Lock should not have been acquired, shard1 is locked", - replicaShard1Lock.isAcquired()); + Future replicaShard1Acquisition = + assertBlocked( + "replicaShard1Lock should not have been acquired, shard1 is locked", + replicaShard1Lock, + shard1Lock, + 1, + 2); // Now ask for a new lock on the collection collLock = @@ -132,19 +184,23 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { null, null); - assertFalse( - "Collection should not have been acquired, shard1 and shard2 locks preventing it", - collLock.isAcquired()); + Future collAcquisition = + assertBlocked( + "Collection should not have been acquired, shard1 and shard2 locks preventing it", + collLock, + shard1Lock, + 0, + 4); - shard1Lock.release(); - assertTrue( + release(shard1Lock); + awaitAcquired( "replicaShard1Lock should have been acquired, as shard1 got released", - replicaShard1Lock.isAcquired()); + replicaShard1Acquisition); assertFalse( "Collection should not have been acquired, shard2 lock is preventing it", - collLock.isAcquired()); + collAcquisition.isDone()); - replicaShard1Lock.release(); + release(replicaShard1Lock); // Request a lock on replica of shard2 DistributedMultiLock replicaShard2Lock = @@ -153,26 +209,30 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) { COLLECTION_NAME, SHARD2_NAME, REPLICA_NAME); - assertFalse( - "replicaShard2Lock should not have been acquired, shard2 is locked", - replicaShard2Lock.isAcquired()); + Future replicaShard2Acquisition = + assertBlocked( + "replicaShard2Lock should not have been acquired, shard2 is locked", + replicaShard2Lock, + shard2Lock, + 0, + 3); - shard2Lock.release(); + release(shard2Lock); - assertTrue( + awaitAcquired( "Collection should have been acquired as shard2 got released and replicaShard2Locks was requested after the collection lock", - collLock.isAcquired()); + collAcquisition); assertFalse( "replicaShard2Lock should not have been acquired, collLock is locked", - replicaShard2Lock.isAcquired()); + replicaShard2Acquisition.isDone()); - collLock.release(); - assertTrue( + release(collLock); + awaitAcquired( "replicaShard2Lock should have been acquired, the collection lock got released", - replicaShard2Lock.isAcquired()); + replicaShard2Acquisition); // Release remaining lock to allow the multithreaded locking to succeed - replicaShard2Lock.release(); + release(replicaShard2Lock); } private void multithreadedTests(CollectionApiLockFactory apiLockingHelper) throws Exception { @@ -183,7 +243,7 @@ private void multithreadedTests(CollectionApiLockFactory apiLockingHelper) throw COLLECTION_NAME, null, null); - assertTrue("Collection should have been acquired", collLock.isAcquired()); + assertAcquired("Collection should have been acquired", collLock); // ...blocks a lock on replica from being acquired final DistributedMultiLock replicaShard1Lock = @@ -192,45 +252,18 @@ private void multithreadedTests(CollectionApiLockFactory apiLockingHelper) throw COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME); - assertFalse( - "replicaShard1Lock should not have been acquired, because collection is locked", - replicaShard1Lock.isAcquired()); - - // Wait for acquisition of the replica lock on another thread (and be notified via a latch) - final CountDownLatch latch = new CountDownLatch(1); - new Thread( - () -> { - replicaShard1Lock.waitUntilAcquired(); - // countDown() will not be called if waitUntilAcquired() threw any kind of exception - latch.countDown(); - }) - .start(); - - // Wait for the thread to start and to get blocked in waitUntilAcquired() (thread start could - // have been checked more reliably using another latch, and verifying the thread is in - // waitUntilAcquired done through that thread stacktrace, but that would be overkill compared to - // the very slight race condition of waiting 30ms, but a race that would not cause the test to - // fail since we're testing... that nothing happened yet). - Thread.sleep(30); - - assertEquals("we should not have been notified that replica was acquired", 1, latch.getCount()); - assertFalse("replica lock should not have been acquired", replicaShard1Lock.isAcquired()); - - collLock.release(); - assertTrue( + Future replicaShard1Acquisition = + assertBlocked( + "replicaShard1Lock should not have been acquired, because collection is locked", + replicaShard1Lock, + collLock, + 0, + 2); + + release(collLock); + awaitAcquired( "replica lock should have been acquired now that collection lock was released", - replicaShard1Lock.isAcquired()); - - // Wait for the Zookeeper watch to fire + the thread to be unblocked and countdown the latch - // We'll wait up to 10 seconds here, so should be safe even if GC is extraordinarily high with a - // pause - int i = 0; - while (i < 1000 && latch.getCount() != 0) { - Thread.sleep(10); - i++; - } - assertEquals( - "we should have been notified that replica lock was acquired", 0, latch.getCount()); + replicaShard1Acquisition); } public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) @@ -239,16 +272,20 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.CREATE), "coll1", null, null); assertEquals("Wrong number of internalLocks", 1, coll1Lock.getCountInternalLocks()); - assertTrue("Lock should be acquired", coll1Lock.isAcquired()); + assertAcquired("Lock should be acquired", coll1Lock); // Test sub-locks at the same level DistributedMultiLock coll1Lock2 = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.RELOAD), "coll1", null, null); assertEquals("Wrong number of internalLocks", 1, coll1Lock2.getCountInternalLocks()); - assertFalse( - "Should not be able to lock coll1 without using a callingLockId", coll1Lock2.isAcquired()); - coll1Lock2.release(); + assertBlocked( + "Should not be able to lock coll1 without using a callingLockId", + coll1Lock2, + coll1Lock, + 0, + 2); + abandon(coll1Lock2); coll1Lock2 = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.RELOAD) @@ -257,8 +294,8 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) null, null); assertEquals("Wrong number of internalLocks", 1, coll1Lock2.getCountInternalLocks()); - assertTrue("Should be able to lock coll1 when using a callingLockId", coll1Lock2.isAcquired()); - coll1Lock2.release(); + assertAcquired("Should be able to lock coll1 when using a callingLockId", coll1Lock2); + release(coll1Lock2); // Test locks underneath DistributedMultiLock shard1Lock = @@ -269,9 +306,8 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "shard1", null); assertEquals("Wrong number of internalLocks", 2, shard1Lock.getCountInternalLocks()); - assertTrue( - "Should be able to lock coll1/shard1 when using a callingLockId on coll1", - shard1Lock.isAcquired()); + assertAcquired( + "Should be able to lock coll1/shard1 when using a callingLockId on coll1", shard1Lock); DistributedMultiLock shard1Lock2 = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.ADDREPLICA) @@ -280,10 +316,13 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "shard1", null); assertEquals("Wrong number of internalLocks", 2, shard1Lock2.getCountInternalLocks()); - assertFalse( + assertBlocked( "Should not be able to lock coll1/shard1 since our callingLockId is only coll1, not shard1, since shard1 is already locked", - shard1Lock2.isAcquired()); - shard1Lock2.release(); + shard1Lock2, + shard1Lock, + 1, + 2); + abandon(shard1Lock2); DistributedMultiLock shard2Lock = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.ADDREPLICA) @@ -292,11 +331,11 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "shard2", null); assertEquals("Wrong number of internalLocks", 2, shard2Lock.getCountInternalLocks()); - assertTrue( + assertAcquired( "Should be able to lock coll1/shard2 when using a callingLockId on coll1, since shard2 has not been locked yet", - shard2Lock.isAcquired()); - shard2Lock.release(); - shard1Lock.release(); + shard2Lock); + release(shard2Lock); + release(shard1Lock); // Test locks 2 underneath DistributedMultiLock replica1Lock = @@ -307,9 +346,9 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "shard1", "replica1"); assertEquals("Wrong number of internalLocks", 3, replica1Lock.getCountInternalLocks()); - assertTrue( + assertAcquired( "Should be able to lock shard1/replica1 when using a callingLockId on coll1", - replica1Lock.isAcquired()); + replica1Lock); DistributedMultiLock replica2Lock = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK) @@ -318,10 +357,13 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "shard1", "replica1"); assertEquals("Wrong number of internalLocks", 3, replica2Lock.getCountInternalLocks()); - assertFalse( + assertBlocked( "Should not be able to lock coll1/shard1/replica1 since our callingLockId is only coll1, not replica1, which is already locked", - replica2Lock.isAcquired()); - replica2Lock.release(); + replica2Lock, + replica1Lock, + 2, + 2); + abandon(replica2Lock); shard1Lock = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.ADDREPLICA) @@ -330,10 +372,13 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "shard1", null); assertEquals("Wrong number of internalLocks", 2, shard1Lock.getCountInternalLocks()); - assertFalse( + assertBlocked( "Should not be able to lock coll1/shard1 since our callingLockId is only coll1, not shard1, which is locked because of a replica task", - shard1Lock.isAcquired()); - shard1Lock.release(); + shard1Lock, + replica1Lock, + 1, + 2); + abandon(shard1Lock); replica2Lock = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.MOCK_REPLICA_TASK) @@ -342,12 +387,12 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "shard1", "replica2"); assertEquals("Wrong number of internalLocks", 3, replica2Lock.getCountInternalLocks()); - assertTrue( + assertAcquired( "Should be able to lock shard2/replica2 when using a callingLockId on coll1, since shard2 has not been locked yet", - replica2Lock.isAcquired()); - replica2Lock.release(); - replica1Lock.release(); - coll1Lock.release(); + replica2Lock); + release(replica2Lock); + release(replica1Lock); + release(coll1Lock); // Test difference at a higher level DistributedMultiLock shard1Lock1 = @@ -357,9 +402,9 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "shard1", null); assertEquals("Wrong number of internalLocks", 2, shard1Lock1.getCountInternalLocks()); - assertTrue( + assertAcquired( "Should be able to lock coll1/shard1 when not using a callingLockId since shard1 has not been locked yet", - shard1Lock1.isAcquired()); + shard1Lock1); shard1Lock2 = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.INSTALLSHARDDATA), @@ -367,9 +412,9 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "shard1", null); assertEquals("Wrong number of internalLocks", 2, shard1Lock2.getCountInternalLocks()); - assertTrue( + assertAcquired( "Should be able to lock coll2/shard1 when not using a callingLockId since shard1 has not been locked yet for coll2", - shard1Lock2.isAcquired()); + shard1Lock2); String badLockId = shard1Lock2.getLockId(); assertThrows( "Should not be able to lock coll1/shard1 since our callingLockId is coll2", @@ -389,21 +434,23 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "shard1", null); assertEquals("Wrong number of internalLocks", 2, shard1Lock3.getCountInternalLocks()); - assertTrue( - "Should be able to lock coll1/shard1 since our callingLockId is coll1/shard1", - shard1Lock3.isAcquired()); - shard1Lock2.release(); - shard1Lock3.release(); + assertAcquired( + "Should be able to lock coll1/shard1 since our callingLockId is coll1/shard1", shard1Lock3); + release(shard1Lock2); + release(shard1Lock3); // Test difference at a higher level coll1Lock2 = apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.RELOAD), "coll1", null, null); assertEquals("Wrong number of internalLocks", 1, coll1Lock2.getCountInternalLocks()); - assertFalse( + assertBlocked( "Should not be able to lock coll1 since we have no callingLockId and shard1 is already locked. Cannot move up", - coll1Lock2.isAcquired()); - coll1Lock2.release(); + coll1Lock2, + shard1Lock1, + 0, + 2); + abandon(coll1Lock2); assertExceptionThrownWithMessageContaining( SolrException.class, List.of("Cannot mirror lock"), @@ -414,7 +461,7 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "coll1", null, null)); - shard1Lock3.release(); + release(shard1Lock3); // Test an unrelated lock assertThrows( @@ -427,6 +474,6 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "coll2", null, null)); - shard1Lock1.release(); + release(shard1Lock1); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/DeleteCollectionLockCleanupTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/DeleteCollectionLockCleanupTest.java index 63977657a8e7..d000f140dfcc 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/DeleteCollectionLockCleanupTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/DeleteCollectionLockCleanupTest.java @@ -69,8 +69,10 @@ public void testDeleteRemovesLegacyLockDirs() throws Exception { } private static String lockDir(DistributedLock lock) { + lock.waitUntilAcquired(); + String lockId = lock.getLockId(); lock.release(); - return parent(lock.getLockId()); + return parent(lockId); } private static String parent(String path) { From 9352a4d6d0f8dfec2530345cdc4a582e3fcc7fce Mon Sep 17 00:00:00 2001 From: Amrit Sarkar Date: Tue, 17 Nov 2026 01:52:49 -0800 Subject: [PATCH 3/9] SOLR-17704: Remove DistributedLock.isAcquired --- .../solr/cloud/ConfigSetApiLockFactory.java | 8 ++++---- .../org/apache/solr/cloud/DistributedLock.java | 2 -- .../apache/solr/cloud/DistributedMultiLock.java | 13 ------------- .../org/apache/solr/cloud/ZkDistributedLock.java | 16 ---------------- .../collections/CollectionApiLockFactory.java | 8 ++++---- 5 files changed, 8 insertions(+), 39 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ConfigSetApiLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ConfigSetApiLockFactory.java index 75dab9e65d98..75d41bdb9a52 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ConfigSetApiLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ConfigSetApiLockFactory.java @@ -39,10 +39,10 @@ public ConfigSetApiLockFactory(DistributedConfigSetLockFactory lockFactory) { *

This method obtains a write lock on {@code configSetName} as well as (when not {@code * null}), a read lock on {@code baseConfigSetName}. * - * @return a lock that once {@link DistributedMultiLock#isAcquired()} guarantees the corresponding - * Config Set API command can execute safely. The returned lock MUST be {@link - * DistributedMultiLock#release()} no matter what once no longer needed as otherwise it would - * prevent other threads from locking. + * @return a lock that once {@link DistributedMultiLock#waitUntilAcquired()} returns guarantees + * the corresponding Config Set API command can execute safely. The returned lock MUST + * be {@link DistributedMultiLock#release()} no matter what once no longer needed as otherwise + * it would prevent other threads from locking. */ public DistributedMultiLock createConfigSetApiLock( String configSetName, String baseConfigSetName) { diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java index c26c5d499f03..30ed3b6cec61 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java @@ -23,8 +23,6 @@ public interface DistributedLock { void release(); - boolean isAcquired(); - String getLockId(); boolean isMirroringLock(); diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java index 97be2293399c..34bdfa8569fc 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java @@ -64,19 +64,6 @@ public void release() { } } - public boolean isAcquired() { - if (isReleased) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, "Released lock can't be tested"); - } - for (DistributedLock lock : locks) { - if (!lock.isAcquired()) { - return false; - } - } - return true; - } - public String getLockId() { return locks.stream().map(DistributedLock::getLockId).collect(Collectors.joining(",")); } diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java index 30b70f66b82b..a5d2a4d9ef44 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java @@ -212,22 +212,6 @@ public void release() { } } - @Override - public boolean isAcquired() { - try { - if (released) { - throw new IllegalStateException( - "Bug. isAcquired() should not be called after release(). " + lockNode); - } - return nodeToWatch() == null; - } catch (KeeperException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } - } - /** * @return Another lock node (complete path) that must go away for us to acquire the lock, or * {@code null} if the lock is ours. diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java index 62cf0dd65af1..a7a1bb51664a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java @@ -52,10 +52,10 @@ public class CollectionApiLockFactory { * executing concurrently with an operation on one of the shards of the collection). See * documentation linked to SOLR-14840 regarding Collection API locking. * - * @return a lock that once {@link DistributedMultiLock#isAcquired()} guarantees the corresponding - * Collection API command can execute safely. The returned lock MUST be {@link - * DistributedMultiLock#release()} no matter what once no longer needed as otherwise it would - * prevent other threads from locking. + * @return a lock that once {@link DistributedMultiLock#waitUntilAcquired()} returns guarantees + * the corresponding Collection API command can execute safely. The returned lock MUST + * be {@link DistributedMultiLock#release()} no matter what once no longer needed as otherwise + * it would prevent other threads from locking. */ DistributedMultiLock createCollectionApiLock( AdminCmdContext adminCmdContext, String collName, String shardId, String replicaName) { From 48d45d48608d53450c902893c602e94cb5a49964 Mon Sep 17 00:00:00 2001 From: Amrit Sarkar Date: Tue, 17 Nov 2026 01:52:49 -0800 Subject: [PATCH 4/9] SOLR-17704: Add tests for lazy lock levels and interrupted waiters --- .../solr/cloud/ZkDistributedLockTest.java | 139 ++++++++++++++++++ .../collections/CollectionApiLockingTest.java | 61 ++++++++ 2 files changed, 200 insertions(+) diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java index 5f2a4fecc1d3..31a1098931ab 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java @@ -322,6 +322,145 @@ public String create(String path, byte[] data, CreateMode createMode) } } + private DistributedCollectionLockFactory collectionLockFactory(SolrZkClient zkClient) { + return new ZkDistributedCollectionLockFactory(zkClient, "/lockTestCollectionRoot"); + } + + private DistributedLock collectionLock( + DistributedCollectionLockFactory factory, boolean write, String callingLockId) { + return factory.createLock( + write, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, callingLockId); + } + + private void assertWriteAcquirable(DistributedCollectionLockFactory factory) throws Exception { + ThreadBoundLock next = bind(collectionLock(factory, true, null)); + next.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); + next.release(); + } + + @Test + public void testInterruptedWaiterLeavesNoNode() throws Exception { + ZkTestServer server = new ZkTestServer(createTempDir("zkData")); + try { + server.run(); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(server.getZkAddress()) + .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + DistributedCollectionLockFactory factory = collectionLockFactory(zkClient); + DistributedLock read = collectionLock(factory, false, null); + ThreadBoundLock readThread = bind(read); + readThread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); + String readId = read.getLockId(); + + ThreadBoundLock waiter = bind(collectionLock(factory, true, null)); + waiter.acquire(); + ThreadBoundLock.waitForLockNodes(zkClient, readId, 2); + + waiter.close(); + ThreadBoundLock.waitForLockNodes(zkClient, readId, 1); + + readThread.release(); + } + } finally { + server.shutdown(); + } + } + + @Test + public void testReleaseWithoutAcquireIsNoOp() throws Exception { + ZkTestServer server = new ZkTestServer(createTempDir("zkData")); + try { + server.run(); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(server.getZkAddress()) + .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + DistributedCollectionLockFactory factory = collectionLockFactory(zkClient); + collectionLock(factory, true, null).release(); + assertWriteAcquirable(factory); + } + } finally { + server.shutdown(); + } + } + + @Test + public void testMirrorOfReleasedLockFails() throws Exception { + ZkTestServer server = new ZkTestServer(createTempDir("zkData")); + try { + server.run(); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(server.getZkAddress()) + .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + DistributedCollectionLockFactory factory = collectionLockFactory(zkClient); + DistributedLock write = collectionLock(factory, true, null); + ThreadBoundLock writeThread = bind(write); + writeThread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); + String releasedId = write.getLockId(); + writeThread.release(); + + DistributedLock mirror = collectionLock(factory, false, releasedId); + SolrException e = expectThrows(SolrException.class, mirror::waitUntilAcquired); + assertTrue(e.getMessage(), e.getMessage().contains("Missing mirrored lock node")); + } + } finally { + server.shutdown(); + } + } + + @Test + public void testReleaseOnOtherThreadFails() throws Exception { + ZkTestServer server = new ZkTestServer(createTempDir("zkData")); + try { + server.run(); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(server.getZkAddress()) + .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + DistributedCollectionLockFactory factory = collectionLockFactory(zkClient); + DistributedLock write = collectionLock(factory, true, null); + ThreadBoundLock holder = bind(write); + holder.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); + + SolrException e = expectThrows(SolrException.class, write::release); + assertTrue(e.getCause() instanceof IllegalMonitorStateException); + + holder.release(); + assertWriteAcquirable(factory); + } + } finally { + server.shutdown(); + } + } + + @Test + public void testRepeatWaitThenSingleReleaseFrees() throws Exception { + ZkTestServer server = new ZkTestServer(createTempDir("zkData")); + try { + server.run(); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(server.getZkAddress()) + .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + DistributedCollectionLockFactory factory = collectionLockFactory(zkClient); + DistributedLock write = collectionLock(factory, true, null); + write.waitUntilAcquired(); + write.waitUntilAcquired(); + write.release(); + assertWriteAcquirable(factory); + } + } finally { + server.shutdown(); + } + } + /** Asserts every node between the lock node and {@code root} is a container. */ private static void assertLockDirsAreContainers(ZkTestServer server, String lockId, String root) { for (String dir = parent(lockId); !dir.equals(root); dir = parent(dir)) { diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java index ed51af1dbe6d..af26ce419b14 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java @@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.cloud.DistributedLock; import org.apache.solr.cloud.DistributedMultiLock; import org.apache.solr.cloud.ThreadBoundLock; import org.apache.solr.cloud.ZkDistributedCollectionLockFactory; @@ -75,6 +76,66 @@ public void monothreadedApiLockTests() throws Exception { } } + @Test + public void testBlockedMultiLockHoldsNoLowerLevelNode() throws Exception { + ZkTestServer server = new ZkTestServer(createTempDir("zkData")); + try { + server.run(); + try (SolrZkClient client = + new SolrZkClient.Builder() + .withUrl(server.getZkAddress()) + .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + zkClient = client; + ZkDistributedCollectionLockFactory factory = + new ZkDistributedCollectionLockFactory(zkClient, "/apiLockTestRoot"); + CollectionApiLockFactory apiLockFactory = new CollectionApiLockFactory(factory); + + try { + DistributedMultiLock collWrite = + apiLockFactory.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.CREATE), + COLLECTION_NAME, + null, + null); + assertAcquired("Collection write should have been acquired", collWrite); + + DistributedMultiLock split = + apiLockFactory.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.SPLITSHARD), + COLLECTION_NAME, + SHARD1_NAME, + null); + Future splitAcquisition = + assertBlocked("Split should be blocked by collection write", split, collWrite, 0, 2); + + DistributedLock shardWrite = + factory.createLock( + true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, null, null); + ThreadBoundLock shardWriteThread = + new ThreadBoundLock(shardWrite::waitUntilAcquired, shardWrite::release); + try { + awaitAcquired( + "Plain shard write should acquire while the split is blocked", + shardWriteThread.acquire()); + shardWriteThread.release(); + } finally { + shardWriteThread.close(); + } + + release(collWrite); + awaitAcquired( + "Split should acquire once the collection write is released", splitAcquisition); + release(split); + } finally { + boundLocks.values().forEach(ThreadBoundLock::close); + } + } + } finally { + server.shutdown(); + } + } + private ThreadBoundLock bound(DistributedMultiLock lock) { return boundLocks.computeIfAbsent( lock, l -> new ThreadBoundLock(l::waitUntilAcquired, l::release)); From ea145a2bdf80f9f860d98ea521e964a6104ff345 Mon Sep 17 00:00:00 2001 From: Amrit Sarkar Date: Tue, 17 Nov 2026 01:52:49 -0800 Subject: [PATCH 5/9] SOLR-17704: Back ZkDistributedLock with Curator recipes --- .../DistributedCollectionLockFactory.java | 12 +- .../DistributedConfigSetLockFactory.java | 8 +- .../apache/solr/cloud/DistributedLock.java | 5 +- .../apache/solr/cloud/ZkDistributedLock.java | 312 +++------ .../solr/cloud/ZkDistributedLockFactory.java | 55 +- .../apache/solr/cloud/ApiLockingTestBase.java | 69 ++ .../solr/cloud/ConfigSetApiLockingTest.java | 90 +-- .../apache/solr/cloud/ThreadBoundLock.java | 78 ++- .../solr/cloud/ZkDistributedLockTest.java | 648 +++++++----------- .../collections/CollectionApiLockingTest.java | 284 +++----- 10 files changed, 614 insertions(+), 947 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/cloud/ApiLockingTestBase.java diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java index f2ab771232ed..b955b49fc167 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java @@ -22,17 +22,15 @@ public interface DistributedCollectionLockFactory { /** - * Create a new lock of the specified type (read or write) entering the "competition" for actually - * getting the lock at the given level for the given path i.e. a lock at {@code collName} or a - * lock at {@code collName/shardId} or a lock at {@code collName/shardId/replicaName}, depending - * on the passed {@code level}. + * Create a new lock of the specified type (read or write) at the given level for the given path + * i.e. a lock at {@code collName} or a lock at {@code collName/shardId} or a lock at {@code + * collName/shardId/replicaName}, depending on the passed {@code level}. * *

The paths are used to define which locks compete with each other (locks of equal paths * compete). * - *

Upon return from this call, the lock has not been acquired but the it had entered the - * lock acquiring "competition", and the caller can decide to wait until the lock is granted by - * calling {@link DistributedLock#waitUntilAcquired()}.
+ *

Upon return, the lock has not been acquired. It enters the competition in {@link + * DistributedLock#waitUntilAcquired()}.
* Separating the lock creation from lock acquisition allows a more deterministic release of the * locks when/if they can't be acquired. * diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedConfigSetLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/DistributedConfigSetLockFactory.java index dcead4344af3..c29593c21604 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedConfigSetLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedConfigSetLockFactory.java @@ -19,12 +19,10 @@ public interface DistributedConfigSetLockFactory { /** - * Create a new lock of the specified type (read or write) entering the "competition" for actually - * getting the lock for the {@code configSetName} + * Create a new lock of the specified type (read or write) for the {@code configSetName} * - *

Upon return from this call, the lock has not been acquired but the it had entered the - * lock acquiring "competition", and the caller can decide to wait until the lock is granted by - * calling {@link DistributedLock#waitUntilAcquired()}.
+ *

Upon return, the lock has not been acquired. It enters the competition in {@link + * DistributedLock#waitUntilAcquired()}.
* Separating the lock creation from lock acquisition allows a more deterministic release of the * locks when/if they can't be acquired. * diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java index 30ed3b6cec61..10290ff3a989 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java @@ -17,7 +17,10 @@ package org.apache.solr.cloud; -/** A lock that can be used across cluster nodes. */ +/** + * A lock that can be used across cluster nodes. {@link #waitUntilAcquired()} and {@link #release()} + * must be called on the same thread. + */ public interface DistributedLock { void waitUntilAcquired(); diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java index a5d2a4d9ef44..935b867c1bb5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java @@ -19,273 +19,145 @@ import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; -import java.util.List; -import java.util.concurrent.CountDownLatch; +import java.util.function.Supplier; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; +import org.apache.curator.utils.ZKPaths; import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.util.StrUtils; -import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; /** - * A lock to be used across cluster nodes based on Zookeeper. Used for the distributed Collection - * API and distributed Config Set API implementations. - * - * @see Zookeeper - * lock recipe + * A lock to be used across cluster nodes based on Zookeeper. {@link Owned} wraps a Curator mutex, + * {@link Mirrored} shares a node held by the caller. */ abstract class ZkDistributedLock implements DistributedLock { - /** End of the lock node name prefix before the sequential part */ - static final char LOCK_PREFIX_SUFFIX = '_'; - - /** Prefix of EPHEMERAL read lock node names */ - static final char READ_LOCK_PREFIX_CHAR = 'R'; - - static final String READ_LOCK_PREFIX = "" + READ_LOCK_PREFIX_CHAR + LOCK_PREFIX_SUFFIX; - - /** Prefix of EPHEMERAL write lock node names */ - static final char WRITE_LOCK_PREFIX_CHAR = 'W'; - static final String WRITE_LOCK_PREFIX = "" + WRITE_LOCK_PREFIX_CHAR + LOCK_PREFIX_SUFFIX; - - /** Read lock. */ - static class Read extends ZkDistributedLock { - protected Read(SolrZkClient zkClient, String lockPath, String lockIdToMirror) - throws KeeperException, InterruptedException { - super(zkClient, lockPath, READ_LOCK_PREFIX, lockIdToMirror); + /** Marker Curator puts in the node name of a write lock. */ + private static final String CURATOR_WRITE_LOCK_NAME = "__WRIT__"; + + /** A lock that competes in ZooKeeper, backed by a Curator mutex. */ + static final class Owned extends ZkDistributedLock { + private final InterProcessMutex mutex; + private final String lockPath; + private final Supplier lockPathSupplier; + private volatile String lockId; + private boolean acquired; + private boolean released; + + Owned(InterProcessMutex mutex, String lockPath, Supplier lockPathSupplier) { + this.mutex = mutex; + this.lockPath = lockPath; + this.lockPathSupplier = lockPathSupplier; } @Override - boolean isBlockedByNodeType(String otherLockName) { - // A read lock is only blocked by a lower numbered write lock - // Lower numbered read locks are ok, they can coexist. - return otherLockName.startsWith(WRITE_LOCK_PREFIX); + public void waitUntilAcquired() { + if (released) { + throw new IllegalStateException( + "Bug. waitUntilAcquired() should not be called after release(). " + this); + } + if (acquired) { + return; + } + try { + mutex.acquire(); + lockId = lockPathSupplier.get(); + acquired = true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SolrException(SERVER_ERROR, e); + } catch (Exception e) { + throw new SolrException(SERVER_ERROR, e); + } } @Override - boolean canMirrorLock(String lockId) { - return true; + public void release() { + released = true; + if (!acquired) { + return; + } + try { + mutex.release(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SolrException(SERVER_ERROR, e); + } catch (Exception e) { + throw new SolrException(SERVER_ERROR, e); + } } - } - /** Write lock. */ - static class Write extends ZkDistributedLock { - protected Write(SolrZkClient zkClient, String lockPath, String lockIdToMirror) - throws KeeperException, InterruptedException { - super(zkClient, lockPath, WRITE_LOCK_PREFIX, lockIdToMirror); + @Override + public String getLockId() { + return lockId; } @Override - boolean isBlockedByNodeType(String otherLockName) { - // A write lock is blocked by another read or write lock with a lower sequence number - return true; + public boolean isMirroringLock() { + return false; } @Override - boolean canMirrorLock(String lockId) { - // Only another Write lock can be mirrored - int lockTypeSuffixIndex = lockId.lastIndexOf(LOCK_PREFIX_SUFFIX) - 1; - if (lockTypeSuffixIndex < 0) { - return false; - } else { - return lockId.charAt(lockTypeSuffixIndex) == WRITE_LOCK_PREFIX_CHAR; - } + public String toString() { + return lockId != null ? lockId : lockPath; } } - private final SolrZkClient zkClient; - private final String lockDir; - private final String lockNode; - protected final long sequence; - protected volatile boolean released = false; - protected final boolean mirrored; + /** A lock that shares the node of another lock held by the caller. */ + static final class Mirrored extends ZkDistributedLock { + private final SolrZkClient zkClient; + private final String lockId; - protected ZkDistributedLock( - SolrZkClient zkClient, String lockDir, String lockNodePrefix, String lockIdToMirror) - throws KeeperException, InterruptedException { - this.zkClient = zkClient; - this.lockDir = lockDir; - - // Create the SEQUENTIAL EPHEMERAL node. We enter the locking rat race here. We MUST eventually - // call release() or we block others. - if (StrUtils.isBlank(lockIdToMirror)) { - lockNode = - zkClient.create( - lockDir - + DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR - + lockNodePrefix, - null, - CreateMode.EPHEMERAL_SEQUENTIAL); - - mirrored = false; - } else { - if (!lockIdToMirror.startsWith(lockDir)) { + Mirrored(SolrZkClient zkClient, String lockDir, boolean isWriteLock, String lockIdToMirror) { + if (!lockIdToMirror.startsWith( + lockDir + DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR)) { throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, + SERVER_ERROR, "Requested lock with path: " + lockDir + " cannot mirror the callingLock with id: " + lockIdToMirror); } - if (!canMirrorLock(lockIdToMirror)) { + if (isWriteLock + && !ZKPaths.getNodeFromPath(lockIdToMirror).contains(CURATOR_WRITE_LOCK_NAME)) { throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Cannot mirror lock " + lockIdToMirror + " with given lockPrefix: " + lockNodePrefix); + SERVER_ERROR, "Cannot mirror lock " + lockIdToMirror + " with a write lock"); } - lockNode = lockIdToMirror; - mirrored = true; - } - sequence = getSequenceFromNodename(lockNode); - } - - private static class DeletedNodeWatcher implements Watcher { - final CountDownLatch latch; - final String nodeBeingWatched; - String errorMessage = null; // non null means error - - DeletedNodeWatcher(String nodeBeingWatched) { - this.latch = new CountDownLatch(1); - this.nodeBeingWatched = nodeBeingWatched; + this.zkClient = zkClient; + this.lockId = lockIdToMirror; } @Override - public void process(WatchedEvent event) { - if (event.getType() == Event.EventType.None) { - return; - } else if (event.getType() != Event.EventType.NodeDeleted) { - synchronized (this) { - errorMessage = - "Received unexpected watch event " + event.getType() + " on " + nodeBeingWatched; + public void waitUntilAcquired() { + try { + if (!zkClient.exists(lockId)) { + throw new SolrException(SERVER_ERROR, "Missing mirrored lock node " + lockId); } + } catch (KeeperException e) { + throw new SolrException(SERVER_ERROR, e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SolrException(SERVER_ERROR, e); } - latch.countDown(); } - void await() throws InterruptedException { - latch.await(); - synchronized (this) { - if (errorMessage != null) { - throw new SolrException(SERVER_ERROR, errorMessage); - } - } - } - } - - @Override - public void waitUntilAcquired() { - try { - if (released) { - throw new IllegalStateException( - "Bug. waitUntilAcquired() should not be called after release(). " + lockNode); - } - String nodeToWatch = nodeToWatch(); - while (nodeToWatch != null) { - final DeletedNodeWatcher watcher = new DeletedNodeWatcher(nodeToWatch); - if (zkClient.exists(nodeToWatch, watcher) != null) { - watcher.await(); - } - nodeToWatch = nodeToWatch(); - } - } catch (KeeperException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } - } + @Override + public void release() {} - @Override - public void release() { - try { - if (!mirrored) { - zkClient.delete(lockNode, -1); - released = true; - } - } catch (KeeperException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + @Override + public String getLockId() { + return lockId; } - } - /** - * @return Another lock node (complete path) that must go away for us to acquire the lock, or - * {@code null} if the lock is ours. - */ - String nodeToWatch() throws KeeperException, InterruptedException { - List locks = zkClient.getChildren(lockDir, null); - boolean foundSelf = false; // For finding bugs or ZK bad behavior - // We deviate from the ZK recipe here: we do not sort the list of nodes, and we stop waiting on - // the first one we find that blocks us. This is done in O(n), whereas sorting is more - // expensive. And we iterate only once over the set of children. Might cause more wakeups than - // needed though (multiple locks watching the same one rather than one another) but lock - // contention expected low (how many concurrent and conflicting Collection API requests a - // reasonable user can issue?) and the code below is simpler. Changing to a sorted "optimal" - // approach implies only changes in this method. - for (String lock : locks) { - long seq = getSequenceFromNodename(lock); - if (seq == sequence) { - foundSelf = true; - } else if (seq < sequence && isBlockedByNodeType(lock)) { - // Return the full path to the node to watch - return lockDir + DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR + lock; - } - // seq is bigger than sequence, can't block us. If the iteration was sorted we could avoid - // iterating over those. - } - if (!foundSelf) { - // If this basic assumption doesn't hold with Zookeeper, we're in deep trouble. And not only - // here. - if (mirrored) { - throw new SolrException(SERVER_ERROR, "Missing mirrored lock node " + lockNode); - } else { - throw new SolrException(SERVER_ERROR, "Missing lock node " + lockNode); - } + @Override + public boolean isMirroringLock() { + return true; } - // Didn't return early on any other blocking lock, means we own it - return null; - } - - /** - * @param otherLockName name of a competing lock node. Used to find out the type of the lock (read - * or write) - * @return {@code true} if a lock node of the given type with a lower sequence number blocks us - * from acquiring the lock. - */ - abstract boolean isBlockedByNodeType(String otherLockName); - - static long getSequenceFromNodename(String lockNode) { - // Javadoc of ZooKeeper.create() specifies "The sequence number is always fixed length of 10 - // digits, 0 padded" - // for sequential nodes - final int SEQUENCE_LENGTH = 10; - // Before the sequence we have our specific char - if (lockNode.charAt(lockNode.length() - SEQUENCE_LENGTH - 1) != LOCK_PREFIX_SUFFIX) { - throw new SolrException(SERVER_ERROR, "Unexpected lock node path " + lockNode); + @Override + public String toString() { + return lockId; } - - return Long.parseLong(lockNode.substring(lockNode.length() - SEQUENCE_LENGTH)); - } - - @Override - public String getLockId() { - return lockNode; - } - - @Override - public boolean isMirroringLock() { - return mirrored; - } - - abstract boolean canMirrorLock(String lockId); - - @Override - public String toString() { - return lockNode; } } diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java index 90d475faac29..78452898be98 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java @@ -17,17 +17,15 @@ package org.apache.solr.cloud; +import org.apache.curator.framework.recipes.locks.InterProcessReadWriteLock; import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; -import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.zookeeper.CreateMode; +import org.apache.solr.common.util.StrUtils; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.Stat; abstract class ZkDistributedLockFactory { - static final int MAX_CREATE_ATTEMPTS = 3; - private final SolrZkClient zkClient; private final String rootPath; @@ -38,26 +36,18 @@ abstract class ZkDistributedLockFactory { protected DistributedLock doCreateLock( boolean isWriteLock, String lockPath, String lockIdToMirror) { - try { - for (int attempt = 1; ; attempt++) { - try { - return isWriteLock - ? new ZkDistributedLock.Write(zkClient, lockPath, lockIdToMirror) - : new ZkDistributedLock.Read(zkClient, lockPath, lockIdToMirror); - } catch (KeeperException.NoNodeException e) { - if (attempt == MAX_CREATE_ATTEMPTS) { - throw e; - } - // Lock dirs are containers, so ZK drops them when empty. Recreate and try again. - makeLockPath(lockPath); - } + if (StrUtils.isBlank(lockIdToMirror)) { + // Empty payload: otherwise Curator stores the local IP address in every lock node + InterProcessReadWriteLock readWriteLock = + new InterProcessReadWriteLock(zkClient.getCuratorFramework(), lockPath, new byte[0]); + if (isWriteLock) { + InterProcessReadWriteLock.WriteLock writeLock = readWriteLock.writeLock(); + return new ZkDistributedLock.Owned(writeLock, lockPath, writeLock::getLockPath); } - } catch (KeeperException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + InterProcessReadWriteLock.ReadLock readLock = readWriteLock.readLock(); + return new ZkDistributedLock.Owned(readLock, lockPath, readLock::getLockPath); } + return new ZkDistributedLock.Mirrored(zkClient, lockPath, isWriteLock, lockIdToMirror); } protected StringBuilder getPathPrefix() { @@ -66,27 +56,6 @@ protected StringBuilder getPathPrefix() { return path; } - private void makeLockPath(String lockPath) throws KeeperException, InterruptedException { - if (!zkClient.exists(rootPath)) { - zkClient.makePath(rootPath, false); - } - StringBuilder path = new StringBuilder(rootPath); - for (String node : - lockPath - .substring(rootPath.length() + 1) - .split(DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR)) { - path.append(DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR).append(node); - try { - zkClient.create(path.toString(), null, CreateMode.CONTAINER); - } catch (KeeperException.NodeExistsException nee) { - // Some other thread (on this or another JVM) beat us to create the node, that's ok. - } catch (KeeperException.NoNodeException nne) { - // A parent got removed while we were here. The caller retries. - return; - } - } - } - /** * Deletes the empty lock dirs under {@code path}, deepest first. Lock nodes and dirs still * holding them are left alone. diff --git a/solr/core/src/test/org/apache/solr/cloud/ApiLockingTestBase.java b/solr/core/src/test/org/apache/solr/cloud/ApiLockingTestBase.java new file mode 100644 index 000000000000..4c97c77bcf6a --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/ApiLockingTestBase.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + +import static org.apache.solr.cloud.ThreadBoundLock.TIMEOUT_MS; +import static org.apache.solr.cloud.ThreadBoundLock.awaitAcquired; + +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.cloud.SolrZkClient; +import org.junit.After; +import org.junit.Before; + +/** Shared ZooKeeper setup and thread-bound lock helpers for the API locking tests. */ +public abstract class ApiLockingTestBase extends SolrTestCaseJ4 { + + protected static final String LOCK_ROOT = "/apiLockTestRoot"; + + private final Map boundLocks = new IdentityHashMap<>(); + private ZkTestServer server; + protected SolrZkClient zkClient; + + @Before + public void startZk() throws Exception { + server = new ZkTestServer(createTempDir("zkData")); + server.run(); + zkClient = + new SolrZkClient.Builder() + .withUrl(server.getZkAddress()) + .withTimeout(TIMEOUT_MS, TimeUnit.MILLISECONDS) + .build(); + } + + @After + public void stopZk() throws Exception { + boundLocks.values().forEach(ThreadBoundLock::close); + zkClient.close(); + server.shutdown(); + } + + protected ThreadBoundLock bound(DistributedMultiLock lock) { + return ThreadBoundLock.bound(boundLocks, lock); + } + + protected void assertAcquired(String message, DistributedMultiLock lock) throws Exception { + awaitAcquired(message, bound(lock).acquire()); + } + + protected void release(DistributedMultiLock lock) throws Exception { + bound(lock).release(); + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java index ec345ef8be52..f3682ed26cc3 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java @@ -17,86 +17,28 @@ package org.apache.solr.cloud; -import java.util.IdentityHashMap; -import java.util.Map; +import static org.apache.solr.cloud.ThreadBoundLock.awaitAcquired; + import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.cloud.SolrZkClient; import org.junit.Test; -public class ConfigSetApiLockingTest extends SolrTestCaseJ4 { +public class ConfigSetApiLockingTest extends ApiLockingTestBase { private static final String CONFIG_SET_NAME_1 = "configSet1"; private static final String CONFIG_SET_NAME_2 = "configSet2"; private static final String CONFIG_SET_NAME_3 = "configSet3"; private static final String BASE_CONFIG_SET_NAME = "baseConfigSet"; - static final int TIMEOUT = 10000; - - private final Map boundLocks = new IdentityHashMap<>(); - private SolrZkClient zkClient; - /** * Tests the Config Set API locking. All tests run from single test method to save on setup time. */ @Test public void monothreadedApiLockTests() throws Exception { - ZkTestServer server = new ZkTestServer(createTempDir("zkData")); - try { - server.run(); - try (SolrZkClient client = - new SolrZkClient.Builder() - .withUrl(server.getZkAddress()) - .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - zkClient = client; - ConfigSetApiLockFactory apiLockFactory = - new ConfigSetApiLockFactory( - new ZkDistributedConfigSetLockFactory(zkClient, "/apiLockTestRoot")); - - try { - monothreadedTests(apiLockFactory); - multithreadedTests(apiLockFactory); - } finally { - boundLocks.values().forEach(ThreadBoundLock::close); - } - } - } finally { - server.shutdown(); - } - } - - private ThreadBoundLock bound(DistributedMultiLock lock) { - return boundLocks.computeIfAbsent( - lock, l -> new ThreadBoundLock(l::waitUntilAcquired, l::release)); - } - - private void assertAcquired(String message, DistributedMultiLock lock) throws Exception { - awaitAcquired(message, bound(lock).acquire()); - } - - private void awaitAcquired(String message, Future acquisition) throws Exception { - try { - acquisition.get(TIMEOUT, TimeUnit.MILLISECONDS); - } catch (TimeoutException e) { - throw new AssertionError(message, e); - } - } - - private Future assertBlocked( - String message, DistributedMultiLock lock, DistributedMultiLock holder, int level, int count) - throws Exception { - Future acquisition = bound(lock).acquire(); - ThreadBoundLock.waitForLockNodes( - zkClient, DistributedMultiLock.splitLockIds(holder.getLockId()).get(level), count); - assertFalse(message, acquisition.isDone()); - return acquisition; - } + ConfigSetApiLockFactory apiLockFactory = + new ConfigSetApiLockFactory(new ZkDistributedConfigSetLockFactory(zkClient, LOCK_ROOT)); - private void release(DistributedMultiLock lock) throws Exception { - bound(lock).release(); + monothreadedTests(apiLockFactory); + multithreadedTests(apiLockFactory); } private void monothreadedTests(ConfigSetApiLockFactory apiLockingHelper) throws Exception { @@ -118,7 +60,8 @@ private void monothreadedTests(ConfigSetApiLockFactory apiLockingHelper) throws DistributedMultiLock csBaseLock = apiLockingHelper.createConfigSetApiLock(BASE_CONFIG_SET_NAME, null); Future csBaseAcquisition = - assertBlocked("csBaseLock should not have been acquired", csBaseLock, cs2Lock, 1, 3); + bound(csBaseLock) + .acquireBlocked("csBaseLock should not have been acquired", zkClient, LOCK_ROOT, 1); release(cs2Lock); assertFalse( @@ -130,12 +73,12 @@ private void monothreadedTests(ConfigSetApiLockFactory apiLockingHelper) throws // Acquiring a lock with a locked base config set should not be possible cs2Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_2, BASE_CONFIG_SET_NAME); Future cs2Acquisition = - assertBlocked( - "cs2Lock should not have been acquired, csBaseLock still held", - cs2Lock, - csBaseLock, - 0, - 2); + bound(cs2Lock) + .acquireBlocked( + "cs2Lock should not have been acquired, csBaseLock still held", + zkClient, + LOCK_ROOT, + 2); release(csBaseLock); awaitAcquired("cs2Lock should now be acquired, csBaseLock was freed", cs2Acquisition); @@ -153,7 +96,8 @@ private void multithreadedTests(ConfigSetApiLockFactory apiLockingHelper) throws DistributedMultiLock csBaseLock = apiLockingHelper.createConfigSetApiLock(BASE_CONFIG_SET_NAME, null); Future csBaseAcquisition = - assertBlocked("csBaseLock should not have been acquired", csBaseLock, cs2Lock, 1, 2); + bound(csBaseLock) + .acquireBlocked("csBaseLock should not have been acquired", zkClient, LOCK_ROOT, 1); release(cs2Lock); awaitAcquired( diff --git a/solr/core/src/test/org/apache/solr/cloud/ThreadBoundLock.java b/solr/core/src/test/org/apache/solr/cloud/ThreadBoundLock.java index 7342f1049a64..c7dfee0c7c89 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ThreadBoundLock.java +++ b/solr/core/src/test/org/apache/solr/cloud/ThreadBoundLock.java @@ -17,15 +17,21 @@ package org.apache.solr.cloud; +import static org.junit.Assert.assertFalse; + import java.io.Closeable; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.SolrNamedThreadFactory; import org.apache.solr.common.util.TimeSource; import org.apache.solr.util.TimeOut; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.data.Stat; /** * Runs a lock's acquire and release on one dedicated thread, since Curator locks belong to the @@ -33,13 +39,13 @@ */ public class ThreadBoundLock implements Closeable { - static final int RELEASE_TIMEOUT_MS = ZkDistributedLockTest.TIMEOUT; - static final int LOCK_NODES_TIMEOUT_MS = ZkDistributedLockTest.TIMEOUT; + public static final int TIMEOUT_MS = 10000; private final ExecutorService executor = ExecutorUtil.newMDCAwareSingleThreadExecutor(new SolrNamedThreadFactory("lockTest")); private final Runnable waitUntilAcquired; private final Runnable release; + private volatile Future lastAcquire; public ThreadBoundLock(Runnable waitUntilAcquired, Runnable release) { this.waitUntilAcquired = waitUntilAcquired; @@ -47,15 +53,75 @@ public ThreadBoundLock(Runnable waitUntilAcquired, Runnable release) { } public Future acquire() { - return executor.submit(waitUntilAcquired); + lastAcquire = executor.submit(waitUntilAcquired); + return lastAcquire; + } + + /** + * Starts an acquire and returns once it has added {@code newNodes} ephemeral nodes under {@code + * root} without completing, failing if the acquire completes instead. + */ + public Future acquireBlocked(String message, SolrZkClient zkClient, String root, int newNodes) + throws Exception { + int expected = countEphemeralNodes(zkClient, root) + newNodes; + Future acquisition = acquire(); + new TimeOut(TIMEOUT_MS, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME) + .waitFor( + message + ": waiter node not seen in ZooKeeper", + () -> { + try { + return acquisition.isDone() || countEphemeralNodes(zkClient, root) == expected; + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + assertFalse(message, acquisition.isDone()); + return acquisition; + } + + private static int countEphemeralNodes(SolrZkClient zkClient, String path) + throws KeeperException, InterruptedException { + int count = 0; + try { + for (String child : zkClient.getChildren(path, null)) { + String childPath = path + "/" + child; + Stat stat = zkClient.exists(childPath, null); + if (stat == null) { + continue; + } + count += stat.getEphemeralOwner() != 0 ? 1 : countEphemeralNodes(zkClient, childPath); + } + } catch (KeeperException.NoNodeException e) { + return count; + } + return count; + } + + public static ThreadBoundLock bound( + Map boundLocks, DistributedMultiLock lock) { + return boundLocks.computeIfAbsent( + lock, l -> new ThreadBoundLock(l::waitUntilAcquired, l::release)); + } + + public static void awaitAcquired(String message, Future acquisition) throws Exception { + try { + acquisition.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + throw new AssertionError(message, e); + } + } + + /** Interrupts a blocked acquire, then releases whatever was acquired, on the lock's thread. */ + public void abandon() throws Exception { + lastAcquire.cancel(true); + release(); } /** Only valid once {@link #acquire()} completed, since a blocked acquire occupies the thread. */ public void release() throws Exception { - executor.submit(release).get(RELEASE_TIMEOUT_MS, TimeUnit.MILLISECONDS); + executor.submit(release).get(TIMEOUT_MS, TimeUnit.MILLISECONDS); } - /** Interrupts a blocked acquire. */ @Override public void close() { ExecutorUtil.shutdownNowAndAwaitTermination(executor); @@ -65,7 +131,7 @@ public void close() { public static void waitForLockNodes(SolrZkClient zkClient, String heldLockId, int count) throws Exception { String dir = heldLockId.substring(0, heldLockId.lastIndexOf('/')); - new TimeOut(LOCK_NODES_TIMEOUT_MS, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME) + new TimeOut(TIMEOUT_MS, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME) .waitFor( "expected " + count + " lock nodes in " + dir, () -> { diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java index 31a1098931ab..b1b653ed230e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java @@ -17,20 +17,18 @@ package org.apache.solr.cloud; +import static org.apache.solr.cloud.ThreadBoundLock.TIMEOUT_MS; + import java.util.ArrayList; import java.util.List; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.params.CollectionParams; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; import org.junit.After; +import org.junit.Before; import org.junit.Test; public class ZkDistributedLockTest extends SolrTestCaseJ4 { @@ -40,14 +38,30 @@ public class ZkDistributedLockTest extends SolrTestCaseJ4 { private final String REPLICA_NAME = "lockReplica"; private static final String CONFIG_SET_NAME = "lockConfigSet"; - - static final int TIMEOUT = 10000; + private static final String COLLECTION_ROOT = "/lockTestCollectionRoot"; + private static final String CONFIG_SET_ROOT = "/lockTestConfigSetRoot"; private final List boundLocks = new ArrayList<>(); + private ZkTestServer server; + private SolrZkClient zkClient; + + @Before + public void startZk() throws Exception { + server = new ZkTestServer(createTempDir("zkData")); + server.run(); + zkClient = + new SolrZkClient.Builder() + .withUrl(server.getZkHost()) + .withTimeout(TIMEOUT_MS, TimeUnit.MILLISECONDS) + .build(); + } + @After - public void closeBoundLocks() { + public void stopZk() throws Exception { boundLocks.forEach(ThreadBoundLock::close); + zkClient.close(); + server.shutdown(); } private ThreadBoundLock bind(DistributedLock lock) { @@ -68,29 +82,17 @@ private ThreadBoundLock bind(DistributedLock lock) { */ @Test public void testSingleLocks() throws Exception { - ZkTestServer server = new ZkTestServer(createTempDir("zkData")); - try { - server.run(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(server.getZkAddress()) - .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - DistributedCollectionLockFactory collLockFactory = - new ZkDistributedCollectionLockFactory(zkClient, "/lockTestCollectionRoot"); - - monothreadedCollectionTests(zkClient, collLockFactory); - multithreadedCollectionTests(zkClient, collLockFactory); - - DistributedConfigSetLockFactory configSetLockFactory = - new ZkDistributedConfigSetLockFactory(zkClient, "/lockTestConfigSetRoot"); - - monothreadedConfigSetTests(zkClient, configSetLockFactory); - multithreadedConfigSetTests(zkClient, configSetLockFactory); - } - } finally { - server.shutdown(); - } + DistributedCollectionLockFactory collLockFactory = + new ZkDistributedCollectionLockFactory(zkClient, COLLECTION_ROOT); + + monothreadedCollectionTests(collLockFactory); + multithreadedCollectionTests(collLockFactory); + + DistributedConfigSetLockFactory configSetLockFactory = + new ZkDistributedConfigSetLockFactory(zkClient, CONFIG_SET_ROOT); + + monothreadedConfigSetTests(configSetLockFactory); + multithreadedConfigSetTests(configSetLockFactory); } /** @@ -99,231 +101,102 @@ public void testSingleLocks() throws Exception { */ @Test public void testLockDirsAreContainers() throws Exception { - ZkTestServer server = new ZkTestServer(createTempDir("zkData")); - try { - server.run(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(server.getZkHost()) - .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - String collRoot = "/lockTestCollectionRoot"; - DistributedCollectionLockFactory collLockFactory = - new ZkDistributedCollectionLockFactory(zkClient, collRoot); - DistributedLock replicaLock = - collLockFactory.createLock( - true, - CollectionParams.LockLevel.REPLICA, - COLLECTION_NAME, - SHARD_NAME, - REPLICA_NAME, - null); - replicaLock.waitUntilAcquired(); - String replicaLockId = replicaLock.getLockId(); - replicaLock.release(); - DistributedLock shardLock = - collLockFactory.createLock( - true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); - shardLock.waitUntilAcquired(); - String shardLockId = shardLock.getLockId(); - shardLock.release(); - DistributedLock collLock = - collLockFactory.createLock( - true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); - collLock.waitUntilAcquired(); - String collLockId = collLock.getLockId(); - collLock.release(); - assertLockDirsAreContainers(server, replicaLockId, collRoot); - assertLockDirsAreContainers(server, shardLockId, collRoot); - assertLockDirsAreContainers(server, collLockId, collRoot); - - String configSetRoot = "/lockTestConfigSetRoot"; - DistributedConfigSetLockFactory configSetLockFactory = - new ZkDistributedConfigSetLockFactory(zkClient, configSetRoot); - DistributedLock configSetLock = configSetLockFactory.createLock(true, CONFIG_SET_NAME); - configSetLock.waitUntilAcquired(); - String configSetLockId = configSetLock.getLockId(); - configSetLock.release(); - assertLockDirsAreContainers(server, configSetLockId, configSetRoot); - - // Simulate ZooKeeper removing the empty containers - zkClient.clean(parent(parent(collLockId))); - DistributedLock lock = - collLockFactory.createLock( - true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); - lock.waitUntilAcquired(); - lock.release(); - } - } finally { - server.shutdown(); - } + DistributedCollectionLockFactory collLockFactory = + new ZkDistributedCollectionLockFactory(zkClient, COLLECTION_ROOT); + DistributedLock replicaLock = + collLockFactory.createLock( + true, + CollectionParams.LockLevel.REPLICA, + COLLECTION_NAME, + SHARD_NAME, + REPLICA_NAME, + null); + replicaLock.waitUntilAcquired(); + String replicaLockId = replicaLock.getLockId(); + replicaLock.release(); + DistributedLock shardLock = + collLockFactory.createLock( + true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); + shardLock.waitUntilAcquired(); + String shardLockId = shardLock.getLockId(); + shardLock.release(); + DistributedLock collLock = + collLockFactory.createLock( + true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); + collLock.waitUntilAcquired(); + String collLockId = collLock.getLockId(); + collLock.release(); + assertLockDirsAreContainers(server, replicaLockId, COLLECTION_ROOT); + assertLockDirsAreContainers(server, shardLockId, COLLECTION_ROOT); + assertLockDirsAreContainers(server, collLockId, COLLECTION_ROOT); + + DistributedConfigSetLockFactory configSetLockFactory = + new ZkDistributedConfigSetLockFactory(zkClient, CONFIG_SET_ROOT); + DistributedLock configSetLock = configSetLockFactory.createLock(true, CONFIG_SET_NAME); + configSetLock.waitUntilAcquired(); + String configSetLockId = configSetLock.getLockId(); + configSetLock.release(); + assertLockDirsAreContainers(server, configSetLockId, CONFIG_SET_ROOT); + + // Simulate ZooKeeper removing the empty containers + zkClient.clean(parent(parent(collLockId))); + DistributedLock lock = + collLockFactory.createLock( + true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); + lock.waitUntilAcquired(); + lock.release(); } /** Lock dirs left as persistent nodes by older versions get removed, live locks are kept. */ @Test public void testDeleteEmptyLockDirs() throws Exception { - ZkTestServer server = new ZkTestServer(createTempDir("zkData")); - try { - server.run(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(server.getZkAddress()) - .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - String root = "/lockTestCollectionRoot"; - ZkDistributedCollectionLockFactory factory = - new ZkDistributedCollectionLockFactory(zkClient, root); - String collLockDir = - lockDir( - factory.createLock( - true, - CollectionParams.LockLevel.COLLECTION, - COLLECTION_NAME, - null, - null, - null)); - String replicaLockDir = - lockDir( - factory.createLock( - true, - CollectionParams.LockLevel.REPLICA, - COLLECTION_NAME, - SHARD_NAME, - REPLICA_NAME, - null)); - String shardLockDir = - lockDir( - factory.createLock( - true, - CollectionParams.LockLevel.SHARD, - COLLECTION_NAME, - SHARD_NAME, - null, - null)); - String otherCollLockDir = - lockDir( - factory.createLock( - true, CollectionParams.LockLevel.COLLECTION, "otherColl", null, null, null)); - String coll = parent(collLockDir); - zkClient.clean(root); - for (String dir : List.of(collLockDir, shardLockDir, replicaLockDir, otherCollLockDir)) { - zkClient.makePath(dir, false); - } - - DistributedLock shardLock = + ZkDistributedCollectionLockFactory factory = + new ZkDistributedCollectionLockFactory(zkClient, COLLECTION_ROOT); + String collLockDir = + lockDir( factory.createLock( - true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); - shardLock.waitUntilAcquired(); - - factory.deleteEmptyLockDirs(COLLECTION_NAME); - assertFalse(zkClient.exists(collLockDir)); - assertFalse(zkClient.exists(replicaLockDir)); - assertTrue(zkClient.exists(shardLock.getLockId())); - - shardLock.release(); - factory.deleteEmptyLockDirs(COLLECTION_NAME); - assertFalse(zkClient.exists(coll)); - assertTrue(zkClient.exists(otherCollLockDir)); - } - } finally { - server.shutdown(); - } - } - - /** A parent lock dir removed while the lock path is being created must not fail the lock. */ - @Test - public void testParentRemovedWhileCreatingLockPath() throws Exception { - ZkTestServer server = new ZkTestServer(createTempDir("zkData")); - try { - server.run(); - String root = "/lockTestCollectionRoot"; - AtomicReference shardLockDir = new AtomicReference<>(); - AtomicBoolean removed = new AtomicBoolean(); - try (SolrZkClient zkClient = - new SolrZkClient( - new SolrZkClient.Builder() - .withUrl(server.getZkAddress()) - .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS)) { - @Override - public String create(String path, byte[] data, CreateMode createMode) - throws KeeperException, InterruptedException { - if (path.equals(shardLockDir.get()) && removed.compareAndSet(false, true)) { - delete(parent(path), -1); - } - return super.create(path, data, createMode); - } - }) { - ZkDistributedCollectionLockFactory factory = - new ZkDistributedCollectionLockFactory(zkClient, root); - shardLockDir.set( - lockDir( - factory.createLock( - true, - CollectionParams.LockLevel.SHARD, - COLLECTION_NAME, - SHARD_NAME, - null, - null))); - zkClient.clean(root); - - DistributedLock lock = + true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null)); + String replicaLockDir = + lockDir( + factory.createLock( + true, + CollectionParams.LockLevel.REPLICA, + COLLECTION_NAME, + SHARD_NAME, + REPLICA_NAME, + null)); + String shardLockDir = + lockDir( factory.createLock( - true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); - assertTrue(removed.get()); - lock.waitUntilAcquired(); - lock.release(); - } - } finally { - server.shutdown(); + true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null)); + String otherCollLockDir = + lockDir( + factory.createLock( + true, CollectionParams.LockLevel.COLLECTION, "otherColl", null, null, null)); + String coll = parent(collLockDir); + zkClient.clean(COLLECTION_ROOT); + for (String dir : List.of(collLockDir, shardLockDir, replicaLockDir, otherCollLockDir)) { + zkClient.makePath(dir, false); } - } - /** Lock creation gives up once the lock dir keeps disappearing on every attempt. */ - @Test - public void testLockCreationGivesUpAfterMaxAttempts() throws Exception { - ZkTestServer server = new ZkTestServer(createTempDir("zkData")); - try { - server.run(); - AtomicInteger lockNodeCreates = new AtomicInteger(); - try (SolrZkClient zkClient = - new SolrZkClient( - new SolrZkClient.Builder() - .withUrl(server.getZkAddress()) - .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS)) { - @Override - public String create(String path, byte[] data, CreateMode createMode) - throws KeeperException, InterruptedException { - if (createMode == CreateMode.EPHEMERAL_SEQUENTIAL - && lockNodeCreates.incrementAndGet() - <= ZkDistributedLockFactory.MAX_CREATE_ATTEMPTS) { - throw new KeeperException.NoNodeException(path); - } - return super.create(path, data, createMode); - } - }) { - ZkDistributedCollectionLockFactory factory = - new ZkDistributedCollectionLockFactory(zkClient, "/lockTestCollectionRoot"); - SolrException e = - expectThrows( - SolrException.class, - () -> - factory.createLock( - true, - CollectionParams.LockLevel.COLLECTION, - COLLECTION_NAME, - null, - null, - null)); - assertTrue(e.getCause() instanceof KeeperException.NoNodeException); - assertEquals(ZkDistributedLockFactory.MAX_CREATE_ATTEMPTS, lockNodeCreates.get()); - } - } finally { - server.shutdown(); - } + DistributedLock shardLock = + factory.createLock( + true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); + shardLock.waitUntilAcquired(); + + factory.deleteEmptyLockDirs(COLLECTION_NAME); + assertFalse(zkClient.exists(collLockDir)); + assertFalse(zkClient.exists(replicaLockDir)); + assertTrue(zkClient.exists(shardLock.getLockId())); + + shardLock.release(); + factory.deleteEmptyLockDirs(COLLECTION_NAME); + assertFalse(zkClient.exists(coll)); + assertTrue(zkClient.exists(otherCollLockDir)); } - private DistributedCollectionLockFactory collectionLockFactory(SolrZkClient zkClient) { - return new ZkDistributedCollectionLockFactory(zkClient, "/lockTestCollectionRoot"); + private DistributedCollectionLockFactory collectionLockFactory() { + return new ZkDistributedCollectionLockFactory(zkClient, COLLECTION_ROOT); } private DistributedLock collectionLock( @@ -334,136 +207,78 @@ private DistributedLock collectionLock( private void assertWriteAcquirable(DistributedCollectionLockFactory factory) throws Exception { ThreadBoundLock next = bind(collectionLock(factory, true, null)); - next.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); + next.acquire().get(TIMEOUT_MS, TimeUnit.MILLISECONDS); next.release(); } @Test public void testInterruptedWaiterLeavesNoNode() throws Exception { - ZkTestServer server = new ZkTestServer(createTempDir("zkData")); - try { - server.run(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(server.getZkAddress()) - .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - DistributedCollectionLockFactory factory = collectionLockFactory(zkClient); - DistributedLock read = collectionLock(factory, false, null); - ThreadBoundLock readThread = bind(read); - readThread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); - String readId = read.getLockId(); - - ThreadBoundLock waiter = bind(collectionLock(factory, true, null)); - waiter.acquire(); - ThreadBoundLock.waitForLockNodes(zkClient, readId, 2); - - waiter.close(); - ThreadBoundLock.waitForLockNodes(zkClient, readId, 1); - - readThread.release(); - } - } finally { - server.shutdown(); - } + DistributedCollectionLockFactory factory = collectionLockFactory(); + DistributedLock read = collectionLock(factory, false, null); + ThreadBoundLock readThread = bind(read); + readThread.acquire().get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + String readId = read.getLockId(); + + ThreadBoundLock waiter = bind(collectionLock(factory, true, null)); + waiter.acquire(); + ThreadBoundLock.waitForLockNodes(zkClient, readId, 2); + + waiter.close(); + ThreadBoundLock.waitForLockNodes(zkClient, readId, 1); + + readThread.release(); } @Test public void testReleaseWithoutAcquireIsNoOp() throws Exception { - ZkTestServer server = new ZkTestServer(createTempDir("zkData")); - try { - server.run(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(server.getZkAddress()) - .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - DistributedCollectionLockFactory factory = collectionLockFactory(zkClient); - collectionLock(factory, true, null).release(); - assertWriteAcquirable(factory); - } - } finally { - server.shutdown(); - } + DistributedCollectionLockFactory factory = collectionLockFactory(); + collectionLock(factory, true, null).release(); + assertWriteAcquirable(factory); } @Test public void testMirrorOfReleasedLockFails() throws Exception { - ZkTestServer server = new ZkTestServer(createTempDir("zkData")); - try { - server.run(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(server.getZkAddress()) - .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - DistributedCollectionLockFactory factory = collectionLockFactory(zkClient); - DistributedLock write = collectionLock(factory, true, null); - ThreadBoundLock writeThread = bind(write); - writeThread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); - String releasedId = write.getLockId(); - writeThread.release(); - - DistributedLock mirror = collectionLock(factory, false, releasedId); - SolrException e = expectThrows(SolrException.class, mirror::waitUntilAcquired); - assertTrue(e.getMessage(), e.getMessage().contains("Missing mirrored lock node")); - } - } finally { - server.shutdown(); - } + DistributedCollectionLockFactory factory = collectionLockFactory(); + DistributedLock write = collectionLock(factory, true, null); + ThreadBoundLock writeThread = bind(write); + writeThread.acquire().get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + String releasedId = write.getLockId(); + writeThread.release(); + + DistributedLock mirror = collectionLock(factory, false, releasedId); + SolrException e = expectThrows(SolrException.class, mirror::waitUntilAcquired); + assertTrue(e.getMessage(), e.getMessage().contains("Missing mirrored lock node")); } @Test public void testReleaseOnOtherThreadFails() throws Exception { - ZkTestServer server = new ZkTestServer(createTempDir("zkData")); - try { - server.run(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(server.getZkAddress()) - .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - DistributedCollectionLockFactory factory = collectionLockFactory(zkClient); - DistributedLock write = collectionLock(factory, true, null); - ThreadBoundLock holder = bind(write); - holder.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); - - SolrException e = expectThrows(SolrException.class, write::release); - assertTrue(e.getCause() instanceof IllegalMonitorStateException); - - holder.release(); - assertWriteAcquirable(factory); - } - } finally { - server.shutdown(); - } + DistributedCollectionLockFactory factory = collectionLockFactory(); + DistributedLock write = collectionLock(factory, true, null); + ThreadBoundLock holder = bind(write); + holder.acquire().get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + + SolrException e = expectThrows(SolrException.class, write::release); + assertTrue(e.getCause() instanceof IllegalMonitorStateException); + + holder.release(); + assertWriteAcquirable(factory); } @Test public void testRepeatWaitThenSingleReleaseFrees() throws Exception { - ZkTestServer server = new ZkTestServer(createTempDir("zkData")); - try { - server.run(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(server.getZkAddress()) - .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - DistributedCollectionLockFactory factory = collectionLockFactory(zkClient); - DistributedLock write = collectionLock(factory, true, null); - write.waitUntilAcquired(); - write.waitUntilAcquired(); - write.release(); - assertWriteAcquirable(factory); - } - } finally { - server.shutdown(); - } + DistributedCollectionLockFactory factory = collectionLockFactory(); + DistributedLock write = collectionLock(factory, true, null); + write.waitUntilAcquired(); + write.waitUntilAcquired(); + write.release(); + assertWriteAcquirable(factory); } - /** Asserts every node between the lock node and {@code root} is a container. */ + /** + * Asserts every node from the lock node's parent up to and including {@code root} is a container. + */ private static void assertLockDirsAreContainers(ZkTestServer server, String lockId, String root) { - for (String dir = parent(lockId); !dir.equals(root); dir = parent(dir)) { + for (String dir = parent(lockId); dir.startsWith(root); dir = parent(dir)) { assertTrue( dir + " should be a container node", server.getZKDatabase().getDataTree().getContainers().contains(dir)); @@ -482,44 +297,46 @@ private static String parent(String path) { return path.substring(0, path.lastIndexOf('/')); } - private void monothreadedCollectionTests( - SolrZkClient zkClient, DistributedCollectionLockFactory factory) throws Exception { + private void monothreadedCollectionTests(DistributedCollectionLockFactory factory) + throws Exception { // Collection level locks DistributedLock collRL1 = factory.createLock( false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); ThreadBoundLock collRL1Thread = bind(collRL1); - collRL1Thread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); + collRL1Thread.acquire().get(TIMEOUT_MS, TimeUnit.MILLISECONDS); DistributedLock collRL2 = factory.createLock( false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); ThreadBoundLock collRL2Thread = bind(collRL2); - collRL2Thread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); + collRL2Thread.acquire().get(TIMEOUT_MS, TimeUnit.MILLISECONDS); DistributedLock collWL3 = factory.createLock( true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); ThreadBoundLock collWL3Thread = bind(collWL3); - Future collWL3Acquired = collWL3Thread.acquire(); - ThreadBoundLock.waitForLockNodes(zkClient, collRL1.getLockId(), 3); - assertFalse( - "collWL3 should not have been acquired, due to collRL1 and collRL2", - collWL3Acquired.isDone()); + Future collWL3Acquired = + collWL3Thread.acquireBlocked( + "collWL3 should not have been acquired, due to collRL1 and collRL2", + zkClient, + COLLECTION_ROOT, + 1); collRL1Thread.release(); collRL2Thread.release(); - collWL3Acquired.get(TIMEOUT, TimeUnit.MILLISECONDS); + collWL3Acquired.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); DistributedLock collRL4 = factory.createLock( false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); ThreadBoundLock collRL4Thread = bind(collRL4); - Future collRL4Acquired = collRL4Thread.acquire(); - ThreadBoundLock.waitForLockNodes(zkClient, collWL3.getLockId(), 2); - assertFalse( - "collRL4 should not have been acquired, due to collWL3 locking the collection", - collRL4Acquired.isDone()); + Future collRL4Acquired = + collRL4Thread.acquireBlocked( + "collRL4 should not have been acquired, due to collWL3 locking the collection", + zkClient, + COLLECTION_ROOT, + 1); // Collection is write locked by collWL3 and collRL4 read lock waiting behind. Now moving to // request shard level locks. These are totally independent of the Collection level locks so @@ -528,17 +345,18 @@ private void monothreadedCollectionTests( factory.createLock( true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); ThreadBoundLock shardWL5Thread = bind(shardWL5); - shardWL5Thread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); + shardWL5Thread.acquire().get(TIMEOUT_MS, TimeUnit.MILLISECONDS); DistributedLock shardWL6 = factory.createLock( true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null, null); ThreadBoundLock shardWL6Thread = bind(shardWL6); - Future shardWL6Acquired = shardWL6Thread.acquire(); - ThreadBoundLock.waitForLockNodes(zkClient, shardWL5.getLockId(), 2); - assertFalse( - "shardWL6 should not have been acquired, shardWL5 is locking that shard", - shardWL6Acquired.isDone()); + Future shardWL6Acquired = + shardWL6Thread.acquireBlocked( + "shardWL6 should not have been acquired, shardWL5 is locking that shard", + zkClient, + COLLECTION_ROOT, + 1); // Get a lock on a Replica. Again this is independent of collection or shard level DistributedLock replicaRL7 = @@ -550,7 +368,7 @@ private void monothreadedCollectionTests( REPLICA_NAME, null); ThreadBoundLock replicaRL7Thread = bind(replicaRL7); - replicaRL7Thread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); + replicaRL7Thread.acquire().get(TIMEOUT_MS, TimeUnit.MILLISECONDS); DistributedLock replicaWL8 = factory.createLock( @@ -561,19 +379,20 @@ private void monothreadedCollectionTests( REPLICA_NAME, null); ThreadBoundLock replicaWL8Thread = bind(replicaWL8); - Future replicaWL8Acquired = replicaWL8Thread.acquire(); - ThreadBoundLock.waitForLockNodes(zkClient, replicaRL7.getLockId(), 2); - assertFalse( - "replicaWL8 should not have been acquired, replicaRL7 is read locking that replica", - replicaWL8Acquired.isDone()); + Future replicaWL8Acquired = + replicaWL8Thread.acquireBlocked( + "replicaWL8 should not have been acquired, replicaRL7 is read locking that replica", + zkClient, + COLLECTION_ROOT, + 1); replicaRL7Thread.release(); - replicaWL8Acquired.get(TIMEOUT, TimeUnit.MILLISECONDS); + replicaWL8Acquired.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); collWL3Thread.release(); - collRL4Acquired.get(TIMEOUT, TimeUnit.MILLISECONDS); + collRL4Acquired.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); shardWL5Thread.release(); - shardWL6Acquired.get(TIMEOUT, TimeUnit.MILLISECONDS); + shardWL6Acquired.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); replicaWL8Thread.release(); @@ -581,83 +400,86 @@ private void monothreadedCollectionTests( collRL4Thread.release(); } - private void multithreadedCollectionTests( - SolrZkClient zkClient, DistributedCollectionLockFactory factory) throws Exception { + private void multithreadedCollectionTests(DistributedCollectionLockFactory factory) + throws Exception { // Acquiring right away a read lock DistributedLock readLock = factory.createLock( false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); ThreadBoundLock readLockThread = bind(readLock); - readLockThread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); + readLockThread.acquire().get(TIMEOUT_MS, TimeUnit.MILLISECONDS); // And now creating a write lock, that can't be acquired just yet, because of the read lock DistributedLock writeLock = factory.createLock( true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null, null); ThreadBoundLock writeLockThread = bind(writeLock); - Future writeLockAcquired = writeLockThread.acquire(); - ThreadBoundLock.waitForLockNodes(zkClient, readLock.getLockId(), 2); - assertFalse("writeLock should not have been acquired", writeLockAcquired.isDone()); + Future writeLockAcquired = + writeLockThread.acquireBlocked( + "writeLock should not have been acquired", zkClient, COLLECTION_ROOT, 1); readLockThread.release(); - writeLockAcquired.get(TIMEOUT, TimeUnit.MILLISECONDS); + writeLockAcquired.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); writeLockThread.release(); } - private void monothreadedConfigSetTests( - SolrZkClient zkClient, DistributedConfigSetLockFactory factory) throws Exception { + private void monothreadedConfigSetTests(DistributedConfigSetLockFactory factory) + throws Exception { DistributedLock configSetRL1 = factory.createLock(false, CONFIG_SET_NAME); ThreadBoundLock configSetRL1Thread = bind(configSetRL1); - configSetRL1Thread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); + configSetRL1Thread.acquire().get(TIMEOUT_MS, TimeUnit.MILLISECONDS); DistributedLock configSetRL2 = factory.createLock(false, CONFIG_SET_NAME); ThreadBoundLock configSetRL2Thread = bind(configSetRL2); - configSetRL2Thread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); + configSetRL2Thread.acquire().get(TIMEOUT_MS, TimeUnit.MILLISECONDS); DistributedLock configSetWL1 = factory.createLock(true, CONFIG_SET_NAME); ThreadBoundLock configSetWL1Thread = bind(configSetWL1); - Future configSetWL1Acquired = configSetWL1Thread.acquire(); - ThreadBoundLock.waitForLockNodes(zkClient, configSetRL1.getLockId(), 3); - assertFalse( - "configSetWL1 should not have been acquired due to configSetRL1 and configSetRL2", - configSetWL1Acquired.isDone()); + Future configSetWL1Acquired = + configSetWL1Thread.acquireBlocked( + "configSetWL1 should not have been acquired due to configSetRL1 and configSetRL2", + zkClient, + CONFIG_SET_ROOT, + 1); configSetRL1Thread.release(); configSetRL2Thread.release(); - configSetWL1Acquired.get(TIMEOUT, TimeUnit.MILLISECONDS); + configSetWL1Acquired.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); DistributedLock configSetRL3 = factory.createLock(false, CONFIG_SET_NAME); ThreadBoundLock configSetRL3Thread = bind(configSetRL3); - Future configSetRL3Acquired = configSetRL3Thread.acquire(); - ThreadBoundLock.waitForLockNodes(zkClient, configSetWL1.getLockId(), 2); - assertFalse( - "configSetRL3 should not have been acquired due to configSetWL1", - configSetRL3Acquired.isDone()); + Future configSetRL3Acquired = + configSetRL3Thread.acquireBlocked( + "configSetRL3 should not have been acquired due to configSetWL1", + zkClient, + CONFIG_SET_ROOT, + 1); configSetWL1Thread.release(); - configSetRL3Acquired.get(TIMEOUT, TimeUnit.MILLISECONDS); + configSetRL3Acquired.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); configSetRL3Thread.release(); } - private void multithreadedConfigSetTests( - SolrZkClient zkClient, DistributedConfigSetLockFactory factory) throws Exception { + private void multithreadedConfigSetTests(DistributedConfigSetLockFactory factory) + throws Exception { // Acquiring right away a read lock DistributedLock configSetRL1 = factory.createLock(false, CONFIG_SET_NAME); ThreadBoundLock configSetRL1Thread = bind(configSetRL1); - configSetRL1Thread.acquire().get(TIMEOUT, TimeUnit.MILLISECONDS); + configSetRL1Thread.acquire().get(TIMEOUT_MS, TimeUnit.MILLISECONDS); // And now creating a write lock, that can't be acquired just yet, because of the read lock DistributedLock configSetWL1 = factory.createLock(true, CONFIG_SET_NAME); ThreadBoundLock configSetWL1Thread = bind(configSetWL1); - Future configSetWL1Acquired = configSetWL1Thread.acquire(); - ThreadBoundLock.waitForLockNodes(zkClient, configSetRL1.getLockId(), 2); - assertFalse( - "configSetWL1 should not have been acquired due to configSetRL1", - configSetWL1Acquired.isDone()); + Future configSetWL1Acquired = + configSetWL1Thread.acquireBlocked( + "configSetWL1 should not have been acquired due to configSetRL1", + zkClient, + CONFIG_SET_ROOT, + 1); configSetRL1Thread.release(); - configSetWL1Acquired.get(TIMEOUT, TimeUnit.MILLISECONDS); + configSetWL1Acquired.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); configSetWL1Thread.release(); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java index af26ce419b14..5b150d0d5148 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java @@ -17,159 +17,83 @@ package org.apache.solr.cloud.api.collections; -import java.util.IdentityHashMap; +import static org.apache.solr.cloud.ThreadBoundLock.awaitAcquired; + import java.util.List; -import java.util.Map; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.cloud.ApiLockingTestBase; import org.apache.solr.cloud.DistributedLock; import org.apache.solr.cloud.DistributedMultiLock; import org.apache.solr.cloud.ThreadBoundLock; import org.apache.solr.cloud.ZkDistributedCollectionLockFactory; -import org.apache.solr.cloud.ZkTestServer; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.params.CollectionParams; import org.junit.Test; -public class CollectionApiLockingTest extends SolrTestCaseJ4 { +public class CollectionApiLockingTest extends ApiLockingTestBase { private static final String COLLECTION_NAME = "lockColl"; final String SHARD1_NAME = "lockShard1"; final String SHARD2_NAME = "lockShard2"; final String REPLICA_NAME = "lockReplica"; - static final int TIMEOUT = 10000; - - private final Map boundLocks = new IdentityHashMap<>(); - private SolrZkClient zkClient; - /** * Tests the Collection API locking. All tests run from single test method to save on setup time. */ @Test public void monothreadedApiLockTests() throws Exception { - ZkTestServer server = new ZkTestServer(createTempDir("zkData")); - try { - server.run(); - try (SolrZkClient client = - new SolrZkClient.Builder() - .withUrl(server.getZkAddress()) - .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - zkClient = client; - CollectionApiLockFactory apiLockFactory = - new CollectionApiLockFactory( - new ZkDistributedCollectionLockFactory(zkClient, "/apiLockTestRoot")); - - try { - monothreadedTests(apiLockFactory); - multithreadedTests(apiLockFactory); - testCallingLockIdSubLocks(apiLockFactory); - } finally { - boundLocks.values().forEach(ThreadBoundLock::close); - } - } - } finally { - server.shutdown(); - } + CollectionApiLockFactory apiLockFactory = + new CollectionApiLockFactory(new ZkDistributedCollectionLockFactory(zkClient, LOCK_ROOT)); + + monothreadedTests(apiLockFactory); + multithreadedTests(apiLockFactory); + testCallingLockIdSubLocks(apiLockFactory); } @Test public void testBlockedMultiLockHoldsNoLowerLevelNode() throws Exception { - ZkTestServer server = new ZkTestServer(createTempDir("zkData")); - try { - server.run(); - try (SolrZkClient client = - new SolrZkClient.Builder() - .withUrl(server.getZkAddress()) - .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - zkClient = client; - ZkDistributedCollectionLockFactory factory = - new ZkDistributedCollectionLockFactory(zkClient, "/apiLockTestRoot"); - CollectionApiLockFactory apiLockFactory = new CollectionApiLockFactory(factory); - - try { - DistributedMultiLock collWrite = - apiLockFactory.createCollectionApiLock( - new AdminCmdContext(CollectionParams.CollectionAction.CREATE), - COLLECTION_NAME, - null, - null); - assertAcquired("Collection write should have been acquired", collWrite); - - DistributedMultiLock split = - apiLockFactory.createCollectionApiLock( - new AdminCmdContext(CollectionParams.CollectionAction.SPLITSHARD), - COLLECTION_NAME, - SHARD1_NAME, - null); - Future splitAcquisition = - assertBlocked("Split should be blocked by collection write", split, collWrite, 0, 2); - - DistributedLock shardWrite = - factory.createLock( - true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, null, null); - ThreadBoundLock shardWriteThread = - new ThreadBoundLock(shardWrite::waitUntilAcquired, shardWrite::release); - try { - awaitAcquired( - "Plain shard write should acquire while the split is blocked", - shardWriteThread.acquire()); - shardWriteThread.release(); - } finally { - shardWriteThread.close(); - } - - release(collWrite); - awaitAcquired( - "Split should acquire once the collection write is released", splitAcquisition); - release(split); - } finally { - boundLocks.values().forEach(ThreadBoundLock::close); - } - } - } finally { - server.shutdown(); - } - } - - private ThreadBoundLock bound(DistributedMultiLock lock) { - return boundLocks.computeIfAbsent( - lock, l -> new ThreadBoundLock(l::waitUntilAcquired, l::release)); - } + ZkDistributedCollectionLockFactory factory = + new ZkDistributedCollectionLockFactory(zkClient, LOCK_ROOT); + CollectionApiLockFactory apiLockFactory = new CollectionApiLockFactory(factory); - private void assertAcquired(String message, DistributedMultiLock lock) throws Exception { - awaitAcquired(message, bound(lock).acquire()); - } + DistributedMultiLock collWrite = + apiLockFactory.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.CREATE), + COLLECTION_NAME, + null, + null); + assertAcquired("Collection write should have been acquired", collWrite); - private void awaitAcquired(String message, Future acquisition) throws Exception { + DistributedMultiLock split = + apiLockFactory.createCollectionApiLock( + new AdminCmdContext(CollectionParams.CollectionAction.SPLITSHARD), + COLLECTION_NAME, + SHARD1_NAME, + null); + Future splitAcquisition = + bound(split) + .acquireBlocked("Split should be blocked by collection write", zkClient, LOCK_ROOT, 1); + + DistributedLock shardWrite = + factory.createLock( + true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, null, null); + ThreadBoundLock shardWriteThread = + new ThreadBoundLock(shardWrite::waitUntilAcquired, shardWrite::release); try { - acquisition.get(TIMEOUT, TimeUnit.MILLISECONDS); - } catch (TimeoutException e) { - throw new AssertionError(message, e); + awaitAcquired( + "Plain shard write should acquire while the split is blocked", + shardWriteThread.acquire()); + shardWriteThread.release(); + } finally { + shardWriteThread.close(); } - } - - private Future assertBlocked( - String message, DistributedMultiLock lock, DistributedMultiLock holder, int level, int count) - throws Exception { - Future acquisition = bound(lock).acquire(); - ThreadBoundLock.waitForLockNodes( - zkClient, DistributedMultiLock.splitLockIds(holder.getLockId()).get(level), count); - assertFalse(message, acquisition.isDone()); - return acquisition; - } - private void release(DistributedMultiLock lock) throws Exception { - bound(lock).release(); + release(collWrite); + awaitAcquired("Split should acquire once the collection write is released", splitAcquisition); + release(split); } - private void abandon(DistributedMultiLock lock) { - bound(lock).close(); - lock.release(); + private void abandon(DistributedMultiLock lock) throws Exception { + bound(lock).abandon(); } private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) throws Exception { @@ -196,7 +120,8 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) throws SHARD1_NAME, null); Future shard1Acquisition = - assertBlocked("Shard1 should not have been acquired", shard1Lock, collLock, 0, 2); + bound(shard1Lock) + .acquireBlocked("Shard1 should not have been acquired", zkClient, LOCK_ROOT, 1); assertEquals( "Lock at shard level expected to need two distributed locks", 2, @@ -211,9 +136,10 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) throws SHARD2_NAME, null); Future shard2Acquisition = - assertBlocked("Shard2 should not have been acquired", shard2Lock, collLock, 0, 3); + bound(shard2Lock) + .acquireBlocked("Shard2 should not have been acquired", zkClient, LOCK_ROOT, 1); - assertAcquired("Collection should still be acquired", collLock); + assertTrue("Collection lock node should still exist", zkClient.exists(collLock.getLockId())); release(collLock); @@ -230,12 +156,12 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) throws SHARD1_NAME, REPLICA_NAME); Future replicaShard1Acquisition = - assertBlocked( - "replicaShard1Lock should not have been acquired, shard1 is locked", - replicaShard1Lock, - shard1Lock, - 1, - 2); + bound(replicaShard1Lock) + .acquireBlocked( + "replicaShard1Lock should not have been acquired, shard1 is locked", + zkClient, + LOCK_ROOT, + 2); // Now ask for a new lock on the collection collLock = @@ -246,12 +172,12 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) throws null); Future collAcquisition = - assertBlocked( - "Collection should not have been acquired, shard1 and shard2 locks preventing it", - collLock, - shard1Lock, - 0, - 4); + bound(collLock) + .acquireBlocked( + "Collection should not have been acquired, shard1 and shard2 locks preventing it", + zkClient, + LOCK_ROOT, + 1); release(shard1Lock); awaitAcquired( @@ -271,12 +197,12 @@ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) throws SHARD2_NAME, REPLICA_NAME); Future replicaShard2Acquisition = - assertBlocked( - "replicaShard2Lock should not have been acquired, shard2 is locked", - replicaShard2Lock, - shard2Lock, - 0, - 3); + bound(replicaShard2Lock) + .acquireBlocked( + "replicaShard2Lock should not have been acquired, shard2 is locked", + zkClient, + LOCK_ROOT, + 1); release(shard2Lock); @@ -314,12 +240,12 @@ private void multithreadedTests(CollectionApiLockFactory apiLockingHelper) throw SHARD1_NAME, REPLICA_NAME); Future replicaShard1Acquisition = - assertBlocked( - "replicaShard1Lock should not have been acquired, because collection is locked", - replicaShard1Lock, - collLock, - 0, - 2); + bound(replicaShard1Lock) + .acquireBlocked( + "replicaShard1Lock should not have been acquired, because collection is locked", + zkClient, + LOCK_ROOT, + 1); release(collLock); awaitAcquired( @@ -340,12 +266,12 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.RELOAD), "coll1", null, null); assertEquals("Wrong number of internalLocks", 1, coll1Lock2.getCountInternalLocks()); - assertBlocked( - "Should not be able to lock coll1 without using a callingLockId", - coll1Lock2, - coll1Lock, - 0, - 2); + bound(coll1Lock2) + .acquireBlocked( + "Should not be able to lock coll1 without using a callingLockId", + zkClient, + LOCK_ROOT, + 1); abandon(coll1Lock2); coll1Lock2 = apiLockingHelper.createCollectionApiLock( @@ -377,12 +303,12 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "shard1", null); assertEquals("Wrong number of internalLocks", 2, shard1Lock2.getCountInternalLocks()); - assertBlocked( - "Should not be able to lock coll1/shard1 since our callingLockId is only coll1, not shard1, since shard1 is already locked", - shard1Lock2, - shard1Lock, - 1, - 2); + bound(shard1Lock2) + .acquireBlocked( + "Should not be able to lock coll1/shard1 since our callingLockId is only coll1, not shard1, since shard1 is already locked", + zkClient, + LOCK_ROOT, + 1); abandon(shard1Lock2); DistributedMultiLock shard2Lock = apiLockingHelper.createCollectionApiLock( @@ -418,12 +344,12 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "shard1", "replica1"); assertEquals("Wrong number of internalLocks", 3, replica2Lock.getCountInternalLocks()); - assertBlocked( - "Should not be able to lock coll1/shard1/replica1 since our callingLockId is only coll1, not replica1, which is already locked", - replica2Lock, - replica1Lock, - 2, - 2); + bound(replica2Lock) + .acquireBlocked( + "Should not be able to lock coll1/shard1/replica1 since our callingLockId is only coll1, not replica1, which is already locked", + zkClient, + LOCK_ROOT, + 2); abandon(replica2Lock); shard1Lock = apiLockingHelper.createCollectionApiLock( @@ -433,12 +359,12 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) "shard1", null); assertEquals("Wrong number of internalLocks", 2, shard1Lock.getCountInternalLocks()); - assertBlocked( - "Should not be able to lock coll1/shard1 since our callingLockId is only coll1, not shard1, which is locked because of a replica task", - shard1Lock, - replica1Lock, - 1, - 2); + bound(shard1Lock) + .acquireBlocked( + "Should not be able to lock coll1/shard1 since our callingLockId is only coll1, not shard1, which is locked because of a replica task", + zkClient, + LOCK_ROOT, + 1); abandon(shard1Lock); replica2Lock = apiLockingHelper.createCollectionApiLock( @@ -505,12 +431,12 @@ public void testCallingLockIdSubLocks(CollectionApiLockFactory apiLockingHelper) apiLockingHelper.createCollectionApiLock( new AdminCmdContext(CollectionParams.CollectionAction.RELOAD), "coll1", null, null); assertEquals("Wrong number of internalLocks", 1, coll1Lock2.getCountInternalLocks()); - assertBlocked( - "Should not be able to lock coll1 since we have no callingLockId and shard1 is already locked. Cannot move up", - coll1Lock2, - shard1Lock1, - 0, - 2); + bound(coll1Lock2) + .acquireBlocked( + "Should not be able to lock coll1 since we have no callingLockId and shard1 is already locked. Cannot move up", + zkClient, + LOCK_ROOT, + 1); abandon(coll1Lock2); assertExceptionThrownWithMessageContaining( SolrException.class, From 6983ba2f4120b1fb9ffef7155f744b1e1838370e Mon Sep 17 00:00:00 2001 From: Amrit Sarkar Date: Tue, 17 Nov 2026 01:52:49 -0800 Subject: [PATCH 6/9] SOLR-17704: Upgrade note and changelog --- changelog/unreleased/SOLR-17704-curator-locks.yml | 8 ++++++++ .../pages/major-changes-in-solr-11.adoc | 13 +++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 changelog/unreleased/SOLR-17704-curator-locks.yml diff --git a/changelog/unreleased/SOLR-17704-curator-locks.yml b/changelog/unreleased/SOLR-17704-curator-locks.yml new file mode 100644 index 000000000000..56fd1043770a --- /dev/null +++ b/changelog/unreleased/SOLR-17704-curator-locks.yml @@ -0,0 +1,8 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Collection and ConfigSet API locks without the Overseer now use Apache Curator; upgrading such clusters needs a full restart or paused API commands +type: changed +authors: + - name: Amrit Sarkar +links: + - name: SOLR-17704 + url: https://issues.apache.org/jira/browse/SOLR-17704 diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-11.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-11.adoc index 184aee185213..dc8e3febfeab 100644 --- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-11.adoc +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-11.adoc @@ -25,6 +25,19 @@ This page highlights the most important changes including new features and chang Before starting an upgrade to this version of Solr, please be sure to review all information about changes from the version you are currently on up to this one, to include the minor version number changes as well. For example, if you are currently using Solr 10.1, you should review changes made in all subsequent 10.x releases in addition to the 11.0-specific changes on this page. +=== Rolling Upgrades + +If the Overseer is disabled (cluster property `overseerEnabled` or `SOLR_CLOUD_OVERSEER_ENABLED` set to `false`), Collection and ConfigSet API locks now use a different ZooKeeper node format. +10.x and 11 nodes don't see each other's locks, so while both versions are running, two conflicting commands on the same collection or configset can run at the same time without an error. + +If the Overseer is disabled, either: + +* Upgrade all nodes with a full cluster restart, or +* Pause Collection and ConfigSet API operations, do a rolling restart, and resume them once every node is on 11. +Replica state changes during the rolling restart are fine. + +Clusters running with the Overseer are not affected. + == Removed Features === Overseer Roles API From 6980231ca515d8710defa3db59295373125e40ce Mon Sep 17 00:00:00 2001 From: Amrit Sarkar Date: Tue, 17 Nov 2026 12:12:47 -0800 Subject: [PATCH 7/9] SOLR-17704: Release every lock of a multi-lock when one release fails --- .../solr/cloud/DistributedMultiLock.java | 14 +++- ...butedCollectionConfigSetCommandRunner.java | 8 +- .../solr/cloud/DistributedMultiLockTest.java | 76 +++++++++++++++++++ 3 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/cloud/DistributedMultiLockTest.java diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java index 34bdfa8569fc..b4616cb2d11f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java @@ -59,8 +59,20 @@ public void waitUntilAcquired() { public void release() { isReleased = true; + RuntimeException failure = null; for (DistributedLock lock : locks) { - lock.release(); + try { + lock.release(); + } catch (RuntimeException e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + } + if (failure != null) { + throw failure; } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java index 09156aecea12..6232269e91b0 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java @@ -554,7 +554,13 @@ public Void call() throws IOException { SolrException.ErrorCode.BAD_REQUEST, "Bug! Unknown Config Set action: " + action); } } finally { - lock.release(); + try { + lock.release(); + } catch (SolrException se) { + if (log.isErrorEnabled()) { + log.error("Error when releasing config set locks for action {}", action, se); + } + } } return null; diff --git a/solr/core/src/test/org/apache/solr/cloud/DistributedMultiLockTest.java b/solr/core/src/test/org/apache/solr/cloud/DistributedMultiLockTest.java new file mode 100644 index 000000000000..8766dc5e5824 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/DistributedMultiLockTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import java.util.List; +import org.apache.solr.SolrTestCase; +import org.apache.solr.common.SolrException; +import org.junit.Test; + +/** Tests for {@link DistributedMultiLock}. */ +public class DistributedMultiLockTest extends SolrTestCase { + + @Test + public void testReleaseFailureStillReleasesOtherLocks() { + StubLock failing = + new StubLock(new SolrException(SolrException.ErrorCode.SERVER_ERROR, "first")); + StubLock alsoFailing = + new StubLock(new SolrException(SolrException.ErrorCode.SERVER_ERROR, "second")); + StubLock healthy = new StubLock(null); + DistributedMultiLock multiLock = + new DistributedMultiLock(List.of(failing, alsoFailing, healthy)); + + SolrException e = expectThrows(SolrException.class, multiLock::release); + + assertEquals("first", e.getMessage()); + assertEquals(1, e.getSuppressed().length); + assertEquals("second", e.getSuppressed()[0].getMessage()); + assertTrue(failing.releaseCalled); + assertTrue(alsoFailing.releaseCalled); + assertTrue(healthy.releaseCalled); + } + + private static class StubLock implements DistributedLock { + private final SolrException releaseFailure; + private boolean releaseCalled; + + StubLock(SolrException releaseFailure) { + this.releaseFailure = releaseFailure; + } + + @Override + public void waitUntilAcquired() {} + + @Override + public void release() { + releaseCalled = true; + if (releaseFailure != null) { + throw releaseFailure; + } + } + + @Override + public String getLockId() { + return null; + } + + @Override + public boolean isMirroringLock() { + return false; + } + } +} From 8f7e0bfe9d87bb9f606e579d42df2cad81f3b9a5 Mon Sep 17 00:00:00 2001 From: Amrit Sarkar Date: Wed, 18 Nov 2026 15:01:19 -0800 Subject: [PATCH 8/9] SOLR-17704: Store the node name in lock nodes, clarify upgrade note --- .../ZkDistributedCollectionLockFactory.java | 5 +-- .../ZkDistributedConfigSetLockFactory.java | 5 +-- .../solr/cloud/ZkDistributedLockFactory.java | 16 +++++---- ...butedCollectionConfigSetCommandRunner.java | 8 +++-- .../solr/cloud/ConfigSetApiLockingTest.java | 3 +- .../solr/cloud/ZkDistributedLockTest.java | 34 +++++++++++++++---- .../AliasCollectionApiLockTest.java | 3 +- .../collections/CollectionApiLockingTest.java | 5 +-- .../DeleteCollectionLockCleanupTest.java | 2 +- .../pages/major-changes-in-solr-11.adoc | 2 +- 10 files changed, 59 insertions(+), 24 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java index 54a17b919ca2..8155d771f16b 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java @@ -36,8 +36,9 @@ public class ZkDistributedCollectionLockFactory extends ZkDistributedLockFactory implements DistributedCollectionLockFactory { - public ZkDistributedCollectionLockFactory(SolrZkClient zkClient, String rootPath) { - super(zkClient, rootPath); + public ZkDistributedCollectionLockFactory( + SolrZkClient zkClient, String rootPath, String nodeName) { + super(zkClient, rootPath, nodeName); } @Override diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java index c3da79044e74..d3ab2f77c5a1 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java @@ -32,8 +32,9 @@ public class ZkDistributedConfigSetLockFactory extends ZkDistributedLockFactory implements DistributedConfigSetLockFactory { - public ZkDistributedConfigSetLockFactory(SolrZkClient zkClient, String rootPath) { - super(zkClient, rootPath); + public ZkDistributedConfigSetLockFactory( + SolrZkClient zkClient, String rootPath, String nodeName) { + super(zkClient, rootPath, nodeName); } @Override diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java index 78452898be98..4edcdcec1a7d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.nio.charset.StandardCharsets; import org.apache.curator.framework.recipes.locks.InterProcessReadWriteLock; import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; import org.apache.solr.common.cloud.SolrZkClient; @@ -28,26 +29,29 @@ abstract class ZkDistributedLockFactory { private final SolrZkClient zkClient; private final String rootPath; + private final byte[] lockNodeData; - ZkDistributedLockFactory(SolrZkClient zkClient, String rootPath) { + ZkDistributedLockFactory(SolrZkClient zkClient, String rootPath, String nodeName) { this.zkClient = zkClient; this.rootPath = rootPath; + this.lockNodeData = nodeName.getBytes(StandardCharsets.UTF_8); } protected DistributedLock doCreateLock( boolean isWriteLock, String lockPath, String lockIdToMirror) { if (StrUtils.isBlank(lockIdToMirror)) { - // Empty payload: otherwise Curator stores the local IP address in every lock node InterProcessReadWriteLock readWriteLock = - new InterProcessReadWriteLock(zkClient.getCuratorFramework(), lockPath, new byte[0]); + new InterProcessReadWriteLock(zkClient.getCuratorFramework(), lockPath, lockNodeData); if (isWriteLock) { InterProcessReadWriteLock.WriteLock writeLock = readWriteLock.writeLock(); return new ZkDistributedLock.Owned(writeLock, lockPath, writeLock::getLockPath); + } else { + InterProcessReadWriteLock.ReadLock readLock = readWriteLock.readLock(); + return new ZkDistributedLock.Owned(readLock, lockPath, readLock::getLockPath); } - InterProcessReadWriteLock.ReadLock readLock = readWriteLock.readLock(); - return new ZkDistributedLock.Owned(readLock, lockPath, readLock::getLockPath); + } else { + return new ZkDistributedLock.Mirrored(zkClient, lockPath, isWriteLock, lockIdToMirror); } - return new ZkDistributedLock.Mirrored(zkClient, lockPath, isWriteLock, lockIdToMirror); } protected StringBuilder getPathPrefix() { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java index 6232269e91b0..0c18afb60a3e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java @@ -407,7 +407,9 @@ public OverseerSolrResponse call() { // below ZkDistributedCollectionLockFactory lockFactory = new ZkDistributedCollectionLockFactory( - ccc.getZkStateReader().getZkClient(), ZK_COLLECTION_LOCKS); + ccc.getZkStateReader().getZkClient(), + ZK_COLLECTION_LOCKS, + coreContainer.getZkController().getNodeName()); DistributedMultiLock lock = new CollectionApiLockFactory(lockFactory) .createCollectionApiLock(adminCmdContext, lockCollName, shardId, replicaName); @@ -527,7 +529,9 @@ public Void call() throws IOException { DistributedMultiLock lock = new ConfigSetApiLockFactory( new ZkDistributedConfigSetLockFactory( - ccc.getZkStateReader().getZkClient(), ZK_CONFIG_SET_LOCKS)) + ccc.getZkStateReader().getZkClient(), + ZK_CONFIG_SET_LOCKS, + coreContainer.getZkController().getNodeName())) .createConfigSetApiLock(configSetName, baseConfigSetName); try { diff --git a/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java index f3682ed26cc3..04475c7a62cd 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java @@ -35,7 +35,8 @@ public class ConfigSetApiLockingTest extends ApiLockingTestBase { @Test public void monothreadedApiLockTests() throws Exception { ConfigSetApiLockFactory apiLockFactory = - new ConfigSetApiLockFactory(new ZkDistributedConfigSetLockFactory(zkClient, LOCK_ROOT)); + new ConfigSetApiLockFactory( + new ZkDistributedConfigSetLockFactory(zkClient, LOCK_ROOT, "testNode")); monothreadedTests(apiLockFactory); multithreadedTests(apiLockFactory); diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java index b1b653ed230e..eb67ba169737 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java @@ -19,6 +19,7 @@ import static org.apache.solr.cloud.ThreadBoundLock.TIMEOUT_MS; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Future; @@ -83,13 +84,13 @@ private ThreadBoundLock bind(DistributedLock lock) { @Test public void testSingleLocks() throws Exception { DistributedCollectionLockFactory collLockFactory = - new ZkDistributedCollectionLockFactory(zkClient, COLLECTION_ROOT); + new ZkDistributedCollectionLockFactory(zkClient, COLLECTION_ROOT, "testNode"); monothreadedCollectionTests(collLockFactory); multithreadedCollectionTests(collLockFactory); DistributedConfigSetLockFactory configSetLockFactory = - new ZkDistributedConfigSetLockFactory(zkClient, CONFIG_SET_ROOT); + new ZkDistributedConfigSetLockFactory(zkClient, CONFIG_SET_ROOT, "testNode"); monothreadedConfigSetTests(configSetLockFactory); multithreadedConfigSetTests(configSetLockFactory); @@ -102,7 +103,7 @@ public void testSingleLocks() throws Exception { @Test public void testLockDirsAreContainers() throws Exception { DistributedCollectionLockFactory collLockFactory = - new ZkDistributedCollectionLockFactory(zkClient, COLLECTION_ROOT); + new ZkDistributedCollectionLockFactory(zkClient, COLLECTION_ROOT, "testNode"); DistributedLock replicaLock = collLockFactory.createLock( true, @@ -131,7 +132,7 @@ public void testLockDirsAreContainers() throws Exception { assertLockDirsAreContainers(server, collLockId, COLLECTION_ROOT); DistributedConfigSetLockFactory configSetLockFactory = - new ZkDistributedConfigSetLockFactory(zkClient, CONFIG_SET_ROOT); + new ZkDistributedConfigSetLockFactory(zkClient, CONFIG_SET_ROOT, "testNode"); DistributedLock configSetLock = configSetLockFactory.createLock(true, CONFIG_SET_NAME); configSetLock.waitUntilAcquired(); String configSetLockId = configSetLock.getLockId(); @@ -147,11 +148,32 @@ public void testLockDirsAreContainers() throws Exception { lock.release(); } + /** Lock nodes record the Solr node that holds them. */ + @Test + public void testLockNodeHoldsNodeName() throws Exception { + DistributedLock collLock = collectionLock(collectionLockFactory(), true, null); + collLock.waitUntilAcquired(); + assertEquals( + "testNode", + new String(zkClient.getData(collLock.getLockId(), null, null), StandardCharsets.UTF_8)); + collLock.release(); + + DistributedLock configSetLock = + new ZkDistributedConfigSetLockFactory(zkClient, CONFIG_SET_ROOT, "testNode") + .createLock(false, CONFIG_SET_NAME); + configSetLock.waitUntilAcquired(); + assertEquals( + "testNode", + new String( + zkClient.getData(configSetLock.getLockId(), null, null), StandardCharsets.UTF_8)); + configSetLock.release(); + } + /** Lock dirs left as persistent nodes by older versions get removed, live locks are kept. */ @Test public void testDeleteEmptyLockDirs() throws Exception { ZkDistributedCollectionLockFactory factory = - new ZkDistributedCollectionLockFactory(zkClient, COLLECTION_ROOT); + new ZkDistributedCollectionLockFactory(zkClient, COLLECTION_ROOT, "testNode"); String collLockDir = lockDir( factory.createLock( @@ -196,7 +218,7 @@ public void testDeleteEmptyLockDirs() throws Exception { } private DistributedCollectionLockFactory collectionLockFactory() { - return new ZkDistributedCollectionLockFactory(zkClient, COLLECTION_ROOT); + return new ZkDistributedCollectionLockFactory(zkClient, COLLECTION_ROOT, "testNode"); } private DistributedLock collectionLock( diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/AliasCollectionApiLockTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/AliasCollectionApiLockTest.java index 26123ed86da8..d397cf4cdfb6 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/AliasCollectionApiLockTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/AliasCollectionApiLockTest.java @@ -61,7 +61,8 @@ public void testAliasedCommandWaitsForCollectionLock() throws Exception { DistributedLock held = new ZkDistributedCollectionLockFactory( cluster.getZkClient(), - DistributedCollectionConfigSetCommandRunner.ZK_COLLECTION_LOCKS) + DistributedCollectionConfigSetCommandRunner.ZK_COLLECTION_LOCKS, + "testNode") .createLock(true, LockLevel.COLLECTION, COLLECTION, null, null, null); held.waitUntilAcquired(); try { diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java index 5b150d0d5148..d49177c448e7 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java @@ -42,7 +42,8 @@ public class CollectionApiLockingTest extends ApiLockingTestBase { @Test public void monothreadedApiLockTests() throws Exception { CollectionApiLockFactory apiLockFactory = - new CollectionApiLockFactory(new ZkDistributedCollectionLockFactory(zkClient, LOCK_ROOT)); + new CollectionApiLockFactory( + new ZkDistributedCollectionLockFactory(zkClient, LOCK_ROOT, "testNode")); monothreadedTests(apiLockFactory); multithreadedTests(apiLockFactory); @@ -52,7 +53,7 @@ public void monothreadedApiLockTests() throws Exception { @Test public void testBlockedMultiLockHoldsNoLowerLevelNode() throws Exception { ZkDistributedCollectionLockFactory factory = - new ZkDistributedCollectionLockFactory(zkClient, LOCK_ROOT); + new ZkDistributedCollectionLockFactory(zkClient, LOCK_ROOT, "testNode"); CollectionApiLockFactory apiLockFactory = new CollectionApiLockFactory(factory); DistributedMultiLock collWrite = diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/DeleteCollectionLockCleanupTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/DeleteCollectionLockCleanupTest.java index d000f140dfcc..a8da6abc6706 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/DeleteCollectionLockCleanupTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/DeleteCollectionLockCleanupTest.java @@ -52,7 +52,7 @@ public void testDeleteRemovesLegacyLockDirs() throws Exception { SolrZkClient zkClient = cluster.getZkClient(); ZkDistributedCollectionLockFactory factory = new ZkDistributedCollectionLockFactory( - zkClient, DistributedCollectionConfigSetCommandRunner.ZK_COLLECTION_LOCKS); + zkClient, DistributedCollectionConfigSetCommandRunner.ZK_COLLECTION_LOCKS, "testNode"); String collLockDir = lockDir(factory.createLock(true, LockLevel.COLLECTION, collection, null, null, null)); String shardLockDir = diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-11.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-11.adoc index dc8e3febfeab..dc3c7cdfaa92 100644 --- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-11.adoc +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-11.adoc @@ -27,7 +27,7 @@ For example, if you are currently using Solr 10.1, you should review changes mad === Rolling Upgrades -If the Overseer is disabled (cluster property `overseerEnabled` or `SOLR_CLOUD_OVERSEER_ENABLED` set to `false`), Collection and ConfigSet API locks now use a different ZooKeeper node format. +If the Overseer is disabled (cluster property `overseerEnabled` or `SOLR_CLOUD_OVERSEER_ENABLED` set to `false`), Collection and ConfigSet API locks now use a different ZooKeeper node format than they did in Solr 10 and previous versions. 10.x and 11 nodes don't see each other's locks, so while both versions are running, two conflicting commands on the same collection or configset can run at the same time without an error. If the Overseer is disabled, either: From bdf7df49bcea88339fc9411fc8ed6ebe5c252ed0 Mon Sep 17 00:00:00 2001 From: Amrit Sarkar Date: Wed, 18 Nov 2026 16:19:39 -0800 Subject: [PATCH 9/9] SOLR-17704: Declare test lock map as IdentityHashMap --- .../src/test/org/apache/solr/cloud/ApiLockingTestBase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/ApiLockingTestBase.java b/solr/core/src/test/org/apache/solr/cloud/ApiLockingTestBase.java index 4c97c77bcf6a..a5c6e9ac1a24 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ApiLockingTestBase.java +++ b/solr/core/src/test/org/apache/solr/cloud/ApiLockingTestBase.java @@ -21,7 +21,6 @@ import static org.apache.solr.cloud.ThreadBoundLock.awaitAcquired; import java.util.IdentityHashMap; -import java.util.Map; import java.util.concurrent.TimeUnit; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.cloud.SolrZkClient; @@ -33,7 +32,8 @@ public abstract class ApiLockingTestBase extends SolrTestCaseJ4 { protected static final String LOCK_ROOT = "/apiLockTestRoot"; - private final Map boundLocks = new IdentityHashMap<>(); + private final IdentityHashMap boundLocks = + new IdentityHashMap<>(); private ZkTestServer server; protected SolrZkClient zkClient;