From ba1742861131b8d97bac1023d8b74b87fd3d71ae Mon Sep 17 00:00:00 2001 From: Fabian Kovacs Date: Thu, 6 Aug 2026 17:12:55 +0200 Subject: [PATCH 1/5] Adapt postQuery to allow setting of external query --- .../conquery/apiv1/QueryProcessor.java | 111 +++++++++++++----- .../models/query/ExecutionManager.java | 12 +- .../resources/api/DatasetQueryResource.java | 5 +- 3 files changed, 89 insertions(+), 39 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 485141efea..cc588af14e 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -9,6 +9,13 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; +import jakarta.inject.Inject; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.Validator; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.ExecutionStatus; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; @@ -175,6 +182,51 @@ public Stream getQueriesFiltered( .filter(Objects::nonNull); } + public List getAllQueries(DatasetId dataset, HttpServletRequest req, Subject subject, boolean allProviders) { + try (Stream allQueries = storage.getAllExecutionIds()) { + return getQueriesFiltered(dataset, RequestAwareUriBuilder.fromRequest(req), subject, allQueries, allProviders).toList(); + } + } + + public Stream getQueriesFiltered( + DatasetId datasetId, + UriBuilder uriBuilder, + Subject subject, + Stream allQueries, + boolean allProviders) { + + return allQueries + // Checks that are possible on the execution id alone (no cache load necessary) + .filter(id -> id.getDataset().equals(datasetId)) + .filter(id -> { + ExecutionManager executionManager = datasetRegistry.get(id.getDataset()).getExecutionManager(); + final ExecutionState state = executionManager.getState(id); + return state == ExecutionState.NEW || state == ExecutionState.DONE; + }) + // Resolve + .map(Id::get) + .filter(Objects::nonNull) // Cautionary + // Checks that are only possible on the execution object (cache might need to fetch some data) + .filter(QueryProcessor::canFrontendRender) + // Ownership check only works on the object not on the id :/ + .filter(exec -> subject.isPermitted(exec, Ability.READ)) + .filter(Predicate.not(ManagedExecution::isSystem)) + + .map(mq -> { + try { + final OverviewExecutionStatus status = mq.buildStatusOverview(subject); + + if (mq.isReadyToDownload()) { + status.setResultUrls(getResultAssets(config.getResultProviders(), mq, uriBuilder, allProviders)); + } + return status; + } catch (Exception e) { + log.error("FAILED building status for {}", mq, e); + } + return null; + }) + .filter(Objects::nonNull); + } /** * Cancel a running query: Sending cancellation to shards, which will cause them to stop executing them, results are not sent back, and incoming results will be discarded. */ @@ -312,7 +364,7 @@ public ExternalUploadResult uploadEntities(Subject subject, DatasetId dataset, E execution = ((ManagedQuery) namespace .getExecutionManager() - .createExecution(query, subject.getId(), namespace, false)); + .createExecution(query, subject.getId(), namespace, false, UUID.randomUUID())); if (upload.getLabel() != null) { execution.setLabel(upload.getLabel()); @@ -345,7 +397,7 @@ public FullExecutionStatus getSingleEntityExport( // TODO make sure that subqueries are also system // TODO do not persist system queries - final EntityPreviewExecution execution = (EntityPreviewExecution) postQuery(dataset, form, subject, true); + final EntityPreviewExecution execution = (EntityPreviewExecution) postQuery(dataset, form, subject, true, Optional.empty()); final ExecutionManager executionManager = namespace.getExecutionManager(); if (executionManager.awaitDone(execution.getId(), 10, TimeUnit.SECONDS) == ExecutionState.RUNNING) { @@ -370,7 +422,7 @@ public FullExecutionStatus getSingleEntityExport( * Creates a query for all datasets, then submits it for execution on the * intended dataset. */ - public ManagedExecution postQuery(DatasetId dataset, QueryDescription queryContent, Subject subject, boolean system) { + public ManagedExecution postQuery(DatasetId dataset, QueryDescription queryContent, Subject subject, boolean system, Optional maybeQueryId) { log.info("Query posted on Dataset[{}] by User[{{}].", dataset, subject.getId()); @@ -405,6 +457,10 @@ public ManagedExecution postQuery(DatasetId dataset, QueryDescription queryConte // After all authorization checks we can now use the actual subject to invoke the query and do not to bubble down the Userish in methods + if (maybeQueryId.map(id -> storage.getExecution(new ManagedExecutionId(dataset, id)) != null).orElse(false)) { + throw new WebApplicationException("Query[%s] already exists.".formatted(maybeQueryId.get()), Response.Status.CONFLICT); + } + ExecutionMetrics.reportNamespacedIds(namespacedIdentifiableCollector.getIdentifiables(), primaryGroupName); ExecutionMetrics.reportQueryClassUsage(queryContent.getClass(), primaryGroupName); @@ -412,38 +468,40 @@ public ManagedExecution postQuery(DatasetId dataset, QueryDescription queryConte final Namespace namespace = datasetRegistry.get(dataset); final ExecutionManager executionManager = namespace.getExecutionManager(); + final Optional reusedId = onlyReusingChecker.getOnlyReused(); - // If this is only a re-executing query, try to execute the underlying query instead. - { - final Optional executionId = onlyReusingChecker.getOnlyReused(); + // Never reuse if + if (maybeQueryId.isEmpty() && reusedId.isPresent()) { + // If this is only a re-executing query, try to execute the underlying query instead. + Optional maybeReused = tryReuse(queryContent, reusedId.get(), namespace, executionManager, subject.getUser()); - final Optional - execution = - executionId.map(id -> tryReuse(queryContent, id, namespace, executionManager, subject.getUser())); - - if (execution.isPresent()) { - return execution.get(); + if (maybeReused.isPresent()) { + return maybeReused.get(); } } // Execute the query - return executionManager.runQuery(namespace, queryContent, subject.getId(), system); + final ManagedExecution execution = executionManager.createExecution(queryContent, subject.getId(), namespace, system, maybeQueryId.orElseGet(UUID::randomUUID)); + + executionManager.execute(execution); + + return execution; } /** * Determine if the submitted query does reuse ONLY another query and restart that instead of creating another one. */ - private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId executionId, Namespace namespace, ExecutionManager executionManager, User user) { + private Optional tryReuse(QueryDescription query, ManagedExecutionId executionId, Namespace namespace, ExecutionManager executionManager, User user) { ManagedExecution execution = storage.getExecution(executionId); if (execution == null) { - return null; + return Optional.empty(); } // Direct reuse only works if the queries are of the same type (As reuse reconstructs the Query for different types) if (!query.getClass().equals(execution.getSubmitted().getClass())) { - return null; + return Optional.empty(); } // If SecondaryIds differ from selected and prior, we cannot reuse them. @@ -452,7 +510,7 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId exe final SecondaryIdDescriptionId reusedSecondaryId = ((SecondaryIdQuery) execution.getSubmitted()).getSecondaryId(); if (!selectedSecondaryId.equals(reusedSecondaryId)) { - return null; + return Optional.empty(); } } @@ -460,7 +518,7 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId exe if (!user.isOwner(execution)) { final ManagedExecution newExecution = - executionManager.createExecution(execution.getSubmitted(), user.getId(), namespace, false); + executionManager.createExecution(execution.getSubmitted(), user.getId(), namespace, false, UUID.randomUUID()); newExecution.setLabel(execution.getLabel()); newExecution.setTags(execution.getTags().clone()); storage.updateExecution(newExecution); @@ -468,17 +526,16 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId exe } final ExecutionState state = execution.getState(); - if (state.equals(ExecutionState.RUNNING)) { - log.trace("The Execution[{}] was already started and its state is: {}", execution.getId(), state); - return execution; - } - - log.trace("Re-executing Query {}", execution); - executionManager.execute(execution); + if (!state.equals(ExecutionState.RUNNING)) { + log.trace("Re-executing Query {}", execution); - return execution; + executionManager.execute(execution); + } else { + log.trace("The Execution[{}] was already started and its state is: {}", execution.getId(), state); + } + return Optional.of(execution); } /** @@ -511,7 +568,7 @@ public Stream> resolveEntities(Subject subject, List getAllQueries(@Auth Subject subject, @Que } @POST - public Response postQuery(@Auth Subject subject, @QueryParam("all-providers") Optional allProviders, @NotNull @Valid QueryDescription query) { + public Response postQuery(@Auth Subject subject, @QueryParam("all-providers") Optional allProviders, @QueryParam("queryId") Optional queryId, @NotNull @Valid QueryDescription query) { subject.authorize(dataset, Ability.READ); - final ManagedExecution execution = processor.postQuery(dataset, query, subject, false); + final ManagedExecution execution = processor.postQuery(dataset, query, subject, false, queryId); return Response.ok(processor.getQueryFullStatus(execution.getId(), subject, From 14bb80d6d945d87ffde0e2fe8e4685fd77b5ac55 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs Date: Mon, 10 Aug 2026 17:38:45 +0200 Subject: [PATCH 2/5] adds capability to just upload a query instead of running it via PUT instead of POST --- .../conquery/apiv1/QueryProcessor.java | 187 ++++++------------ .../resources/api/DatasetQueryResource.java | 60 ++++-- .../com/bakdata/conquery/util/QueryUtils.java | 38 +++- .../integration/tests/ReusedQueryTest.java | 1 - .../tests/endpoints/apiEndpointInfo.json | 5 + 5 files changed, 138 insertions(+), 153 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index cc588af14e..eb1a9de5ec 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -34,7 +34,6 @@ import com.bakdata.conquery.models.auth.AuthorizationHelper; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Subject; -import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; import com.bakdata.conquery.models.common.Range; @@ -47,7 +46,10 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.identifiable.ids.Id; -import com.bakdata.conquery.models.identifiable.ids.specific.*; +import com.bakdata.conquery.models.identifiable.ids.specific.ConnectorId; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; +import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.*; import com.bakdata.conquery.models.query.preview.EntityPreviewExecution; @@ -64,12 +66,6 @@ import com.bakdata.conquery.util.QueryUtils; import com.bakdata.conquery.util.QueryUtils.NamespacedIdentifiableCollector; import com.bakdata.conquery.util.io.IdColumnUtil; -import jakarta.inject.Inject; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.validation.Validator; -import jakarta.ws.rs.BadRequestException; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.UriBuilder; import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -136,50 +132,29 @@ public static List getResultAssets( } - public List getAllQueries(DatasetId dataset, HttpServletRequest req, Subject subject, boolean allProviders) { - try (Stream allQueries = storage.getAllExecutionIds()) { - return getQueriesFiltered(dataset, RequestAwareUriBuilder.fromRequest(req), subject, allQueries, allProviders).toList(); - } - } + private static List visitQuery(QueryDescription queryContent, ExecutionManager executionManager, String primaryGroupName) { + final List visitors = new ArrayList<>(); - public Stream getQueriesFiltered( - DatasetId datasetId, - UriBuilder uriBuilder, - Subject subject, - Stream allQueries, - boolean allProviders) { + // This maps works as long as we have query visitors that are not configured in anyway. + // So adding a visitor twice would replace the previous one but both would have yielded the same result. + // For the future a better data structure might be desired that also regards similar QueryVisitors of different configuration + queryContent.addVisitors(visitors); - return allQueries - // Checks that are possible on the execution id alone (no cache load necessary) - .filter(id -> id.getDataset().equals(datasetId)) - .filter(id -> { - ExecutionManager executionManager = datasetRegistry.get(id.getDataset()).getExecutionManager(); - final ExecutionState state = executionManager.getState(id); - return state == ExecutionState.NEW || state == ExecutionState.DONE; - }) - // Resolve - .map(Id::get) - .filter(Objects::nonNull) // Cautionary - // Checks that are only possible on the execution object (cache might need to fetch some data) - .filter(QueryProcessor::canFrontendRender) - // Ownership check only works on the object not on the id :/ - .filter(exec -> subject.isPermitted(exec, Ability.READ)) - .filter(Predicate.not(ManagedExecution::isSystem)) + // Initialize checks that need to traverse the query tree + visitors.add(new QueryUtils.OnlyReusingChecker(executionManager)); + visitors.add(new NamespacedIdentifiableCollector()); + visitors.add(new ExecutionMetrics.QueryMetricsReporter(primaryGroupName)); - .map(mq -> { - try { - final OverviewExecutionStatus status = mq.buildStatusOverview(subject); - if (mq.isReadyToDownload()) { - status.setResultUrls(getResultAssets(config.getResultProviders(), mq, uriBuilder, allProviders)); - } - return status; - } catch (Exception e) { - log.error("FAILED building status for {}", mq, e); - } - return null; - }) - .filter(Objects::nonNull); + // Chain all Consumers + Consumer consumerChain = QueryUtils.getNoOpEntryPoint(); + for (QueryVisitor visitor : visitors) { + consumerChain = consumerChain.andThen(visitor); + } + + // Apply consumers to the query tree + queryContent.visit(consumerChain); + return visitors; } public List getAllQueries(DatasetId dataset, HttpServletRequest req, Subject subject, boolean allProviders) { @@ -227,6 +202,7 @@ public Stream getQueriesFiltered( }) .filter(Objects::nonNull); } + /** * Cancel a running query: Sending cancellation to shards, which will cause them to stop executing them, results are not sent back, and incoming results will be discarded. */ @@ -315,7 +291,6 @@ public void deleteQuery(Subject subject, ManagedExecutionId executionId) { storage.removeExecution(executionId); } - public FullExecutionStatus getQueryFullStatus(ManagedExecutionId queryId, Subject subject, UriBuilder url, Boolean allProviders, boolean await) { final Namespace namespace = datasetRegistry.get(queryId.getDataset()); @@ -397,7 +372,8 @@ public FullExecutionStatus getSingleEntityExport( // TODO make sure that subqueries are also system // TODO do not persist system queries - final EntityPreviewExecution execution = (EntityPreviewExecution) postQuery(dataset, form, subject, true, Optional.empty()); + final EntityPreviewExecution execution = (EntityPreviewExecution) createQuery(dataset, form, subject, true, Optional.empty()); + runExecution(execution); final ExecutionManager executionManager = namespace.getExecutionManager(); if (executionManager.awaitDone(execution.getId(), 10, TimeUnit.SECONDS) == ExecutionState.RUNNING) { @@ -422,42 +398,24 @@ public FullExecutionStatus getSingleEntityExport( * Creates a query for all datasets, then submits it for execution on the * intended dataset. */ - public ManagedExecution postQuery(DatasetId dataset, QueryDescription queryContent, Subject subject, boolean system, Optional maybeQueryId) { + public ManagedExecution createQuery(DatasetId dataset, QueryDescription queryContent, Subject subject, boolean system, Optional maybeQueryId) { log.info("Query posted on Dataset[{}] by User[{{}].", dataset, subject.getId()); - // This maps works as long as we have query visitors that are not configured in anyway. - // So adding a visitor twice would replace the previous one but both would have yielded the same result. - // For the future a better data structure might be desired that also regards similar QueryVisitors of different configuration - final List visitors = new ArrayList<>(); - queryContent.addVisitors(visitors); - // Initialize checks that need to traverse the query tree - final QueryUtils.OnlyReusingChecker onlyReusingChecker = new QueryUtils.OnlyReusingChecker(); - visitors.add(onlyReusingChecker); - final NamespacedIdentifiableCollector namespacedIdentifiableCollector = new NamespacedIdentifiableCollector(); - visitors.add(namespacedIdentifiableCollector); + final Namespace namespace = datasetRegistry.get(dataset); + final ExecutionManager executionManager = namespace.getExecutionManager(); final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(subject, storage).map(Group::getName).orElse("none"); - final ExecutionMetrics.QueryMetricsReporter queryMetricsReporter = new ExecutionMetrics.QueryMetricsReporter(primaryGroupName); - visitors.add(queryMetricsReporter); - - - // Chain all Consumers - Consumer consumerChain = QueryUtils.getNoOpEntryPoint(); - for (QueryVisitor visitor : visitors) { - consumerChain = consumerChain.andThen(visitor); - } - - // Apply consumers to the query tree - queryContent.visit(consumerChain); + final List visitors = visitQuery(queryContent, executionManager, primaryGroupName); + final NamespacedIdentifiableCollector namespacedIdentifiableCollector = QueryUtils.getVisitor(visitors, NamespacedIdentifiableCollector.class); + final QueryUtils.OnlyReusingChecker onlyReusingChecker = QueryUtils.getVisitor(visitors, QueryUtils.OnlyReusingChecker.class); queryContent.authorize(subject, dataset, visitors, storage); - // After all authorization checks we can now use the actual subject to invoke the query and do not to bubble down the Userish in methods - + // After all authorization checks we can now use the actual subject to invoke the query and do not to bubble down the User-ish in methods - if (maybeQueryId.map(id -> storage.getExecution(new ManagedExecutionId(dataset, id)) != null).orElse(false)) { + if (maybeQueryId.isPresent() && storage.getExecution(new ManagedExecutionId(dataset, maybeQueryId.get())) != null) { throw new WebApplicationException("Query[%s] already exists.".formatted(maybeQueryId.get()), Response.Status.CONFLICT); } @@ -465,77 +423,49 @@ public ManagedExecution postQuery(DatasetId dataset, QueryDescription queryConte ExecutionMetrics.reportQueryClassUsage(queryContent.getClass(), primaryGroupName); - final Namespace namespace = datasetRegistry.get(dataset); - final ExecutionManager executionManager = namespace.getExecutionManager(); - - final Optional reusedId = onlyReusingChecker.getOnlyReused(); - - // Never reuse if - if (maybeQueryId.isEmpty() && reusedId.isPresent()) { - // If this is only a re-executing query, try to execute the underlying query instead. - Optional maybeReused = tryReuse(queryContent, reusedId.get(), namespace, executionManager, subject.getUser()); - - if (maybeReused.isPresent()) { - return maybeReused.get(); - } - } - // Execute the query - final ManagedExecution execution = executionManager.createExecution(queryContent, subject.getId(), namespace, system, maybeQueryId.orElseGet(UUID::randomUUID)); + final UUID queryId = maybeQueryId.orElseGet(UUID::randomUUID); - executionManager.execute(execution); + final Optional maybeReused = onlyReusingChecker.getOnlyReused(queryContent); - return execution; - } - /** - * Determine if the submitted query does reuse ONLY another query and restart that instead of creating another one. - */ - private Optional tryReuse(QueryDescription query, ManagedExecutionId executionId, Namespace namespace, ExecutionManager executionManager, User user) { - - ManagedExecution execution = storage.getExecution(executionId); - - if (execution == null) { - return Optional.empty(); + // Never reuse if a query-Id is provided + // If this is only a re-executing query, try to execute the underlying query instead. + if (maybeQueryId.isPresent() || maybeReused.isEmpty()) { + return executionManager.createExecution(queryContent, subject.getId(), namespace, system, queryId); } - // Direct reuse only works if the queries are of the same type (As reuse reconstructs the Query for different types) - if (!query.getClass().equals(execution.getSubmitted().getClass())) { - return Optional.empty(); - } - - // If SecondaryIds differ from selected and prior, we cannot reuse them. - if (query instanceof SecondaryIdQuery secondaryIdQuery) { - final SecondaryIdDescriptionId selectedSecondaryId = secondaryIdQuery.getSecondaryId(); - final SecondaryIdDescriptionId reusedSecondaryId = ((SecondaryIdQuery) execution.getSubmitted()).getSecondaryId(); - - if (!selectedSecondaryId.equals(reusedSecondaryId)) { - return Optional.empty(); - } - } + ManagedExecution execution = maybeReused.get(); // If the user is not the owner of the execution, we definitely create a new Execution, so the owner can cancel it - if (!user.isOwner(execution)) { + if (!subject.isOwner(execution)) { final ManagedExecution newExecution = - executionManager.createExecution(execution.getSubmitted(), user.getId(), namespace, false, UUID.randomUUID()); + executionManager.createExecution(execution.getSubmitted(), subject.getId(), namespace, false, queryId); + newExecution.setLabel(execution.getLabel()); newExecution.setTags(execution.getTags().clone()); + storage.updateExecution(newExecution); - execution = newExecution; + return newExecution; } final ExecutionState state = execution.getState(); if (!state.equals(ExecutionState.RUNNING)) { - log.trace("Re-executing Query {}", execution); - - executionManager.execute(execution); - } else { - log.trace("The Execution[{}] was already started and its state is: {}", execution.getId(), state); + log.trace("Re-executing Query {}", execution.getId()); } - return Optional.of(execution); + return execution; + + } + + public void runExecution(ManagedExecution execution) { + if (execution.getState().equals(ExecutionState.RUNNING)) { + log.trace("The Execution[{}] is already RUNNING.", execution.getId()); + return; + } + execution.getNamespace().getExecutionManager().execute(execution); } /** @@ -568,7 +498,8 @@ public Stream> resolveEntities(Subject subject, List> resolveEntities(@Auth Subject subject, @Valid } - @POST @Path("/upload") public ExternalUploadResult upload(@Auth Subject subject, @Valid ExternalUpload upload) { @@ -95,20 +88,51 @@ public List getAllQueries(@Auth Subject subject, @Que return processor.getAllQueries(dataset, servletRequest, subject, allProviders.orElse(false)); } + /*** + * Create and run the submitted query + */ @POST public Response postQuery(@Auth Subject subject, @QueryParam("all-providers") Optional allProviders, @QueryParam("queryId") Optional queryId, @NotNull @Valid QueryDescription query) { + if (queryId.isPresent() && queryId.get().version() != 4) { + throw new BadRequestException("Invalid query id: UUID version 4 required."); + } subject.authorize(dataset, Ability.READ); - final ManagedExecution execution = processor.postQuery(dataset, query, subject, false, queryId); + final ManagedExecution execution = processor.createQuery(dataset, query, subject, false, queryId); + processor.runExecution(execution); return Response.ok(processor.getQueryFullStatus(execution.getId(), - subject, - RequestAwareUriBuilder.fromRequest(servletRequest), - allProviders.orElse(false), - false - )) - .status(Response.Status.CREATED) - .build(); + subject, + RequestAwareUriBuilder.fromRequest(servletRequest), + allProviders.orElse(false), + false + )) + .status(Response.Status.CREATED) + .build(); } + + /** + * Only create the query. + */ + @PUT + public Response putQuery(@Auth Subject subject, @QueryParam("all-providers") Optional allProviders, @QueryParam("queryId") Optional queryId, @NotNull @Valid QueryDescription query) { + if (queryId.isPresent() && queryId.get().version() != 4) { + throw new BadRequestException("Invalid query id: UUID version 4 required."); + } + + subject.authorize(dataset, Ability.READ); + + final ManagedExecution execution = processor.createQuery(dataset, query, subject, false, queryId); + + return Response.ok(processor.getQueryFullStatus(execution.getId(), + subject, + RequestAwareUriBuilder.fromRequest(servletRequest), + allProviders.orElse(false), + false + )) + .status(Response.Status.CREATED) + .build(); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java b/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java index 7198fcc577..d725db1759 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java +++ b/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java @@ -17,6 +17,7 @@ import c10n.C10N; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; @@ -26,19 +27,18 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.identifiable.NamespacedIdentifiable; import com.bakdata.conquery.models.identifiable.ids.NamespacedId; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; -import com.bakdata.conquery.models.query.NamespacedIdentifiableHolding; -import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.QueryExecutionContext; -import com.bakdata.conquery.models.query.Visitable; +import com.bakdata.conquery.models.query.*; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.visitor.QueryVisitor; import com.google.common.base.Strings; import com.google.common.collect.MoreCollectors; +import lombok.Data; import lombok.Getter; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; @@ -184,11 +184,15 @@ public boolean resolvesExternalIds() { /** * Test if this query is only reusing a different query (ie not combining it with other elements, or changing its secondaryId) */ + @Data public static class OnlyReusingChecker implements QueryVisitor { + private final ExecutionManager storage; + private CQReusedQuery reusedQuery = null; private boolean containsOthersElements = false; + @Override public void accept(Visitable element) { if(containsOthersElements){ @@ -213,12 +217,34 @@ public void accept(Visitable element) { } } - public Optional getOnlyReused() { + public Optional getOnlyReused(QueryDescription query) { if (containsOthersElements || reusedQuery == null) { return Optional.empty(); } - return Optional.of(reusedQuery.getQueryId()); + ManagedExecutionId executionId = getReusedQuery().getQueryId(); + ManagedQuery execution = (ManagedQuery) storage.getExecution(executionId); + + if (execution == null) { + return Optional.empty(); + } + + // Direct reuse only works if the queries are of the same type (As reuse reconstructs the Query for different types) + if (!query.getClass().equals(execution.getSubmitted().getClass())) { + return Optional.empty(); + } + + // If SecondaryIds differ from selected and prior, we cannot reuse them. + if (query instanceof SecondaryIdQuery secondaryIdQuery) { + final SecondaryIdDescriptionId selectedSecondaryId = secondaryIdQuery.getSecondaryId(); + final SecondaryIdDescriptionId reusedSecondaryId = ((SecondaryIdQuery) execution.getSubmitted()).getSecondaryId(); + + if (!selectedSecondaryId.equals(reusedSecondaryId)) { + return Optional.empty(); + } + } + + return Optional.of(execution); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java index c5d5d7dcd9..d40662f612 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java @@ -211,7 +211,6 @@ public void execute(String name, TestConquery testConquery) throws Exception { { // Reuse by another user (create a copy of the actual query) - final SecondaryIdQuery reused = new SecondaryIdQuery(); reused.setRoot(new CQReusedQuery(execution.getId())); diff --git a/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json b/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json index fb3684bacc..ad8af6a90f 100644 --- a/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json +++ b/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json @@ -14,6 +14,11 @@ "path": "/datasets/{dataset}/queries", "clazz": "DatasetQueryResource" }, + { + "method": "PUT", + "path": "/datasets/{dataset}/queries", + "clazz": "DatasetQueryResource" + }, { "method": "GET", "path": "/datasets/{dataset}/entity-preview", From 4be40b19c166d1f57f17bff610d9c282461a1279 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs Date: Wed, 26 Aug 2026 17:27:43 +0200 Subject: [PATCH 3/5] Adds validation class for queryId=UUIDv4 --- .../conquery/apiv1/QueryProcessor.java | 13 ++++++----- .../resources/api/DatasetQueryResource.java | 17 ++++---------- .../util/validation/UUID4Validator.java | 13 +++++++++++ .../conquery/util/validation/ValidUUID4.java | 23 +++++++++++++++++++ 4 files changed, 48 insertions(+), 18 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/util/validation/UUID4Validator.java create mode 100644 backend/src/main/java/com/bakdata/conquery/util/validation/ValidUUID4.java diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index eb1a9de5ec..a4016b0298 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -372,7 +372,7 @@ public FullExecutionStatus getSingleEntityExport( // TODO make sure that subqueries are also system // TODO do not persist system queries - final EntityPreviewExecution execution = (EntityPreviewExecution) createQuery(dataset, form, subject, true, Optional.empty()); + final EntityPreviewExecution execution = (EntityPreviewExecution) createExecution(dataset, form, subject, true, Optional.empty()); runExecution(execution); final ExecutionManager executionManager = namespace.getExecutionManager(); @@ -398,7 +398,7 @@ public FullExecutionStatus getSingleEntityExport( * Creates a query for all datasets, then submits it for execution on the * intended dataset. */ - public ManagedExecution createQuery(DatasetId dataset, QueryDescription queryContent, Subject subject, boolean system, Optional maybeQueryId) { + public ManagedExecution createExecution(DatasetId dataset, QueryDescription queryContent, Subject subject, boolean system, Optional maybeQueryId) { log.info("Query posted on Dataset[{}] by User[{{}].", dataset, subject.getId()); @@ -413,9 +413,10 @@ public ManagedExecution createQuery(DatasetId dataset, QueryDescription queryCon final QueryUtils.OnlyReusingChecker onlyReusingChecker = QueryUtils.getVisitor(visitors, QueryUtils.OnlyReusingChecker.class); queryContent.authorize(subject, dataset, visitors, storage); - // After all authorization checks we can now use the actual subject to invoke the query and do not to bubble down the User-ish in methods - if (maybeQueryId.isPresent() && storage.getExecution(new ManagedExecutionId(dataset, maybeQueryId.get())) != null) { + // After all authorization checks we can now use the actual subject to invoke the query and do not to bubble down the User-ish in method + + if (maybeQueryId.filter(id -> storage.getExecution(new ManagedExecutionId(dataset, id)) != null).isPresent()) { throw new WebApplicationException("Query[%s] already exists.".formatted(maybeQueryId.get()), Response.Status.CONFLICT); } @@ -453,7 +454,7 @@ public ManagedExecution createQuery(DatasetId dataset, QueryDescription queryCon final ExecutionState state = execution.getState(); if (!state.equals(ExecutionState.RUNNING)) { - log.trace("Re-executing Query {}", execution.getId()); + log.trace("Reusing Query {}", execution.getId()); } return execution; @@ -498,7 +499,7 @@ public Stream> resolveEntities(Subject subject, List getAllQueries(@Auth Subject subject, @Que * Create and run the submitted query */ @POST - public Response postQuery(@Auth Subject subject, @QueryParam("all-providers") Optional allProviders, @QueryParam("queryId") Optional queryId, @NotNull @Valid QueryDescription query) { - if (queryId.isPresent() && queryId.get().version() != 4) { - throw new BadRequestException("Invalid query id: UUID version 4 required."); - } - + public Response postQuery(@Auth Subject subject, @QueryParam("all-providers") Optional allProviders, @QueryParam("queryId") Optional<@ValidUUID4 UUID> queryId, @NotNull @Valid QueryDescription query) { subject.authorize(dataset, Ability.READ); - final ManagedExecution execution = processor.createQuery(dataset, query, subject, false, queryId); + final ManagedExecution execution = processor.createExecution(dataset, query, subject, false, queryId); processor.runExecution(execution); return Response.ok(processor.getQueryFullStatus(execution.getId(), @@ -116,14 +113,10 @@ public Response postQuery(@Auth Subject subject, @QueryParam("all-providers") Op * Only create the query. */ @PUT - public Response putQuery(@Auth Subject subject, @QueryParam("all-providers") Optional allProviders, @QueryParam("queryId") Optional queryId, @NotNull @Valid QueryDescription query) { - if (queryId.isPresent() && queryId.get().version() != 4) { - throw new BadRequestException("Invalid query id: UUID version 4 required."); - } - + public Response putQuery(@Auth Subject subject, @QueryParam("all-providers") Optional allProviders, @QueryParam("queryId") Optional<@ValidUUID4 UUID> queryId, @NotNull @Valid QueryDescription query) { subject.authorize(dataset, Ability.READ); - final ManagedExecution execution = processor.createQuery(dataset, query, subject, false, queryId); + final ManagedExecution execution = processor.createExecution(dataset, query, subject, false, queryId); return Response.ok(processor.getQueryFullStatus(execution.getId(), subject, diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/UUID4Validator.java b/backend/src/main/java/com/bakdata/conquery/util/validation/UUID4Validator.java new file mode 100644 index 0000000000..ff6a18d3b5 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/util/validation/UUID4Validator.java @@ -0,0 +1,13 @@ +package com.bakdata.conquery.util.validation; + +import java.util.UUID; +import jakarta.validation.ConstraintValidator; +import jakarta.validation.ConstraintValidatorContext; + +public class UUID4Validator implements ConstraintValidator { + + @Override + public boolean isValid(UUID id, ConstraintValidatorContext context) { + return id == null || id.version() != 4; + } +} \ No newline at end of file diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/ValidUUID4.java b/backend/src/main/java/com/bakdata/conquery/util/validation/ValidUUID4.java new file mode 100644 index 0000000000..ddb88c2b0d --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/util/validation/ValidUUID4.java @@ -0,0 +1,23 @@ +package com.bakdata.conquery.util.validation; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import static java.lang.annotation.ElementType.*; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Target({ANNOTATION_TYPE, FIELD, TYPE_USE}) +@Retention(RUNTIME) +@Constraint(validatedBy = UUID4Validator.class) +@Documented +public @interface ValidUUID4 { + String message() default "UUID is not version 4"; + + Class[] groups() default {}; + + @SuppressWarnings("UnusedDeclaration") Class[] payload() default {}; +} \ No newline at end of file From 205f20f9ce1a08e6ec934e9331eefe4a7c31879e Mon Sep 17 00:00:00 2001 From: Fabian Kovacs Date: Thu, 27 Aug 2026 13:08:15 +0200 Subject: [PATCH 4/5] cleanup --- .../conquery/apiv1/QueryProcessor.java | 2 +- .../com/bakdata/conquery/util/QueryUtils.java | 48 ++++++++----------- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index a4016b0298..890aa5c2b2 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -427,7 +427,7 @@ public ManagedExecution createExecution(DatasetId dataset, QueryDescription quer final UUID queryId = maybeQueryId.orElseGet(UUID::randomUUID); - final Optional maybeReused = onlyReusingChecker.getOnlyReused(queryContent); + final Optional maybeReused = onlyReusingChecker.getOnlyReused(queryContent); // Never reuse if a query-Id is provided diff --git a/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java b/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java index d725db1759..8fe11bac0b 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java +++ b/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java @@ -1,13 +1,6 @@ package com.bakdata.conquery.util; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Predicate; @@ -27,7 +20,6 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.identifiable.NamespacedIdentifiable; import com.bakdata.conquery.models.identifiable.ids.NamespacedId; @@ -48,11 +40,13 @@ public class QueryUtils { private static final int MAX_CONCEPT_LABEL_CONCAT_LENGTH = 70; + /** * Provides a starting operator for consumer chains, that does nothing. */ public static Consumer getNoOpEntryPoint() { - return (whatever) -> {}; + return (whatever) -> { + }; } /** @@ -66,8 +60,9 @@ public static T getVisitor(List visitors, * Create label for children preferring the children's user label if provided. */ public static String createUserMultiLabel(List elements, String delimiter, String postfix, Locale locale) { - return elements.stream().map(elt -> elt.userLabel(locale)).collect(Collectors.joining(delimiter,"", postfix)); + return elements.stream().map(elt -> elt.userLabel(locale)).collect(Collectors.joining(delimiter, "", postfix)); } + /** * Create label for children the children's default label ignoring user provided label. */ @@ -87,7 +82,7 @@ public static String makeQueryLabel(final Visitable query, PrintSettings cfg, Ma final Map, List> sortedContents = Visitable.stream(query) - .collect(Collectors.groupingBy(Visitable::getClass)); + .collect(Collectors.groupingBy(Visitable::getClass)); int sbStartSize = sb.length(); @@ -118,15 +113,15 @@ public static String makeQueryLabel(final Visitable query, PrintSettings cfg, Ma final AtomicInteger length = new AtomicInteger(); sortedContents.get(CQConcept.class) - .stream() - .map(CQConcept.class::cast) + .stream() + .map(CQConcept.class::cast) - .map(c -> makeLabelWithRootAndChild(c, cfg)) - .filter(Predicate.not(Strings::isNullOrEmpty)) - .distinct() + .map(c -> makeLabelWithRootAndChild(c, cfg)) + .filter(Predicate.not(Strings::isNullOrEmpty)) + .distinct() - .takeWhile(elem -> length.addAndGet(elem.length()) < MAX_CONCEPT_LABEL_CONCAT_LENGTH) - .forEach(label -> sb.append(label).append(" ")); + .takeWhile(elem -> length.addAndGet(elem.length()) < MAX_CONCEPT_LABEL_CONCAT_LENGTH) + .forEach(label -> sb.append(label).append(" ")); // Last entry will output one Space that we don't want if (!sb.isEmpty()) { @@ -158,7 +153,7 @@ private static String makeLabelWithRootAndChild(CQConcept cqConcept, PrintSettin // Concat everything with dashes return label.replace(" ", "-"); } - + /** * Checks if the query requires to resolve external ids. */ @@ -195,7 +190,7 @@ public static class OnlyReusingChecker implements QueryVisitor { @Override public void accept(Visitable element) { - if(containsOthersElements){ + if (containsOthersElements) { return; } @@ -205,8 +200,7 @@ public void accept(Visitable element) { if (reusedQuery == null) { reusedQuery = (CQReusedQuery) element; - } - else { + } else { containsOthersElements = true; } return; @@ -217,7 +211,7 @@ public void accept(Visitable element) { } } - public Optional getOnlyReused(QueryDescription query) { + public Optional getOnlyReused(QueryDescription query) { if (containsOthersElements || reusedQuery == null) { return Optional.empty(); } @@ -276,16 +270,16 @@ public static class AvailableSecondaryIdCollector implements QueryVisitor { @Override public void accept(Visitable element) { - if(element instanceof final CQConcept cqConcept){ + if (element instanceof final CQConcept cqConcept) { // Excluded Concepts are not available - if(cqConcept.isExcludeFromSecondaryId()){ + if (cqConcept.isExcludeFromSecondaryId()) { return; } for (Connector connector : cqConcept.getConcept().getConnectors()) { for (Column column : connector.getResolvedTable().getColumns()) { - if(column.getSecondaryId() == null){ + if (column.getSecondaryId() == null) { continue; } From bdeebb75e3574acab155a772efad869c96c86165 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs Date: Tue, 1 Sep 2026 15:44:42 +0200 Subject: [PATCH 5/5] Adds test for valid UUID and no reuse if queryId is present --- .../util/validation/UUID4Validator.java | 2 +- .../integration/common/IntegrationUtils.java | 14 +++++-- .../json/AbstractQueryEngineTest.java | 2 +- .../conquery/integration/json/FormTest.java | 2 +- .../tests/ConceptPermissionTest.java | 4 +- .../tests/ExternalFormBackendTest.java | 2 +- .../integration/tests/ImportUpdateTest.java | 4 +- .../tests/QueryStatisticsTest.java | 2 +- .../integration/tests/ReusedQueryTest.java | 38 ++++++++++++++----- .../ConceptUpdateAndDeletionTest.java | 10 ++--- .../tests/deletion/DatasetDeletionTest.java | 8 ++-- .../tests/deletion/ImportDeletionTest.java | 8 ++-- .../tests/deletion/TableDeletionTest.java | 8 ++-- 13 files changed, 66 insertions(+), 38 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/UUID4Validator.java b/backend/src/main/java/com/bakdata/conquery/util/validation/UUID4Validator.java index ff6a18d3b5..e522f65226 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/validation/UUID4Validator.java +++ b/backend/src/main/java/com/bakdata/conquery/util/validation/UUID4Validator.java @@ -8,6 +8,6 @@ public class UUID4Validator implements ConstraintValidator { @Override public boolean isValid(UUID id, ConstraintValidatorContext context) { - return id == null || id.version() != 4; + return id == null || id.version() == 4; } } \ No newline at end of file diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java index 84a4ae5832..d1eb2f6845 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java @@ -6,10 +6,12 @@ import java.net.URI; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.stream.Stream; import javax.annotation.Nullable; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; +import jakarta.ws.rs.client.WebTarget; import jakarta.ws.rs.core.GenericType; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; @@ -49,12 +51,18 @@ public static Query parseQuery(StandaloneSupport support, JsonNode rawQuery) thr /** * Send a query onto the conquery instance and assert the result's size. */ - public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, Object query, long expectedSize, ExecutionState expectedState, @Nullable User user, int expectedResponseCode) { + public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, Object query, UUID uuid, long expectedSize, ExecutionState expectedState, @Nullable User user, int expectedResponseCode) { final URI postQueryURI = getPostQueryURI(conquery); // Submit Query - Invocation.Builder request = conquery.getClient() - .target(postQueryURI) + WebTarget webTarget = conquery.getClient() + .target(postQueryURI); + + if (uuid != null) { + webTarget = webTarget.queryParam("queryId", uuid); + } + + Invocation.Builder request = webTarget .request(MediaType.APPLICATION_JSON_TYPE); if (user != null) { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index a1a53fda87..65f762afd8 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -46,7 +46,7 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException final User testUser = standaloneSupport.getTestUser(); - final ManagedExecutionId executionId = IntegrationUtils.assertQueryResult(standaloneSupport, query, -1, ExecutionState.DONE, testUser, 201); + final ManagedExecutionId executionId = IntegrationUtils.assertQueryResult(standaloneSupport, query, null, -1, ExecutionState.DONE, testUser, 201); final ManagedExecution execution = standaloneSupport.getMetaStorage().getExecution(executionId); execution.initExecutable(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java index 8c2a592dd9..bc0ef4afaf 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java @@ -79,7 +79,7 @@ public boolean isWithConcepts() { public void executeTest(StandaloneSupport support) throws Exception { - final ManagedExecutionId managedExecutionId = IntegrationUtils.assertQueryResult(support, form, -1, ExecutionState.DONE, support.getTestUser(), 201); + final ManagedExecutionId managedExecutionId = IntegrationUtils.assertQueryResult(support, form, null, -1, ExecutionState.DONE, support.getTestUser(), 201); log.info("{} QUERIES EXECUTED", getLabel()); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java index f213b70ccb..5be7ea5744 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java @@ -64,7 +64,7 @@ public void execute(StandaloneSupport conquery) throws Exception { // The lone concept that is used in the test. Concept concept = allConcepts.iterator().next(); - IntegrationUtils.assertQueryResult(conquery, query, -1, ExecutionState.FAILED, user, 403); + IntegrationUtils.assertQueryResult(conquery, query, null, -1, ExecutionState.FAILED, user, 403); // Add the necessary Permission { @@ -75,7 +75,7 @@ public void execute(StandaloneSupport conquery) throws Exception { } // Only assert permissions - IntegrationUtils.assertQueryResult(conquery, query, -1, ExecutionState.DONE, user, 201); + IntegrationUtils.assertQueryResult(conquery, query, null, -1, ExecutionState.DONE, user, 201); conquery.waitUntilWorkDone(); // Clean up diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java index 5ddfbe603e..d35bbf2053 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java @@ -102,7 +102,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { managedExecutionId = IntegrationUtils.assertQueryResult(support, String.format("{\"type\": \"%s\", \"testProp\": \"testVal\"}", externalFormId), - -1, + null, -1, ExecutionState.DONE, testUser, 201 diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ImportUpdateTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ImportUpdateTest.java index e168ed6586..0ffa5a164f 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ImportUpdateTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ImportUpdateTest.java @@ -116,7 +116,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { assertThat(namespace.getNumberOfEntities()).isEqualTo(4); //assert that the query can be executed after the import - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } @@ -207,7 +207,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { } assertThat(namespace.getNumberOfEntities()).isEqualTo(9); // Issue a query and assert that it has more content. - IntegrationUtils.assertQueryResult(conquery, query, 4L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 4L, ExecutionState.DONE, conquery.getTestUser(), 201); } } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/QueryStatisticsTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/QueryStatisticsTest.java index 4e88141ebd..03f88a4b14 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/QueryStatisticsTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/QueryStatisticsTest.java @@ -51,7 +51,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { conquery.waitUntilWorkDone(); final ManagedExecutionId executionId = - IntegrationUtils.assertQueryResult(conquery, test.getQuery(), 6, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, test.getQuery(), null, 6, ExecutionState.DONE, conquery.getTestUser(), 201); final UriBuilder uriBuilder = HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), QueryResource.class, "getDescription"); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java index d40662f612..c4546f59b1 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.MediaType; @@ -84,7 +85,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final long expectedSize = 3L; - final ManagedExecutionId id = IntegrationUtils.assertQueryResult(conquery, query, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); + final ManagedExecutionId id = IntegrationUtils.assertQueryResult(conquery, query, null, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); assertThat(id).isNotNull(); @@ -96,7 +97,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final ConceptQuery reused = new ConceptQuery(new CQReusedQuery(execution.getId())); - IntegrationUtils.assertQueryResult(conquery, reused, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, reused, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } // Reuse by API @@ -124,7 +125,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { reused.setSecondaryId(query.getSecondaryId()); - IntegrationUtils.assertQueryResult(conquery, reused, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, reused, null, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); } // Reuse in SecondaryId, but do exclude @@ -160,7 +161,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { reused.setSecondaryId(query.getSecondaryId()); - IntegrationUtils.assertQueryResult(conquery, reused, 1L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, reused, null, 1L, ExecutionState.DONE, conquery.getTestUser(), 201); } // Reuse Multiple times with different query types @@ -170,7 +171,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { reused1.setSecondaryId(query.getSecondaryId()); - final ManagedExecutionId reused1Id = IntegrationUtils.assertQueryResult(conquery, reused1, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); + final ManagedExecutionId reused1Id = IntegrationUtils.assertQueryResult(conquery, reused1, null, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); final ManagedQuery execution1 = (ManagedQuery) metaStorage.getExecution(reused1Id); { final SecondaryIdQuery reused2 = new SecondaryIdQuery(); @@ -180,7 +181,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { final ManagedExecutionId reused2Id = - IntegrationUtils.assertQueryResult(conquery, reused2, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, reused2, null, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); + final ManagedQuery execution2 = (ManagedQuery) metaStorage.getExecution(reused2Id); assertThat(reused2Id) @@ -190,7 +192,25 @@ public void execute(String name, TestConquery testConquery) throws Exception { // Now we change to ConceptQuery final ConceptQuery reused3 = new ConceptQuery(new CQReusedQuery(execution2.getId())); - IntegrationUtils.assertQueryResult(conquery, reused3, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, reused3, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + } + + { + // Would reuse, but id was explicitly provided, so it's new ID. + final SecondaryIdQuery reused2 = new SecondaryIdQuery(); + reused2.setRoot(new CQReusedQuery(execution1.getId())); + + reused2.setSecondaryId(query.getSecondaryId()); + + UUID uuid = UUID.randomUUID(); + + final ManagedExecutionId + reused2Id = + IntegrationUtils.assertQueryResult(conquery, reused2, uuid, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); + + assertThat(reused2Id) + .as("Query should be reused.") + .isEqualTo(new ManagedExecutionId(conquery.getDataset(), uuid)); } { @@ -202,7 +222,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final ManagedExecutionId executionId = - IntegrationUtils.assertQueryResult(conquery, reusedDiffId, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, reusedDiffId, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); assertThat(executionId) .as("Query should NOT be reused.") @@ -224,7 +244,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { execution.createPermission(Set.of(Ability.READ)) )); - ManagedExecutionId copyId = IntegrationUtils.assertQueryResult(conquery, reused, expectedSize, ExecutionState.DONE, shareHolder, 201); + ManagedExecutionId copyId = IntegrationUtils.assertQueryResult(conquery, reused, null, expectedSize, ExecutionState.DONE, shareHolder, 201); ManagedExecution copy = metaStorage.getExecution(copyId); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java index 2edf7b948c..660951a8f9 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java @@ -104,7 +104,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { } } log.info("Executing query before update"); - IntegrationUtils.assertQueryResult(conquery, query, 1L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 1L, ExecutionState.DONE, conquery.getTestUser(), 201); conquery.waitUntilWorkDone(); log.info("Query before update executed"); } @@ -152,7 +152,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after update"); // Assert that it now contains 2 instead of 1. - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); conquery.waitUntilWorkDone(); log.info("Query after update executed"); } @@ -200,7 +200,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after restart."); // Re-assert state. - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); conquery.waitUntilWorkDone(); } } @@ -247,7 +247,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after deletion (EXPECTING AN EXCEPTION IN THE LOGS!)"); // Issue a query and assert that it is failing. - IntegrationUtils.assertQueryResult(conquery, query, 0L, ExecutionState.FAILED, conquery.getTestUser(), 404); + IntegrationUtils.assertQueryResult(conquery, query, null, 0L, ExecutionState.FAILED, conquery.getTestUser(), 404); } @@ -288,7 +288,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after restart (EXPECTING AN EXCEPTION IN THE LOGS!)"); // Issue a query and assert that it is failing. - IntegrationUtils.assertQueryResult(conquery, query, 0L, ExecutionState.FAILED, conquery.getTestUser(), 404); + IntegrationUtils.assertQueryResult(conquery, query, null, 0L, ExecutionState.FAILED, conquery.getTestUser(), 404); } } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java index 52e77e2a6a..d110e9a665 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java @@ -94,7 +94,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query before deletion"); - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } // Delete Dataset. @@ -143,7 +143,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { // Try to execute the query after deletion - IntegrationUtils.assertQueryResult(conquery, query, 0, ExecutionState.FAILED, conquery.getTestUser(), 404); + IntegrationUtils.assertQueryResult(conquery, query, null, 0, ExecutionState.FAILED, conquery.getTestUser(), 404); } } @@ -184,7 +184,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final Query query2 = IntegrationUtils.parseQuery(conqueryReimport, test.getRawQuery()); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conqueryReimport, query2, 2L, ExecutionState.DONE, conqueryReimport.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conqueryReimport, query2, null, 2L, ExecutionState.DONE, conqueryReimport.getTestUser(), 201); } @@ -220,7 +220,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final Query query3 = IntegrationUtils.parseQuery(conqueryRestart, test.getRawQuery()); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conqueryRestart, query3, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conqueryRestart, query3, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java index dc4448a7c1..8196315aea 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java @@ -132,7 +132,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query before deletion"); - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } // Delete the import. @@ -199,7 +199,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after deletion"); // Issue a query and assert that it has less content. - IntegrationUtils.assertQueryResult(conquery, query, 1L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 1L, ExecutionState.DONE, conquery.getTestUser(), 201); } conquery.waitUntilWorkDone(); @@ -278,7 +278,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after re-import"); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } // Finally, restart conquery and assert again, that the data is correct. @@ -317,7 +317,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after re-import"); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conquery2, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery2, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java index bf70a1f8fa..f9c8f4e7a0 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java @@ -107,7 +107,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query before deletion"); - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } // Delete the import. @@ -195,7 +195,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after deletion. Expecting a failure here."); - IntegrationUtils.assertQueryResult(conquery, query, 0L, ExecutionState.FAILED, conquery.getTestUser(), 404); + IntegrationUtils.assertQueryResult(conquery, query, null, 0L, ExecutionState.FAILED, conquery.getTestUser(), 404); } conquery.waitUntilWorkDone(); @@ -260,7 +260,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after re-import"); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } // Finally, restart conquery and assert again, that the data is correct. @@ -300,7 +300,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after re-import and restart"); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conquery2, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery2, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } } }