diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 485141efea..890aa5c2b2 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -9,6 +9,13 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; +import jakarta.inject.Inject; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.Validator; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.ExecutionStatus; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; @@ -27,7 +34,6 @@ import com.bakdata.conquery.models.auth.AuthorizationHelper; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Subject; -import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; import com.bakdata.conquery.models.common.Range; @@ -40,7 +46,10 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.identifiable.ids.Id; -import com.bakdata.conquery.models.identifiable.ids.specific.*; +import com.bakdata.conquery.models.identifiable.ids.specific.ConnectorId; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; +import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.*; import com.bakdata.conquery.models.query.preview.EntityPreviewExecution; @@ -57,12 +66,6 @@ import com.bakdata.conquery.util.QueryUtils; import com.bakdata.conquery.util.QueryUtils.NamespacedIdentifiableCollector; import com.bakdata.conquery.util.io.IdColumnUtil; -import jakarta.inject.Inject; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.validation.Validator; -import jakarta.ws.rs.BadRequestException; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.UriBuilder; import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -129,6 +132,31 @@ public static List getResultAssets( } + private static List visitQuery(QueryDescription queryContent, ExecutionManager executionManager, String primaryGroupName) { + final List visitors = new ArrayList<>(); + + // This maps works as long as we have query visitors that are not configured in anyway. + // So adding a visitor twice would replace the previous one but both would have yielded the same result. + // For the future a better data structure might be desired that also regards similar QueryVisitors of different configuration + queryContent.addVisitors(visitors); + + // Initialize checks that need to traverse the query tree + visitors.add(new QueryUtils.OnlyReusingChecker(executionManager)); + visitors.add(new NamespacedIdentifiableCollector()); + visitors.add(new ExecutionMetrics.QueryMetricsReporter(primaryGroupName)); + + + // Chain all Consumers + Consumer consumerChain = QueryUtils.getNoOpEntryPoint(); + for (QueryVisitor visitor : visitors) { + consumerChain = consumerChain.andThen(visitor); + } + + // Apply consumers to the query tree + queryContent.visit(consumerChain); + return visitors; + } + public List getAllQueries(DatasetId dataset, HttpServletRequest req, Subject subject, boolean allProviders) { try (Stream allQueries = storage.getAllExecutionIds()) { return getQueriesFiltered(dataset, RequestAwareUriBuilder.fromRequest(req), subject, allQueries, allProviders).toList(); @@ -263,7 +291,6 @@ public void deleteQuery(Subject subject, ManagedExecutionId executionId) { storage.removeExecution(executionId); } - public FullExecutionStatus getQueryFullStatus(ManagedExecutionId queryId, Subject subject, UriBuilder url, Boolean allProviders, boolean await) { final Namespace namespace = datasetRegistry.get(queryId.getDataset()); @@ -312,7 +339,7 @@ public ExternalUploadResult uploadEntities(Subject subject, DatasetId dataset, E execution = ((ManagedQuery) namespace .getExecutionManager() - .createExecution(query, subject.getId(), namespace, false)); + .createExecution(query, subject.getId(), namespace, false, UUID.randomUUID())); if (upload.getLabel() != null) { execution.setLabel(upload.getLabel()); @@ -345,7 +372,8 @@ public FullExecutionStatus getSingleEntityExport( // TODO make sure that subqueries are also system // TODO do not persist system queries - final EntityPreviewExecution execution = (EntityPreviewExecution) postQuery(dataset, form, subject, true); + final EntityPreviewExecution execution = (EntityPreviewExecution) createExecution(dataset, form, subject, true, Optional.empty()); + runExecution(execution); final ExecutionManager executionManager = namespace.getExecutionManager(); if (executionManager.awaitDone(execution.getId(), 10, TimeUnit.SECONDS) == ExecutionState.RUNNING) { @@ -370,117 +398,77 @@ public FullExecutionStatus getSingleEntityExport( * Creates a query for all datasets, then submits it for execution on the * intended dataset. */ - public ManagedExecution postQuery(DatasetId dataset, QueryDescription queryContent, Subject subject, boolean system) { + public ManagedExecution createExecution(DatasetId dataset, QueryDescription queryContent, Subject subject, boolean system, Optional maybeQueryId) { log.info("Query posted on Dataset[{}] by User[{{}].", dataset, subject.getId()); - // This maps works as long as we have query visitors that are not configured in anyway. - // So adding a visitor twice would replace the previous one but both would have yielded the same result. - // For the future a better data structure might be desired that also regards similar QueryVisitors of different configuration - final List visitors = new ArrayList<>(); - queryContent.addVisitors(visitors); - // Initialize checks that need to traverse the query tree - final QueryUtils.OnlyReusingChecker onlyReusingChecker = new QueryUtils.OnlyReusingChecker(); - visitors.add(onlyReusingChecker); - final NamespacedIdentifiableCollector namespacedIdentifiableCollector = new NamespacedIdentifiableCollector(); - visitors.add(namespacedIdentifiableCollector); + final Namespace namespace = datasetRegistry.get(dataset); + final ExecutionManager executionManager = namespace.getExecutionManager(); final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(subject, storage).map(Group::getName).orElse("none"); - final ExecutionMetrics.QueryMetricsReporter queryMetricsReporter = new ExecutionMetrics.QueryMetricsReporter(primaryGroupName); - visitors.add(queryMetricsReporter); - - - // Chain all Consumers - Consumer consumerChain = QueryUtils.getNoOpEntryPoint(); - for (QueryVisitor visitor : visitors) { - consumerChain = consumerChain.andThen(visitor); - } - - // Apply consumers to the query tree - queryContent.visit(consumerChain); + final List visitors = visitQuery(queryContent, executionManager, primaryGroupName); + final NamespacedIdentifiableCollector namespacedIdentifiableCollector = QueryUtils.getVisitor(visitors, NamespacedIdentifiableCollector.class); + final QueryUtils.OnlyReusingChecker onlyReusingChecker = QueryUtils.getVisitor(visitors, QueryUtils.OnlyReusingChecker.class); queryContent.authorize(subject, dataset, visitors, storage); - // After all authorization checks we can now use the actual subject to invoke the query and do not to bubble down the Userish in methods + // After all authorization checks we can now use the actual subject to invoke the query and do not to bubble down the User-ish in method + + if (maybeQueryId.filter(id -> storage.getExecution(new ManagedExecutionId(dataset, id)) != null).isPresent()) { + throw new WebApplicationException("Query[%s] already exists.".formatted(maybeQueryId.get()), Response.Status.CONFLICT); + } ExecutionMetrics.reportNamespacedIds(namespacedIdentifiableCollector.getIdentifiables(), primaryGroupName); ExecutionMetrics.reportQueryClassUsage(queryContent.getClass(), primaryGroupName); - final Namespace namespace = datasetRegistry.get(dataset); - final ExecutionManager executionManager = namespace.getExecutionManager(); - - - // If this is only a re-executing query, try to execute the underlying query instead. - { - final Optional executionId = onlyReusingChecker.getOnlyReused(); - - final Optional - execution = - executionId.map(id -> tryReuse(queryContent, id, namespace, executionManager, subject.getUser())); - - if (execution.isPresent()) { - return execution.get(); - } - } - // Execute the query - return executionManager.runQuery(namespace, queryContent, subject.getId(), system); - } - - /** - * Determine if the submitted query does reuse ONLY another query and restart that instead of creating another one. - */ - private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId executionId, Namespace namespace, ExecutionManager executionManager, User user) { + final UUID queryId = maybeQueryId.orElseGet(UUID::randomUUID); - ManagedExecution execution = storage.getExecution(executionId); + final Optional maybeReused = onlyReusingChecker.getOnlyReused(queryContent); - if (execution == null) { - return null; - } - // Direct reuse only works if the queries are of the same type (As reuse reconstructs the Query for different types) - if (!query.getClass().equals(execution.getSubmitted().getClass())) { - return null; + // Never reuse if a query-Id is provided + // If this is only a re-executing query, try to execute the underlying query instead. + if (maybeQueryId.isPresent() || maybeReused.isEmpty()) { + return executionManager.createExecution(queryContent, subject.getId(), namespace, system, queryId); } - // If SecondaryIds differ from selected and prior, we cannot reuse them. - if (query instanceof SecondaryIdQuery secondaryIdQuery) { - final SecondaryIdDescriptionId selectedSecondaryId = secondaryIdQuery.getSecondaryId(); - final SecondaryIdDescriptionId reusedSecondaryId = ((SecondaryIdQuery) execution.getSubmitted()).getSecondaryId(); - - if (!selectedSecondaryId.equals(reusedSecondaryId)) { - return null; - } - } + ManagedExecution execution = maybeReused.get(); // If the user is not the owner of the execution, we definitely create a new Execution, so the owner can cancel it - if (!user.isOwner(execution)) { + if (!subject.isOwner(execution)) { final ManagedExecution newExecution = - executionManager.createExecution(execution.getSubmitted(), user.getId(), namespace, false); + executionManager.createExecution(execution.getSubmitted(), subject.getId(), namespace, false, queryId); + newExecution.setLabel(execution.getLabel()); newExecution.setTags(execution.getTags().clone()); + storage.updateExecution(newExecution); - execution = newExecution; + return newExecution; } final ExecutionState state = execution.getState(); - if (state.equals(ExecutionState.RUNNING)) { - log.trace("The Execution[{}] was already started and its state is: {}", execution.getId(), state); - return execution; - } - - log.trace("Re-executing Query {}", execution); - executionManager.execute(execution); + if (!state.equals(ExecutionState.RUNNING)) { + log.trace("Reusing Query {}", execution.getId()); + } return execution; } + public void runExecution(ManagedExecution execution) { + if (execution.getState().equals(ExecutionState.RUNNING)) { + log.trace("The Execution[{}] is already RUNNING.", execution.getId()); + return; + } + execution.getNamespace().getExecutionManager().execute(execution); + } + /** * Execute a basic query on a single concept and return only the included entities Id's. */ @@ -511,7 +499,8 @@ public Stream> resolveEntities(Subject subject, List> resolveEntities(@Auth Subject subject, @Valid } - @POST @Path("/upload") public ExternalUploadResult upload(@Auth Subject subject, @Valid ExternalUpload upload) { @@ -94,20 +89,43 @@ public List getAllQueries(@Auth Subject subject, @Que return processor.getAllQueries(dataset, servletRequest, subject, allProviders.orElse(false)); } + /*** + * Create and run the submitted query + */ @POST - public Response postQuery(@Auth Subject subject, @QueryParam("all-providers") Optional allProviders, @NotNull @Valid QueryDescription query) { + public Response postQuery(@Auth Subject subject, @QueryParam("all-providers") Optional allProviders, @QueryParam("queryId") Optional<@ValidUUID4 UUID> queryId, @NotNull @Valid QueryDescription query) { + subject.authorize(dataset, Ability.READ); + final ManagedExecution execution = processor.createExecution(dataset, query, subject, false, queryId); + processor.runExecution(execution); + + return Response.ok(processor.getQueryFullStatus(execution.getId(), + subject, + RequestAwareUriBuilder.fromRequest(servletRequest), + allProviders.orElse(false), + false + )) + .status(Response.Status.CREATED) + .build(); + } + + /** + * Only create the query. + */ + @PUT + public Response putQuery(@Auth Subject subject, @QueryParam("all-providers") Optional allProviders, @QueryParam("queryId") Optional<@ValidUUID4 UUID> queryId, @NotNull @Valid QueryDescription query) { subject.authorize(dataset, Ability.READ); - final ManagedExecution execution = processor.postQuery(dataset, query, subject, false); + final ManagedExecution execution = processor.createExecution(dataset, query, subject, false, queryId); return Response.ok(processor.getQueryFullStatus(execution.getId(), - subject, - RequestAwareUriBuilder.fromRequest(servletRequest), - allProviders.orElse(false), - false - )) - .status(Response.Status.CREATED) - .build(); + subject, + RequestAwareUriBuilder.fromRequest(servletRequest), + allProviders.orElse(false), + false + )) + .status(Response.Status.CREATED) + .build(); } + } diff --git a/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java b/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java index 7198fcc577..8fe11bac0b 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java +++ b/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java @@ -1,13 +1,6 @@ package com.bakdata.conquery.util; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Predicate; @@ -17,6 +10,7 @@ import c10n.C10N; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; @@ -31,14 +25,12 @@ import com.bakdata.conquery.models.identifiable.ids.NamespacedId; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; -import com.bakdata.conquery.models.query.NamespacedIdentifiableHolding; -import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.QueryExecutionContext; -import com.bakdata.conquery.models.query.Visitable; +import com.bakdata.conquery.models.query.*; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.visitor.QueryVisitor; import com.google.common.base.Strings; import com.google.common.collect.MoreCollectors; +import lombok.Data; import lombok.Getter; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; @@ -48,11 +40,13 @@ public class QueryUtils { private static final int MAX_CONCEPT_LABEL_CONCAT_LENGTH = 70; + /** * Provides a starting operator for consumer chains, that does nothing. */ public static Consumer getNoOpEntryPoint() { - return (whatever) -> {}; + return (whatever) -> { + }; } /** @@ -66,8 +60,9 @@ public static T getVisitor(List visitors, * Create label for children preferring the children's user label if provided. */ public static String createUserMultiLabel(List elements, String delimiter, String postfix, Locale locale) { - return elements.stream().map(elt -> elt.userLabel(locale)).collect(Collectors.joining(delimiter,"", postfix)); + return elements.stream().map(elt -> elt.userLabel(locale)).collect(Collectors.joining(delimiter, "", postfix)); } + /** * Create label for children the children's default label ignoring user provided label. */ @@ -87,7 +82,7 @@ public static String makeQueryLabel(final Visitable query, PrintSettings cfg, Ma final Map, List> sortedContents = Visitable.stream(query) - .collect(Collectors.groupingBy(Visitable::getClass)); + .collect(Collectors.groupingBy(Visitable::getClass)); int sbStartSize = sb.length(); @@ -118,15 +113,15 @@ public static String makeQueryLabel(final Visitable query, PrintSettings cfg, Ma final AtomicInteger length = new AtomicInteger(); sortedContents.get(CQConcept.class) - .stream() - .map(CQConcept.class::cast) + .stream() + .map(CQConcept.class::cast) - .map(c -> makeLabelWithRootAndChild(c, cfg)) - .filter(Predicate.not(Strings::isNullOrEmpty)) - .distinct() + .map(c -> makeLabelWithRootAndChild(c, cfg)) + .filter(Predicate.not(Strings::isNullOrEmpty)) + .distinct() - .takeWhile(elem -> length.addAndGet(elem.length()) < MAX_CONCEPT_LABEL_CONCAT_LENGTH) - .forEach(label -> sb.append(label).append(" ")); + .takeWhile(elem -> length.addAndGet(elem.length()) < MAX_CONCEPT_LABEL_CONCAT_LENGTH) + .forEach(label -> sb.append(label).append(" ")); // Last entry will output one Space that we don't want if (!sb.isEmpty()) { @@ -158,7 +153,7 @@ private static String makeLabelWithRootAndChild(CQConcept cqConcept, PrintSettin // Concat everything with dashes return label.replace(" ", "-"); } - + /** * Checks if the query requires to resolve external ids. */ @@ -184,14 +179,18 @@ public boolean resolvesExternalIds() { /** * Test if this query is only reusing a different query (ie not combining it with other elements, or changing its secondaryId) */ + @Data public static class OnlyReusingChecker implements QueryVisitor { + private final ExecutionManager storage; + private CQReusedQuery reusedQuery = null; private boolean containsOthersElements = false; + @Override public void accept(Visitable element) { - if(containsOthersElements){ + if (containsOthersElements) { return; } @@ -201,8 +200,7 @@ public void accept(Visitable element) { if (reusedQuery == null) { reusedQuery = (CQReusedQuery) element; - } - else { + } else { containsOthersElements = true; } return; @@ -213,12 +211,34 @@ public void accept(Visitable element) { } } - public Optional getOnlyReused() { + public Optional getOnlyReused(QueryDescription query) { if (containsOthersElements || reusedQuery == null) { return Optional.empty(); } - return Optional.of(reusedQuery.getQueryId()); + ManagedExecutionId executionId = getReusedQuery().getQueryId(); + ManagedQuery execution = (ManagedQuery) storage.getExecution(executionId); + + if (execution == null) { + return Optional.empty(); + } + + // Direct reuse only works if the queries are of the same type (As reuse reconstructs the Query for different types) + if (!query.getClass().equals(execution.getSubmitted().getClass())) { + return Optional.empty(); + } + + // If SecondaryIds differ from selected and prior, we cannot reuse them. + if (query instanceof SecondaryIdQuery secondaryIdQuery) { + final SecondaryIdDescriptionId selectedSecondaryId = secondaryIdQuery.getSecondaryId(); + final SecondaryIdDescriptionId reusedSecondaryId = ((SecondaryIdQuery) execution.getSubmitted()).getSecondaryId(); + + if (!selectedSecondaryId.equals(reusedSecondaryId)) { + return Optional.empty(); + } + } + + return Optional.of(execution); } } @@ -250,16 +270,16 @@ public static class AvailableSecondaryIdCollector implements QueryVisitor { @Override public void accept(Visitable element) { - if(element instanceof final CQConcept cqConcept){ + if (element instanceof final CQConcept cqConcept) { // Excluded Concepts are not available - if(cqConcept.isExcludeFromSecondaryId()){ + if (cqConcept.isExcludeFromSecondaryId()) { return; } for (Connector connector : cqConcept.getConcept().getConnectors()) { for (Column column : connector.getResolvedTable().getColumns()) { - if(column.getSecondaryId() == null){ + if (column.getSecondaryId() == null) { continue; } diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/UUID4Validator.java b/backend/src/main/java/com/bakdata/conquery/util/validation/UUID4Validator.java new file mode 100644 index 0000000000..e522f65226 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/util/validation/UUID4Validator.java @@ -0,0 +1,13 @@ +package com.bakdata.conquery.util.validation; + +import java.util.UUID; +import jakarta.validation.ConstraintValidator; +import jakarta.validation.ConstraintValidatorContext; + +public class UUID4Validator implements ConstraintValidator { + + @Override + public boolean isValid(UUID id, ConstraintValidatorContext context) { + return id == null || id.version() == 4; + } +} \ No newline at end of file diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/ValidUUID4.java b/backend/src/main/java/com/bakdata/conquery/util/validation/ValidUUID4.java new file mode 100644 index 0000000000..ddb88c2b0d --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/util/validation/ValidUUID4.java @@ -0,0 +1,23 @@ +package com.bakdata.conquery.util.validation; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import static java.lang.annotation.ElementType.*; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Target({ANNOTATION_TYPE, FIELD, TYPE_USE}) +@Retention(RUNTIME) +@Constraint(validatedBy = UUID4Validator.class) +@Documented +public @interface ValidUUID4 { + String message() default "UUID is not version 4"; + + Class[] groups() default {}; + + @SuppressWarnings("UnusedDeclaration") Class[] payload() default {}; +} \ No newline at end of file diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java index 84a4ae5832..d1eb2f6845 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java @@ -6,10 +6,12 @@ import java.net.URI; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.stream.Stream; import javax.annotation.Nullable; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; +import jakarta.ws.rs.client.WebTarget; import jakarta.ws.rs.core.GenericType; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; @@ -49,12 +51,18 @@ public static Query parseQuery(StandaloneSupport support, JsonNode rawQuery) thr /** * Send a query onto the conquery instance and assert the result's size. */ - public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, Object query, long expectedSize, ExecutionState expectedState, @Nullable User user, int expectedResponseCode) { + public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, Object query, UUID uuid, long expectedSize, ExecutionState expectedState, @Nullable User user, int expectedResponseCode) { final URI postQueryURI = getPostQueryURI(conquery); // Submit Query - Invocation.Builder request = conquery.getClient() - .target(postQueryURI) + WebTarget webTarget = conquery.getClient() + .target(postQueryURI); + + if (uuid != null) { + webTarget = webTarget.queryParam("queryId", uuid); + } + + Invocation.Builder request = webTarget .request(MediaType.APPLICATION_JSON_TYPE); if (user != null) { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index a1a53fda87..65f762afd8 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -46,7 +46,7 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException final User testUser = standaloneSupport.getTestUser(); - final ManagedExecutionId executionId = IntegrationUtils.assertQueryResult(standaloneSupport, query, -1, ExecutionState.DONE, testUser, 201); + final ManagedExecutionId executionId = IntegrationUtils.assertQueryResult(standaloneSupport, query, null, -1, ExecutionState.DONE, testUser, 201); final ManagedExecution execution = standaloneSupport.getMetaStorage().getExecution(executionId); execution.initExecutable(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java index 8c2a592dd9..bc0ef4afaf 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java @@ -79,7 +79,7 @@ public boolean isWithConcepts() { public void executeTest(StandaloneSupport support) throws Exception { - final ManagedExecutionId managedExecutionId = IntegrationUtils.assertQueryResult(support, form, -1, ExecutionState.DONE, support.getTestUser(), 201); + final ManagedExecutionId managedExecutionId = IntegrationUtils.assertQueryResult(support, form, null, -1, ExecutionState.DONE, support.getTestUser(), 201); log.info("{} QUERIES EXECUTED", getLabel()); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java index f213b70ccb..5be7ea5744 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java @@ -64,7 +64,7 @@ public void execute(StandaloneSupport conquery) throws Exception { // The lone concept that is used in the test. Concept concept = allConcepts.iterator().next(); - IntegrationUtils.assertQueryResult(conquery, query, -1, ExecutionState.FAILED, user, 403); + IntegrationUtils.assertQueryResult(conquery, query, null, -1, ExecutionState.FAILED, user, 403); // Add the necessary Permission { @@ -75,7 +75,7 @@ public void execute(StandaloneSupport conquery) throws Exception { } // Only assert permissions - IntegrationUtils.assertQueryResult(conquery, query, -1, ExecutionState.DONE, user, 201); + IntegrationUtils.assertQueryResult(conquery, query, null, -1, ExecutionState.DONE, user, 201); conquery.waitUntilWorkDone(); // Clean up diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java index 5ddfbe603e..d35bbf2053 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java @@ -102,7 +102,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { managedExecutionId = IntegrationUtils.assertQueryResult(support, String.format("{\"type\": \"%s\", \"testProp\": \"testVal\"}", externalFormId), - -1, + null, -1, ExecutionState.DONE, testUser, 201 diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ImportUpdateTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ImportUpdateTest.java index e168ed6586..0ffa5a164f 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ImportUpdateTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ImportUpdateTest.java @@ -116,7 +116,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { assertThat(namespace.getNumberOfEntities()).isEqualTo(4); //assert that the query can be executed after the import - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } @@ -207,7 +207,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { } assertThat(namespace.getNumberOfEntities()).isEqualTo(9); // Issue a query and assert that it has more content. - IntegrationUtils.assertQueryResult(conquery, query, 4L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 4L, ExecutionState.DONE, conquery.getTestUser(), 201); } } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/QueryStatisticsTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/QueryStatisticsTest.java index 4e88141ebd..03f88a4b14 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/QueryStatisticsTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/QueryStatisticsTest.java @@ -51,7 +51,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { conquery.waitUntilWorkDone(); final ManagedExecutionId executionId = - IntegrationUtils.assertQueryResult(conquery, test.getQuery(), 6, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, test.getQuery(), null, 6, ExecutionState.DONE, conquery.getTestUser(), 201); final UriBuilder uriBuilder = HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), QueryResource.class, "getDescription"); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java index c5d5d7dcd9..c4546f59b1 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.MediaType; @@ -84,7 +85,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final long expectedSize = 3L; - final ManagedExecutionId id = IntegrationUtils.assertQueryResult(conquery, query, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); + final ManagedExecutionId id = IntegrationUtils.assertQueryResult(conquery, query, null, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); assertThat(id).isNotNull(); @@ -96,7 +97,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final ConceptQuery reused = new ConceptQuery(new CQReusedQuery(execution.getId())); - IntegrationUtils.assertQueryResult(conquery, reused, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, reused, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } // Reuse by API @@ -124,7 +125,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { reused.setSecondaryId(query.getSecondaryId()); - IntegrationUtils.assertQueryResult(conquery, reused, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, reused, null, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); } // Reuse in SecondaryId, but do exclude @@ -160,7 +161,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { reused.setSecondaryId(query.getSecondaryId()); - IntegrationUtils.assertQueryResult(conquery, reused, 1L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, reused, null, 1L, ExecutionState.DONE, conquery.getTestUser(), 201); } // Reuse Multiple times with different query types @@ -170,7 +171,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { reused1.setSecondaryId(query.getSecondaryId()); - final ManagedExecutionId reused1Id = IntegrationUtils.assertQueryResult(conquery, reused1, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); + final ManagedExecutionId reused1Id = IntegrationUtils.assertQueryResult(conquery, reused1, null, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); final ManagedQuery execution1 = (ManagedQuery) metaStorage.getExecution(reused1Id); { final SecondaryIdQuery reused2 = new SecondaryIdQuery(); @@ -180,7 +181,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { final ManagedExecutionId reused2Id = - IntegrationUtils.assertQueryResult(conquery, reused2, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, reused2, null, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); + final ManagedQuery execution2 = (ManagedQuery) metaStorage.getExecution(reused2Id); assertThat(reused2Id) @@ -190,7 +192,25 @@ public void execute(String name, TestConquery testConquery) throws Exception { // Now we change to ConceptQuery final ConceptQuery reused3 = new ConceptQuery(new CQReusedQuery(execution2.getId())); - IntegrationUtils.assertQueryResult(conquery, reused3, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, reused3, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + } + + { + // Would reuse, but id was explicitly provided, so it's new ID. + final SecondaryIdQuery reused2 = new SecondaryIdQuery(); + reused2.setRoot(new CQReusedQuery(execution1.getId())); + + reused2.setSecondaryId(query.getSecondaryId()); + + UUID uuid = UUID.randomUUID(); + + final ManagedExecutionId + reused2Id = + IntegrationUtils.assertQueryResult(conquery, reused2, uuid, expectedSize, ExecutionState.DONE, conquery.getTestUser(), 201); + + assertThat(reused2Id) + .as("Query should be reused.") + .isEqualTo(new ManagedExecutionId(conquery.getDataset(), uuid)); } { @@ -202,7 +222,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final ManagedExecutionId executionId = - IntegrationUtils.assertQueryResult(conquery, reusedDiffId, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, reusedDiffId, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); assertThat(executionId) .as("Query should NOT be reused.") @@ -211,7 +231,6 @@ public void execute(String name, TestConquery testConquery) throws Exception { { // Reuse by another user (create a copy of the actual query) - final SecondaryIdQuery reused = new SecondaryIdQuery(); reused.setRoot(new CQReusedQuery(execution.getId())); @@ -225,7 +244,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { execution.createPermission(Set.of(Ability.READ)) )); - ManagedExecutionId copyId = IntegrationUtils.assertQueryResult(conquery, reused, expectedSize, ExecutionState.DONE, shareHolder, 201); + ManagedExecutionId copyId = IntegrationUtils.assertQueryResult(conquery, reused, null, expectedSize, ExecutionState.DONE, shareHolder, 201); ManagedExecution copy = metaStorage.getExecution(copyId); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java index 2edf7b948c..660951a8f9 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java @@ -104,7 +104,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { } } log.info("Executing query before update"); - IntegrationUtils.assertQueryResult(conquery, query, 1L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 1L, ExecutionState.DONE, conquery.getTestUser(), 201); conquery.waitUntilWorkDone(); log.info("Query before update executed"); } @@ -152,7 +152,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after update"); // Assert that it now contains 2 instead of 1. - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); conquery.waitUntilWorkDone(); log.info("Query after update executed"); } @@ -200,7 +200,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after restart."); // Re-assert state. - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); conquery.waitUntilWorkDone(); } } @@ -247,7 +247,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after deletion (EXPECTING AN EXCEPTION IN THE LOGS!)"); // Issue a query and assert that it is failing. - IntegrationUtils.assertQueryResult(conquery, query, 0L, ExecutionState.FAILED, conquery.getTestUser(), 404); + IntegrationUtils.assertQueryResult(conquery, query, null, 0L, ExecutionState.FAILED, conquery.getTestUser(), 404); } @@ -288,7 +288,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after restart (EXPECTING AN EXCEPTION IN THE LOGS!)"); // Issue a query and assert that it is failing. - IntegrationUtils.assertQueryResult(conquery, query, 0L, ExecutionState.FAILED, conquery.getTestUser(), 404); + IntegrationUtils.assertQueryResult(conquery, query, null, 0L, ExecutionState.FAILED, conquery.getTestUser(), 404); } } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java index 52e77e2a6a..d110e9a665 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java @@ -94,7 +94,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query before deletion"); - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } // Delete Dataset. @@ -143,7 +143,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { // Try to execute the query after deletion - IntegrationUtils.assertQueryResult(conquery, query, 0, ExecutionState.FAILED, conquery.getTestUser(), 404); + IntegrationUtils.assertQueryResult(conquery, query, null, 0, ExecutionState.FAILED, conquery.getTestUser(), 404); } } @@ -184,7 +184,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final Query query2 = IntegrationUtils.parseQuery(conqueryReimport, test.getRawQuery()); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conqueryReimport, query2, 2L, ExecutionState.DONE, conqueryReimport.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conqueryReimport, query2, null, 2L, ExecutionState.DONE, conqueryReimport.getTestUser(), 201); } @@ -220,7 +220,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final Query query3 = IntegrationUtils.parseQuery(conqueryRestart, test.getRawQuery()); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conqueryRestart, query3, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conqueryRestart, query3, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java index dc4448a7c1..8196315aea 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java @@ -132,7 +132,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query before deletion"); - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } // Delete the import. @@ -199,7 +199,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after deletion"); // Issue a query and assert that it has less content. - IntegrationUtils.assertQueryResult(conquery, query, 1L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 1L, ExecutionState.DONE, conquery.getTestUser(), 201); } conquery.waitUntilWorkDone(); @@ -278,7 +278,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after re-import"); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } // Finally, restart conquery and assert again, that the data is correct. @@ -317,7 +317,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after re-import"); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conquery2, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery2, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java index bf70a1f8fa..f9c8f4e7a0 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java @@ -107,7 +107,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query before deletion"); - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } // Delete the import. @@ -195,7 +195,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after deletion. Expecting a failure here."); - IntegrationUtils.assertQueryResult(conquery, query, 0L, ExecutionState.FAILED, conquery.getTestUser(), 404); + IntegrationUtils.assertQueryResult(conquery, query, null, 0L, ExecutionState.FAILED, conquery.getTestUser(), 404); } conquery.waitUntilWorkDone(); @@ -260,7 +260,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after re-import"); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } // Finally, restart conquery and assert again, that the data is correct. @@ -300,7 +300,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after re-import and restart"); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conquery2, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult(conquery2, query, null, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); } } } diff --git a/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json b/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json index fb3684bacc..ad8af6a90f 100644 --- a/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json +++ b/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json @@ -14,6 +14,11 @@ "path": "/datasets/{dataset}/queries", "clazz": "DatasetQueryResource" }, + { + "method": "PUT", + "path": "/datasets/{dataset}/queries", + "clazz": "DatasetQueryResource" + }, { "method": "GET", "path": "/datasets/{dataset}/entity-preview",